From 4c51dacfd97c677233cb79dbde08f52aa258b3bb Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Tue, 12 Jun 2018 13:33:58 -0700 Subject: [PATCH 001/148] groundwork for firestore --- .../cloud/firestore_v1beta1/__init__.py | 2 + .../cloud/firestore_v1beta1/collection.py | 6 ++ .../cloud/firestore_v1beta1/document.py | 5 ++ .../google/cloud/firestore_v1beta1/query.py | 13 +++++ .../google/cloud/firestore_v1beta1/watch.py | 58 +++++++++++++++++++ 5 files changed, 84 insertions(+) create mode 100644 firestore/google/cloud/firestore_v1beta1/watch.py diff --git a/firestore/google/cloud/firestore_v1beta1/__init__.py b/firestore/google/cloud/firestore_v1beta1/__init__.py index 1ae905bfdee1..d3bd90405f12 100644 --- a/firestore/google/cloud/firestore_v1beta1/__init__.py +++ b/firestore/google/cloud/firestore_v1beta1/__init__.py @@ -34,6 +34,7 @@ from google.cloud.firestore_v1beta1.query import Query from google.cloud.firestore_v1beta1.transaction import Transaction from google.cloud.firestore_v1beta1.transaction import transactional +from google.cloud.firestore_v1beta1.watch import Watch __all__ = [ @@ -53,6 +54,7 @@ 'Transaction', 'transactional', 'types', + 'Watch', 'WriteBatch', 'WriteOption', ] diff --git a/firestore/google/cloud/firestore_v1beta1/collection.py b/firestore/google/cloud/firestore_v1beta1/collection.py index 81e3dba448c3..c49c3e4080af 100644 --- a/firestore/google/cloud/firestore_v1beta1/collection.py +++ b/firestore/google/cloud/firestore_v1beta1/collection.py @@ -371,6 +371,12 @@ def get(self, transaction=None): query = query_mod.Query(self) return query.get(transaction=transaction) + def onSnapshot(options, callback): + ''' + given options and the callback, monitor this collection for changes + ''' + raise NotImplemented + def _auto_id(): """Generate a "random" automatically generated ID. diff --git a/firestore/google/cloud/firestore_v1beta1/document.py b/firestore/google/cloud/firestore_v1beta1/document.py index b3069bdf4753..830d09e4e0bb 100644 --- a/firestore/google/cloud/firestore_v1beta1/document.py +++ b/firestore/google/cloud/firestore_v1beta1/document.py @@ -422,6 +422,11 @@ def get(self, field_paths=None, transaction=None): [self], field_paths=field_paths, transaction=transaction) return _consume_single_get(snapshot_generator) + def onSnapshot(options, callback): + ''' + given options and the callback, monitor this document for changes + ''' + raise NotImplemented class DocumentSnapshot(object): """A snapshot of document data in a Firestore database. diff --git a/firestore/google/cloud/firestore_v1beta1/query.py b/firestore/google/cloud/firestore_v1beta1/query.py index a4d0243a8724..909eb914e2ea 100644 --- a/firestore/google/cloud/firestore_v1beta1/query.py +++ b/firestore/google/cloud/firestore_v1beta1/query.py @@ -601,6 +601,19 @@ def get(self, transaction=None): else: yield snapshot + def onSnapshot(callback, options): + ''' + db.collection("cities").where("state", "==", "CA") + .onSnapshot(function(querySnapshot) { + var cities = []; + querySnapshot.forEach(function(doc) { + cities.push(doc.data().name); + }); + console.log("Current cities in CA: ", cities.join(", ")); + }); + ''' + raise NotImplemented + def _enum_from_op_string(op_string): """Convert a string representation of a binary operator to an enum. diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py new file mode 100644 index 000000000000..b79762c205d8 --- /dev/null +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -0,0 +1,58 @@ +# Copyright 2017 Google LLC All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Python client for Google Cloud Firestore Watch.""" + +WATCH_TARGET_ID = 0x5079 # "Py" + +class Watch(object): + pass + +''' +You can listen to a document with the onSnapshot() method. An initial call +using the callback you provide creates a document snapshot immediately with the +\current contents of the single document. Then, each time the contents change, +another call updates the document snapshot. + +db.collection("cities") + .onSnapshot + + +Internal: Count: 1, Average: 4.0 +Get Realtime Updates with Cloud Firestore +You can listen to a document with the onSnapshot() method. An initial call using +the callback you provide creates a document snapshot immediately with the +current contents of the single document. Then, each time the contents change, +another call updates the document snapshot. + +Note: Realtime listeners are not yet supported in the Python, Go, or PHP client +libraries. + +db.collection("cities").doc("SF") + .onSnapshot(function(doc) { + console.log("Current data: ", doc.data()); + }); +test.firestore.js + +Events for local changes +Local writes in your app will invoke snapshot listeners immediately. This is +because of an important feature called "latency compensation." When you perform +a write, your listeners will be notified with the new data before the data is +sent to the backend. + +Retrieved documents have a metadata.hasPendingWrites property that indicates +whether the document has local changes that haven't been written to the backend +yet. You can use this property to determine the source of events received by +your snapshot listener: +''' \ No newline at end of file From 972b62fa7c414f54bd2e40b6add8a0aacbefac93 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Mon, 18 Jun 2018 12:57:40 -0700 Subject: [PATCH 002/148] Use chemelnucfin sample (pythonification of nodejs) as base --- .../google/cloud/firestore_v1beta1/watch.py | 406 +++++++++++++++++- 1 file changed, 400 insertions(+), 6 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index b79762c205d8..7c653ee29b5d 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -12,12 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Python client for Google Cloud Firestore Watch.""" - -WATCH_TARGET_ID = 0x5079 # "Py" +import logging -class Watch(object): - pass +"""Python client for Google Cloud Firestore Watch.""" ''' You can listen to a document with the onSnapshot() method. An initial call @@ -55,4 +52,401 @@ class Watch(object): whether the document has local changes that haven't been written to the backend yet. You can use this property to determine the source of events received by your snapshot listener: -''' \ No newline at end of file +''' + +_LOGGER = logging.getLogger(__name__) + +WATCH_TARGET_ID = 0xf0 + +GRPC_STATUS_CODE = { + 'OK': 0, + 'CANCELLED': 1, + 'UNKNOWN': 2, + 'INVALID_ARGUMENT': 3, + 'DEADLINE_EXCEEDED': 4, + 'NOT_FOUND': 5, + 'ALREADY_EXISTS': 6, + 'PERMISSION_DENIED': 7, + 'UNAUTHENTICATED': 16, + 'RESOURCE_EXHAUSTED': 8, + 'FAILED_PRECONDITION': 9, + 'ABORTED': 10, + 'OUT_OF_RANGE': 11, + 'UNIMPLEMENTED': 12, + 'INTERNAL': 13, + 'UNAVAILABLE': 14, + 'DATA_LOSS': 15, + 'DO_NOT_USE': -1 +} + + +def is_permanent_error(self, error): + try: + if (error.code == GRPC_STATUS_CODE['CANCELLED'] or + error.code == GRPC_STATUS_CODE['UNKNOWN'] or + error.code == GRPC_STATUS_CODE['DEADLINE_EXCEEDED'] or + error.code == GRPC_STATUS_CODE['RESOURCE_EXHAUSTED'] or + error.code == GRPC_STATUS_CODE['INTERNAL'] or + error.code == GRPC_STATUS_CODE['UNAVAILABLE'] or + error.code == GRPC_STATUS_CODE['UNAUTHENTICATED'] + ): + return False + else: + return True + except AttributeError: + _LOGGER.error("Unable to determine error code") + return False + + +class Watch(object): + def __init__(self, firestore, target, comparator): + self._firestore = firestore + self._api = firestore.api + self._targets = target + self._comparator = comparator + self._backoff = ExponentialBackOff() + + @classmethod + def for_document(cls, document_ref): + return cls(document_ref.firestore, + {documents: {documents: [document_ref.formatted_name],}, + target_id: WATCH_TARGET_ID, + }, + DOCUMENT_WATCH_COMPARATOR) + + @classmethod + def for_query(cls, query): + return cls(query.firestore, + {query: query.to_proto(), + target_id: WATCH_TARGET_ID + } + query.comparator()) + + def on_snapshot(self, on_next, on_error): + doc_tree = rbtree(self.comparator) + doc_map = {} + change_map = {} + + current = False + has_pushed = False + is_active = True + + REMOVED = {} + + request = { database: self._firestore.formatted_name, + add_target: self._targets + } + + stream = through.obj() + + current_stream = None + + def reset_docs(): + log() + change_map.clear() + del resume_token + for snapshot in doc_tree: + change_map.set(snapshot.ref.formatted_name, REMOVED) + current = False + + def close_stream(err): + if current_stream is not None: + current_stream.unpipe(stream) + current_stream.end() + current_stream = None + stream.end() + + if is_active: + is_active = False + _LOGGER.error('Invoking on_error: ', err) + on_error(err) + + def maybe_reopen_stream(err): + if is_active and not is_permanent_error(err): + _LOGGER.error('Stream ended, re-opening after retryable error: ', err) + request.add_target.resume_token = resume_token + change_map.clear() + + if is_resource_exhausted_error(err): + self._backoff.reset_to_max() + reset_stream() + else: + _LOGGER.error('Stream ended, sending error: ', err) + close_stream(err) + + def reset_stream(): + _LOGGER.info('Opening new stream') + if current_stream: + current_stream.unpipe(stream) + current_stream.end() + current_stream = None + init_stream() + + def init_stream(): + self._backoff.back_off_and_wait() + if not is_active: + _LOGGER.info('Not initializing inactive stream') + return + + backend_stream = self._firestore.read_write_stream( + self._api.Firestore._listen.bind(self._api.Firestore), + request, + ) + + + if not is_active: + _LOGGER.info('Closing inactive stream') + backend_stream.end() + _LOGGER.info('Opened new stream') + current_stream = backend_stream + + def on_error(err): + maybe_reopen_stream(err) + + current_stream.on('error')(on_error) + + def on_end(): + err = Error('Stream ended unexpectedly') + err.code = GRPC_STATUS_CODE['UNKNOWN'] + maybe_reopen_stream(err) + + current_stream.on('end')(on_end) + current_stream.pipe(stream) + current_stream.resume() + + current_stream.catch(close_stream) + + def affects_target(target_ids, current_id): + for target_id in target_ids: + if target_id == current_id: + return True + return False + + def extract_changes(doc_map, changes, read_time): + deletes = [] + adds = [] + updates = [] + + for value, name in changes: + if value == REMOVED: + if doc_map.has(name): + deletes.append(name) + elif doc_map.has(name): + value.read_time = read_time + upates.append(value.build()) + else: + value.read_time = read_time + adds.append(value.build()) + return deletes, adds, updates + + def compute_snapshot(doc_tree, doc_map, changes): + if len(doc_tree) != doc_map: + raise ValueError('The document tree and document map should' + 'have the same number of entries.') + updated_tree = doc_tree + updated_map = doc_map + + def delete_doc(name): + """ raises KeyError if name not in updated_map""" + old_document = updated_map.pop(name) # Raises KeyError + existing = updated_tree.find(old_document) + old_index = existing.index + updated_tree = existing.remove() + return DocumentChange('removed', + old_document, + old_index, + -1) + + def add_doc(new_document): + name = new_document.ref.formatted_name + if name in updated_map: + raise ValueError('Document to add already exists') + updated_tree = updated_tree.insert(new_document, null) + new_index = updated_tree.find(new_document).index + updated_map[name] = new_document + return DocumentChange('added', + new_document, + -1, + new_index) + + def modify_doc(new_document): + name = new_document.ref.formattedName + if not name in updated_map: + raise ValueError('Document to modify does not exsit') + old_document = updated_map[name] + if old_document.update_time != new_document.update_time): + remove_change = delete_doc(name) + add_change = add_doc(new_document) + return DocumentChange('modified', + new_document, + remove_change.old_index, + add_change.new_index) + return None + + applied_changes = [] + + def compartor_sort(name1, name2): + return self._comparator(updated_map[name1], updated_map[name2]) + changes.deletes.sort(comparator_sort) + + + for name in changes.deletes: + changes.delete_doc(name) + if change: + applied_changes.push(change) + + changes.adds.sort(self._compartor) + + for snapshot in changes.adds: + change = add_doc(snapshot) + if change: + applied_changes.push(change) + + changes.updates.sort(self._compartor) + + for snapshot in changes.updates: + change = modify_doc(snapshot) + if change: + applied_changes.push(change) + + if not len(updated_tree) == len(updated_map): + raise RuntimeError('The update document tree and document ' + 'map should have the same number of ' + 'entries') + + + return {updated_tree, updated_map, applied_changes) + + def push(read_time, next_resume_token): + changes = extract_changes(doc_map, change_map, read_time) + diff = compute_snapshot(doc_tree, doc_map, changes) + + if not has_pushed or len(diff.applied_changes) > 0: + _LOGGER.info('Sending snapshot with %d changes and %d documents' + % (len(diff.applied_changes), len(updated_tree))) + + next(read_time, diff.updatedTree.keys, diff.applied_changes) + + doc_tree = diff.updated_tree + doc_map = diff.updated_map + change_map.clear() + resume_token = next_resume_token + + def current_size(): + changes = extract_changes(doc_map, change_map): + return doc_map.size + len(changes.adds) - len(changes.deletes) + + init_stream() + + def proto(): + if proto.target_change: + _LOGGER.log('Processing target change') + change = proto.target_change + no_target_ids = not target_ids + if change.target_change_type == 'NO_CHANGE': + if no_target_ids and change.read_time and current: + push(DocumentSnapshot.to_ISO_time(change.read_time) + change.resume_token) + elif change.target_change_type == 'ADD': + if WATCH_TARGET_ID != change.target_ids[0]: + raise ValueError('Unexpected target ID sent by server') + elif change.target_change_type == 'REMOVE': + code = 13 + message = 'internal error' + if change.cause: + code = change.cause.code + message = change.cause.message + close_stream(Error('Error ' + code + ': ' + message)) + elif change.target_change_type == 'RESET': + reset_docs() + elif change.target_change_type == 'CURRENT': + current = true + else: + close_stream(Error('Unknown target change type: ' + str(change))) + + stream.on('data', proto) # ?? + + if change.resume_token and affects_target(change.target_ids, WATCH_TARGET_ID): + self._backoff.reset() + + elif proto.document_change: + _LOGGER.info('Processing change event') + + target_ids = proto.document_change.target_ids + removed_target_ids = proto.document_change.removed_target_ids + + changed = False + + removed = False + for target_id in target_ids: + if target_id == WATCH_TARGET_ID: + changed = True + + for target_id in removed_target_ids: + if removed_target_ids == WATCH_TARGET_ID: + removed = True + + document = proto.document_change.document + name = document.name + + if changed: + _LOGGER.info('Received document change') + snapshot = DocumentSnapshot.Builder() + snapshot.ref = DocumentReference(self._firestore, + ResourcePath.from_slash_separated_string(name)) + snapshot.fields_proto = document.fields + snapshot.create_time = DocumentSnapshot.to_ISO_time(document.create_time) + snapshot.update_time = DocumentSnapshot.to_ISO_time(document.update_time) + change_map[name] = snapshot + elif removed: + _LOGGER.info('Received document remove') + change_map[name] = REMOVED + elif proto.document_delete + _LOGGER.info('Processing remove event') + name = proto.document_delete.document + change_map[name] = REMOVED + elif proto.document_remove: + _LOGGER.info('Processing remove event') + name = proto.document_remove.document + change_map[name] = REMOVED + elif proto.filter: + _LOGGER.info('Processing filter update') + if proto.filter.count != current_size(): + reset_docs() + reset_stream() + else: + close_stream(Error('Unknown listen response type: ' + str(proto))) + + def on_end(): + _LOGGER.info('Processing stream end') + if current_stream: + current_stream.end() + + on('end', on_end) + + def initialize(): + return {} + + def end_stream(): + _LOGGER.info('Ending stream') + is_active = False + on_next = initialize + on_error = initialize + stream.end() + + return end_stream + + + + + + + + + + + + + + + \ No newline at end of file From a66ce738347e475472ff750cf8b81b071a27dad6 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Mon, 18 Jun 2018 13:10:26 -0700 Subject: [PATCH 003/148] syntactic and style fixes --- .../google/cloud/firestore_v1beta1/watch.py | 131 +++++++----------- 1 file changed, 52 insertions(+), 79 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 7c653ee29b5d..62b33a88bf8b 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -13,50 +13,14 @@ # limitations under the License. import logging +from google.firestore.v1beta1 import DocumentChange -"""Python client for Google Cloud Firestore Watch.""" - -''' -You can listen to a document with the onSnapshot() method. An initial call -using the callback you provide creates a document snapshot immediately with the -\current contents of the single document. Then, each time the contents change, -another call updates the document snapshot. - -db.collection("cities") - .onSnapshot - - -Internal: Count: 1, Average: 4.0 -Get Realtime Updates with Cloud Firestore -You can listen to a document with the onSnapshot() method. An initial call using -the callback you provide creates a document snapshot immediately with the -current contents of the single document. Then, each time the contents change, -another call updates the document snapshot. - -Note: Realtime listeners are not yet supported in the Python, Go, or PHP client -libraries. -db.collection("cities").doc("SF") - .onSnapshot(function(doc) { - console.log("Current data: ", doc.data()); - }); -test.firestore.js - -Events for local changes -Local writes in your app will invoke snapshot listeners immediately. This is -because of an important feature called "latency compensation." When you perform -a write, your listeners will be notified with the new data before the data is -sent to the backend. - -Retrieved documents have a metadata.hasPendingWrites property that indicates -whether the document has local changes that haven't been written to the backend -yet. You can use this property to determine the source of events received by -your snapshot listener: -''' +"""Python client for Google Cloud Firestore Watch.""" _LOGGER = logging.getLogger(__name__) -WATCH_TARGET_ID = 0xf0 +WATCH_TARGET_ID = 0x5079 # "Py" GRPC_STATUS_CODE = { 'OK': 0, @@ -83,13 +47,12 @@ def is_permanent_error(self, error): try: if (error.code == GRPC_STATUS_CODE['CANCELLED'] or - error.code == GRPC_STATUS_CODE['UNKNOWN'] or - error.code == GRPC_STATUS_CODE['DEADLINE_EXCEEDED'] or - error.code == GRPC_STATUS_CODE['RESOURCE_EXHAUSTED'] or - error.code == GRPC_STATUS_CODE['INTERNAL'] or - error.code == GRPC_STATUS_CODE['UNAVAILABLE'] or - error.code == GRPC_STATUS_CODE['UNAUTHENTICATED'] - ): + error.code == GRPC_STATUS_CODE['UNKNOWN'] or + error.code == GRPC_STATUS_CODE['DEADLINE_EXCEEDED'] or + error.code == GRPC_STATUS_CODE['RESOURCE_EXHAUSTED'] or + error.code == GRPC_STATUS_CODE['INTERNAL'] or + error.code == GRPC_STATUS_CODE['UNAVAILABLE'] or + error.code == GRPC_STATUS_CODE['UNAUTHENTICATED']): return False else: return True @@ -98,6 +61,11 @@ def is_permanent_error(self, error): return False +def document_watch_comparator(doc1, doc2): + assert doc1 == doc2, 'Document watches only support one document.' + return 0 + + class Watch(object): def __init__(self, firestore, target, comparator): self._firestore = firestore @@ -109,17 +77,19 @@ def __init__(self, firestore, target, comparator): @classmethod def for_document(cls, document_ref): return cls(document_ref.firestore, - {documents: {documents: [document_ref.formatted_name],}, - target_id: WATCH_TARGET_ID, + { + documents: {documents: [document_ref.formatted_name]}, + target_id: WATCH_TARGET_ID }, - DOCUMENT_WATCH_COMPARATOR) - + document_watch_comparator) + @classmethod def for_query(cls, query): return cls(query.firestore, - {query: query.to_proto(), - target_id: WATCH_TARGET_ID - } + { + query: query.to_proto(), + target_id: WATCH_TARGET_ID + }, query.comparator()) def on_snapshot(self, on_next, on_error): @@ -132,10 +102,9 @@ def on_snapshot(self, on_next, on_error): is_active = True REMOVED = {} - - request = { database: self._firestore.formatted_name, - add_target: self._targets - } + + request = {database: self._firestore.formatted_name, + add_target: self._targets} stream = through.obj() @@ -163,7 +132,8 @@ def close_stream(err): def maybe_reopen_stream(err): if is_active and not is_permanent_error(err): - _LOGGER.error('Stream ended, re-opening after retryable error: ', err) + _LOGGER.error( + 'Stream ended, re-opening after retryable error: ', err) request.add_target.resume_token = resume_token change_map.clear() @@ -193,18 +163,17 @@ def init_stream(): request, ) - if not is_active: _LOGGER.info('Closing inactive stream') backend_stream.end() _LOGGER.info('Opened new stream') current_stream = backend_stream - + def on_error(err): maybe_reopen_stream(err) current_stream.on('error')(on_error) - + def on_end(): err = Error('Stream ended unexpectedly') err.code = GRPC_STATUS_CODE['UNKNOWN'] @@ -215,7 +184,7 @@ def on_end(): current_stream.resume() current_stream.catch(close_stream) - + def affects_target(target_ids, current_id): for target_id in target_ids: if target_id == current_id: @@ -243,7 +212,7 @@ def compute_snapshot(doc_tree, doc_map, changes): if len(doc_tree) != doc_map: raise ValueError('The document tree and document map should' 'have the same number of entries.') - updated_tree = doc_tree + updated_tree = doc_tree updated_map = doc_map def delete_doc(name): @@ -271,10 +240,10 @@ def add_doc(new_document): def modify_doc(new_document): name = new_document.ref.formattedName - if not name in updated_map: + if name not in updated_map: raise ValueError('Document to modify does not exsit') old_document = updated_map[name] - if old_document.update_time != new_document.update_time): + if old_document.update_time != new_document.update_time: remove_change = delete_doc(name) add_change = add_doc(new_document) return DocumentChange('modified', @@ -289,7 +258,6 @@ def compartor_sort(name1, name2): return self._comparator(updated_map[name1], updated_map[name2]) changes.deletes.sort(comparator_sort) - for name in changes.deletes: changes.delete_doc(name) if change: @@ -314,16 +282,16 @@ def compartor_sort(name1, name2): 'map should have the same number of ' 'entries') - - return {updated_tree, updated_map, applied_changes) + return {updated_tree, updated_map, applied_changes} def push(read_time, next_resume_token): changes = extract_changes(doc_map, change_map, read_time) diff = compute_snapshot(doc_tree, doc_map, changes) if not has_pushed or len(diff.applied_changes) > 0: - _LOGGER.info('Sending snapshot with %d changes and %d documents' - % (len(diff.applied_changes), len(updated_tree))) + _LOGGER.info( + 'Sending snapshot with %d changes and %d documents' + % (len(diff.applied_changes), len(updated_tree))) next(read_time, diff.updatedTree.keys, diff.applied_changes) @@ -333,7 +301,7 @@ def push(read_time, next_resume_token): resume_token = next_resume_token def current_size(): - changes = extract_changes(doc_map, change_map): + changes = extract_changes(doc_map, change_map) return doc_map.size + len(changes.adds) - len(changes.deletes) init_stream() @@ -345,7 +313,7 @@ def proto(): no_target_ids = not target_ids if change.target_change_type == 'NO_CHANGE': if no_target_ids and change.read_time and current: - push(DocumentSnapshot.to_ISO_time(change.read_time) + push(DocumentSnapshot.to_ISO_time(change.read_time), change.resume_token) elif change.target_change_type == 'ADD': if WATCH_TARGET_ID != change.target_ids[0]: @@ -362,11 +330,13 @@ def proto(): elif change.target_change_type == 'CURRENT': current = true else: - close_stream(Error('Unknown target change type: ' + str(change))) + close_stream( + Error('Unknown target change type: ' + str(change))) stream.on('data', proto) # ?? - if change.resume_token and affects_target(change.target_ids, WATCH_TARGET_ID): + if change.resume_token and \ + affects_target(change.target_ids, WATCH_TARGET_ID): self._backoff.reset() elif proto.document_change: @@ -392,16 +362,19 @@ def proto(): if changed: _LOGGER.info('Received document change') snapshot = DocumentSnapshot.Builder() - snapshot.ref = DocumentReference(self._firestore, - ResourcePath.from_slash_separated_string(name)) + snapshot.ref = DocumentReference( + self._firestore, + ResourcePath.from_slash_separated_string(name)) snapshot.fields_proto = document.fields - snapshot.create_time = DocumentSnapshot.to_ISO_time(document.create_time) - snapshot.update_time = DocumentSnapshot.to_ISO_time(document.update_time) + snapshot.create_time = DocumentSnapshot.to_ISO_time( + document.create_time) + snapshot.update_time = DocumentSnapshot.to_ISO_time( + document.update_time) change_map[name] = snapshot elif removed: _LOGGER.info('Received document remove') change_map[name] = REMOVED - elif proto.document_delete + elif proto.document_delete: _LOGGER.info('Processing remove event') name = proto.document_delete.document change_map[name] = REMOVED From 1390eb827843a8f2f66490ab147df3c916b9ff48 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Wed, 27 Jun 2018 14:39:35 -0700 Subject: [PATCH 004/148] hold work --- firestore/google/cloud/firestore.py | 2 + .../google/cloud/firestore_v1beta1/bidi.py | 559 +++++++ .../cloud/firestore_v1beta1/document.py | 17 +- .../google/cloud/firestore_v1beta1/watch.py | 747 +++++---- firestore/tests/system.py | 1344 +++++++++-------- 5 files changed, 1666 insertions(+), 1003 deletions(-) create mode 100644 firestore/google/cloud/firestore_v1beta1/bidi.py diff --git a/firestore/google/cloud/firestore.py b/firestore/google/cloud/firestore.py index b7bec0c3adf5..a03ae65ea798 100644 --- a/firestore/google/cloud/firestore.py +++ b/firestore/google/cloud/firestore.py @@ -31,6 +31,7 @@ from google.cloud.firestore_v1beta1 import Transaction from google.cloud.firestore_v1beta1 import transactional from google.cloud.firestore_v1beta1 import types +from google.cloud.firestore_v1beta1 import Watch from google.cloud.firestore_v1beta1 import WriteBatch from google.cloud.firestore_v1beta1 import WriteOption @@ -52,6 +53,7 @@ 'Transaction', 'transactional', 'types', + 'Watch', 'WriteBatch', 'WriteOption', ] diff --git a/firestore/google/cloud/firestore_v1beta1/bidi.py b/firestore/google/cloud/firestore_v1beta1/bidi.py new file mode 100644 index 000000000000..00877e70058e --- /dev/null +++ b/firestore/google/cloud/firestore_v1beta1/bidi.py @@ -0,0 +1,559 @@ +# Copyright 2017, Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Bi-directional streaming RPC helpers.""" + +import logging +import threading + +from six.moves import queue + +from google.api_core import exceptions + +_LOGGER = logging.getLogger(__name__) +_BIDIRECTIONAL_CONSUMER_NAME = 'Thread-ConsumeBidirectionalStream' + + +class _RequestQueueGenerator(object): + """A helper for sending requests to a gRPC stream from a Queue. + + This generator takes requests off a given queue and yields them to gRPC. + + This helper is useful when you have an indeterminate, indefinite, or + otherwise open-ended set of requests to send through a request-streaming + (or bidirectional) RPC. + + The reason this is necessary is because gRPC takes an iterator as the + request for request-streaming RPCs. gRPC consumes this iterator in another + thread to allow it to block while generating requests for the stream. + However, if the generator blocks indefinitely gRPC will not be able to + clean up the thread as it'll be blocked on `next(iterator)` and not be able + to check the channel status to stop iterating. This helper mitigates that + by waiting on the queue with a timeout and checking the RPC state before + yielding. + + Finally, it allows for retrying without swapping queues because if it does + pull an item off the queue when the RPC is inactive, it'll immediately put + it back and then exit. This is necessary because yielding the item in this + case will cause gRPC to discard it. In practice, this means that the order + of messages is not guaranteed. If such a thing is necessary it would be + easy to use a priority queue. + + Example:: + + requests = request_queue_generator(q) + call = stub.StreamingRequest(iter(requests)) + requests.call = call + + for response in call: + print(response) + q.put(...) + + Note that it is possible to accomplish this behavior without "spinning" + (using a queue timeout). One possible way would be to use more threads to + multiplex the grpc end event with the queue, another possible way is to + use selectors and a custom event/queue object. Both of these approaches + are significant from an engineering perspective for small benefit - the + CPU consumed by spinning is pretty minuscule. + + Args: + queue (queue.Queue): The request queue. + period (float): The number of seconds to wait for items from the queue + before checking if the RPC is cancelled. In practice, this + determines the maximum amount of time the request consumption + thread will live after the RPC is cancelled. + initial_request (Union[protobuf.Message, + Callable[None, protobuf.Message]]): The initial request to + yield. This is done independently of the request queue to allow fo + easily restarting streams that require some initial configuration + request. + """ + def __init__(self, queue, period=1, initial_request=None): + self._queue = queue + self._period = period + self._initial_request = initial_request + self.call = None + + def _is_active(self): + # Note: there is a possibility that this starts *before* the call + # property is set. So we have to check if self.call is set before + # seeing if it's active. + if self.call is not None and not self.call.is_active(): + return False + else: + return True + + def __iter__(self): + if self._initial_request is not None: + if callable(self._initial_request): + yield self._initial_request() + else: + yield self._initial_request + + while True: + try: + item = self._queue.get(timeout=self._period) + except queue.Empty: + if not self._is_active(): + _LOGGER.debug( + 'Empty queue and inactive call, exiting request ' + 'generator.') + return + else: + # call is still active, keep waiting for queue items. + continue + + # The consumer explicitly sent "None", indicating that the request + # should end. + if item is None: + _LOGGER.debug('Cleanly exiting request generator.') + return + + if not self._is_active(): + # We have an item, but the call is closed. We should put the + # item back on the queue so that the next call can consume it. + self._queue.put(item) + _LOGGER.debug( + 'Inactive call, replacing item on queue and exiting ' + 'request generator.') + return + + yield item + + +class BidiRpc(object): + """A helper for consuming a bi-directional streaming RPC. + + This maps gRPC's built-in interface which uses a request iterator and a + response iterator into a socket-like :func:`send` and :func:`recv`. This + is a more useful pattern for long-running or asymmetric streams (streams + where there is not a direct correlation between the requests and + responses). + + Example:: + + initial_request = example_pb2.StreamingRpcRequest( + setting='example') + rpc = BidiRpc(stub.StreamingRpc, initial_request=initial_request) + + rpc.open() + + while rpc.is_active(): + print(rpc.recv()) + rpc.send(example_pb2.StreamingRpcRequest( + data='example')) + + This does *not* retry the stream on errors. See :class:`ResumableBidiRpc`. + + Args: + start_rpc (grpc.StreamStreamMultiCallable): The gRPC method used to + start the RPC. + initial_request (Union[protobuf.Message, + Callable[None, protobuf.Message]]): The initial request to + yield. This is useful if an initial request is needed to start the + stream. + """ + def __init__(self, start_rpc, initial_request=None): + self._start_rpc = start_rpc + self._initial_request = initial_request + self._request_queue = queue.Queue() + self._request_generator = None + self._is_active = False + self._callbacks = [] + self.call = None + + def add_done_callback(self, callback): + """Adds a callback that will be called when the RPC terminates. + + This occurs when the RPC errors or is successfully terminated. + + Args: + callback (Callable[[grpc.Future], None]): The callback to execute. + It will be provided with the same gRPC future as the underlying + stream which will also be a :class:`grpc.Call`. + """ + self._callbacks.append(callback) + + def _on_call_done(self, future): + for callback in self._callbacks: + callback(future) + + def open(self): + """Opens the stream.""" + if self.is_active: + raise ValueError('Can not open an already open stream.') + + request_generator = _RequestQueueGenerator( + self._request_queue, initial_request=self._initial_request) + call = self._start_rpc(iter(request_generator)) + + request_generator.call = call + + # TODO: api_core should expose the future interface for wrapped + # callables as well. + if hasattr(call, '_wrapped'): # pragma: NO COVER + call._wrapped.add_done_callback(self._on_call_done) + else: + call.add_done_callback(self._on_call_done) + + self._request_generator = request_generator + self.call = call + + def close(self): + """Closes the stream.""" + if self.call is None: + return + + self._request_queue.put(None) + self.call.cancel() + self._request_generator = None + # Don't set self.call to None. Keep it around so that send/recv can + # raise the error. + + def send(self, request): + """Queue a message to be sent on the stream. + + Send is non-blocking. + + If the underlying RPC has been closed, this will raise. + + Args: + request (protobuf.Message): The request to send. + """ + if self.call is None: + raise ValueError( + 'Can not send() on an RPC that has never been open()ed.') + + # Don't use self.is_active(), as ResumableBidiRpc will overload it + # to mean something semantically different. + if self.call.is_active(): + self._request_queue.put(request) + else: + # calling next should cause the call to raise. + next(self.call) + + def recv(self): + """Wait for a message to be returned from the stream. + + Recv is blocking. + + If the underlying RPC has been closed, this will raise. + + Returns: + protobuf.Message: The received message. + """ + if self.call is None: + raise ValueError( + 'Can not recv() on an RPC that has never been open()ed.') + + return next(self.call) + + @property + def is_active(self): + """bool: True if this stream is currently open and active.""" + return self.call is not None and self.call.is_active() + + @property + def pending_requests(self): + """int: Returns an estimate of the number of queued requests.""" + return self._request_queue.qsize() + + +class ResumableBidiRpc(BidiRpc): + """A :class:`BidiRpc` that can automatically resume the stream on errors. + + It uses the ``should_recover`` arg to determine if it should re-establish + the stream on error. + + Example:: + + def should_recover(exc): + return ( + isinstance(exc, grpc.RpcError) and + exc.code() == grpc.StatusCode.UNVAILABLE) + + initial_request = example_pb2.StreamingRpcRequest( + setting='example') + + rpc = ResumeableBidiRpc( + stub.StreamingRpc, + initial_request=initial_request, + should_recover=should_recover) + + rpc.open() + + while rpc.is_active(): + print(rpc.recv()) + rpc.send(example_pb2.StreamingRpcRequest( + data='example')) + + Args: + start_rpc (grpc.StreamStreamMultiCallable): The gRPC method used to + start the RPC. + initial_request (Union[protobuf.Message, + Callable[None, protobuf.Message]]): The initial request to + yield. This is useful if an initial request is needed to start the + stream. + should_recover (Callable[[Exception], bool]): A function that returns + True if the stream should be recovered. This will be called + whenever an error is encountered on the stream. + """ + def __init__(self, start_rpc, should_recover, initial_request=None): + super(ResumableBidiRpc, self).__init__(start_rpc, initial_request) + self._should_recover = should_recover + self._operational_lock = threading.RLock() + self._finalized = False + self._finalize_lock = threading.Lock() + + def _finalize(self, result): + with self._finalize_lock: + if self._finalized: + return + + for callback in self._callbacks: + callback(result) + + self._finalized = True + + def _on_call_done(self, future): + # Unlike the base class, we only execute the callbacks on a terminal + # error, not for errors that we can recover from. Note that grpc's + # "future" here is also a grpc.RpcError. + if not self._should_recover(future): + self._finalize(future) + else: + _LOGGER.debug('Re-opening stream from gRPC callback.') + self._reopen() + + def _reopen(self): + with self._operational_lock: + # Another thread already managed to re-open this stream. + if self.call is not None and self.call.is_active(): + _LOGGER.debug('Stream was already re-established.') + return + + self.call = None + # Request generator should exit cleanly since the RPC its bound to + # has exited. + self.request_generator = None + + # Note: we do not currently do any sort of backoff here. The + # assumption is that re-establishing the stream under normal + # circumstances will happen in intervals greater than 60s. + # However, it is possible in a degenerative case that the server + # closes the stream rapidly which would lead to thrashing here, + # but hopefully in those cases the server would return a non- + # retryable error. + + try: + self.open() + # If re-opening or re-calling the method fails for any reason, + # consider it a terminal error and finalize the stream. + except Exception as exc: + self._finalize(exc) + raise + + _LOGGER.info('Re-established stream') + + def _recoverable(self, method, *args, **kwargs): + """Wraps a method to recover the stream and retry on error. + + If a retryable error occurs while making the call, then the stream will + be re-opened and the method will be retried. This happens indefinitely + so long as the error is a retryable one. If an error occurs while + re-opening the stream, then this method will raise immediately and + trigger finalization of this object. + + Args: + method (Callable[..., Any]): The method to call. + args: The args to pass to the method. + kwargs: The kwargs to pass to the method. + """ + while True: + try: + return method(*args, **kwargs) + + except Exception as exc: + _LOGGER.debug('Call to retryable %r caused %s.', method, exc) + if not self._should_recover(exc): + self.close() + _LOGGER.debug('Not retrying %r due to %s.', method, exc) + self._finalize(exc) + raise exc + + _LOGGER.debug('Re-opening stream from retryable %r.', method) + self._reopen() + + def send(self, request): + return self._recoverable( + super(ResumableBidiRpc, self).send, request) + + def recv(self): + return self._recoverable( + super(ResumableBidiRpc, self).recv) + + @property + def is_active(self): + """bool: True if this stream is currently open and active.""" + # Use the operational lock. It's entirely possible for something + # to check the active state *while* the RPC is being retried. + # Also, use finalized to track the actual terminal state here. + # This is because if the stream is re-established by the gRPC thread + # it's technically possible to check this between when gRPC marks the + # RPC as inactive and when gRPC executes our callback that re-opens + # the stream. + with self._operational_lock: + return self.call is not None and not self._finalized + + +class BackgroundConsumer(object): + """A bi-directional stream consumer that runs in a separate thread. + + This maps the consumption of a stream into a callback-based model. It also + provides :func:`pause` and :func:`resume` to allow for flow-control. + + Example:: + + def should_recover(exc): + return ( + isinstance(exc, grpc.RpcError) and + exc.code() == grpc.StatusCode.UNVAILABLE) + + initial_request = example_pb2.StreamingRpcRequest( + setting='example') + + rpc = ResumeableBidiRpc( + stub.StreamingRpc, + initial_request=initial_request, + should_recover=should_recover) + + def on_response(response): + print(response) + + consumer = BackgroundConsumer(rpc, on_response) + consume.start() + + Note that error handling *must* be done by using the provided + ``bidi_rpc``'s ``add_done_callback``. This helper will automatically exit + whenever the RPC itself exits and will not provide any error details. + + Args: + bidi_rpc (BidiRpc): The RPC to consume. Should not have been + ``open()``ed yet. + on_response (Callable[[protobuf.Message], None]): The callback to + be called for every response on the stream. + """ + def __init__(self, bidi_rpc, on_response): + self._bidi_rpc = bidi_rpc + self._on_response = on_response + self._paused = False + self._wake = threading.Condition() + self._thread = None + self._operational_lock = threading.Lock() + + def _on_call_done(self, future): + # Resume the thread if it's paused, this prevents blocking forever + # when the RPC has terminated. + self.resume() + + def _thread_main(self): + try: + self._bidi_rpc.add_done_callback(self._on_call_done) + self._bidi_rpc.open() + + while self._bidi_rpc.is_active: + # Do not allow the paused status to change at all during this + # section. There is a condition where we could be resumed + # between checking if we are paused and calling wake.wait(), + # which means that we will miss the notification to wake up + # (oops!) and wait for a notification that will never come. + # Keeping the lock throughout avoids that. + # In the future, we could use `Condition.wait_for` if we drop + # Python 2.7. + with self._wake: + if self._paused: + _LOGGER.debug('paused, waiting for waking.') + self._wake.wait() + _LOGGER.debug('woken.') + + _LOGGER.debug('waiting for recv.') + response = self._bidi_rpc.recv() + _LOGGER.debug('recved response.') + self._on_response(response) + + except exceptions.GoogleAPICallError as exc: + _LOGGER.debug( + '%s caught error %s and will exit. Generally this is due to ' + 'the RPC itself being cancelled and the error will be ' + 'surfaced to the calling code.', + _BIDIRECTIONAL_CONSUMER_NAME, exc, exc_info=True) + + except Exception as exc: + _LOGGER.exception( + '%s caught unexpected exception %s and will exit.', + _BIDIRECTIONAL_CONSUMER_NAME, exc) + + else: + _LOGGER.error( + 'The bidirectional RPC unexpectedly exited. This is a truly ' + 'exceptional case. Please file a bug with your logs.') + + _LOGGER.info('%s exiting', _BIDIRECTIONAL_CONSUMER_NAME) + + def start(self): + """Start the background thread and begin consuming the thread.""" + with self._operational_lock: + thread = threading.Thread( + name=_BIDIRECTIONAL_CONSUMER_NAME, + target=self._thread_main) + thread.daemon = True + thread.start() + self._thread = thread + _LOGGER.debug('Started helper thread %s', thread.name) + + def stop(self): + """Stop consuming the stream and shutdown the background thread.""" + with self._operational_lock: + self._bidi_rpc.close() + + if self._thread is not None: + # Resume the thread to wake it up in case it is sleeping. + self.resume() + self._thread.join() + + self._thread = None + + @property + def is_active(self): + """bool: True if the background thread is active.""" + return self._thread is not None and self._thread.is_alive() + + def pause(self): + """Pauses the response stream. + + This does *not* pause the request stream. + """ + with self._wake: + self._paused = True + + def resume(self): + """Resumes the response stream.""" + with self._wake: + self._paused = False + self._wake.notifyAll() + + @property + def is_paused(self): + """bool: True if the response stream is paused.""" + return self._paused diff --git a/firestore/google/cloud/firestore_v1beta1/document.py b/firestore/google/cloud/firestore_v1beta1/document.py index 830d09e4e0bb..62d63363c278 100644 --- a/firestore/google/cloud/firestore_v1beta1/document.py +++ b/firestore/google/cloud/firestore_v1beta1/document.py @@ -18,7 +18,8 @@ import copy from google.cloud.firestore_v1beta1 import _helpers - +from google.cloud.firestore_v1beta1.watch import Watch +from google.cloud.firestore_v1beta1.proto.firestore_pb2 import Target class DocumentReference(object): """A reference to a document in a Firestore database. @@ -422,11 +423,21 @@ def get(self, field_paths=None, transaction=None): [self], field_paths=field_paths, transaction=transaction) return _consume_single_get(snapshot_generator) - def onSnapshot(options, callback): + def on_snapshot(self, options, callback): ''' given options and the callback, monitor this document for changes ''' - raise NotImplemented + #google.firestore.v1beta1.Target.DocumentsTarget + documentsTarget = Target.DocumentsTarget( + documents=[self._document_path]) + + Watch( + self._client, + Target( + documents=documentsTarget + ), + None) + class DocumentSnapshot(object): """A snapshot of document data in a Firestore database. diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 62b33a88bf8b..cf374c984cae 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -13,8 +13,19 @@ # limitations under the License. import logging -from google.firestore.v1beta1 import DocumentChange +#from google.cloud.firestore_v1beta1 import DocumentReference, DocumentSnapshot + +#from google.cloud.firestore_v1beta1.document import DocumentReference +#from google.cloud.firestore_v1beta1.document import DocumentSnapshot +#import google.cloud.firestore_v1beta1.client as client +from google.cloud.firestore_v1beta1.bidi import BidiRpc, ResumableBidiRpc +from google.cloud.firestore_v1beta1.proto import firestore_pb2 + +#from bidi import BidiRpc, ResumableBidiRpc +import time +import random +import grpc """Python client for Google Cloud Firestore Watch.""" @@ -66,20 +77,98 @@ def document_watch_comparator(doc1, doc2): return 0 +class ExponentialBackOff(object): + _INITIAL_SLEEP = 1.0 + """float: Initial "max" for sleep interval.""" + _MAX_SLEEP = 30.0 + """float: Eventual "max" sleep time.""" + _MULTIPLIER = 2.0 + """float: Multiplier for exponential backoff.""" + + def __init__(self, initial_sleep=_INITIAL_SLEEP, max_sleep=_MAX_SLEEP, + multiplier=_MULTIPLIER): + self.initial_sleep = self.current_sleep = initial_sleep + self.max_sleep = max_sleep + self.multipler = multiplier + + def back_off(self): + self.current_sleep = self._sleep(self.current_sleep, + self.max_sleep, + self.multipler) + + def reset_to_max(self): + self.current_sleep = self.max_sleep + + def reset(self): + self.current_sleep = self._INITIAL_SLEEP + + def _sleep(self, current_sleep, max_sleep=_MAX_SLEEP, + multiplier=_MULTIPLIER): + """Sleep and produce a new sleep time. + + .. _Exponential Backoff And Jitter: https://www.awsarchitectureblog.com/\ + 2015/03/backoff.html + + Select a duration between zero and ``current_sleep``. It might seem + counterintuitive to have so much jitter, but + `Exponential Backoff And Jitter`_ argues that "full jitter" is + the best strategy. + + Args: + current_sleep (float): The current "max" for sleep interval. + max_sleep (Optional[float]): Eventual "max" sleep time + multiplier (Optional[float]): Multiplier for exponential backoff. + + Returns: + float: Newly doubled ``current_sleep`` or ``max_sleep`` (whichever + is smaller) + """ + actual_sleep = random.uniform(0.0, self.current_sleep) + time.sleep(actual_sleep) + return min(self.multiplier * self.current_sleep, self.max_sleep) + class Watch(object): - def __init__(self, firestore, target, comparator): + def __init__(self, + firestore, #: client.Client, + target, + comparator): + self._firestore = firestore - self._api = firestore.api + self._api = firestore._firestore_api self._targets = target self._comparator = comparator self._backoff = ExponentialBackOff() + def should_recover(exc): + return ( + isinstance(exc, grpc.RpcError) and + exc.code() == grpc.StatusCode.UNVAILABLE) + + initial_request = firestore_pb2.ListenRequest( + #database=firestore.database_root_path, + add_target=target + # database, add_taret, remove_target, labels + ) + + rpc = ResumableBidiRpc( + # self._api.firestore_stub.Listen, + #firestore_pb2.BetaFirestoreStub.Listen, + self._api.firestore_stub.Listen, + initial_request=initial_request, + should_recover=should_recover) + + rpc.open() + + while rpc.is_active: + print(rpc.recv()) + @classmethod def for_document(cls, document_ref): return cls(document_ref.firestore, { - documents: {documents: [document_ref.formatted_name]}, - target_id: WATCH_TARGET_ID + 'documents': { + 'documents': [document_ref._document_path]}, + 'target_id': WATCH_TARGET_ID }, document_watch_comparator) @@ -87,339 +176,329 @@ def for_document(cls, document_ref): def for_query(cls, query): return cls(query.firestore, { - query: query.to_proto(), - target_id: WATCH_TARGET_ID + 'query': query.to_proto(), + 'target_id': WATCH_TARGET_ID }, query.comparator()) - def on_snapshot(self, on_next, on_error): - doc_tree = rbtree(self.comparator) - doc_map = {} - change_map = {} - - current = False - has_pushed = False - is_active = True - - REMOVED = {} - - request = {database: self._firestore.formatted_name, - add_target: self._targets} - - stream = through.obj() - - current_stream = None - - def reset_docs(): - log() - change_map.clear() - del resume_token - for snapshot in doc_tree: - change_map.set(snapshot.ref.formatted_name, REMOVED) - current = False - - def close_stream(err): - if current_stream is not None: - current_stream.unpipe(stream) - current_stream.end() - current_stream = None - stream.end() - - if is_active: - is_active = False - _LOGGER.error('Invoking on_error: ', err) - on_error(err) - - def maybe_reopen_stream(err): - if is_active and not is_permanent_error(err): - _LOGGER.error( - 'Stream ended, re-opening after retryable error: ', err) - request.add_target.resume_token = resume_token - change_map.clear() - - if is_resource_exhausted_error(err): - self._backoff.reset_to_max() - reset_stream() - else: - _LOGGER.error('Stream ended, sending error: ', err) - close_stream(err) - - def reset_stream(): - _LOGGER.info('Opening new stream') - if current_stream: - current_stream.unpipe(stream) - current_stream.end() - current_stream = None - init_stream() - - def init_stream(): - self._backoff.back_off_and_wait() - if not is_active: - _LOGGER.info('Not initializing inactive stream') - return - - backend_stream = self._firestore.read_write_stream( - self._api.Firestore._listen.bind(self._api.Firestore), - request, - ) - - if not is_active: - _LOGGER.info('Closing inactive stream') - backend_stream.end() - _LOGGER.info('Opened new stream') - current_stream = backend_stream - - def on_error(err): - maybe_reopen_stream(err) - - current_stream.on('error')(on_error) - - def on_end(): - err = Error('Stream ended unexpectedly') - err.code = GRPC_STATUS_CODE['UNKNOWN'] - maybe_reopen_stream(err) - - current_stream.on('end')(on_end) - current_stream.pipe(stream) - current_stream.resume() - - current_stream.catch(close_stream) - - def affects_target(target_ids, current_id): - for target_id in target_ids: - if target_id == current_id: - return True - return False - - def extract_changes(doc_map, changes, read_time): - deletes = [] - adds = [] - updates = [] - - for value, name in changes: - if value == REMOVED: - if doc_map.has(name): - deletes.append(name) - elif doc_map.has(name): - value.read_time = read_time - upates.append(value.build()) - else: - value.read_time = read_time - adds.append(value.build()) - return deletes, adds, updates - - def compute_snapshot(doc_tree, doc_map, changes): - if len(doc_tree) != doc_map: - raise ValueError('The document tree and document map should' - 'have the same number of entries.') - updated_tree = doc_tree - updated_map = doc_map - - def delete_doc(name): - """ raises KeyError if name not in updated_map""" - old_document = updated_map.pop(name) # Raises KeyError - existing = updated_tree.find(old_document) - old_index = existing.index - updated_tree = existing.remove() - return DocumentChange('removed', - old_document, - old_index, - -1) - - def add_doc(new_document): - name = new_document.ref.formatted_name - if name in updated_map: - raise ValueError('Document to add already exists') - updated_tree = updated_tree.insert(new_document, null) - new_index = updated_tree.find(new_document).index - updated_map[name] = new_document - return DocumentChange('added', - new_document, - -1, - new_index) - - def modify_doc(new_document): - name = new_document.ref.formattedName - if name not in updated_map: - raise ValueError('Document to modify does not exsit') - old_document = updated_map[name] - if old_document.update_time != new_document.update_time: - remove_change = delete_doc(name) - add_change = add_doc(new_document) - return DocumentChange('modified', - new_document, - remove_change.old_index, - add_change.new_index) - return None - - applied_changes = [] - - def compartor_sort(name1, name2): - return self._comparator(updated_map[name1], updated_map[name2]) - changes.deletes.sort(comparator_sort) - - for name in changes.deletes: - changes.delete_doc(name) - if change: - applied_changes.push(change) - - changes.adds.sort(self._compartor) - - for snapshot in changes.adds: - change = add_doc(snapshot) - if change: - applied_changes.push(change) - - changes.updates.sort(self._compartor) - - for snapshot in changes.updates: - change = modify_doc(snapshot) - if change: - applied_changes.push(change) - - if not len(updated_tree) == len(updated_map): - raise RuntimeError('The update document tree and document ' - 'map should have the same number of ' - 'entries') - - return {updated_tree, updated_map, applied_changes} - - def push(read_time, next_resume_token): - changes = extract_changes(doc_map, change_map, read_time) - diff = compute_snapshot(doc_tree, doc_map, changes) - - if not has_pushed or len(diff.applied_changes) > 0: - _LOGGER.info( - 'Sending snapshot with %d changes and %d documents' - % (len(diff.applied_changes), len(updated_tree))) - - next(read_time, diff.updatedTree.keys, diff.applied_changes) - - doc_tree = diff.updated_tree - doc_map = diff.updated_map - change_map.clear() - resume_token = next_resume_token - - def current_size(): - changes = extract_changes(doc_map, change_map) - return doc_map.size + len(changes.adds) - len(changes.deletes) - - init_stream() - - def proto(): - if proto.target_change: - _LOGGER.log('Processing target change') - change = proto.target_change - no_target_ids = not target_ids - if change.target_change_type == 'NO_CHANGE': - if no_target_ids and change.read_time and current: - push(DocumentSnapshot.to_ISO_time(change.read_time), - change.resume_token) - elif change.target_change_type == 'ADD': - if WATCH_TARGET_ID != change.target_ids[0]: - raise ValueError('Unexpected target ID sent by server') - elif change.target_change_type == 'REMOVE': - code = 13 - message = 'internal error' - if change.cause: - code = change.cause.code - message = change.cause.message - close_stream(Error('Error ' + code + ': ' + message)) - elif change.target_change_type == 'RESET': - reset_docs() - elif change.target_change_type == 'CURRENT': - current = true - else: - close_stream( - Error('Unknown target change type: ' + str(change))) - - stream.on('data', proto) # ?? - - if change.resume_token and \ - affects_target(change.target_ids, WATCH_TARGET_ID): - self._backoff.reset() - - elif proto.document_change: - _LOGGER.info('Processing change event') - - target_ids = proto.document_change.target_ids - removed_target_ids = proto.document_change.removed_target_ids - - changed = False - - removed = False - for target_id in target_ids: - if target_id == WATCH_TARGET_ID: - changed = True - - for target_id in removed_target_ids: - if removed_target_ids == WATCH_TARGET_ID: - removed = True - - document = proto.document_change.document - name = document.name - - if changed: - _LOGGER.info('Received document change') - snapshot = DocumentSnapshot.Builder() - snapshot.ref = DocumentReference( - self._firestore, - ResourcePath.from_slash_separated_string(name)) - snapshot.fields_proto = document.fields - snapshot.create_time = DocumentSnapshot.to_ISO_time( - document.create_time) - snapshot.update_time = DocumentSnapshot.to_ISO_time( - document.update_time) - change_map[name] = snapshot - elif removed: - _LOGGER.info('Received document remove') - change_map[name] = REMOVED - elif proto.document_delete: - _LOGGER.info('Processing remove event') - name = proto.document_delete.document - change_map[name] = REMOVED - elif proto.document_remove: - _LOGGER.info('Processing remove event') - name = proto.document_remove.document - change_map[name] = REMOVED - elif proto.filter: - _LOGGER.info('Processing filter update') - if proto.filter.count != current_size(): - reset_docs() - reset_stream() - else: - close_stream(Error('Unknown listen response type: ' + str(proto))) - - def on_end(): - _LOGGER.info('Processing stream end') - if current_stream: - current_stream.end() - - on('end', on_end) - - def initialize(): - return {} - - def end_stream(): - _LOGGER.info('Ending stream') - is_active = False - on_next = initialize - on_error = initialize - stream.end() - - return end_stream - - - + # def on_snapshot(self, on_next, on_error): + # doc_dict = {} + # doc_map = {} + # change_map = {} + + # current = False + # has_pushed = False + # is_active = True + + # REMOVED = {} + + # request = {'database': self._firestore.formatted_name, + # 'add_target': self._targets} + + # stream = through.obj() # TODO: fix through (node holdover) + + # current_stream = None + + # def reset_docs(): + # log() + # change_map.clear() + # del resume_token + # for snapshot in doc_dict: + # change_map.set(snapshot.ref.formatted_name, REMOVED) + # current = False + + # def close_stream(err): + # if current_stream is not None: + # current_stream.unpipe(stream) + # current_stream.end() + # current_stream = None + # stream.end() + + # if is_active: + # is_active = False + # _LOGGER.error('Invoking on_error: ', err) + # on_error(err) + + # def maybe_reopen_stream(err): + # if is_active and not is_permanent_error(err): + # _LOGGER.error( + # 'Stream ended, re-opening after retryable error: ', err) + # request.add_target.resume_token = resume_token + # change_map.clear() + + # if is_resource_exhausted_error(err): + # self._backoff.reset_to_max() + # reset_stream() + # else: + # _LOGGER.error('Stream ended, sending error: ', err) + # close_stream(err) + + # def reset_stream(): + # _LOGGER.info('Opening new stream') + # if current_stream: + # current_stream.unpipe(stream) + # current_stream.end() + # current_stream = None + # init_stream() + + # def init_stream(): + # self._backoff.back_off() + # if not is_active: + # _LOGGER.info('Not initializing inactive stream') + # return + + # backend_stream = self._firestore.read_write_stream( + # self._api.Firestore._listen.bind(self._api.Firestore), + # request, + # ) + + # if not is_active: + # _LOGGER.info('Closing inactive stream') + # backend_stream.end() + # _LOGGER.info('Opened new stream') + # current_stream = backend_stream + + # def on_error(err): + # maybe_reopen_stream(err) + + # current_stream.on('error')(on_error) + + # def on_end(): + # err = Exception('Stream ended unexpectedly') + # err.code = GRPC_STATUS_CODE['UNKNOWN'] + # maybe_reopen_stream(err) + + # current_stream.on('end')(on_end) + # current_stream.pipe(stream) + # current_stream.resume() + + # current_stream.catch(close_stream) + + # def affects_target(target_ids, current_id): + # for target_id in target_ids: + # if target_id == current_id: + # return True + # return False + + # def extract_changes(doc_map, changes, read_time): + # deletes = [] + # adds = [] + # updates = [] + + # for value, name in changes: + # if value == REMOVED: + # if doc_map.has(name): + # deletes.append(name) + # elif doc_map.has(name): + # value.read_time = read_time + # updates.append(value.build()) + # else: + # value.read_time = read_time + # adds.append(value.build()) + # return deletes, adds, updates + + # def compute_snapshot(doc_dict, doc_map, changes): + # if len(doc_dict) != doc_map: + # raise ValueError('The document tree and document map should' + # 'have the same number of entries.') + # updated_dict = doc_dict + # updated_map = doc_map + + # def delete_doc(name): + # """ raises KeyError if name not in updated_map""" + # old_document = updated_map.pop(name) # Raises KeyError + # existing = updated_dict.find(old_document) + # old_index = existing.index + # updated_dict = existing.remove() + # return DocumentChange('removed', + # old_document, + # old_index, + # -1) + + # def add_doc(new_document): + # name = new_document.ref.formatted_name + # if name in updated_map: + # raise ValueError('Document to add already exists') + # updated_dict = updated_dict.insert(new_document, null) + # new_index = updated_dict.find(new_document).index + # updated_map[name] = new_document + # return DocumentChange('added', + # new_document, + # -1, + # new_index) + + # def modify_doc(new_document): + # name = new_document.ref.formattedName + # if name not in updated_map: + # raise ValueError('Document to modify does not exsit') + # old_document = updated_map[name] + # if old_document.update_time != new_document.update_time: + # remove_change = delete_doc(name) + # add_change = add_doc(new_document) + # return DocumentChange('modified', + # new_document, + # remove_change.old_index, + # add_change.new_index) + # return None + + # applied_changes = [] + + # def comparator_sort(name1, name2): + # return self._comparator(updated_map[name1], updated_map[name2]) + + # changes.deletes.sort(comparator_sort) + + # for name in changes.deletes: + # changes.delete_doc(name) + # if change: + # applied_changes.push(change) + + # changes.adds.sort(self._compartor) + + # for snapshot in changes.adds: + # change = add_doc(snapshot) + # if change: + # applied_changes.push(change) + + # changes.updates.sort(self._compartor) + + # for snapshot in changes.updates: + # change = modify_doc(snapshot) + # if change: + # applied_changes.push(change) + + # if not len(updated_dict) == len(updated_map): + # raise RuntimeError('The update document tree and document ' + # 'map should have the same number of ' + # 'entries') + + # return {updated_dict, updated_map, applied_changes} + + # def push(read_time, next_resume_token): + # changes = extract_changes(doc_map, change_map, read_time) + # diff = compute_snapshot(doc_dict, doc_map, changes) + + # if not has_pushed or len(diff.applied_changes) > 0: + # _LOGGER.info( + # 'Sending snapshot with %d changes and %d documents' + # % (len(diff.applied_changes), len(updated_dict))) + + # next(read_time, diff.updatedTree.keys, diff.applied_changes) + + # doc_dict = diff.updated_dict + # doc_map = diff.updated_map + # change_map.clear() + # resume_token = next_resume_token + + # def current_size(): + # changes = extract_changes(doc_map, change_map) + # return doc_map.size + len(changes.adds) - len(changes.deletes) + + # init_stream() + + # def proto(): + # if proto.target_change: + # _LOGGER.log('Processing target change') + # change = proto.target_change + # no_target_ids = not target_ids + # if change.target_change_type == 'NO_CHANGE': + # if no_target_ids and change.read_time and current: + # push(DocumentSnapshot.to_ISO_time(change.read_time), + # change.resume_token) + # elif change.target_change_type == 'ADD': + # if WATCH_TARGET_ID != change.target_ids[0]: + # raise ValueError('Unexpected target ID sent by server') + # elif change.target_change_type == 'REMOVE': + # code = 13 + # message = 'internal error' + # if change.cause: + # code = change.cause.code + # message = change.cause.message + # close_stream(Error('Error ' + code + ': ' + message)) + # elif change.target_change_type == 'RESET': + # reset_docs() + # elif change.target_change_type == 'CURRENT': + # current = true + # else: + # close_stream( + # Exception('Unknown target change type: ' + str(change))) + + # stream.on('data', proto) # ?? + + # if change.resume_token and \ + # affects_target(change.target_ids, WATCH_TARGET_ID): + # self._backoff.reset() + + # elif proto.document_change: + # _LOGGER.info('Processing change event') + + # target_ids = proto.document_change.target_ids + # removed_target_ids = proto.document_change.removed_target_ids + + # changed = False + + # removed = False + # for target_id in target_ids: + # if target_id == WATCH_TARGET_ID: + # changed = True + + # for target_id in removed_target_ids: + # if removed_target_ids == WATCH_TARGET_ID: + # removed = True + + # document = proto.document_change.document + # name = document.name + + # if changed: + # _LOGGER.info('Received document change') + # snapshot = DocumentSnapshot.Builder() + # snapshot.ref = DocumentReference( + # self._firestore, + # ResourcePath.from_slash_separated_string(name)) + # snapshot.fields_proto = document.fields + # snapshot.create_time = DocumentSnapshot.to_ISO_time( + # document.create_time) + # snapshot.update_time = DocumentSnapshot.to_ISO_time( + # document.update_time) + # change_map[name] = snapshot + # elif removed: + # _LOGGER.info('Received document remove') + # change_map[name] = REMOVED + # elif proto.document_delete: + # _LOGGER.info('Processing remove event') + # name = proto.document_delete.document + # change_map[name] = REMOVED + # elif proto.document_remove: + # _LOGGER.info('Processing remove event') + # name = proto.document_remove.document + # change_map[name] = REMOVED + # elif proto.filter: + # _LOGGER.info('Processing filter update') + # if proto.filter.count != current_size(): + # reset_docs() + # reset_stream() + # else: + # close_stream(Error('Unknown listen response type: ' + str(proto))) + + # def on_end(): + # _LOGGER.info('Processing stream end') + # if current_stream: + # current_stream.end() + + # on('end', on_end) + + # def initialize(): + # return {} + + # def end_stream(): + # _LOGGER.info('Ending stream') + # is_active = False + # on_next = initialize + # on_error = initialize + # stream.end() + + # return end_stream - - - - - - - - \ No newline at end of file diff --git a/firestore/tests/system.py b/firestore/tests/system.py index 65348673b3a4..5926c79014c1 100644 --- a/firestore/tests/system.py +++ b/firestore/tests/system.py @@ -57,669 +57,681 @@ def cleanup(): document.delete() -def test_create_document(client, cleanup): - now = datetime.datetime.utcnow().replace(tzinfo=UTC) - document_id = 'shun' + unique_resource_id('-') - document = client.document('collek', document_id) - # Add to clean-up before API request (in case ``create()`` fails). - cleanup(document) - - data = { - 'now': firestore.SERVER_TIMESTAMP, - 'eenta-ger': 11, - 'bites': b'\xe2\x98\x83 \xe2\x9b\xb5', - 'also': { - 'nestednow': firestore.SERVER_TIMESTAMP, - 'quarter': 0.25, - }, - } - write_result = document.create(data) - updated = _pb_timestamp_to_datetime(write_result.update_time) - delta = updated - now - # Allow a bit of clock skew, but make sure timestamps are close. - assert -300.0 < delta.total_seconds() < 300.0 - - with pytest.raises(AlreadyExists): - document.create(data) - - # Verify the server times. - snapshot = document.get() - stored_data = snapshot.to_dict() - server_now = stored_data['now'] - - delta = updated - server_now - # NOTE: We could check the ``transform_results`` from the write result - # for the document transform, but this value gets dropped. Instead - # we make sure the timestamps are close. - assert 0.0 <= delta.total_seconds() < 5.0 - expected_data = { - 'now': server_now, - 'eenta-ger': data['eenta-ger'], - 'bites': data['bites'], - 'also': { - 'nestednow': server_now, - 'quarter': data['also']['quarter'], - }, - } - assert stored_data == expected_data - - -def test_cannot_use_foreign_key(client, cleanup): - document_id = 'cannot' + unique_resource_id('-') - document = client.document('foreign-key', document_id) - # Add to clean-up before API request (in case ``create()`` fails). - cleanup(document) - - other_client = firestore.Client( - project='other-prahj', - credentials=client._credentials, - database='dee-bee') - assert other_client._database_string != client._database_string - fake_doc = other_client.document('foo', 'bar') - with pytest.raises(InvalidArgument): - document.create({'ref': fake_doc}) - - -def assert_timestamp_less(timestamp_pb1, timestamp_pb2): - dt_val1 = _pb_timestamp_to_datetime(timestamp_pb1) - dt_val2 = _pb_timestamp_to_datetime(timestamp_pb2) - assert dt_val1 < dt_val2 - - -def test_no_document(client, cleanup): - document_id = 'no_document' + unique_resource_id('-') - document = client.document('abcde', document_id) - snapshot = document.get() - assert snapshot.to_dict() is None - - -def test_document_set(client, cleanup): - document_id = 'for-set' + unique_resource_id('-') - document = client.document('i-did-it', document_id) - # Add to clean-up before API request (in case ``set()`` fails). - cleanup(document) - - # 0. Make sure the document doesn't exist yet - snapshot = document.get() - assert snapshot.to_dict() is None - - # 1. Use ``create()`` to create the document. - data1 = {'foo': 88} - write_result1 = document.create(data1) - snapshot1 = document.get() - assert snapshot1.to_dict() == data1 - # Make sure the update is what created the document. - assert snapshot1.create_time == snapshot1.update_time - assert snapshot1.update_time == write_result1.update_time - - # 2. Call ``set()`` again to overwrite. - data2 = {'bar': None} - write_result2 = document.set(data2) - snapshot2 = document.get() - assert snapshot2.to_dict() == data2 - # Make sure the create time hasn't changed. - assert snapshot2.create_time == snapshot1.create_time - assert snapshot2.update_time == write_result2.update_time - - -def test_document_integer_field(client, cleanup): - document_id = 'for-set' + unique_resource_id('-') - document = client.document('i-did-it', document_id) - # Add to clean-up before API request (in case ``set()`` fails). - cleanup(document) - - data1 = { - '1a': { - '2b': '3c', - 'ab': '5e'}, - '6f': { - '7g': '8h', - 'cd': '0j'} - } - document.create(data1) - - data2 = {'1a.ab': '4d', '6f.7g': '9h'} - option2 = client.write_option(exists=True) - document.update(data2, option=option2) - snapshot = document.get() - expected = { - '1a': { - '2b': '3c', - 'ab': '4d'}, - '6f': { - '7g': '9h', - 'cd': '0j'} - } - assert snapshot.to_dict() == expected - - -def test_document_set_merge(client, cleanup): - document_id = 'for-set' + unique_resource_id('-') - document = client.document('i-did-it', document_id) - # Add to clean-up before API request (in case ``set()`` fails). - cleanup(document) - - # 0. Make sure the document doesn't exist yet - snapshot = document.get() - assert not snapshot.exists - - # 1. Use ``create()`` to create the document. - data1 = {'name': 'Sam', - 'address': {'city': 'SF', - 'state': 'CA'}} - write_result1 = document.create(data1) - snapshot1 = document.get() - assert snapshot1.to_dict() == data1 - # Make sure the update is what created the document. - assert snapshot1.create_time == snapshot1.update_time - assert snapshot1.update_time == write_result1.update_time - - # 2. Call ``set()`` to merge - data2 = {'address': {'city': 'LA'}} - write_result2 = document.set(data2, merge=True) - snapshot2 = document.get() - assert snapshot2.to_dict() == {'name': 'Sam', - 'address': {'city': 'LA', - 'state': 'CA'}} - # Make sure the create time hasn't changed. - assert snapshot2.create_time == snapshot1.create_time - assert snapshot2.update_time == write_result2.update_time - - -def test_update_document(client, cleanup): - document_id = 'for-update' + unique_resource_id('-') - document = client.document('made', document_id) - # Add to clean-up before API request (in case ``create()`` fails). - cleanup(document) - - # 0. Try to update before the document exists. - with pytest.raises(NotFound) as exc_info: - document.update({'not': 'there'}) - assert exc_info.value.message.startswith(MISSING_DOCUMENT) - assert document_id in exc_info.value.message - - # 1. Try to update before the document exists (now with an option). - option1 = client.write_option(exists=True) - with pytest.raises(NotFound) as exc_info: - document.update({'still': 'not-there'}, option=option1) - assert exc_info.value.message.startswith(MISSING_DOCUMENT) - assert document_id in exc_info.value.message - - # 2. Update and create the document (with an option). - data = { - 'foo': { - 'bar': 'baz', - }, - 'scoop': { - 'barn': 981, - }, - 'other': True, - } - option2 = client.write_option(exists=False) - write_result2 = document.update(data, option=option2) - - # 3. Send an update without a field path (no option). - field_updates3 = {'foo': {'quux': 800}} - write_result3 = document.update(field_updates3) - assert_timestamp_less(write_result2.update_time, write_result3.update_time) - snapshot3 = document.get() - expected3 = { - 'foo': field_updates3['foo'], - 'scoop': data['scoop'], - 'other': data['other'], - } - assert snapshot3.to_dict() == expected3 - - # 4. Send an update **with** a field path and a delete and a valid - # "last timestamp" option. - field_updates4 = { - 'scoop.silo': None, - 'other': firestore.DELETE_FIELD, - } - option4 = client.write_option(last_update_time=snapshot3.update_time) - write_result4 = document.update(field_updates4, option=option4) - assert_timestamp_less(write_result3.update_time, write_result4.update_time) - snapshot4 = document.get() - expected4 = { - 'foo': field_updates3['foo'], - 'scoop': { - 'barn': data['scoop']['barn'], - 'silo': field_updates4['scoop.silo'], - }, - } - assert snapshot4.to_dict() == expected4 - - # 5. Call ``update()`` with invalid (in the past) "last timestamp" option. - assert_timestamp_less(option4._last_update_time, snapshot4.update_time) - with pytest.raises(FailedPrecondition) as exc_info: - document.update({'bad': 'time-past'}, option=option4) - - # 6. Call ``update()`` with invalid (in future) "last timestamp" option. - timestamp_pb = timestamp_pb2.Timestamp( - seconds=snapshot4.update_time.nanos + 3600, - nanos=snapshot4.update_time.nanos, - ) - option6 = client.write_option(last_update_time=timestamp_pb) - with pytest.raises(FailedPrecondition) as exc_info: - document.update({'bad': 'time-future'}, option=option6) - - -def check_snapshot(snapshot, document, data, write_result): - assert snapshot.reference is document - assert snapshot.to_dict() == data - assert snapshot.exists - assert snapshot.create_time == write_result.update_time - assert snapshot.update_time == write_result.update_time - - -def test_document_get(client, cleanup): - now = datetime.datetime.utcnow().replace(tzinfo=UTC) - document_id = 'for-get' + unique_resource_id('-') - document = client.document('created', document_id) - # Add to clean-up before API request (in case ``create()`` fails). - cleanup(document) - - # First make sure it doesn't exist. - assert not document.get().exists - - ref_doc = client.document('top', 'middle1', 'middle2', 'bottom') - data = { - 'turtle': 'power', - 'cheese': 19.5, - 'fire': 199099299, - 'referee': ref_doc, - 'gio': firestore.GeoPoint(45.5, 90.0), - 'deep': [ - u'some', - b'\xde\xad\xbe\xef', - ], - 'map': { - 'ice': True, - 'water': None, - 'vapor': { - 'deeper': now, - }, - }, - } - write_result = document.create(data) - snapshot = document.get() - check_snapshot(snapshot, document, data, write_result) - assert_timestamp_less(snapshot.create_time, snapshot.read_time) - - -def test_document_delete(client, cleanup): - document_id = 'deleted' + unique_resource_id('-') - document = client.document('here-to-be', document_id) - # Add to clean-up before API request (in case ``create()`` fails). - cleanup(document) - document.create({'not': 'much'}) - - # 1. Call ``delete()`` with invalid (in the past) "last timestamp" option. - snapshot1 = document.get() - timestamp_pb = timestamp_pb2.Timestamp( - seconds=snapshot1.update_time.nanos - 3600, - nanos=snapshot1.update_time.nanos, - ) - option1 = client.write_option(last_update_time=timestamp_pb) - with pytest.raises(FailedPrecondition): - document.delete(option=option1) - - # 2. Call ``delete()`` with invalid (in future) "last timestamp" option. - timestamp_pb = timestamp_pb2.Timestamp( - seconds=snapshot1.update_time.nanos + 3600, - nanos=snapshot1.update_time.nanos, - ) - option2 = client.write_option(last_update_time=timestamp_pb) - with pytest.raises(FailedPrecondition): - document.delete(option=option2) - - # 3. Actually ``delete()`` the document. - delete_time3 = document.delete() - - # 4. ``delete()`` again, even though we know the document is gone. - delete_time4 = document.delete() - assert_timestamp_less(delete_time3, delete_time4) - - -def test_collection_add(client, cleanup): - collection1 = client.collection('collek') - collection2 = client.collection('collek', 'shun', 'child') - explicit_doc_id = 'hula' + unique_resource_id('-') - - # Auto-ID at top-level. - data1 = {'foo': 'bar'} - update_time1, document_ref1 = collection1.add(data1) - cleanup(document_ref1) - snapshot1 = document_ref1.get() - assert snapshot1.to_dict() == data1 - assert snapshot1.create_time == update_time1 - assert snapshot1.update_time == update_time1 - assert RANDOM_ID_REGEX.match(document_ref1.id) - - # Explicit ID at top-level. - data2 = {'baz': 999} - update_time2, document_ref2 = collection1.add( - data2, document_id=explicit_doc_id) - cleanup(document_ref2) - snapshot2 = document_ref2.get() - assert snapshot2.to_dict() == data2 - assert snapshot2.create_time == update_time2 - assert snapshot2.update_time == update_time2 - assert document_ref2.id == explicit_doc_id - - # Auto-ID for nested collection. - data3 = {'quux': b'\x00\x01\x02\x03'} - update_time3, document_ref3 = collection2.add(data3) - cleanup(document_ref3) - snapshot3 = document_ref3.get() - assert snapshot3.to_dict() == data3 - assert snapshot3.create_time == update_time3 - assert snapshot3.update_time == update_time3 - assert RANDOM_ID_REGEX.match(document_ref3.id) - - # Explicit for nested collection. - data4 = {'kazaam': None, 'bad': False} - update_time4, document_ref4 = collection2.add( - data4, document_id=explicit_doc_id) - cleanup(document_ref4) - snapshot4 = document_ref4.get() - assert snapshot4.to_dict() == data4 - assert snapshot4.create_time == update_time4 - assert snapshot4.update_time == update_time4 - assert document_ref4.id == explicit_doc_id - - -def test_query_get(client, cleanup): - sub_collection = 'child' + unique_resource_id('-') - collection = client.collection('collek', 'shun', sub_collection) - - stored = {} - num_vals = 5 - allowed_vals = six.moves.xrange(num_vals) - for a_val in allowed_vals: - for b_val in allowed_vals: - document_data = { - 'a': a_val, - 'b': b_val, - 'stats': { - 'sum': a_val + b_val, - 'product': a_val * b_val, - }, - } - _, doc_ref = collection.add(document_data) - # Add to clean-up. - cleanup(doc_ref) - stored[doc_ref.id] = document_data - - # 0. Limit to snapshots where ``a==1``. - query0 = collection.where('a', '==', 1) - values0 = { - snapshot.id: snapshot.to_dict() - for snapshot in query0.get() - } - assert len(values0) == num_vals - for key, value in six.iteritems(values0): - assert stored[key] == value - assert value['a'] == 1 - - # 1. Order by ``b``. - query1 = collection.order_by('b', direction=query0.DESCENDING) - values1 = [ - (snapshot.id, snapshot.to_dict()) - for snapshot in query1.get() - ] - assert len(values1) == len(stored) - b_vals1 = [] - for key, value in values1: - assert stored[key] == value - b_vals1.append(value['b']) - # Make sure the ``b``-values are in DESCENDING order. - assert sorted(b_vals1, reverse=True) == b_vals1 - - # 2. Limit to snapshots where ``stats.sum > 1`` (a field path). - query2 = collection.where('stats.sum', '>', 4) - values2 = { - snapshot.id: snapshot.to_dict() - for snapshot in query2.get() - } - assert len(values2) == 10 - ab_pairs2 = set() - for key, value in six.iteritems(values2): - assert stored[key] == value - ab_pairs2.add((value['a'], value['b'])) - - expected_ab_pairs = set([ - (a_val, b_val) - for a_val in allowed_vals - for b_val in allowed_vals - if a_val + b_val > 4 - ]) - assert expected_ab_pairs == ab_pairs2 - - # 3. Use a start and end cursor. - query3 = collection.start_at({'a': num_vals - 2}) - query3 = query3.order_by('a') - query3 = query3.end_before({'a': num_vals - 1}) - values3 = [ - (snapshot.id, snapshot.to_dict()) - for snapshot in query3.get() - ] - assert len(values3) == num_vals - for key, value in values3: - assert stored[key] == value - assert value['a'] == num_vals - 2 - b_vals1.append(value['b']) - - # 4. Send a query with no results. - query4 = collection.where('b', '==', num_vals + 100) - values4 = list(query4.get()) - assert len(values4) == 0 - - # 5. Select a subset of fields. - query5 = collection.where('b', '<=', 1) - query5 = query5.select(['a', 'stats.product']) - values5 = { - snapshot.id: snapshot.to_dict() - for snapshot in query5.get() - } - assert len(values5) == num_vals * 2 # a ANY, b in (0, 1) - for key, value in six.iteritems(values5): - expected = { - 'a': stored[key]['a'], - 'stats': { - 'product': stored[key]['stats']['product'], - }, - } - assert expected == value - - # 6. Add multiple filters via ``where()``. - query6 = collection.where('stats.product', '>', 5) - query6 = query6.where('stats.product', '<', 10) - values6 = { - snapshot.id: snapshot.to_dict() - for snapshot in query6.get() - } - - matching_pairs = [ - (a_val, b_val) - for a_val in allowed_vals - for b_val in allowed_vals - if 5 < a_val * b_val < 10 - ] - assert len(values6) == len(matching_pairs) - for key, value in six.iteritems(values6): - assert stored[key] == value - pair = (value['a'], value['b']) - assert pair in matching_pairs - - # 7. Skip the first three results, when ``b==2`` - query7 = collection.where('b', '==', 2) - offset = 3 - query7 = query7.offset(offset) - values7 = { - snapshot.id: snapshot.to_dict() - for snapshot in query7.get() - } - # NOTE: We don't check the ``a``-values, since that would require - # an ``order_by('a')``, which combined with the ``b == 2`` - # filter would necessitate an index. - assert len(values7) == num_vals - offset - for key, value in six.iteritems(values7): - assert stored[key] == value - assert value['b'] == 2 - - -def test_query_unary(client, cleanup): - collection_name = 'unary' + unique_resource_id('-') - collection = client.collection(collection_name) - field_name = 'foo' - - _, document0 = collection.add({field_name: None}) - # Add to clean-up. - cleanup(document0) - - nan_val = float('nan') - _, document1 = collection.add({field_name: nan_val}) - # Add to clean-up. - cleanup(document1) - - # 0. Query for null. - query0 = collection.where(field_name, '==', None) - values0 = list(query0.get()) - assert len(values0) == 1 - snapshot0 = values0[0] - assert snapshot0.reference._path == document0._path - assert snapshot0.to_dict() == {field_name: None} - - # 1. Query for a NAN. - query1 = collection.where(field_name, '==', nan_val) - values1 = list(query1.get()) - assert len(values1) == 1 - snapshot1 = values1[0] - assert snapshot1.reference._path == document1._path - data1 = snapshot1.to_dict() - assert len(data1) == 1 - assert math.isnan(data1[field_name]) - - -def test_get_all(client, cleanup): - collection_name = 'get-all' + unique_resource_id('-') - - document1 = client.document(collection_name, 'a') - document2 = client.document(collection_name, 'b') - document3 = client.document(collection_name, 'c') - # Add to clean-up before API requests (in case ``create()`` fails). - cleanup(document1) - cleanup(document3) - - data1 = { - 'a': { - 'b': 2, - 'c': 3, - }, - 'd': 4, - 'e': 0, - } - write_result1 = document1.create(data1) - data3 = { - 'a': { - 'b': 5, - 'c': 6, - }, - 'd': 7, - 'e': 100, - } - write_result3 = document3.create(data3) - - # 0. Get 3 unique documents, one of which is missing. - snapshots = list(client.get_all( - [document1, document2, document3])) - - assert snapshots[0].exists - assert snapshots[1].exists - assert not snapshots[2].exists - snapshots = [snapshot for snapshot in snapshots if snapshot.exists] - id_attr = operator.attrgetter('id') - snapshots.sort(key=id_attr) - - snapshot1, snapshot3 = snapshots - check_snapshot(snapshot1, document1, data1, write_result1) - check_snapshot(snapshot3, document3, data3, write_result3) - - # 1. Get 2 colliding documents. - document1_also = client.document(collection_name, 'a') - snapshots = list(client.get_all([document1, document1_also])) - - assert len(snapshots) == 1 - assert document1 is not document1_also - check_snapshot(snapshots[0], document1_also, data1, write_result1) - - # 2. Use ``field_paths`` / projection in ``get_all()``. - snapshots = list(client.get_all( - [document1, document3], field_paths=['a.b', 'd'])) - - assert len(snapshots) == 2 - snapshots.sort(key=id_attr) - - snapshot1, snapshot3 = snapshots - restricted1 = { - 'a': {'b': data1['a']['b']}, - 'd': data1['d'], - } - check_snapshot(snapshot1, document1, restricted1, write_result1) - restricted3 = { - 'a': {'b': data3['a']['b']}, - 'd': data3['d'], - } - check_snapshot(snapshot3, document3, restricted3, write_result3) - - -def test_batch(client, cleanup): - collection_name = 'batch' + unique_resource_id('-') - - document1 = client.document(collection_name, 'abc') - document2 = client.document(collection_name, 'mno') - document3 = client.document(collection_name, 'xyz') - # Add to clean-up before API request (in case ``create()`` fails). - cleanup(document1) - cleanup(document2) - cleanup(document3) - - data2 = { - 'some': { - 'deep': 'stuff', - 'and': 'here', - }, - 'water': 100.0, - } - document2.create(data2) - document3.create({'other': 19}) - - batch = client.batch() - data1 = {'all': True} - batch.create(document1, data1) - new_value = 'there' - batch.update(document2, {'some.and': new_value}) - batch.delete(document3) - write_results = batch.commit() - - assert len(write_results) == 3 - - write_result1 = write_results[0] - write_result2 = write_results[1] - write_result3 = write_results[2] - assert not write_result3.HasField('update_time') - - snapshot1 = document1.get() - assert snapshot1.to_dict() == data1 - assert snapshot1.create_time == write_result1.update_time - assert snapshot1.update_time == write_result1.update_time - - snapshot2 = document2.get() - assert snapshot2.to_dict() != data2 - data2['some']['and'] = new_value - assert snapshot2.to_dict() == data2 - assert_timestamp_less(snapshot2.create_time, write_result2.update_time) - assert snapshot2.update_time == write_result2.update_time - - assert not document3.get().exists +def test_watch_document(client, cleanup): + # Add a new document + db = client + doc_ref = db.collection(u'users').document(u'alovelace') + doc_ref.set({ + u'first': u'Ada', + u'last': u'Lovelace', + u'born': 1815 + }) + doc_ref.on_snapshot(None, None) + + +# def test_create_document(client, cleanup): +# now = datetime.datetime.utcnow().replace(tzinfo=UTC) +# document_id = 'shun' + unique_resource_id('-') +# document = client.document('collek', document_id) +# # Add to clean-up before API request (in case ``create()`` fails). +# cleanup(document) + +# data = { +# 'now': firestore.SERVER_TIMESTAMP, +# 'eenta-ger': 11, +# 'bites': b'\xe2\x98\x83 \xe2\x9b\xb5', +# 'also': { +# 'nestednow': firestore.SERVER_TIMESTAMP, +# 'quarter': 0.25, +# }, +# } +# write_result = document.create(data) +# updated = _pb_timestamp_to_datetime(write_result.update_time) +# delta = updated - now +# # Allow a bit of clock skew, but make sure timestamps are close. +# assert -300.0 < delta.total_seconds() < 300.0 + +# with pytest.raises(AlreadyExists): +# document.create(data) + +# # Verify the server times. +# snapshot = document.get() +# stored_data = snapshot.to_dict() +# server_now = stored_data['now'] + +# delta = updated - server_now +# # NOTE: We could check the ``transform_results`` from the write result +# # for the document transform, but this value gets dropped. Instead +# # we make sure the timestamps are close. +# assert 0.0 <= delta.total_seconds() < 5.0 +# expected_data = { +# 'now': server_now, +# 'eenta-ger': data['eenta-ger'], +# 'bites': data['bites'], +# 'also': { +# 'nestednow': server_now, +# 'quarter': data['also']['quarter'], +# }, +# } +# assert stored_data == expected_data + + +# def test_cannot_use_foreign_key(client, cleanup): +# document_id = 'cannot' + unique_resource_id('-') +# document = client.document('foreign-key', document_id) +# # Add to clean-up before API request (in case ``create()`` fails). +# cleanup(document) + +# other_client = firestore.Client( +# project='other-prahj', +# credentials=client._credentials, +# database='dee-bee') +# assert other_client._database_string != client._database_string +# fake_doc = other_client.document('foo', 'bar') +# with pytest.raises(InvalidArgument): +# document.create({'ref': fake_doc}) + + +# def assert_timestamp_less(timestamp_pb1, timestamp_pb2): +# dt_val1 = _pb_timestamp_to_datetime(timestamp_pb1) +# dt_val2 = _pb_timestamp_to_datetime(timestamp_pb2) +# assert dt_val1 < dt_val2 + + +# def test_no_document(client, cleanup): +# document_id = 'no_document' + unique_resource_id('-') +# document = client.document('abcde', document_id) +# snapshot = document.get() +# assert snapshot.to_dict() is None + + +# def test_document_set(client, cleanup): +# document_id = 'for-set' + unique_resource_id('-') +# document = client.document('i-did-it', document_id) +# # Add to clean-up before API request (in case ``set()`` fails). +# cleanup(document) + +# # 0. Make sure the document doesn't exist yet +# snapshot = document.get() +# assert snapshot.to_dict() is None + +# # 1. Use ``create()`` to create the document. +# data1 = {'foo': 88} +# write_result1 = document.create(data1) +# snapshot1 = document.get() +# assert snapshot1.to_dict() == data1 +# # Make sure the update is what created the document. +# assert snapshot1.create_time == snapshot1.update_time +# assert snapshot1.update_time == write_result1.update_time + +# # 2. Call ``set()`` again to overwrite. +# data2 = {'bar': None} +# write_result2 = document.set(data2) +# snapshot2 = document.get() +# assert snapshot2.to_dict() == data2 +# # Make sure the create time hasn't changed. +# assert snapshot2.create_time == snapshot1.create_time +# assert snapshot2.update_time == write_result2.update_time + + +# def test_document_integer_field(client, cleanup): +# document_id = 'for-set' + unique_resource_id('-') +# document = client.document('i-did-it', document_id) +# # Add to clean-up before API request (in case ``set()`` fails). +# cleanup(document) + +# data1 = { +# '1a': { +# '2b': '3c', +# 'ab': '5e'}, +# '6f': { +# '7g': '8h', +# 'cd': '0j'} +# } +# document.create(data1) + +# data2 = {'1a.ab': '4d', '6f.7g': '9h'} +# option2 = client.write_option(exists=True) +# document.update(data2, option=option2) +# snapshot = document.get() +# expected = { +# '1a': { +# '2b': '3c', +# 'ab': '4d'}, +# '6f': { +# '7g': '9h', +# 'cd': '0j'} +# } +# assert snapshot.to_dict() == expected + + +# def test_document_set_merge(client, cleanup): +# document_id = 'for-set' + unique_resource_id('-') +# document = client.document('i-did-it', document_id) +# # Add to clean-up before API request (in case ``set()`` fails). +# cleanup(document) + +# # 0. Make sure the document doesn't exist yet +# snapshot = document.get() +# assert not snapshot.exists + +# # 1. Use ``create()`` to create the document. +# data1 = {'name': 'Sam', +# 'address': {'city': 'SF', +# 'state': 'CA'}} +# write_result1 = document.create(data1) +# snapshot1 = document.get() +# assert snapshot1.to_dict() == data1 +# # Make sure the update is what created the document. +# assert snapshot1.create_time == snapshot1.update_time +# assert snapshot1.update_time == write_result1.update_time + +# # 2. Call ``set()`` to merge +# data2 = {'address': {'city': 'LA'}} +# write_result2 = document.set(data2, merge=True) +# snapshot2 = document.get() +# assert snapshot2.to_dict() == {'name': 'Sam', +# 'address': {'city': 'LA', +# 'state': 'CA'}} +# # Make sure the create time hasn't changed. +# assert snapshot2.create_time == snapshot1.create_time +# assert snapshot2.update_time == write_result2.update_time + + +# def test_update_document(client, cleanup): +# document_id = 'for-update' + unique_resource_id('-') +# document = client.document('made', document_id) +# # Add to clean-up before API request (in case ``create()`` fails). +# cleanup(document) + +# # 0. Try to update before the document exists. +# with pytest.raises(NotFound) as exc_info: +# document.update({'not': 'there'}) +# assert exc_info.value.message.startswith(MISSING_DOCUMENT) +# assert document_id in exc_info.value.message + +# # 1. Try to update before the document exists (now with an option). +# option1 = client.write_option(exists=True) +# with pytest.raises(NotFound) as exc_info: +# document.update({'still': 'not-there'}, option=option1) +# assert exc_info.value.message.startswith(MISSING_DOCUMENT) +# assert document_id in exc_info.value.message + +# # 2. Update and create the document (with an option). +# data = { +# 'foo': { +# 'bar': 'baz', +# }, +# 'scoop': { +# 'barn': 981, +# }, +# 'other': True, +# } +# option2 = client.write_option(exists=False) +# write_result2 = document.update(data, option=option2) + +# # 3. Send an update without a field path (no option). +# field_updates3 = {'foo': {'quux': 800}} +# write_result3 = document.update(field_updates3) +# assert_timestamp_less(write_result2.update_time, write_result3.update_time) +# snapshot3 = document.get() +# expected3 = { +# 'foo': field_updates3['foo'], +# 'scoop': data['scoop'], +# 'other': data['other'], +# } +# assert snapshot3.to_dict() == expected3 + +# # 4. Send an update **with** a field path and a delete and a valid +# # "last timestamp" option. +# field_updates4 = { +# 'scoop.silo': None, +# 'other': firestore.DELETE_FIELD, +# } +# option4 = client.write_option(last_update_time=snapshot3.update_time) +# write_result4 = document.update(field_updates4, option=option4) +# assert_timestamp_less(write_result3.update_time, write_result4.update_time) +# snapshot4 = document.get() +# expected4 = { +# 'foo': field_updates3['foo'], +# 'scoop': { +# 'barn': data['scoop']['barn'], +# 'silo': field_updates4['scoop.silo'], +# }, +# } +# assert snapshot4.to_dict() == expected4 + +# # 5. Call ``update()`` with invalid (in the past) "last timestamp" option. +# assert_timestamp_less(option4._last_update_time, snapshot4.update_time) +# with pytest.raises(FailedPrecondition) as exc_info: +# document.update({'bad': 'time-past'}, option=option4) + +# # 6. Call ``update()`` with invalid (in future) "last timestamp" option. +# timestamp_pb = timestamp_pb2.Timestamp( +# seconds=snapshot4.update_time.nanos + 3600, +# nanos=snapshot4.update_time.nanos, +# ) +# option6 = client.write_option(last_update_time=timestamp_pb) +# with pytest.raises(FailedPrecondition) as exc_info: +# document.update({'bad': 'time-future'}, option=option6) + + +# def check_snapshot(snapshot, document, data, write_result): +# assert snapshot.reference is document +# assert snapshot.to_dict() == data +# assert snapshot.exists +# assert snapshot.create_time == write_result.update_time +# assert snapshot.update_time == write_result.update_time + + +# def test_document_get(client, cleanup): +# now = datetime.datetime.utcnow().replace(tzinfo=UTC) +# document_id = 'for-get' + unique_resource_id('-') +# document = client.document('created', document_id) +# # Add to clean-up before API request (in case ``create()`` fails). +# cleanup(document) + +# # First make sure it doesn't exist. +# assert not document.get().exists + +# ref_doc = client.document('top', 'middle1', 'middle2', 'bottom') +# data = { +# 'turtle': 'power', +# 'cheese': 19.5, +# 'fire': 199099299, +# 'referee': ref_doc, +# 'gio': firestore.GeoPoint(45.5, 90.0), +# 'deep': [ +# u'some', +# b'\xde\xad\xbe\xef', +# ], +# 'map': { +# 'ice': True, +# 'water': None, +# 'vapor': { +# 'deeper': now, +# }, +# }, +# } +# write_result = document.create(data) +# snapshot = document.get() +# check_snapshot(snapshot, document, data, write_result) +# assert_timestamp_less(snapshot.create_time, snapshot.read_time) + + +# def test_document_delete(client, cleanup): +# document_id = 'deleted' + unique_resource_id('-') +# document = client.document('here-to-be', document_id) +# # Add to clean-up before API request (in case ``create()`` fails). +# cleanup(document) +# document.create({'not': 'much'}) + +# # 1. Call ``delete()`` with invalid (in the past) "last timestamp" option. +# snapshot1 = document.get() +# timestamp_pb = timestamp_pb2.Timestamp( +# seconds=snapshot1.update_time.nanos - 3600, +# nanos=snapshot1.update_time.nanos, +# ) +# option1 = client.write_option(last_update_time=timestamp_pb) +# with pytest.raises(FailedPrecondition): +# document.delete(option=option1) + +# # 2. Call ``delete()`` with invalid (in future) "last timestamp" option. +# timestamp_pb = timestamp_pb2.Timestamp( +# seconds=snapshot1.update_time.nanos + 3600, +# nanos=snapshot1.update_time.nanos, +# ) +# option2 = client.write_option(last_update_time=timestamp_pb) +# with pytest.raises(FailedPrecondition): +# document.delete(option=option2) + +# # 3. Actually ``delete()`` the document. +# delete_time3 = document.delete() + +# # 4. ``delete()`` again, even though we know the document is gone. +# delete_time4 = document.delete() +# assert_timestamp_less(delete_time3, delete_time4) + + +# def test_collection_add(client, cleanup): +# collection1 = client.collection('collek') +# collection2 = client.collection('collek', 'shun', 'child') +# explicit_doc_id = 'hula' + unique_resource_id('-') + +# # Auto-ID at top-level. +# data1 = {'foo': 'bar'} +# update_time1, document_ref1 = collection1.add(data1) +# cleanup(document_ref1) +# snapshot1 = document_ref1.get() +# assert snapshot1.to_dict() == data1 +# assert snapshot1.create_time == update_time1 +# assert snapshot1.update_time == update_time1 +# assert RANDOM_ID_REGEX.match(document_ref1.id) + +# # Explicit ID at top-level. +# data2 = {'baz': 999} +# update_time2, document_ref2 = collection1.add( +# data2, document_id=explicit_doc_id) +# cleanup(document_ref2) +# snapshot2 = document_ref2.get() +# assert snapshot2.to_dict() == data2 +# assert snapshot2.create_time == update_time2 +# assert snapshot2.update_time == update_time2 +# assert document_ref2.id == explicit_doc_id + +# # Auto-ID for nested collection. +# data3 = {'quux': b'\x00\x01\x02\x03'} +# update_time3, document_ref3 = collection2.add(data3) +# cleanup(document_ref3) +# snapshot3 = document_ref3.get() +# assert snapshot3.to_dict() == data3 +# assert snapshot3.create_time == update_time3 +# assert snapshot3.update_time == update_time3 +# assert RANDOM_ID_REGEX.match(document_ref3.id) + +# # Explicit for nested collection. +# data4 = {'kazaam': None, 'bad': False} +# update_time4, document_ref4 = collection2.add( +# data4, document_id=explicit_doc_id) +# cleanup(document_ref4) +# snapshot4 = document_ref4.get() +# assert snapshot4.to_dict() == data4 +# assert snapshot4.create_time == update_time4 +# assert snapshot4.update_time == update_time4 +# assert document_ref4.id == explicit_doc_id + + +# def test_query_get(client, cleanup): +# sub_collection = 'child' + unique_resource_id('-') +# collection = client.collection('collek', 'shun', sub_collection) + +# stored = {} +# num_vals = 5 +# allowed_vals = six.moves.xrange(num_vals) +# for a_val in allowed_vals: +# for b_val in allowed_vals: +# document_data = { +# 'a': a_val, +# 'b': b_val, +# 'stats': { +# 'sum': a_val + b_val, +# 'product': a_val * b_val, +# }, +# } +# _, doc_ref = collection.add(document_data) +# # Add to clean-up. +# cleanup(doc_ref) +# stored[doc_ref.id] = document_data + +# # 0. Limit to snapshots where ``a==1``. +# query0 = collection.where('a', '==', 1) +# values0 = { +# snapshot.id: snapshot.to_dict() +# for snapshot in query0.get() +# } +# assert len(values0) == num_vals +# for key, value in six.iteritems(values0): +# assert stored[key] == value +# assert value['a'] == 1 + +# # 1. Order by ``b``. +# query1 = collection.order_by('b', direction=query0.DESCENDING) +# values1 = [ +# (snapshot.id, snapshot.to_dict()) +# for snapshot in query1.get() +# ] +# assert len(values1) == len(stored) +# b_vals1 = [] +# for key, value in values1: +# assert stored[key] == value +# b_vals1.append(value['b']) +# # Make sure the ``b``-values are in DESCENDING order. +# assert sorted(b_vals1, reverse=True) == b_vals1 + +# # 2. Limit to snapshots where ``stats.sum > 1`` (a field path). +# query2 = collection.where('stats.sum', '>', 4) +# values2 = { +# snapshot.id: snapshot.to_dict() +# for snapshot in query2.get() +# } +# assert len(values2) == 10 +# ab_pairs2 = set() +# for key, value in six.iteritems(values2): +# assert stored[key] == value +# ab_pairs2.add((value['a'], value['b'])) + +# expected_ab_pairs = set([ +# (a_val, b_val) +# for a_val in allowed_vals +# for b_val in allowed_vals +# if a_val + b_val > 4 +# ]) +# assert expected_ab_pairs == ab_pairs2 + +# # 3. Use a start and end cursor. +# query3 = collection.start_at({'a': num_vals - 2}) +# query3 = query3.order_by('a') +# query3 = query3.end_before({'a': num_vals - 1}) +# values3 = [ +# (snapshot.id, snapshot.to_dict()) +# for snapshot in query3.get() +# ] +# assert len(values3) == num_vals +# for key, value in values3: +# assert stored[key] == value +# assert value['a'] == num_vals - 2 +# b_vals1.append(value['b']) + +# # 4. Send a query with no results. +# query4 = collection.where('b', '==', num_vals + 100) +# values4 = list(query4.get()) +# assert len(values4) == 0 + +# # 5. Select a subset of fields. +# query5 = collection.where('b', '<=', 1) +# query5 = query5.select(['a', 'stats.product']) +# values5 = { +# snapshot.id: snapshot.to_dict() +# for snapshot in query5.get() +# } +# assert len(values5) == num_vals * 2 # a ANY, b in (0, 1) +# for key, value in six.iteritems(values5): +# expected = { +# 'a': stored[key]['a'], +# 'stats': { +# 'product': stored[key]['stats']['product'], +# }, +# } +# assert expected == value + +# # 6. Add multiple filters via ``where()``. +# query6 = collection.where('stats.product', '>', 5) +# query6 = query6.where('stats.product', '<', 10) +# values6 = { +# snapshot.id: snapshot.to_dict() +# for snapshot in query6.get() +# } + +# matching_pairs = [ +# (a_val, b_val) +# for a_val in allowed_vals +# for b_val in allowed_vals +# if 5 < a_val * b_val < 10 +# ] +# assert len(values6) == len(matching_pairs) +# for key, value in six.iteritems(values6): +# assert stored[key] == value +# pair = (value['a'], value['b']) +# assert pair in matching_pairs + +# # 7. Skip the first three results, when ``b==2`` +# query7 = collection.where('b', '==', 2) +# offset = 3 +# query7 = query7.offset(offset) +# values7 = { +# snapshot.id: snapshot.to_dict() +# for snapshot in query7.get() +# } +# # NOTE: We don't check the ``a``-values, since that would require +# # an ``order_by('a')``, which combined with the ``b == 2`` +# # filter would necessitate an index. +# assert len(values7) == num_vals - offset +# for key, value in six.iteritems(values7): +# assert stored[key] == value +# assert value['b'] == 2 + + +# def test_query_unary(client, cleanup): +# collection_name = 'unary' + unique_resource_id('-') +# collection = client.collection(collection_name) +# field_name = 'foo' + +# _, document0 = collection.add({field_name: None}) +# # Add to clean-up. +# cleanup(document0) + +# nan_val = float('nan') +# _, document1 = collection.add({field_name: nan_val}) +# # Add to clean-up. +# cleanup(document1) + +# # 0. Query for null. +# query0 = collection.where(field_name, '==', None) +# values0 = list(query0.get()) +# assert len(values0) == 1 +# snapshot0 = values0[0] +# assert snapshot0.reference._path == document0._path +# assert snapshot0.to_dict() == {field_name: None} + +# # 1. Query for a NAN. +# query1 = collection.where(field_name, '==', nan_val) +# values1 = list(query1.get()) +# assert len(values1) == 1 +# snapshot1 = values1[0] +# assert snapshot1.reference._path == document1._path +# data1 = snapshot1.to_dict() +# assert len(data1) == 1 +# assert math.isnan(data1[field_name]) + + +# def test_get_all(client, cleanup): +# collection_name = 'get-all' + unique_resource_id('-') + +# document1 = client.document(collection_name, 'a') +# document2 = client.document(collection_name, 'b') +# document3 = client.document(collection_name, 'c') +# # Add to clean-up before API requests (in case ``create()`` fails). +# cleanup(document1) +# cleanup(document3) + +# data1 = { +# 'a': { +# 'b': 2, +# 'c': 3, +# }, +# 'd': 4, +# 'e': 0, +# } +# write_result1 = document1.create(data1) +# data3 = { +# 'a': { +# 'b': 5, +# 'c': 6, +# }, +# 'd': 7, +# 'e': 100, +# } +# write_result3 = document3.create(data3) + +# # 0. Get 3 unique documents, one of which is missing. +# snapshots = list(client.get_all( +# [document1, document2, document3])) + +# assert snapshots[0].exists +# assert snapshots[1].exists +# assert not snapshots[2].exists +# snapshots = [snapshot for snapshot in snapshots if snapshot.exists] +# id_attr = operator.attrgetter('id') +# snapshots.sort(key=id_attr) + +# snapshot1, snapshot3 = snapshots +# check_snapshot(snapshot1, document1, data1, write_result1) +# check_snapshot(snapshot3, document3, data3, write_result3) + +# # 1. Get 2 colliding documents. +# document1_also = client.document(collection_name, 'a') +# snapshots = list(client.get_all([document1, document1_also])) + +# assert len(snapshots) == 1 +# assert document1 is not document1_also +# check_snapshot(snapshots[0], document1_also, data1, write_result1) + +# # 2. Use ``field_paths`` / projection in ``get_all()``. +# snapshots = list(client.get_all( +# [document1, document3], field_paths=['a.b', 'd'])) + +# assert len(snapshots) == 2 +# snapshots.sort(key=id_attr) + +# snapshot1, snapshot3 = snapshots +# restricted1 = { +# 'a': {'b': data1['a']['b']}, +# 'd': data1['d'], +# } +# check_snapshot(snapshot1, document1, restricted1, write_result1) +# restricted3 = { +# 'a': {'b': data3['a']['b']}, +# 'd': data3['d'], +# } +# check_snapshot(snapshot3, document3, restricted3, write_result3) + + +# def test_batch(client, cleanup): +# collection_name = 'batch' + unique_resource_id('-') + +# document1 = client.document(collection_name, 'abc') +# document2 = client.document(collection_name, 'mno') +# document3 = client.document(collection_name, 'xyz') +# # Add to clean-up before API request (in case ``create()`` fails). +# cleanup(document1) +# cleanup(document2) +# cleanup(document3) + +# data2 = { +# 'some': { +# 'deep': 'stuff', +# 'and': 'here', +# }, +# 'water': 100.0, +# } +# document2.create(data2) +# document3.create({'other': 19}) + +# batch = client.batch() +# data1 = {'all': True} +# batch.create(document1, data1) +# new_value = 'there' +# batch.update(document2, {'some.and': new_value}) +# batch.delete(document3) +# write_results = batch.commit() + +# assert len(write_results) == 3 + +# write_result1 = write_results[0] +# write_result2 = write_results[1] +# write_result3 = write_results[2] +# assert not write_result3.HasField('update_time') + +# snapshot1 = document1.get() +# assert snapshot1.to_dict() == data1 +# assert snapshot1.create_time == write_result1.update_time +# assert snapshot1.update_time == write_result1.update_time + +# snapshot2 = document2.get() +# assert snapshot2.to_dict() != data2 +# data2['some']['and'] = new_value +# assert snapshot2.to_dict() == data2 +# assert_timestamp_less(snapshot2.create_time, write_result2.update_time) +# assert snapshot2.update_time == write_result2.update_time + +# assert not document3.get().exists From 45f48d610bf4a229f2ce6eaf129b156ad0876de8 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Wed, 27 Jun 2018 15:48:35 -0700 Subject: [PATCH 005/148] use helper for_document --- .../google/cloud/firestore_v1beta1/document.py | 16 +++++++--------- .../google/cloud/firestore_v1beta1/watch.py | 8 ++++---- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/document.py b/firestore/google/cloud/firestore_v1beta1/document.py index 62d63363c278..6ebb38e2a365 100644 --- a/firestore/google/cloud/firestore_v1beta1/document.py +++ b/firestore/google/cloud/firestore_v1beta1/document.py @@ -428,15 +428,13 @@ def on_snapshot(self, options, callback): given options and the callback, monitor this document for changes ''' #google.firestore.v1beta1.Target.DocumentsTarget - documentsTarget = Target.DocumentsTarget( - documents=[self._document_path]) - - Watch( - self._client, - Target( - documents=documentsTarget - ), - None) + # documentsTarget = Target.DocumentsTarget( + # documents=[self._document_path]) + Watch.for_document(self) + # Watch( + # self._client, + # Target(documents=documentsTarget), + # None) class DocumentSnapshot(object): diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index cf374c984cae..60835260eb21 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -145,9 +145,9 @@ def should_recover(exc): exc.code() == grpc.StatusCode.UNVAILABLE) initial_request = firestore_pb2.ListenRequest( - #database=firestore.database_root_path, + database=firestore._database_string, add_target=target - # database, add_taret, remove_target, labels + # database, add_target, remove_target, labels ) rpc = ResumableBidiRpc( @@ -164,7 +164,7 @@ def should_recover(exc): @classmethod def for_document(cls, document_ref): - return cls(document_ref.firestore, + return cls(document_ref._client, { 'documents': { 'documents': [document_ref._document_path]}, @@ -174,7 +174,7 @@ def for_document(cls, document_ref): @classmethod def for_query(cls, query): - return cls(query.firestore, + return cls(query._client, { 'query': query.to_proto(), 'target_id': WATCH_TARGET_ID From 4758b6bd27b83a804d8dd79c216c9c9900f7e932 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Tue, 3 Jul 2018 15:31:20 -0700 Subject: [PATCH 006/148] Staging changes to firestore for watch. still incomplete but returning document changes --- .../google/cloud/firestore_v1beta1/bidi.py | 3 +- .../cloud/firestore_v1beta1/document.py | 11 +- .../google/cloud/firestore_v1beta1/watch.py | 584 +++++++----------- firestore/tests/system.py | 32 +- 4 files changed, 267 insertions(+), 363 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/bidi.py b/firestore/google/cloud/firestore_v1beta1/bidi.py index 00877e70058e..e7629fc5df8e 100644 --- a/firestore/google/cloud/firestore_v1beta1/bidi.py +++ b/firestore/google/cloud/firestore_v1beta1/bidi.py @@ -196,6 +196,7 @@ def open(self): request_generator = _RequestQueueGenerator( self._request_queue, initial_request=self._initial_request) + print('request generator created') call = self._start_rpc(iter(request_generator)) request_generator.call = call @@ -442,7 +443,7 @@ def on_response(response): print(response) consumer = BackgroundConsumer(rpc, on_response) - consume.start() + consumer.start() Note that error handling *must* be done by using the provided ``bidi_rpc``'s ``add_done_callback``. This helper will automatically exit diff --git a/firestore/google/cloud/firestore_v1beta1/document.py b/firestore/google/cloud/firestore_v1beta1/document.py index 6ebb38e2a365..6c72a1bf48a5 100644 --- a/firestore/google/cloud/firestore_v1beta1/document.py +++ b/firestore/google/cloud/firestore_v1beta1/document.py @@ -423,18 +423,11 @@ def get(self, field_paths=None, transaction=None): [self], field_paths=field_paths, transaction=transaction) return _consume_single_get(snapshot_generator) - def on_snapshot(self, options, callback): + def on_snapshot(self, callback): ''' given options and the callback, monitor this document for changes ''' - #google.firestore.v1beta1.Target.DocumentsTarget - # documentsTarget = Target.DocumentsTarget( - # documents=[self._document_path]) - Watch.for_document(self) - # Watch( - # self._client, - # Target(documents=documentsTarget), - # None) + Watch.for_document(self, callback) class DocumentSnapshot(object): diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 60835260eb21..acf154971dea 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -13,16 +13,15 @@ # limitations under the License. import logging +import threading -#from google.cloud.firestore_v1beta1 import DocumentReference, DocumentSnapshot - -#from google.cloud.firestore_v1beta1.document import DocumentReference -#from google.cloud.firestore_v1beta1.document import DocumentSnapshot #import google.cloud.firestore_v1beta1.client as client -from google.cloud.firestore_v1beta1.bidi import BidiRpc, ResumableBidiRpc +from google.cloud.firestore_v1beta1.bidi import BidiRpc, ResumableBidiRpc, BackgroundConsumer from google.cloud.firestore_v1beta1.proto import firestore_pb2 +from google.api_core import exceptions + -#from bidi import BidiRpc, ResumableBidiRpc +# from bidi import BidiRpc, ResumableBidiRpc import time import random import grpc @@ -53,6 +52,21 @@ 'DATA_LOSS': 15, 'DO_NOT_USE': -1 } +_RPC_ERROR_THREAD_NAME = 'Thread-OnRpcTerminated' +_RETRYABLE_STREAM_ERRORS = ( + exceptions.DeadlineExceeded, + exceptions.ServiceUnavailable, + exceptions.InternalServerError, + exceptions.Unknown, + exceptions.GatewayTimeout +) + + +def _maybe_wrap_exception(exception): + """Wraps a gRPC exception class, if needed.""" + if isinstance(exception, grpc.RpcError): + return exceptions.from_grpc_error(exception) + return exception def is_permanent_error(self, error): @@ -127,12 +141,24 @@ def _sleep(self, current_sleep, max_sleep=_MAX_SLEEP, time.sleep(actual_sleep) return min(self.multiplier * self.current_sleep, self.max_sleep) + class Watch(object): def __init__(self, - firestore, #: client.Client, + document_reference, + firestore, target, - comparator): - + comparator, + on_response): + """ + Args: + firestore: + target: ß + comparator: + on_response: Callback method that reveives a + `google.cloud.firestore_v1beta1.types.ListenResponse` object to + be acted on. + """ + self._document_reference = document_reference self._firestore = firestore self._api = firestore._firestore_api self._targets = target @@ -145,360 +171,218 @@ def should_recover(exc): exc.code() == grpc.StatusCode.UNVAILABLE) initial_request = firestore_pb2.ListenRequest( - database=firestore._database_string, - add_target=target + database=self._firestore._database_string, + add_target=self._targets # database, add_target, remove_target, labels ) rpc = ResumableBidiRpc( - # self._api.firestore_stub.Listen, - #firestore_pb2.BetaFirestoreStub.Listen, self._api.firestore_stub.Listen, initial_request=initial_request, should_recover=should_recover) - rpc.open() + rpc.add_done_callback(self._on_rpc_done) + + def consumer_callback(response): + processed_response = self.process_response(response) + if processed_response: + _LOGGER.debug("running provided callback") + on_response(processed_response) + + self._consumer = BackgroundConsumer(rpc, consumer_callback) + self._consumer.start() + + def _on_rpc_done(self, future): + """Triggered whenever the underlying RPC terminates without recovery. - while rpc.is_active: - print(rpc.recv()) + This is typically triggered from one of two threads: the background + consumer thread (when calling ``recv()`` produces a non-recoverable + error) or the grpc management thread (when cancelling the RPC). + + This method is *non-blocking*. It will start another thread to deal + with shutting everything down. This is to prevent blocking in the + background consumer and preventing it from being ``joined()``. + """ + # TODO: look at pushing this down into the background consumer + _LOGGER.info( + 'RPC termination has signaled shutdown.') + future = _maybe_wrap_exception(future) + thread = threading.Thread( + name=_RPC_ERROR_THREAD_NAME, + target=self.close, + kwargs={'reason': future}) + thread.daemon = True + thread.start() @classmethod - def for_document(cls, document_ref): - return cls(document_ref._client, + def for_document(cls, document_ref, on_response): + """ + Creates a watch snapshot listener for a document. on_response receives + a DocumentChange object, but may also start to get targetChange and such + soon + """ + + + return cls(document_ref, + document_ref._client, { 'documents': { 'documents': [document_ref._document_path]}, 'target_id': WATCH_TARGET_ID }, - document_watch_comparator) - - @classmethod - def for_query(cls, query): - return cls(query._client, - { - 'query': query.to_proto(), - 'target_id': WATCH_TARGET_ID - }, - query.comparator()) - - - # def on_snapshot(self, on_next, on_error): - # doc_dict = {} - # doc_map = {} - # change_map = {} - - # current = False - # has_pushed = False - # is_active = True - - # REMOVED = {} - - # request = {'database': self._firestore.formatted_name, - # 'add_target': self._targets} - - # stream = through.obj() # TODO: fix through (node holdover) - - # current_stream = None - - # def reset_docs(): - # log() - # change_map.clear() - # del resume_token - # for snapshot in doc_dict: - # change_map.set(snapshot.ref.formatted_name, REMOVED) - # current = False - - # def close_stream(err): - # if current_stream is not None: - # current_stream.unpipe(stream) - # current_stream.end() - # current_stream = None - # stream.end() - - # if is_active: - # is_active = False - # _LOGGER.error('Invoking on_error: ', err) - # on_error(err) - - # def maybe_reopen_stream(err): - # if is_active and not is_permanent_error(err): - # _LOGGER.error( - # 'Stream ended, re-opening after retryable error: ', err) - # request.add_target.resume_token = resume_token - # change_map.clear() - - # if is_resource_exhausted_error(err): - # self._backoff.reset_to_max() - # reset_stream() - # else: - # _LOGGER.error('Stream ended, sending error: ', err) - # close_stream(err) - - # def reset_stream(): - # _LOGGER.info('Opening new stream') - # if current_stream: - # current_stream.unpipe(stream) - # current_stream.end() - # current_stream = None - # init_stream() - - # def init_stream(): - # self._backoff.back_off() - # if not is_active: - # _LOGGER.info('Not initializing inactive stream') - # return - - # backend_stream = self._firestore.read_write_stream( - # self._api.Firestore._listen.bind(self._api.Firestore), - # request, - # ) - - # if not is_active: - # _LOGGER.info('Closing inactive stream') - # backend_stream.end() - # _LOGGER.info('Opened new stream') - # current_stream = backend_stream - - # def on_error(err): - # maybe_reopen_stream(err) - - # current_stream.on('error')(on_error) - - # def on_end(): - # err = Exception('Stream ended unexpectedly') - # err.code = GRPC_STATUS_CODE['UNKNOWN'] - # maybe_reopen_stream(err) - - # current_stream.on('end')(on_end) - # current_stream.pipe(stream) - # current_stream.resume() - - # current_stream.catch(close_stream) - - # def affects_target(target_ids, current_id): - # for target_id in target_ids: - # if target_id == current_id: - # return True - # return False - - # def extract_changes(doc_map, changes, read_time): - # deletes = [] - # adds = [] - # updates = [] - - # for value, name in changes: - # if value == REMOVED: - # if doc_map.has(name): - # deletes.append(name) - # elif doc_map.has(name): - # value.read_time = read_time - # updates.append(value.build()) - # else: - # value.read_time = read_time - # adds.append(value.build()) - # return deletes, adds, updates - - # def compute_snapshot(doc_dict, doc_map, changes): - # if len(doc_dict) != doc_map: - # raise ValueError('The document tree and document map should' - # 'have the same number of entries.') - # updated_dict = doc_dict - # updated_map = doc_map - - # def delete_doc(name): - # """ raises KeyError if name not in updated_map""" - # old_document = updated_map.pop(name) # Raises KeyError - # existing = updated_dict.find(old_document) - # old_index = existing.index - # updated_dict = existing.remove() - # return DocumentChange('removed', - # old_document, - # old_index, - # -1) - - # def add_doc(new_document): - # name = new_document.ref.formatted_name - # if name in updated_map: - # raise ValueError('Document to add already exists') - # updated_dict = updated_dict.insert(new_document, null) - # new_index = updated_dict.find(new_document).index - # updated_map[name] = new_document - # return DocumentChange('added', - # new_document, - # -1, - # new_index) - - # def modify_doc(new_document): - # name = new_document.ref.formattedName - # if name not in updated_map: - # raise ValueError('Document to modify does not exsit') - # old_document = updated_map[name] - # if old_document.update_time != new_document.update_time: - # remove_change = delete_doc(name) - # add_change = add_doc(new_document) - # return DocumentChange('modified', - # new_document, - # remove_change.old_index, - # add_change.new_index) - # return None - - # applied_changes = [] - - # def comparator_sort(name1, name2): - # return self._comparator(updated_map[name1], updated_map[name2]) - - # changes.deletes.sort(comparator_sort) - - # for name in changes.deletes: - # changes.delete_doc(name) - # if change: - # applied_changes.push(change) - - # changes.adds.sort(self._compartor) - - # for snapshot in changes.adds: - # change = add_doc(snapshot) - # if change: - # applied_changes.push(change) - - # changes.updates.sort(self._compartor) - - # for snapshot in changes.updates: - # change = modify_doc(snapshot) - # if change: - # applied_changes.push(change) - - # if not len(updated_dict) == len(updated_map): - # raise RuntimeError('The update document tree and document ' - # 'map should have the same number of ' - # 'entries') - - # return {updated_dict, updated_map, applied_changes} - - # def push(read_time, next_resume_token): - # changes = extract_changes(doc_map, change_map, read_time) - # diff = compute_snapshot(doc_dict, doc_map, changes) - - # if not has_pushed or len(diff.applied_changes) > 0: - # _LOGGER.info( - # 'Sending snapshot with %d changes and %d documents' - # % (len(diff.applied_changes), len(updated_dict))) - - # next(read_time, diff.updatedTree.keys, diff.applied_changes) - - # doc_dict = diff.updated_dict - # doc_map = diff.updated_map - # change_map.clear() - # resume_token = next_resume_token - - # def current_size(): - # changes = extract_changes(doc_map, change_map) - # return doc_map.size + len(changes.adds) - len(changes.deletes) - - # init_stream() - - # def proto(): - # if proto.target_change: - # _LOGGER.log('Processing target change') - # change = proto.target_change - # no_target_ids = not target_ids - # if change.target_change_type == 'NO_CHANGE': - # if no_target_ids and change.read_time and current: - # push(DocumentSnapshot.to_ISO_time(change.read_time), - # change.resume_token) - # elif change.target_change_type == 'ADD': - # if WATCH_TARGET_ID != change.target_ids[0]: - # raise ValueError('Unexpected target ID sent by server') - # elif change.target_change_type == 'REMOVE': - # code = 13 - # message = 'internal error' - # if change.cause: - # code = change.cause.code - # message = change.cause.message - # close_stream(Error('Error ' + code + ': ' + message)) - # elif change.target_change_type == 'RESET': - # reset_docs() - # elif change.target_change_type == 'CURRENT': - # current = true - # else: - # close_stream( - # Exception('Unknown target change type: ' + str(change))) - - # stream.on('data', proto) # ?? - - # if change.resume_token and \ - # affects_target(change.target_ids, WATCH_TARGET_ID): - # self._backoff.reset() - - # elif proto.document_change: - # _LOGGER.info('Processing change event') - - # target_ids = proto.document_change.target_ids - # removed_target_ids = proto.document_change.removed_target_ids - - # changed = False - - # removed = False - # for target_id in target_ids: - # if target_id == WATCH_TARGET_ID: - # changed = True - - # for target_id in removed_target_ids: - # if removed_target_ids == WATCH_TARGET_ID: - # removed = True - - # document = proto.document_change.document - # name = document.name - - # if changed: - # _LOGGER.info('Received document change') - # snapshot = DocumentSnapshot.Builder() - # snapshot.ref = DocumentReference( - # self._firestore, - # ResourcePath.from_slash_separated_string(name)) - # snapshot.fields_proto = document.fields - # snapshot.create_time = DocumentSnapshot.to_ISO_time( - # document.create_time) - # snapshot.update_time = DocumentSnapshot.to_ISO_time( - # document.update_time) - # change_map[name] = snapshot - # elif removed: - # _LOGGER.info('Received document remove') - # change_map[name] = REMOVED - # elif proto.document_delete: - # _LOGGER.info('Processing remove event') - # name = proto.document_delete.document - # change_map[name] = REMOVED - # elif proto.document_remove: - # _LOGGER.info('Processing remove event') - # name = proto.document_remove.document - # change_map[name] = REMOVED - # elif proto.filter: - # _LOGGER.info('Processing filter update') - # if proto.filter.count != current_size(): - # reset_docs() - # reset_stream() - # else: - # close_stream(Error('Unknown listen response type: ' + str(proto))) - - # def on_end(): - # _LOGGER.info('Processing stream end') - # if current_stream: - # current_stream.end() - - # on('end', on_end) - - # def initialize(): - # return {} - - # def end_stream(): - # _LOGGER.info('Ending stream') - # is_active = False - # on_next = initialize - # on_error = initialize - # stream.end() - - # return end_stream - - - + document_watch_comparator, + on_response) + + # @classmethod + # def for_query(cls, query, on_response): + # return cls(query._client, + # { + # 'query': query.to_proto(), + # 'target_id': WATCH_TARGET_ID + # }, + # query.comparator(), + # on_response) + + def process_response(self, proto): + """ + Args: + listen_response(`google.cloud.firestore_v1beta1.types.ListenResponse`): + Callback method that reveives a object to + """ + _LOGGER.debug('process_response') + TargetChange = firestore_pb2.TargetChange + + # TODO FIGURE OUT CONDITIONAL OF THIS + _LOGGER.debug(f"STATE: document_change: {proto.document_change} target_change: {proto.target_change} target_change_type: {proto.target_change.target_change_type}") + if str(proto.document_change): + _LOGGER.debug("Document Change") + if str(proto.target_change): + _LOGGER.debug("Target Change") + + if str(proto.target_change): + _LOGGER.info('process_response: Processing target change') + change = proto.target_change + + notarget_ids = change.target_ids is None or len(change.target_ids) + if change.target_change_type == TargetChange.NO_CHANGE: + _LOGGER.info("process_response: " + "Processing target change NO_CHANGE") + # if notarget_ids and change.read_time and current) { + # // This means everything is up-to-date, so emit the current set of + # // docs as a snapshot, if there were changes. + # push( + # DocumentSnapshot.toISOTime(change.readTime), + # change.resumeToken + # ); + # } + elif change.target_change_type == TargetChange.ADD: + _LOGGER.info('process_response: Processing target change ADD') + assert WATCH_TARGET_ID == change.target_ids[0], 'Unexpected target ID sent by server' + + elif change.target_change_type == TargetChange.REMOVE: + _LOGGER.info("process_response: " + "Processing target change REMOVE") + # let code = 13; + # let message = 'internal error'; + # if (change.cause) { + # code = change.cause.code; + # message = change.cause.message; + # } + # // @todo: Surface a .code property on the exception. + # closeStream(new Error('Error ' + code + ': ' + message)); + elif change.target_change_type == TargetChange.RESET: + _LOGGER.info("process_response: " + "Processing target change RESET") + # // Whatever changes have happened so far no longer matter. + # resetDocs(); + elif change.target_change_type == TargetChange.CURRENT: + _LOGGER.info("process_response: " + "Processing target change CURRENT") + # current = True + else: + _LOGGER.info('process_response: Processing target change ELSE') + _LOGGER.info('process_response: Unknown target change ' + + str(change.target_change_type)) + + # closeStream( + # new Error('Unknown target change type: ' + JSON.stringify(change)) + + # if ( + # change.resumeToken and affectsTarget(change.target_ids, WATCH_TARGET_ID) + # ) { + # this._backoff.reset(); + # } + + elif str(proto.document_change): + _LOGGER.debug('Watch.onSnapshot Processing document_change event') + + # No other target_ids can show up here, but we still need to see if the + # targetId was in the added list or removed list. + target_ids = proto.document_change.target_ids or [] + removed_target_ids = proto.document_change.removed_target_ids or [] + changed = False + removed = False + + for target in target_ids: + if target == WATCH_TARGET_ID: + changed = True + + for target in removed_target_ids: + if target == WATCH_TARGET_ID: + removed = True + + document = proto.document_change.document + # name = document.name + + if changed: + _LOGGER.debug('Received document change') + + # reference = DocumentReference( + # self._firestore, + # ResourcePath.fromSlashSeparatedString(name)) + reference = self._document_reference + #create_time = DocumentSnapshot.toISOTime(document.create_time) + #update_time = DocumentSnapshot.toISOTime(document.update_time) + #read_time = DocumentSnapshot.toISOTime(document.read_time) + create_time = document.create_time + update_time = document.update_time + + #read_time = document.read_time + + # TODO: other clients seem to return snapshots + # snapshot = DocumentSnapshot( + # reference, + # document.fields, # DATA? + # exists=True, + # read_time=read_time, + # create_time=create_time, + # update_time=update_time) + # #changeMap.set(name, snapshot); + # return snapshot + return document + + elif removed: + _LOGGER.debug('Watch.onSnapshot Received document remove') + # changeMap.set(name, REMOVED); + + + # Document Delete or Document Remove? + elif (proto.document_delete or proto.document_remove): + _LOGGER.debug('Watch.onSnapshot Processing remove event') + # const name = (proto.document_delete || proto.document_remove).document + # changeMap.set(name, REMOVED); + + elif (proto.filter): + _LOGGER.debug('Watch.onSnapshot Processing filter update') + # if (proto.filter.count !== currentSize()) { + # // We need to remove all the current results. + # resetDocs(); + # // The filter didn't match, so re-issue the query. + # resetStream(); + + else: + _LOGGER.debug("UNKNOWN TYPE. UHOH") + # closeStream( + # new Error('Unknown listen response type: ' + JSON.stringify(proto)) + # ) + \ No newline at end of file diff --git a/firestore/tests/system.py b/firestore/tests/system.py index 5926c79014c1..ee42a5cf2bc2 100644 --- a/firestore/tests/system.py +++ b/firestore/tests/system.py @@ -32,6 +32,7 @@ from google.cloud import firestore from test_utils.system import unique_resource_id +from time import sleep FIRESTORE_CREDS = os.environ.get('FIRESTORE_APPLICATION_CREDENTIALS') FIRESTORE_PROJECT = os.environ.get('GCLOUD_PROJECT') @@ -58,15 +59,40 @@ def cleanup(): def test_watch_document(client, cleanup): - # Add a new document db = client - doc_ref = db.collection(u'users').document(u'alovelace') + doc_ref = db.collection(u'users').document( + u'alovelace' + unique_resource_id()) + + # Initial setting + doc_ref.set({ + u'first': u'Jane', + u'last': u'Doe', + u'born': 1900 + }) + + sleep(1) + + # Setup listener + def on_response(response): + on_response.called_count += 1 + print(f'Response: {response}') + print(type(response)) + + on_response.called_count = 0 + + doc_ref.on_snapshot(on_response) + + # Alter document doc_ref.set({ u'first': u'Ada', u'last': u'Lovelace', u'born': 1815 }) - doc_ref.on_snapshot(None, None) + + sleep(1) + if on_response.called_count != 1: + raise AssertionError("Failed to get exactly one document change") + # def test_create_document(client, cleanup): From ff7aad7b969046c3eb15461d163b480b489a36a2 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Mon, 9 Jul 2018 17:16:09 -0700 Subject: [PATCH 007/148] returning watch result now --- .../cloud/firestore_v1beta1/document.py | 6 +- .../google/cloud/firestore_v1beta1/watch.py | 221 ++++++++++-------- .../pubsub_v1/subscriber/_protocol/bidi.py | 2 +- 3 files changed, 128 insertions(+), 101 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/document.py b/firestore/google/cloud/firestore_v1beta1/document.py index 6c72a1bf48a5..cbbf6e02f13f 100644 --- a/firestore/google/cloud/firestore_v1beta1/document.py +++ b/firestore/google/cloud/firestore_v1beta1/document.py @@ -19,7 +19,7 @@ from google.cloud.firestore_v1beta1 import _helpers from google.cloud.firestore_v1beta1.watch import Watch -from google.cloud.firestore_v1beta1.proto.firestore_pb2 import Target + class DocumentReference(object): """A reference to a document in a Firestore database. @@ -427,8 +427,8 @@ def on_snapshot(self, callback): ''' given options and the callback, monitor this document for changes ''' - Watch.for_document(self, callback) - + Watch.for_document(self, callback, DocumentSnapshot) + class DocumentSnapshot(object): """A snapshot of document data in a Firestore database. diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index acf154971dea..d2faaecdf8bb 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -14,11 +14,13 @@ import logging import threading +from enum import Enum -#import google.cloud.firestore_v1beta1.client as client -from google.cloud.firestore_v1beta1.bidi import BidiRpc, ResumableBidiRpc, BackgroundConsumer +from google.cloud.firestore_v1beta1.bidi import ResumableBidiRpc +from google.cloud.firestore_v1beta1.bidi import BackgroundConsumer from google.cloud.firestore_v1beta1.proto import firestore_pb2 from google.api_core import exceptions +from google.protobuf import json_format # from bidi import BidiRpc, ResumableBidiRpc @@ -142,21 +144,36 @@ def _sleep(self, current_sleep, max_sleep=_MAX_SLEEP, return min(self.multiplier * self.current_sleep, self.max_sleep) +class WatchChangeType(Enum): + ADDED = 0 + MODIFIED = 1 + REMOVED = 2 + + +class WatchResult(object): + def __init__(self, snapshot, name, change_type): + self.snapshot = snapshot + self.name = name + self.change_type = change_type + + class Watch(object): - def __init__(self, + def __init__(self, document_reference, firestore, target, comparator, - on_response): + on_snapshot, + DocumentSnapshotCls): """ Args: firestore: - target: ß + target: comparator: - on_response: Callback method that reveives a - `google.cloud.firestore_v1beta1.types.ListenResponse` object to - be acted on. + on_snapshot: Callback method that receives two arguments, + list(snapshots) and + list(tuple(document_id, change_type)) + DocumentSnapshotCls: instance of the DocumentSnapshot class """ self._document_reference = document_reference self._firestore = firestore @@ -164,6 +181,7 @@ def __init__(self, self._targets = target self._comparator = comparator self._backoff = ExponentialBackOff() + self.DocumentSnapshot = DocumentSnapshotCls def should_recover(exc): return ( @@ -173,23 +191,22 @@ def should_recover(exc): initial_request = firestore_pb2.ListenRequest( database=self._firestore._database_string, add_target=self._targets - # database, add_target, remove_target, labels ) - rpc = ResumableBidiRpc( + self.rpc = ResumableBidiRpc( self._api.firestore_stub.Listen, initial_request=initial_request, should_recover=should_recover) - rpc.add_done_callback(self._on_rpc_done) + self.rpc.add_done_callback(self._on_rpc_done) def consumer_callback(response): processed_response = self.process_response(response) if processed_response: _LOGGER.debug("running provided callback") - on_response(processed_response) + on_snapshot(processed_response) - self._consumer = BackgroundConsumer(rpc, consumer_callback) + self._consumer = BackgroundConsumer(self.rpc, consumer_callback) self._consumer.start() def _on_rpc_done(self, future): @@ -215,14 +232,19 @@ def _on_rpc_done(self, future): thread.start() @classmethod - def for_document(cls, document_ref, on_response): + def for_document(cls, document_ref, on_snapshot, snapshot_class_instance): """ - Creates a watch snapshot listener for a document. on_response receives + Creates a watch snapshot listener for a document. on_snapshot receives a DocumentChange object, but may also start to get targetChange and such soon - """ + Args: + document_ref: Reference to Document + on_snapshot: callback to be called on snapshot + snapshot_class_instance: instance of snapshot cls to make snapshots with to + pass to on_snapshot + """ return cls(document_ref, document_ref._client, { @@ -231,90 +253,93 @@ def for_document(cls, document_ref, on_response): 'target_id': WATCH_TARGET_ID }, document_watch_comparator, - on_response) + on_snapshot, + snapshot_class_instance) # @classmethod - # def for_query(cls, query, on_response): + # def for_query(cls, query, on_snapshot): # return cls(query._client, # { # 'query': query.to_proto(), # 'target_id': WATCH_TARGET_ID # }, # query.comparator(), - # on_response) - + # on_snapshot) + def process_response(self, proto): """ Args: listen_response(`google.cloud.firestore_v1beta1.types.ListenResponse`): - Callback method that reveives a object to + Callback method that receives a object to """ - _LOGGER.debug('process_response') TargetChange = firestore_pb2.TargetChange - - # TODO FIGURE OUT CONDITIONAL OF THIS - _LOGGER.debug(f"STATE: document_change: {proto.document_change} target_change: {proto.target_change} target_change_type: {proto.target_change.target_change_type}") - if str(proto.document_change): - _LOGGER.debug("Document Change") - if str(proto.target_change): - _LOGGER.debug("Target Change") if str(proto.target_change): - _LOGGER.info('process_response: Processing target change') - change = proto.target_change + _LOGGER.debug('process_response: Processing target change') + + change = proto.target_change # google.cloud.firestore_v1beta1.types.TargetChange notarget_ids = change.target_ids is None or len(change.target_ids) if change.target_change_type == TargetChange.NO_CHANGE: - _LOGGER.info("process_response: " - "Processing target change NO_CHANGE") - # if notarget_ids and change.read_time and current) { - # // This means everything is up-to-date, so emit the current set of - # // docs as a snapshot, if there were changes. - # push( - # DocumentSnapshot.toISOTime(change.readTime), - # change.resumeToken - # ); - # } + _LOGGER.debug("process_response: target change: NO_CHANGE") + if notarget_ids and change.read_time: # and current: # current is used to reflect if the local copy of tree is accurate? + # This means everything is up-to-date, so emit the current set of + # docs as a snapshot, if there were changes. + # push( + # DocumentSnapshot.toISOTime(change.readTime), + # change.resumeToken + # ); + # } + # For now, we can do nothing here since there isn't anything to do + # eventually it seems it makes sens to record this as a snapshot? + # TODO : node calls the callback with no change? + pass elif change.target_change_type == TargetChange.ADD: - _LOGGER.info('process_response: Processing target change ADD') + _LOGGER.debug("process_response: target change: ADD") assert WATCH_TARGET_ID == change.target_ids[0], 'Unexpected target ID sent by server' + # TODO : do anything here? + return WatchResult( + None, + self._document_reference.id, + WatchChangeType.ADDED) elif change.target_change_type == TargetChange.REMOVE: - _LOGGER.info("process_response: " - "Processing target change REMOVE") - # let code = 13; - # let message = 'internal error'; - # if (change.cause) { - # code = change.cause.code; - # message = change.cause.message; - # } - # // @todo: Surface a .code property on the exception. - # closeStream(new Error('Error ' + code + ': ' + message)); + _LOGGER.debug("process_response: target change: REMOVE") + + code = 13 + message = 'internal error' + if change.cause: + code = change.cause.code + message = change.cause.message + + # TODO: Surface a .code property on the exception. + raise Exception('Error ' + code + ': ' + message) elif change.target_change_type == TargetChange.RESET: - _LOGGER.info("process_response: " - "Processing target change RESET") + _LOGGER.debug("process_response: target change: RESET") + # // Whatever changes have happened so far no longer matter. - # resetDocs(); + # resetDocs(); # TODO + # TODO : do something here? elif change.target_change_type == TargetChange.CURRENT: - _LOGGER.info("process_response: " - "Processing target change CURRENT") - # current = True + _LOGGER.debug("process_response: target change: CURRENT") + + # current = True # TODO + # TODO: do something here? else: - _LOGGER.info('process_response: Processing target change ELSE') _LOGGER.info('process_response: Unknown target change ' + str(change.target_change_type)) # closeStream( - # new Error('Unknown target change type: ' + JSON.stringify(change)) + # new Error('Unknown target change type: ' + JSON.stringify(change)) + # TODO : make this exit the inner function and stop processing? + raise Exception('Unknown target change type: ' + str(change)) - # if ( - # change.resumeToken and affectsTarget(change.target_ids, WATCH_TARGET_ID) - # ) { - # this._backoff.reset(); - # } + if change.resume_token and self._affects_target(change.target_ids, + WATCH_TARGET_ID): + self._backoff.reset() elif str(proto.document_change): - _LOGGER.debug('Watch.onSnapshot Processing document_change event') + _LOGGER.debug('process_response: Processing document change') # No other target_ids can show up here, but we still need to see if the # targetId was in the added list or removed list. @@ -331,47 +356,40 @@ def process_response(self, proto): if target == WATCH_TARGET_ID: removed = True - document = proto.document_change.document - # name = document.name - if changed: _LOGGER.debug('Received document change') - # reference = DocumentReference( - # self._firestore, - # ResourcePath.fromSlashSeparatedString(name)) - reference = self._document_reference - #create_time = DocumentSnapshot.toISOTime(document.create_time) - #update_time = DocumentSnapshot.toISOTime(document.update_time) - #read_time = DocumentSnapshot.toISOTime(document.read_time) - create_time = document.create_time - update_time = document.update_time - - #read_time = document.read_time - - # TODO: other clients seem to return snapshots - # snapshot = DocumentSnapshot( - # reference, - # document.fields, # DATA? - # exists=True, - # read_time=read_time, - # create_time=create_time, - # update_time=update_time) - # #changeMap.set(name, snapshot); - # return snapshot - return document - + # google.cloud.firestore_v1beta1.types.DocumentChange + document_change = proto.document_change + # google.cloud.firestore_v1beta1.types.Document + document = document_change.document + + data = json_format.MessageToDict(document) + + snapshot = self.DocumentSnapshot( + reference=self._document_reference, + data=data['fields'], + exists=True, + read_time=None, + create_time=document.create_time, + update_time=document.update_time) + + return WatchResult(snapshot, + self._document_reference.id, + WatchChangeType.MODIFIED) + elif removed: _LOGGER.debug('Watch.onSnapshot Received document remove') # changeMap.set(name, REMOVED); - # Document Delete or Document Remove? elif (proto.document_delete or proto.document_remove): _LOGGER.debug('Watch.onSnapshot Processing remove event') # const name = (proto.document_delete || proto.document_remove).document # changeMap.set(name, REMOVED); - + return WatchResult(None, + self._document_reference.id, + WatchChangeType.REMOVED) elif (proto.filter): _LOGGER.debug('Watch.onSnapshot Processing filter update') # if (proto.filter.count !== currentSize()) { @@ -379,10 +397,19 @@ def process_response(self, proto): # resetDocs(); # // The filter didn't match, so re-issue the query. # resetStream(); - + else: _LOGGER.debug("UNKNOWN TYPE. UHOH") # closeStream( # new Error('Unknown listen response type: ' + JSON.stringify(proto)) # ) - \ No newline at end of file + + def _affects_target(self, target_ids, current_id): + if target_ids is None or len(target_ids) == 0: + return True + + for target_id in target_ids: + if target_id == current_id: + return True + + return False diff --git a/pubsub/google/cloud/pubsub_v1/subscriber/_protocol/bidi.py b/pubsub/google/cloud/pubsub_v1/subscriber/_protocol/bidi.py index 00877e70058e..331347836ca2 100644 --- a/pubsub/google/cloud/pubsub_v1/subscriber/_protocol/bidi.py +++ b/pubsub/google/cloud/pubsub_v1/subscriber/_protocol/bidi.py @@ -442,7 +442,7 @@ def on_response(response): print(response) consumer = BackgroundConsumer(rpc, on_response) - consume.start() + consumer.start() Note that error handling *must* be done by using the provided ``bidi_rpc``'s ``add_done_callback``. This helper will automatically exit From c59766988c732f976f8327025f4b1d395b73f034 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Tue, 10 Jul 2018 16:33:22 -0700 Subject: [PATCH 008/148] broken currently, but nearing handling of multiple documents in collection --- .../google/cloud/firestore_v1beta1/watch.py | 479 ++++++++++++++---- 1 file changed, 380 insertions(+), 99 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index d2faaecdf8bb..a266c255a2a6 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -13,7 +13,9 @@ # limitations under the License. import logging +import collections import threading +import datetime from enum import Enum from google.cloud.firestore_v1beta1.bidi import ResumableBidiRpc @@ -63,6 +65,62 @@ exceptions.GatewayTimeout ) +DocTreeEntry = collections.namedtuple('DocTreeEntry', ['value', 'index']) + + +class WatchDocTree(object): + def __init__(self): + self._dict = {} + self._index = 0 + + def keys(self): + return list(self._dict.keys()) + + def insert(self, key, value): + self._dict[key] = DocTreeEntry(value, self._index) + self._index += 1 + return self + + def find(self, key): + return self._dict[key] + + def remove(self, key): + del self._dict[key] + return self + + def __len__(self): + return len(self._dict) + + +class ChangeType(Enum): + ADDED = 0 + MODIFIED = 1 + REMOVED = 2 + + +class DocumentChange(object): + def __init__(self, type, document, old_index, new_index): + """DocumentChange + + Args: + type (ChangeType): + document (document.DocumentSnapshot): + old_index (int): + new_index (int): + """ + # TODO: spec indicated an isEqual param also + self.type = type + self.document = document + self.old_index = old_index + self.new_index = new_index + + +class WatchResult(object): + def __init__(self, snapshot, name, change_type): + self.snapshot = snapshot + self.name = name + self.change_type = change_type + def _maybe_wrap_exception(exception): """Wraps a gRPC exception class, if needed.""" @@ -122,8 +180,8 @@ def _sleep(self, current_sleep, max_sleep=_MAX_SLEEP, multiplier=_MULTIPLIER): """Sleep and produce a new sleep time. - .. _Exponential Backoff And Jitter: https://www.awsarchitectureblog.com/\ - 2015/03/backoff.html + .. _Exponential Backoff And Jitter: + https://www.awsarchitectureblog.com/2015/03/backoff.html Select a duration between zero and ``current_sleep``. It might seem counterintuitive to have so much jitter, but @@ -144,35 +202,30 @@ def _sleep(self, current_sleep, max_sleep=_MAX_SLEEP, return min(self.multiplier * self.current_sleep, self.max_sleep) -class WatchChangeType(Enum): - ADDED = 0 - MODIFIED = 1 - REMOVED = 2 - - -class WatchResult(object): - def __init__(self, snapshot, name, change_type): - self.snapshot = snapshot - self.name = name - self.change_type = change_type - - class Watch(object): def __init__(self, document_reference, firestore, target, comparator, - on_snapshot, + snapshot_callback, DocumentSnapshotCls): """ Args: firestore: target: comparator: - on_snapshot: Callback method that receives two arguments, - list(snapshots) and - list(tuple(document_id, change_type)) + snapshot_callback: Callback method to process snapshots. + Args: + docs (List(DocumentSnapshot)): A callback that returns the + ordered list of documents stored in this snapshot. + changes (List(str)): A callback that returns the list of + changed documents since the last snapshot delivered for + this watch. + read_time (string): The ISO 8601 time at which this + snapshot was obtained. + # TODO: Go had an err here and node.js provided size. + # TODO: do we want to include either? DocumentSnapshotCls: instance of the DocumentSnapshot class """ self._document_reference = document_reference @@ -180,8 +233,8 @@ def __init__(self, self._api = firestore._firestore_api self._targets = target self._comparator = comparator - self._backoff = ExponentialBackOff() self.DocumentSnapshot = DocumentSnapshotCls + self._snapshot_callback = snapshot_callback def should_recover(exc): return ( @@ -200,13 +253,32 @@ def should_recover(exc): self.rpc.add_done_callback(self._on_rpc_done) - def consumer_callback(response): - processed_response = self.process_response(response) - if processed_response: - _LOGGER.debug("running provided callback") - on_snapshot(processed_response) + # Initialize state for on_snapshot + # The sorted tree of QueryDocumentSnapshots as sent in the last + # snapshot. We only look at the keys. + # TODO: using ordered dict right now but not great maybe + self.doc_tree = WatchDocTree() # TODO: rbtree(this._comparator) + + # A map of document names to QueryDocumentSnapshots for the last sent + # snapshot. + self.doc_map = {} + + # The accumulates map of document changes (keyed by document name) for + # the current snapshot. + self.change_map = {} - self._consumer = BackgroundConsumer(self.rpc, consumer_callback) + # The current state of the query results. + self.current = False + + # We need this to track whether we've pushed an initial set of changes, + # since we should push those even when there are no changes, if there + # aren't docs. + self.has_pushed = False + + # The server assigns and updates the resume token. + self.resume_token = None + + self._consumer = BackgroundConsumer(self.rpc, self.on_snapshot) self._consumer.start() def _on_rpc_done(self, future): @@ -232,17 +304,18 @@ def _on_rpc_done(self, future): thread.start() @classmethod - def for_document(cls, document_ref, on_snapshot, snapshot_class_instance): + def for_document(cls, document_ref, snapshot_callback, + snapshot_class_instance): """ - Creates a watch snapshot listener for a document. on_snapshot receives - a DocumentChange object, but may also start to get targetChange and such - soon + Creates a watch snapshot listener for a document. snapshot_callback + receives a DocumentChange object, but may also start to get + targetChange and such soon Args: document_ref: Reference to Document - on_snapshot: callback to be called on snapshot - snapshot_class_instance: instance of snapshot cls to make snapshots with to - pass to on_snapshot + snapshot_callback: callback to be called on snapshot + snapshot_class_instance: instance of snapshot cls to make + snapshots with to pass to snapshot_callback """ return cls(document_ref, @@ -253,21 +326,25 @@ def for_document(cls, document_ref, on_snapshot, snapshot_class_instance): 'target_id': WATCH_TARGET_ID }, document_watch_comparator, - on_snapshot, + snapshot_callback, snapshot_class_instance) # @classmethod - # def for_query(cls, query, on_snapshot): + # def for_query(cls, query, snapshot_callback): # return cls(query._client, # { # 'query': query.to_proto(), # 'target_id': WATCH_TARGET_ID # }, # query.comparator(), - # on_snapshot) + # snapshot_callback) - def process_response(self, proto): + def on_snapshot(self, proto): """ + Called everytime there is a response from listen. Collect changes + and 'push' the changes in a batch to the customer when we receive + 'current' from the listen response. + Args: listen_response(`google.cloud.firestore_v1beta1.types.ListenResponse`): Callback method that receives a object to @@ -275,36 +352,34 @@ def process_response(self, proto): TargetChange = firestore_pb2.TargetChange if str(proto.target_change): - _LOGGER.debug('process_response: Processing target change') + _LOGGER.debug('on_snapshot: target change') - change = proto.target_change # google.cloud.firestore_v1beta1.types.TargetChange + # google.cloud.firestore_v1beta1.types.TargetChange + change = proto.target_change - notarget_ids = change.target_ids is None or len(change.target_ids) + no_target_ids = change.target_ids is None or \ + len(change.target_ids) == 0 if change.target_change_type == TargetChange.NO_CHANGE: - _LOGGER.debug("process_response: target change: NO_CHANGE") - if notarget_ids and change.read_time: # and current: # current is used to reflect if the local copy of tree is accurate? - # This means everything is up-to-date, so emit the current set of - # docs as a snapshot, if there were changes. - # push( - # DocumentSnapshot.toISOTime(change.readTime), - # change.resumeToken - # ); - # } - # For now, we can do nothing here since there isn't anything to do - # eventually it seems it makes sens to record this as a snapshot? - # TODO : node calls the callback with no change? - pass + _LOGGER.debug('on_snapshot: target change: NO_CHANGE') + if no_target_ids and change.read_time and self.current: + # TargetChange.CURRENT followed by TargetChange.NO_CHANGE + # signals a consistent state. Invoke the onSnapshot + # callback as specified by the user. + self.push(change.read_time, change.resume_token) elif change.target_change_type == TargetChange.ADD: - _LOGGER.debug("process_response: target change: ADD") - assert WATCH_TARGET_ID == change.target_ids[0], 'Unexpected target ID sent by server' - # TODO : do anything here? - - return WatchResult( - None, - self._document_reference.id, - WatchChangeType.ADDED) + _LOGGER.debug("on_snapshot: target change: ADD") + assert WATCH_TARGET_ID == change.target_ids[0], \ + 'Unexpected target ID sent by server' + # TODO : do anything here? Node didn't so I think this isn't + # the right thing to do + # wr = WatchResult( + # None, + # self._document_reference.id, + # ChangeType.ADDED) + # self._snapshot_callback(wr) + elif change.target_change_type == TargetChange.REMOVE: - _LOGGER.debug("process_response: target change: REMOVE") + _LOGGER.debug("on_snapshot: target change: REMOVE") code = 13 message = 'internal error' @@ -315,34 +390,34 @@ def process_response(self, proto): # TODO: Surface a .code property on the exception. raise Exception('Error ' + code + ': ' + message) elif change.target_change_type == TargetChange.RESET: - _LOGGER.debug("process_response: target change: RESET") - - # // Whatever changes have happened so far no longer matter. - # resetDocs(); # TODO - # TODO : do something here? + # Whatever changes have happened so far no longer matter. + _LOGGER.debug("on_snapshot: target change: RESET") + self._reset_docs() elif change.target_change_type == TargetChange.CURRENT: - _LOGGER.debug("process_response: target change: CURRENT") - - # current = True # TODO - # TODO: do something here? + _LOGGER.debug("on_snapshot: target change: CURRENT") + self.current = True else: - _LOGGER.info('process_response: Unknown target change ' + + _LOGGER.info('on_snapshot: Unknown target change ' + str(change.target_change_type)) + self._consumer.stop() # closeStream( - # new Error('Unknown target change type: ' + JSON.stringify(change)) + # new Error('Unknown target change type: ' + + # JSON.stringify(change)) # TODO : make this exit the inner function and stop processing? raise Exception('Unknown target change type: ' + str(change)) if change.resume_token and self._affects_target(change.target_ids, WATCH_TARGET_ID): - self._backoff.reset() + # TODO: they node version resets backoff here. We allow + # bidi rpc to do its thing. + pass elif str(proto.document_change): - _LOGGER.debug('process_response: Processing document change') + _LOGGER.debug('on_snapshot: document change') - # No other target_ids can show up here, but we still need to see if the - # targetId was in the added list or removed list. + # No other target_ids can show up here, but we still need to see + # if the targetId was in the added list or removed list. target_ids = proto.document_change.target_ids or [] removed_target_ids = proto.document_change.removed_target_ids or [] changed = False @@ -357,7 +432,7 @@ def process_response(self, proto): removed = True if changed: - _LOGGER.debug('Received document change') + _LOGGER.debug('on_snapshot: document change: CHANGED') # google.cloud.firestore_v1beta1.types.DocumentChange document_change = proto.document_change @@ -374,35 +449,215 @@ def process_response(self, proto): create_time=document.create_time, update_time=document.update_time) - return WatchResult(snapshot, - self._document_reference.id, - WatchChangeType.MODIFIED) + self.change_map[document.name] = snapshot + # TODO: ensure we call this later, on current returend. + # wr = WatchResult(snapshot, + # self._document_reference.id, + # ChangeType.MODIFIED) + # self._snapshot_callback(wr) elif removed: - _LOGGER.debug('Watch.onSnapshot Received document remove') - # changeMap.set(name, REMOVED); + _LOGGER.debug('on_snapshot: document change: REMOVED') + self.change_map[document.name] = ChangeType.REMOVED - # Document Delete or Document Remove? elif (proto.document_delete or proto.document_remove): - _LOGGER.debug('Watch.onSnapshot Processing remove event') - # const name = (proto.document_delete || proto.document_remove).document - # changeMap.set(name, REMOVED); - return WatchResult(None, - self._document_reference.id, - WatchChangeType.REMOVED) + _LOGGER.debug('on_snapshot: document change: DELETE/REMOVE') + name = (proto.document_delete or proto.document_remove).document + self.change_map[name] = ChangeType.REMOVED + # wr = WatchResult(None, + # self._document_reference.id, + # ChangeType.REMOVED) + # self._snapshot_callback(wr) + elif (proto.filter): - _LOGGER.debug('Watch.onSnapshot Processing filter update') - # if (proto.filter.count !== currentSize()) { - # // We need to remove all the current results. - # resetDocs(); - # // The filter didn't match, so re-issue the query. - # resetStream(); + _LOGGER.debug('on_snapshot: filter update') + if proto.filter.count != self._current_size(): + # We need to remove all the current results. + self._reset_docs() + # The filter didn't match, so re-issue the query. + # TODO: reset stream method? + # self._reset_stream(); else: _LOGGER.debug("UNKNOWN TYPE. UHOH") - # closeStream( - # new Error('Unknown listen response type: ' + JSON.stringify(proto)) - # ) + self._consumer.stop() + raise Exception( + 'Unknown listen response type: ' + proto) + # TODO: can we stop but raise an error? + # closeStream( + # new Error('Unknown listen response type: ' + + # JSON.stringify(proto)) + # ) + + def push(self, read_time, next_resume_token): + """ + Assembles a new snapshot from the current set of changes and invokes + the user's callback. Clears the current changes on completion. + """ + # TODO: may need to lock here to avoid races on collecting snapshots + # and sending them to the user. + + deletes, adds, updates = Watch._extract_changes( + self.doc_map, self.change_map, read_time) + updated_tree, updated_map, appliedChanges = \ + Watch._compute_snapshot( + self.doc_tree, self.doc_map, deletes, adds, updates) +# _LOGGER.debug(f"""push +# self.doc_map {self.doc_map} +# self.change_map {self.change_map} +# read_time {read_time} +# deletes {deletes} +# adds {adds} +# updates {updates} +# updated_tree {updated_tree} +# """) + if not self.has_pushed or len(appliedChanges): + _LOGGER.debug( + f'Sending snapshot with {len(appliedChanges)} changes' + f' and {len(updated_tree)} documents') + + _LOGGER.debug(f"updatedTree:{updated_tree}") + self._snapshot_callback( + updated_tree.keys(), + appliedChanges, + datetime.datetime.fromtimestamp(read_time.seconds) + ) + self.has_pushed = True + + self.doc_tree = updated_tree + self.doc_map = updated_map + self.change_map.clear() + self.resume_token = next_resume_token + + def _extract_changes(doc_map, changes, read_time): + deletes = [] + adds = [] + updates = [] + + for name, value in changes.items(): + if value == ChangeType.REMOVED: + if name in doc_map: + deletes.append(name) + elif name in doc_map: + value.read_time = read_time + updates.append(value) + else: + value.read_time = read_time + adds.append(value) + _LOGGER.debug(f'deletes:{len(deletes)} adds:{len(adds)}') + return (deletes, adds, updates) + + def _compute_snapshot(doc_tree, doc_map, delete_changes, add_changes, + update_changes): + # TODO: ACTUALLY NEED TO CALCULATE + # return {updated_tree, updated_map, appliedChanges}; + # return doc_tree, doc_map, changes + + updated_tree = doc_tree + updated_map = doc_map + + assert len(doc_tree) == len(doc_map), \ + 'The document tree and document map should have the same ' + \ + 'number of entries.' + + def delete_doc(name, updated_tree, updated_map): + """ + Applies a document delete to the document tree and document map. + Returns the corresponding DocumentChange event. + """ + assert name in updated_map(name), 'Document to delete does not exist' + old_document = updated_map.get(name) + existing = updated_tree.find(old_document) + old_index = existing.index + # TODO: was existing.remove returning tree (presumably immuatable?) + updated_tree = updated_tree.remove(old_document) + updated_map.delete(name) + return (DocumentChange(ChangeType.REMOVED, + old_document, + old_index, + -1), + updated_tree, updated_map) + + def add_doc(new_document, updated_tree, updated_map): + """ + Applies a document add to the document tree and the document map. + Returns the corresponding DocumentChange event. + """ + name = new_document.reference._document_path + assert name not in updated_map, 'Document to add already exists' + updated_tree = updated_tree.insert(new_document, None) + new_index = updated_tree.find(new_document).index + updated_map[name] = new_document + return (DocumentChange(ChangeType.ADDED, + new_document, + -1, + new_index), + updated_tree, updated_map) + + def modify_doc(new_document, updated_tree, updated_map): + """ + Applies a document modification to the document tree and the + document map. + Returns the DocumentChange event for successful modifications. + """ + name = new_document.ref.formattedName + assert updated_map.has(name), 'Document to modify does not exist' + oldDocument = updated_map.get(name) + if oldDocument.updateTime != new_document.updateTime: + removeChange, updated_tree, updated_map = delete_doc( + name, updated_tree, updated_map) + addChange, updated_tree, updated_map = add_doc( + new_document, updated_tree, updated_map) + return (DocumentChange(ChangeType.MODIFIED, + new_document, + removeChange.old_index, + addChange.new_index), + updated_tree, updated_map) + + return None + + # Process the sorted changes in the order that is expected by our + # clients (removals, additions, and then modifications). We also need + # to sort the individual changes to assure that old_index/new_index + # keep incrementing. + appliedChanges = [] + + # Deletes are sorted based on the order of the existing document. + + # TODO: SORT + # delete_changes.sort( + # lambda name1, name2: + # self._comparator(updated_map.get(name1), updated_map.get(name2))) + + for name in delete_changes: + change, updated_tree, updated_map = delete_doc( + name, updated_tree, updated_map) + if change: + appliedChanges.append(change) + + # TODO: SORT + # add_changes.sort(self._comparator) + _LOGGER.debug('walk over add_changes') + for snapshot in add_changes: + _LOGGER.debug('in add_changes') + change, updated_tree, updated_map = add_doc( + snapshot, updated_tree, updated_map) + if change: + appliedChanges.append(change) + + # TODO: SORT + # update_changes.sort(self._comparator) + for snapshot in update_changes: + change, updated_tree, updated_map = modify_doc( + snapshot, updated_tree, updated_map) + if change: + appliedChanges.append(change) + + assert len(updated_tree) == len(updated_map), \ + 'The update document ' + \ + 'tree and document map should have the same number of entries.' + _LOGGER.debug(f"tree:{updated_tree}, map:{updated_map}, applied:{appliedChanges}") + return (updated_tree, updated_map, appliedChanges) def _affects_target(self, target_ids, current_id): if target_ids is None or len(target_ids) == 0: @@ -413,3 +668,29 @@ def _affects_target(self, target_ids, current_id): return True return False + + def _current_size(self): + """ + Returns the current count of all documents, including the changes from + the current changeMap. + """ + deletes, adds, _ = Watch._extract_changes(self.docMap, self.changeMap) + return self.docMap.size + len(adds) - len(deletes) + + def _reset_docs(self): + """ + Helper to clear the docs on RESET or filter mismatch. + """ + _LOGGER.debug("resetting documents") + self.change_map.clear() + self.resume_token = None + + # TODO: mark each document as deleted. If documents are not delete + # they will be sent again by the server. + # docTree.forEach(snapshot => { + # // Mark each document as deleted. If documents are not deleted, + # // they + # // will be send again by the server. + # changeMap.set(snapshot.ref.formattedName, REMOVED); + + self.current = False From 048b5fafbbb10e04eca2c961e5064e38a25e5db6 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Tue, 10 Jul 2018 17:18:54 -0700 Subject: [PATCH 009/148] small fixes. seems mostly working --- firestore/google/cloud/firestore_v1beta1/watch.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index a266c255a2a6..bfb9b8d848db 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -76,7 +76,15 @@ def __init__(self): def keys(self): return list(self._dict.keys()) + def _copy(self): + wdt = WatchDocTree() + wdt._dict = self._dict.copy() + wdt._index = self._index + self = wdt + return self + def insert(self, key, value): + self = self._copy() self._dict[key] = DocTreeEntry(value, self._index) self._index += 1 return self @@ -85,6 +93,7 @@ def find(self, key): return self._dict[key] def remove(self, key): + self = self._copy() del self._dict[key] return self @@ -565,13 +574,13 @@ def delete_doc(name, updated_tree, updated_map): Applies a document delete to the document tree and document map. Returns the corresponding DocumentChange event. """ - assert name in updated_map(name), 'Document to delete does not exist' + assert name in updated_map, 'Document to delete does not exist' old_document = updated_map.get(name) existing = updated_tree.find(old_document) old_index = existing.index # TODO: was existing.remove returning tree (presumably immuatable?) updated_tree = updated_tree.remove(old_document) - updated_map.delete(name) + del updated_map[name] return (DocumentChange(ChangeType.REMOVED, old_document, old_index, From a2fdd186b6cbca8c0fa36580d4d023d39f51adbf Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Wed, 11 Jul 2018 09:07:47 -0700 Subject: [PATCH 010/148] fix variable name, remove things from doc map on remove --- firestore/google/cloud/firestore_v1beta1/watch.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index bfb9b8d848db..15ef7e8b0a90 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -609,7 +609,7 @@ def modify_doc(new_document, updated_tree, updated_map): document map. Returns the DocumentChange event for successful modifications. """ - name = new_document.ref.formattedName + name = new_document.reference.formattedName assert updated_map.has(name), 'Document to modify does not exist' oldDocument = updated_map.get(name) if oldDocument.updateTime != new_document.updateTime: @@ -696,10 +696,8 @@ def _reset_docs(self): # TODO: mark each document as deleted. If documents are not delete # they will be sent again by the server. - # docTree.forEach(snapshot => { - # // Mark each document as deleted. If documents are not deleted, - # // they - # // will be send again by the server. - # changeMap.set(snapshot.ref.formattedName, REMOVED); + for snapshot in self.doc_tree: + document_name = snapshot.reference.formattedName + self.change_map[document_name] = ChangeType.REMOVED self.current = False From 6a0294eb86e9df06af0494c59dc687e94bd24c01 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Tue, 14 Aug 2018 10:47:43 -0700 Subject: [PATCH 011/148] improve doc strings and comment out yet to be done methods --- .../cloud/firestore_v1beta1/collection.py | 31 +++++++++++++--- .../cloud/firestore_v1beta1/document.py | 30 +++++++++++++-- .../google/cloud/firestore_v1beta1/query.py | 37 +++++++++++++------ .../google/cloud/firestore_v1beta1/watch.py | 27 +++++++++----- firestore/tests/system.py | 1 + 5 files changed, 96 insertions(+), 30 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/collection.py b/firestore/google/cloud/firestore_v1beta1/collection.py index c49c3e4080af..ed5c06baa10d 100644 --- a/firestore/google/cloud/firestore_v1beta1/collection.py +++ b/firestore/google/cloud/firestore_v1beta1/collection.py @@ -371,11 +371,32 @@ def get(self, transaction=None): query = query_mod.Query(self) return query.get(transaction=transaction) - def onSnapshot(options, callback): - ''' - given options and the callback, monitor this collection for changes - ''' - raise NotImplemented + # def on_snapshot(self, callback): + # """Monitor the documents in this collection. + # + # This starts a watch on this collection using a background thread. The + # provided callback is run on the snapshot of the documents. + # + # Args: + # callback(CollectionSnapshot): a callback to run when a change occurs + # + # Example: + # from google.cloud import firestore + # + # db = firestore.Client() + # collection_ref = db.collection(u'users') + # + # def on_snapshot(collection_snapshot): + # for doc in collection_snapshot.docs: + # print(u'{} => {}'.format(doc.id, doc.to_dict())) + # + # # Watch this collection + # collection_watch = collection_ref.on_snapshot(on_snapshot) + # + # # Terminate this watch + # collection_watch.unsubscribe() + # """ + # raise NotImplemented def _auto_id(): diff --git a/firestore/google/cloud/firestore_v1beta1/document.py b/firestore/google/cloud/firestore_v1beta1/document.py index 7021b20202df..fd9eed00b012 100644 --- a/firestore/google/cloud/firestore_v1beta1/document.py +++ b/firestore/google/cloud/firestore_v1beta1/document.py @@ -443,9 +443,33 @@ def collections(self, page_size=None): return iterator def on_snapshot(self, callback): - ''' - given options and the callback, monitor this document for changes - ''' + """Watch this document. + + This starts a watch on this document using a background thread. The + provided callback is run on the snapshot. + + Args: + callback(DocumentSnapshot): a callback to run when a change occurs + + Example: + from google.cloud import firestore + + db = firestore.Client() + collection_ref = db.collection(u'users') + + def on_snapshot(document_snapshot): + doc = document_snapshot + print(u'{} => {}'.format(doc.id, doc.to_dict())) + + doc_ref = db.collection(u'users').document( + u'alovelace' + unique_resource_id()) + + # Watch this document + doc_watch = doc_ref.on_snapshot(on_snapshot) + + # Terminate this watch + doc_watch.unsubscribe() + """ Watch.for_document(self, callback, DocumentSnapshot) diff --git a/firestore/google/cloud/firestore_v1beta1/query.py b/firestore/google/cloud/firestore_v1beta1/query.py index 909eb914e2ea..05a6ba4e44ca 100644 --- a/firestore/google/cloud/firestore_v1beta1/query.py +++ b/firestore/google/cloud/firestore_v1beta1/query.py @@ -601,19 +601,32 @@ def get(self, transaction=None): else: yield snapshot - def onSnapshot(callback, options): - ''' - db.collection("cities").where("state", "==", "CA") - .onSnapshot(function(querySnapshot) { - var cities = []; - querySnapshot.forEach(function(doc) { - cities.push(doc.data().name); - }); - console.log("Current cities in CA: ", cities.join(", ")); - }); - ''' - raise NotImplemented + # def on_snapshot(self, callback): + # """Monitor the documents in this collection that match this query. + # This starts a watch on this query using a background thread. The + # provided callback is run on the snapshot of the documents. + + # Args: + # callback(QuerySnapshot): a callback to run when a change occurs + + # Example: + # from google.cloud import firestore + + # db = firestore.Client() + # query_ref = db.collection(u'users').where("user", "==", u'ada') + + # def on_snapshot(query_snapshot): + # for doc in query_snapshot.docs: + # print(u'{} => {}'.format(doc.id, doc.to_dict())) + + # # Watch this query + # query_watch = query_ref.on_snapshot(on_snapshot) + + # # Terminate this watch + # query_watch.unsubscribe() + # """ + # raise NotImplemented def _enum_from_op_string(op_string): """Convert a string representation of a binary operator to an enum. diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 15ef7e8b0a90..6d8d201aa71e 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -312,6 +312,9 @@ def _on_rpc_done(self, future): thread.daemon = True thread.start() + def unsubscribe(self): + self.rpc.close() + @classmethod def for_document(cls, document_ref, snapshot_callback, snapshot_class_instance): @@ -346,7 +349,19 @@ def for_document(cls, document_ref, snapshot_callback, # 'target_id': WATCH_TARGET_ID # }, # query.comparator(), - # snapshot_callback) + # snapshot_callback, + # snapshot_class_instance) + + # @classmethod + # def for_collection(cls, collection_ref, snapshot_callback): + # return cls(collection_ref._client, + # { + # 'collection': collection_ref.to_proto(), + # 'target_id': WATCH_TARGET_ID + # }, + # document_watch_comparator, + # snapshot_callback, + # snapshot_class_instance) def on_snapshot(self, proto): """ @@ -511,15 +526,7 @@ def push(self, read_time, next_resume_token): updated_tree, updated_map, appliedChanges = \ Watch._compute_snapshot( self.doc_tree, self.doc_map, deletes, adds, updates) -# _LOGGER.debug(f"""push -# self.doc_map {self.doc_map} -# self.change_map {self.change_map} -# read_time {read_time} -# deletes {deletes} -# adds {adds} -# updates {updates} -# updated_tree {updated_tree} -# """) + if not self.has_pushed or len(appliedChanges): _LOGGER.debug( f'Sending snapshot with {len(appliedChanges)} changes' diff --git a/firestore/tests/system.py b/firestore/tests/system.py index fa45be650371..a0060797e221 100644 --- a/firestore/tests/system.py +++ b/firestore/tests/system.py @@ -747,6 +747,7 @@ def test_batch(client, cleanup): assert not document3.get().exists + def test_watch_document(client, cleanup): db = client doc_ref = db.collection(u'users').document( From fd889d5b828c8a03545a76da64511ca8649940cd Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Tue, 14 Aug 2018 13:58:14 -0400 Subject: [PATCH 012/148] remove fstrings for 2.7 compat --- firestore/google/cloud/firestore_v1beta1/watch.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 6d8d201aa71e..57332a858982 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -529,10 +529,10 @@ def push(self, read_time, next_resume_token): if not self.has_pushed or len(appliedChanges): _LOGGER.debug( - f'Sending snapshot with {len(appliedChanges)} changes' - f' and {len(updated_tree)} documents') + 'Sending snapshot with {len(appliedChanges)} changes' + ' and {len(updated_tree)} documents') - _LOGGER.debug(f"updatedTree:{updated_tree}") + _LOGGER.debug("updatedTree:{updated_tree}") self._snapshot_callback( updated_tree.keys(), appliedChanges, @@ -560,7 +560,7 @@ def _extract_changes(doc_map, changes, read_time): else: value.read_time = read_time adds.append(value) - _LOGGER.debug(f'deletes:{len(deletes)} adds:{len(adds)}') + _LOGGER.debug('deletes:{len(deletes)} adds:{len(adds)}') return (deletes, adds, updates) def _compute_snapshot(doc_tree, doc_map, delete_changes, add_changes, @@ -672,7 +672,7 @@ def modify_doc(new_document, updated_tree, updated_map): assert len(updated_tree) == len(updated_map), \ 'The update document ' + \ 'tree and document map should have the same number of entries.' - _LOGGER.debug(f"tree:{updated_tree}, map:{updated_map}, applied:{appliedChanges}") + _LOGGER.debug("tree:{updated_tree}, map:{updated_map}, applied:{appliedChanges}") return (updated_tree, updated_map, appliedChanges) def _affects_target(self, target_ids, current_id): From 83755b834d039dbec299ddfaa0165f5fe5250ad9 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Tue, 14 Aug 2018 11:02:26 -0700 Subject: [PATCH 013/148] be more specific on snapshot type --- firestore/google/cloud/firestore_v1beta1/collection.py | 5 +++-- firestore/google/cloud/firestore_v1beta1/document.py | 3 ++- firestore/google/cloud/firestore_v1beta1/query.py | 5 +++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/collection.py b/firestore/google/cloud/firestore_v1beta1/collection.py index ed5c06baa10d..090da0cb0151 100644 --- a/firestore/google/cloud/firestore_v1beta1/collection.py +++ b/firestore/google/cloud/firestore_v1beta1/collection.py @@ -378,7 +378,8 @@ def get(self, transaction=None): # provided callback is run on the snapshot of the documents. # # Args: - # callback(CollectionSnapshot): a callback to run when a change occurs + # callback(~.firestore.collection.CollectionSnapshot): a callback + # to run when a change occurs. # # Example: # from google.cloud import firestore @@ -387,7 +388,7 @@ def get(self, transaction=None): # collection_ref = db.collection(u'users') # # def on_snapshot(collection_snapshot): - # for doc in collection_snapshot.docs: + # for doc in collection_snapshot.documents: # print(u'{} => {}'.format(doc.id, doc.to_dict())) # # # Watch this collection diff --git a/firestore/google/cloud/firestore_v1beta1/document.py b/firestore/google/cloud/firestore_v1beta1/document.py index fd9eed00b012..cd8c82f859d4 100644 --- a/firestore/google/cloud/firestore_v1beta1/document.py +++ b/firestore/google/cloud/firestore_v1beta1/document.py @@ -449,7 +449,8 @@ def on_snapshot(self, callback): provided callback is run on the snapshot. Args: - callback(DocumentSnapshot): a callback to run when a change occurs + callback(~.firestore.document.DocumentSnapshot):a callback to run + when a change occurs Example: from google.cloud import firestore diff --git a/firestore/google/cloud/firestore_v1beta1/query.py b/firestore/google/cloud/firestore_v1beta1/query.py index 05a6ba4e44ca..88299039c448 100644 --- a/firestore/google/cloud/firestore_v1beta1/query.py +++ b/firestore/google/cloud/firestore_v1beta1/query.py @@ -608,7 +608,8 @@ def get(self, transaction=None): # provided callback is run on the snapshot of the documents. # Args: - # callback(QuerySnapshot): a callback to run when a change occurs + # callback(~.firestore.query.QuerySnapshot): a callback to run when + # a change occurs. # Example: # from google.cloud import firestore @@ -617,7 +618,7 @@ def get(self, transaction=None): # query_ref = db.collection(u'users').where("user", "==", u'ada') # def on_snapshot(query_snapshot): - # for doc in query_snapshot.docs: + # for doc in query_snapshot.documents: # print(u'{} => {}'.format(doc.id, doc.to_dict())) # # Watch this query From a327dfe83e27022ea865227f2544169175b448f9 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Mon, 20 Aug 2018 11:01:41 -0400 Subject: [PATCH 014/148] unit tests for watch module --- .../google/cloud/firestore_v1beta1/query.py | 1 + .../google/cloud/firestore_v1beta1/watch.py | 235 ++++------ firestore/tests/system.py | 2 +- firestore/tests/unit/test_watch.py | 439 ++++++++++++++++++ 4 files changed, 542 insertions(+), 135 deletions(-) create mode 100644 firestore/tests/unit/test_watch.py diff --git a/firestore/google/cloud/firestore_v1beta1/query.py b/firestore/google/cloud/firestore_v1beta1/query.py index 88299039c448..2a88ad054678 100644 --- a/firestore/google/cloud/firestore_v1beta1/query.py +++ b/firestore/google/cloud/firestore_v1beta1/query.py @@ -629,6 +629,7 @@ def get(self, transaction=None): # """ # raise NotImplemented + def _enum_from_op_string(op_string): """Convert a string representation of a binary operator to an enum. diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 57332a858982..a44742b75111 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -26,8 +26,6 @@ # from bidi import BidiRpc, ResumableBidiRpc -import time -import random import grpc """Python client for Google Cloud Firestore Watch.""" @@ -138,87 +136,28 @@ def _maybe_wrap_exception(exception): return exception -def is_permanent_error(self, error): - try: - if (error.code == GRPC_STATUS_CODE['CANCELLED'] or - error.code == GRPC_STATUS_CODE['UNKNOWN'] or - error.code == GRPC_STATUS_CODE['DEADLINE_EXCEEDED'] or - error.code == GRPC_STATUS_CODE['RESOURCE_EXHAUSTED'] or - error.code == GRPC_STATUS_CODE['INTERNAL'] or - error.code == GRPC_STATUS_CODE['UNAVAILABLE'] or - error.code == GRPC_STATUS_CODE['UNAUTHENTICATED']): - return False - else: - return True - except AttributeError: - _LOGGER.error("Unable to determine error code") - return False - - def document_watch_comparator(doc1, doc2): assert doc1 == doc2, 'Document watches only support one document.' return 0 -class ExponentialBackOff(object): - _INITIAL_SLEEP = 1.0 - """float: Initial "max" for sleep interval.""" - _MAX_SLEEP = 30.0 - """float: Eventual "max" sleep time.""" - _MULTIPLIER = 2.0 - """float: Multiplier for exponential backoff.""" - - def __init__(self, initial_sleep=_INITIAL_SLEEP, max_sleep=_MAX_SLEEP, - multiplier=_MULTIPLIER): - self.initial_sleep = self.current_sleep = initial_sleep - self.max_sleep = max_sleep - self.multipler = multiplier - - def back_off(self): - self.current_sleep = self._sleep(self.current_sleep, - self.max_sleep, - self.multipler) - - def reset_to_max(self): - self.current_sleep = self.max_sleep - - def reset(self): - self.current_sleep = self._INITIAL_SLEEP - - def _sleep(self, current_sleep, max_sleep=_MAX_SLEEP, - multiplier=_MULTIPLIER): - """Sleep and produce a new sleep time. - - .. _Exponential Backoff And Jitter: - https://www.awsarchitectureblog.com/2015/03/backoff.html - - Select a duration between zero and ``current_sleep``. It might seem - counterintuitive to have so much jitter, but - `Exponential Backoff And Jitter`_ argues that "full jitter" is - the best strategy. - - Args: - current_sleep (float): The current "max" for sleep interval. - max_sleep (Optional[float]): Eventual "max" sleep time - multiplier (Optional[float]): Multiplier for exponential backoff. - - Returns: - float: Newly doubled ``current_sleep`` or ``max_sleep`` (whichever - is smaller) - """ - actual_sleep = random.uniform(0.0, self.current_sleep) - time.sleep(actual_sleep) - return min(self.multiplier * self.current_sleep, self.max_sleep) +class Watch(object): + threading = threading # FBO unit tests + BackgroundConsumer = BackgroundConsumer # FBO unit tests + ResumableBidiRpc = ResumableBidiRpc # FBO unit tests + MessageToDict = json_format.MessageToDict # FBO unit tests -class Watch(object): def __init__(self, document_reference, firestore, target, comparator, snapshot_callback, - DocumentSnapshotCls): + DocumentSnapshotCls, + BackgroundConsumer=None, # FBO unit testing + ResumableBidiRpc=None, # FBO unit testing + ): """ Args: firestore: @@ -234,7 +173,7 @@ def __init__(self, read_time (string): The ISO 8601 time at which this snapshot was obtained. # TODO: Go had an err here and node.js provided size. - # TODO: do we want to include either? + # TODO: do we want to include either? DocumentSnapshotCls: instance of the DocumentSnapshot class """ self._document_reference = document_reference @@ -255,6 +194,9 @@ def should_recover(exc): add_target=self._targets ) + if ResumableBidiRpc is None: + ResumableBidiRpc = self.ResumableBidiRpc # FBO unit tests + self.rpc = ResumableBidiRpc( self._api.firestore_stub.Listen, initial_request=initial_request, @@ -278,7 +220,7 @@ def should_recover(exc): # The current state of the query results. self.current = False - + # We need this to track whether we've pushed an initial set of changes, # since we should push those even when there are no changes, if there # aren't docs. @@ -286,6 +228,8 @@ def should_recover(exc): # The server assigns and updates the resume token. self.resume_token = None + if BackgroundConsumer is None: # FBO unit tests + BackgroundConsumer = self.BackgroundConsumer self._consumer = BackgroundConsumer(self.rpc, self.on_snapshot) self._consumer.start() @@ -305,14 +249,14 @@ def _on_rpc_done(self, future): _LOGGER.info( 'RPC termination has signaled shutdown.') future = _maybe_wrap_exception(future) - thread = threading.Thread( + thread = self.threading.Thread( name=_RPC_ERROR_THREAD_NAME, target=self.close, kwargs={'reason': future}) thread.daemon = True thread.start() - def unsubscribe(self): + def unsubscribe(self): # XXX should this be aliased to close? self.rpc.close() @classmethod @@ -326,7 +270,7 @@ def for_document(cls, document_ref, snapshot_callback, Args: document_ref: Reference to Document snapshot_callback: callback to be called on snapshot - snapshot_class_instance: instance of snapshot cls to make + snapshot_class_instance: instance of snapshot cls to make snapshots with to pass to snapshot_callback """ @@ -363,6 +307,52 @@ def for_document(cls, document_ref, snapshot_callback, # snapshot_callback, # snapshot_class_instance) + def _on_snapshot_target_change_no_change(self, proto): + _LOGGER.debug('on_snapshot: target change: NO_CHANGE') + change = proto.target_change + + no_target_ids = (change.target_ids is None or + len(change.target_ids) == 0) + if no_target_ids and change.read_time and self.current: + # TargetChange.CURRENT followed by TargetChange.NO_CHANGE + # signals a consistent state. Invoke the onSnapshot + # callback as specified by the user. + self.push(change.read_time, change.resume_token) + + def _on_snapshot_target_change_add(self, proto): + _LOGGER.debug("on_snapshot: target change: ADD") + assert WATCH_TARGET_ID == proto.target_change.target_ids[0], \ + 'Unexpected target ID sent by server' + # TODO : do anything here? Node didn't so I think this isn't + # the right thing to do + # wr = WatchResult( + # None, + # self._document_reference.id, + # ChangeType.ADDED) + # self._snapshot_callback(wr) + + def _on_snapshot_target_change_remove(self, proto): + _LOGGER.debug("on_snapshot: target change: REMOVE") + change = proto.target_change + + code = 13 + message = 'internal error' + if change.cause: + code = change.cause.code + message = change.cause.message + + # TODO: Surface a .code property on the exception. + raise Exception('Error %s: %s' % (code, message)) # XXX Exception? + + def _on_snapshot_target_change_reset(self, proto): + # Whatever changes have happened so far no longer matter. + _LOGGER.debug("on_snapshot: target change: RESET") + self._reset_docs() + + def _on_snapshot_target_change_current(self, proto): + _LOGGER.debug("on_snapshot: target change: CURRENT") + self.current = True + def on_snapshot(self, proto): """ Called everytime there is a response from listen. Collect changes @@ -375,72 +365,47 @@ def on_snapshot(self, proto): """ TargetChange = firestore_pb2.TargetChange - if str(proto.target_change): - _LOGGER.debug('on_snapshot: target change') - - # google.cloud.firestore_v1beta1.types.TargetChange - change = proto.target_change - - no_target_ids = change.target_ids is None or \ - len(change.target_ids) == 0 - if change.target_change_type == TargetChange.NO_CHANGE: - _LOGGER.debug('on_snapshot: target change: NO_CHANGE') - if no_target_ids and change.read_time and self.current: - # TargetChange.CURRENT followed by TargetChange.NO_CHANGE - # signals a consistent state. Invoke the onSnapshot - # callback as specified by the user. - self.push(change.read_time, change.resume_token) - elif change.target_change_type == TargetChange.ADD: - _LOGGER.debug("on_snapshot: target change: ADD") - assert WATCH_TARGET_ID == change.target_ids[0], \ - 'Unexpected target ID sent by server' - # TODO : do anything here? Node didn't so I think this isn't - # the right thing to do - # wr = WatchResult( - # None, - # self._document_reference.id, - # ChangeType.ADDED) - # self._snapshot_callback(wr) + target_changetype_dispatch = { + TargetChange.NO_CHANGE: self._on_snapshot_target_change_no_change, + TargetChange.ADD: self._on_snapshot_target_change_add, + TargetChange.REMOVE: self._on_snapshot_target_change_remove, + TargetChange.RESET: self._on_snapshot_target_change_reset, + TargetChange.CURRENT: self._on_snapshot_target_change_current, + } - elif change.target_change_type == TargetChange.REMOVE: - _LOGGER.debug("on_snapshot: target change: REMOVE") + target_change = proto.target_change - code = 13 - message = 'internal error' - if change.cause: - code = change.cause.code - message = change.cause.message - - # TODO: Surface a .code property on the exception. - raise Exception('Error ' + code + ': ' + message) - elif change.target_change_type == TargetChange.RESET: - # Whatever changes have happened so far no longer matter. - _LOGGER.debug("on_snapshot: target change: RESET") - self._reset_docs() - elif change.target_change_type == TargetChange.CURRENT: - _LOGGER.debug("on_snapshot: target change: CURRENT") - self.current = True - else: + if str(target_change): # XXX why if str + _LOGGER.debug('on_snapshot: target change') + target_change_type = target_change.target_change_type + meth = target_changetype_dispatch.get(target_change_type) + if meth is None: _LOGGER.info('on_snapshot: Unknown target change ' + - str(change.target_change_type)) - + str(target_change_type)) self._consumer.stop() # closeStream( # new Error('Unknown target change type: ' + # JSON.stringify(change)) # TODO : make this exit the inner function and stop processing? - raise Exception('Unknown target change type: ' + str(change)) + raise Exception('Unknown target change type: %s ' % + str(target_change_type)) # XXX Exception? + else: + meth(proto) + + # XXX this is currently a no-op + # affects_target = self._affects_target( + # target_change.target_ids, WATCH_TARGET_ID + # ) - if change.resume_token and self._affects_target(change.target_ids, - WATCH_TARGET_ID): - # TODO: they node version resets backoff here. We allow - # bidi rpc to do its thing. - pass + # if target_change.resume_token and affects_target: + # # TODO: they node version resets backoff here. We allow + # # bidi rpc to do its thing. + # pass - elif str(proto.document_change): + elif str(proto.document_change): # XXX why if str _LOGGER.debug('on_snapshot: document change') - # No other target_ids can show up here, but we still need to see + # No other target_ids can show up here, but we still need to see # if the targetId was in the added list or removed list. target_ids = proto.document_change.target_ids or [] removed_target_ids = proto.document_change.removed_target_ids or [] @@ -463,7 +428,7 @@ def on_snapshot(self, proto): # google.cloud.firestore_v1beta1.types.Document document = document_change.document - data = json_format.MessageToDict(document) + data = self.MessageToDict(document) snapshot = self.DocumentSnapshot( reference=self._document_reference, @@ -482,6 +447,7 @@ def on_snapshot(self, proto): elif removed: _LOGGER.debug('on_snapshot: document change: REMOVED') + document = proto.document_change.document self.change_map[document.name] = ChangeType.REMOVED elif (proto.document_delete or proto.document_remove): @@ -506,7 +472,8 @@ def on_snapshot(self, proto): _LOGGER.debug("UNKNOWN TYPE. UHOH") self._consumer.stop() raise Exception( - 'Unknown listen response type: ' + proto) + 'Unknown listen response type: %s' % proto + ) # XXX Exception? # TODO: can we stop but raise an error? # closeStream( # new Error('Unknown listen response type: ' + @@ -610,6 +577,7 @@ def add_doc(new_document, updated_tree, updated_map): new_index), updated_tree, updated_map) + # XXX modify_doc is broken via formattedName def modify_doc(new_document, updated_tree, updated_map): """ Applies a document modification to the document tree and the @@ -672,7 +640,6 @@ def modify_doc(new_document, updated_tree, updated_map): assert len(updated_tree) == len(updated_map), \ 'The update document ' + \ 'tree and document map should have the same number of entries.' - _LOGGER.debug("tree:{updated_tree}, map:{updated_map}, applied:{appliedChanges}") return (updated_tree, updated_map, appliedChanges) def _affects_target(self, target_ids, current_id): @@ -685,7 +652,7 @@ def _affects_target(self, target_ids, current_id): return False - def _current_size(self): + def _current_size(self): # XXX broken, no docMap or changeMap """ Returns the current count of all documents, including the changes from the current changeMap. @@ -693,7 +660,7 @@ def _current_size(self): deletes, adds, _ = Watch._extract_changes(self.docMap, self.changeMap) return self.docMap.size + len(adds) - len(deletes) - def _reset_docs(self): + def _reset_docs(self): # XXX broken via formattedName """ Helper to clear the docs on RESET or filter mismatch. """ diff --git a/firestore/tests/system.py b/firestore/tests/system.py index a0060797e221..8833a8993ec3 100644 --- a/firestore/tests/system.py +++ b/firestore/tests/system.py @@ -765,7 +765,7 @@ def test_watch_document(client, cleanup): # Setup listener def on_response(response): on_response.called_count += 1 - print(f'Response: {response}') + print('Response: %s' % response) print(type(response)) on_response.called_count = 0 diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py new file mode 100644 index 000000000000..b2e93b799b10 --- /dev/null +++ b/firestore/tests/unit/test_watch.py @@ -0,0 +1,439 @@ +import unittest +import mock +from google.cloud.firestore_v1beta1.proto import firestore_pb2 + + +class TestWatchDocTree(unittest.TestCase): + def _makeOne(self): + from google.cloud.firestore_v1beta1.watch import WatchDocTree + return WatchDocTree() + + def test_insert_and_keys(self): + inst = self._makeOne() + inst = inst.insert('b', 1) + inst = inst.insert('a', 2) + self.assertEqual(sorted(inst.keys()), ['a', 'b']) + + def test_remove_and_keys(self): + inst = self._makeOne() + inst = inst.insert('b', 1) + inst = inst.insert('a', 2) + inst = inst.remove('a') + self.assertEqual(sorted(inst.keys()), ['b']) + + def test_insert_and_find(self): + inst = self._makeOne() + inst = inst.insert('b', 1) + inst = inst.insert('a', 2) + val = inst.find('a') + self.assertEqual(val.value, 2) + + def test___len__(self): + inst = self._makeOne() + inst = inst.insert('b', 1) + inst = inst.insert('a', 2) + self.assertEqual(len(inst), 2) + + +class TestDocumentChange(unittest.TestCase): + def _makeOne(self, type, document, old_index, new_index): + from google.cloud.firestore_v1beta1.watch import DocumentChange + return DocumentChange(type, document, old_index, new_index) + + def test_ctor(self): + inst = self._makeOne('type', 'document', 'old_index', 'new_index') + self.assertEqual(inst.type, 'type') + self.assertEqual(inst.document, 'document') + self.assertEqual(inst.old_index, 'old_index') + self.assertEqual(inst.new_index, 'new_index') + + +class TestWatchResult(unittest.TestCase): + def _makeOne(self, snapshot, name, change_type): + from google.cloud.firestore_v1beta1.watch import WatchResult + return WatchResult(snapshot, name, change_type) + + def test_ctor(self): + inst = self._makeOne('snapshot', 'name', 'change_type') + self.assertEqual(inst.snapshot, 'snapshot') + self.assertEqual(inst.name, 'name') + self.assertEqual(inst.change_type, 'change_type') + + +class Test_maybe_wrap_exception(unittest.TestCase): + def _callFUT(self, exc): + from google.cloud.firestore_v1beta1.watch import _maybe_wrap_exception + return _maybe_wrap_exception(exc) + + def test_is_grpc_error(self): + import grpc + from google.api_core.exceptions import GoogleAPICallError + exc = grpc.RpcError() + result = self._callFUT(exc) + self.assertEqual(result.__class__, GoogleAPICallError) + + def test_is_not_grpc_error(self): + exc = ValueError() + result = self._callFUT(exc) + self.assertEqual(result.__class__, ValueError) + + +class Test_document_watch_comparator(unittest.TestCase): + def _callFUT(self, doc1, doc2): + from google.cloud.firestore_v1beta1.watch import ( + document_watch_comparator, + ) + return document_watch_comparator(doc1, doc2) + + def test_same_doc(self): + result = self._callFUT(1, 1) + self.assertEqual(result, 0) + + def test_diff_doc(self): + self.assertRaises(AssertionError, self._callFUT, 1, 2) + + +class TestWatch(unittest.TestCase): + def _makeOne( + self, + document_reference=None, + firestore=None, + target=None, + comparator=None, + snapshot_callback=None, + snapshot_class=None, + ): + from google.cloud.firestore_v1beta1.watch import Watch + if document_reference is None: + document_reference = DummyDocumentReference() + if firestore is None: + firestore = DummyFirestore() + if target is None: + WATCH_TARGET_ID = 0x5079 # "Py" + target = { + 'documents': { + 'documents': ['/']}, + 'target_id': WATCH_TARGET_ID + } + if comparator is None: + comparator = self._document_watch_comparator + if snapshot_callback is None: + snapshot_callback = self._snapshot_callback + if snapshot_class is None: + snapshot_class = DummyDocumentSnapshot + + inst = Watch( + document_reference, + firestore, + target, + comparator, + snapshot_callback, + snapshot_class, + BackgroundConsumer=DummyBackgroundConsumer, + ResumableBidiRpc=DummyRpc, + ) + return inst + + def _document_watch_comparator(self, doc1, doc2): + return 0 + + def _snapshot_callback(self, docs, changes, read_time): + return True + + def test_ctor(self): + inst = self._makeOne() + self.assertTrue(inst._consumer.started) + self.assertTrue(inst.rpc.callbacks, [inst._on_rpc_done]) + + def dont_test__on_rpc_done(self): # XXX fails + inst = self._makeOne() + threading = DummyThreading() + inst.threading = threading + inst._on_rpc_done(True) # no close method, fails + from google.cloud.firestore_v1beta1.watch import _RPC_ERROR_THREAD_NAME + self.assertTrue(threading.threads[_RPC_ERROR_THREAD_NAME].started) + + def test_unsubscribe(self): + inst = self._makeOne() + inst.unsubscribe() + self.assertTrue(inst.rpc.closed) + + def test_for_document(self): + from google.cloud.firestore_v1beta1.watch import Watch + docref = DummyDocumentReference() + snapshot_callback = self._snapshot_callback + snapshot_class_instance = DummyDocumentSnapshot + modulename = 'google.cloud.firestore_v1beta1.watch' + with mock.patch( + '%s.Watch.ResumableBidiRpc' % modulename, + DummyRpc, + ): + with mock.patch( + '%s.Watch.BackgroundConsumer' % modulename, + DummyBackgroundConsumer, + ): + inst = Watch.for_document( + docref, + snapshot_callback, + snapshot_class_instance + ) + self.assertTrue(inst._consumer.started) + self.assertTrue(inst.rpc.callbacks, [inst._on_rpc_done]) + + def test_on_snapshot_target_no_change_no_target_ids_not_current(self): + inst = self._makeOne() + proto = DummyProto() + inst.on_snapshot(proto) # nothing to assert, no mutations, no rtnval + + def test_on_snapshot_target_no_change_no_target_ids_current(self): + inst = self._makeOne() + proto = DummyProto() + proto.target_change.read_time = 1 + inst.current = True + + def push(read_time, next_resume_token): + inst._read_time = read_time + inst._next_resume_token = next_resume_token + + inst.push = push + inst.on_snapshot(proto) + self.assertEqual(inst._read_time, 1) + self.assertEqual(inst._next_resume_token, None) + + def test_on_snapshot_target_add(self): + inst = self._makeOne() + proto = DummyProto() + proto.target_change.target_change_type = firestore_pb2.TargetChange.ADD + proto.target_change.target_ids = [1] # not "Py" + with self.assertRaises(Exception) as exc: + inst.on_snapshot(proto) + self.assertEqual( + str(exc.exception), + 'Unexpected target ID sent by server' + ) + + def test_on_snapshot_target_remove(self): + inst = self._makeOne() + proto = DummyProto() + target_change = proto.target_change + target_change.target_change_type = firestore_pb2.TargetChange.REMOVE + with self.assertRaises(Exception) as exc: + inst.on_snapshot(proto) + self.assertEqual(str(exc.exception), 'Error 1: hi') + + def test_on_snapshot_target_reset(self): + inst = self._makeOne() + + def reset(): + inst._docs_reset = True + + inst._reset_docs = reset + proto = DummyProto() + target_change = proto.target_change + target_change.target_change_type = firestore_pb2.TargetChange.RESET + inst.on_snapshot(proto) + self.assertTrue(inst._docs_reset) + + def test_on_snapshot_target_current(self): + inst = self._makeOne() + inst.current = False + proto = DummyProto() + target_change = proto.target_change + target_change.target_change_type = firestore_pb2.TargetChange.CURRENT + inst.on_snapshot(proto) + self.assertTrue(inst.current) + + def test_on_snapshot_target_unknown(self): + inst = self._makeOne() + proto = DummyProto() + proto.target_change.target_change_type = 'unknown' + with self.assertRaises(Exception) as exc: + inst.on_snapshot(proto) + self.assertTrue(inst._consumer.stopped) + self.assertEqual( + str(exc.exception), + 'Unknown target change type: unknown ' + ) + + def test_on_snapshot_document_change_removed(self): + from google.cloud.firestore_v1beta1.watch import ( + WATCH_TARGET_ID, + ChangeType, + ) + inst = self._makeOne() + proto = DummyProto() + proto.target_change = '' + proto.document_change.removed_target_ids = [WATCH_TARGET_ID] + + class DummyDocument: + name = 'fred' + + proto.document_change.document = DummyDocument() + inst.on_snapshot(proto) + self.assertTrue(inst.change_map['fred'] is ChangeType.REMOVED) + + def test_on_snapshot_document_change_changed(self): + from google.cloud.firestore_v1beta1.watch import WATCH_TARGET_ID + inst = self._makeOne() + + def message_to_dict(document): + return {'fields': None} + + inst.MessageToDict = message_to_dict + proto = DummyProto() + proto.target_change = '' + proto.document_change.target_ids = [WATCH_TARGET_ID] + + class DummyDocument: + name = 'fred' + create_time = None + update_time = None + + proto.document_change.document = DummyDocument() + inst.on_snapshot(proto) + self.assertEqual(inst.change_map['fred'].data, None) + + def test_on_snapshot_document_removed(self): + from google.cloud.firestore_v1beta1.watch import ChangeType + inst = self._makeOne() + proto = DummyProto() + proto.target_change = '' + proto.document_change = '' + + class DummyRemove(object): + document = 'fred' + + remove = DummyRemove() + proto.document_remove = remove + proto.document_delete = None + inst.on_snapshot(proto) + self.assertTrue(inst.change_map['fred'] is ChangeType.REMOVED) + + def dont_test_on_snapshot_filter_update(self): # XXX _current_size broken + inst = self._makeOne() + proto = DummyProto() + proto.target_change = '' + proto.document_change = '' + proto.document_remove = None + proto.document_delete = None + + class DummyFilter(object): + count = 999 + + proto.filter = DummyFilter() + + def reset(): + self._docs_reset = True + + proto._reset_docs = reset + inst.on_snapshot(proto) + self.assertTrue(inst._docs_reset) + + def test_on_snapshot_unknown_listen_type(self): + inst = self._makeOne() + proto = DummyProto() + proto.target_change = '' + proto.document_change = '' + proto.document_remove = None + proto.document_delete = None + proto.filter = '' + with self.assertRaises(Exception) as exc: + inst.on_snapshot(proto) + self.assertTrue( + str(exc.exception).startswith('Unknown listen response type'), + str(exc.exception) + ) + + +class DummyFirestoreStub(object): + def Listen(self): + pass + + +class DummyFirestoreClient(object): + def __init__(self): + self.firestore_stub = DummyFirestoreStub() + + +class DummyDocumentReference(object): + def __init__(self): + self._client = DummyFirestore() + _document_path = '/' + + +class DummyFirestore(object): + _firestore_api = DummyFirestoreClient() + _database_string = '' + + +class DummyDocumentSnapshot(object): + def __init__(self, **kw): + self.__dict__.update(kw) + + +class DummyBackgroundConsumer(object): + started = False + stopped = False + + def __init__(self, rpc, on_snapshot): + self.rpc = rpc + self.on_snapshot = on_snapshot + + def start(self): + self.started = True + + def stop(self): + self.stopped = True + + +class DummyThread(object): + started = False + + def start(self): + self.started = True + + +class DummyThreading(object): + def __init__(self): + self.threads = {} + + def Thread(self, name, target, kwargs): + thread = DummyThread(name, target, kwargs) + self.threads[name] = thread + return thread + + +class DummyRpc(object): + def __init__(self, listen, initial_request, should_recover): + self.listen = listen + self.initial_request = initial_request + self.should_recover = should_recover + self.closed = False + self.callbacks = [] + + def add_done_callback(self, callback): + self.callbacks.append(callback) + + def close(self): + self.closed = True + + +class DummyCause(object): + code = 1 + message = 'hi' + + +class DummyChange(object): + def __init__(self): + self.target_ids = [] + self.removed_target_ids = [] + self.read_time = 0 + self.target_change_type = firestore_pb2.TargetChange.NO_CHANGE + self.resume_token = None + self.cause = DummyCause() + + +class DummyProto(object): + def __init__(self): + self.target_change = DummyChange() + self.document_change = DummyChange() From a26699d78ba6a0c8c28b8a95ac15b11cd963cce5 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Mon, 20 Aug 2018 14:19:31 -0400 Subject: [PATCH 015/148] these must be staticmethods; improve performance in _affects_target --- firestore/google/cloud/firestore_v1beta1/watch.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index a44742b75111..369fb9e40cea 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -146,7 +146,7 @@ class Watch(object): threading = threading # FBO unit tests BackgroundConsumer = BackgroundConsumer # FBO unit tests ResumableBidiRpc = ResumableBidiRpc # FBO unit tests - MessageToDict = json_format.MessageToDict # FBO unit tests + MessageToDict = staticmethod(json_format.MessageToDict) # FBO unit tests def __init__(self, document_reference, @@ -512,6 +512,7 @@ def push(self, read_time, next_resume_token): self.change_map.clear() self.resume_token = next_resume_token + @staticmethod def _extract_changes(doc_map, changes, read_time): deletes = [] adds = [] @@ -530,6 +531,7 @@ def _extract_changes(doc_map, changes, read_time): _LOGGER.debug('deletes:{len(deletes)} adds:{len(adds)}') return (deletes, adds, updates) + @staticmethod def _compute_snapshot(doc_tree, doc_map, delete_changes, add_changes, update_changes): # TODO: ACTUALLY NEED TO CALCULATE @@ -646,9 +648,8 @@ def _affects_target(self, target_ids, current_id): if target_ids is None or len(target_ids) == 0: return True - for target_id in target_ids: - if target_id == current_id: - return True + if current_id in target_ids: + return True return False From cf85c920593813c79d534f17c73d411dada266ef Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Mon, 20 Aug 2018 14:46:15 -0400 Subject: [PATCH 016/148] make system test for watch pass --- firestore/tests/system.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/firestore/tests/system.py b/firestore/tests/system.py index 8833a8993ec3..a39d98571563 100644 --- a/firestore/tests/system.py +++ b/firestore/tests/system.py @@ -763,10 +763,8 @@ def test_watch_document(client, cleanup): sleep(1) # Setup listener - def on_response(response): + def on_response(*arg): on_response.called_count += 1 - print('Response: %s' % response) - print(type(response)) on_response.called_count = 0 From 9b0a4c28901b1930ba862278a007301137c23c9f Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Mon, 20 Aug 2018 15:12:37 -0400 Subject: [PATCH 017/148] containment check instead of iteration --- firestore/google/cloud/firestore_v1beta1/watch.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 369fb9e40cea..4cf214a8afd2 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -412,13 +412,11 @@ def on_snapshot(self, proto): changed = False removed = False - for target in target_ids: - if target == WATCH_TARGET_ID: - changed = True + if WATCH_TARGET_ID in target_ids: + changed = True - for target in removed_target_ids: - if target == WATCH_TARGET_ID: - removed = True + if WATCH_TARGET_ID in removed_target_ids: + removed = True if changed: _LOGGER.debug('on_snapshot: document change: CHANGED') From 5bd6374dbb486172754d9fe5e1606a30c8ab70ba Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Mon, 20 Aug 2018 15:55:10 -0400 Subject: [PATCH 018/148] fix filter update test --- .../google/cloud/firestore_v1beta1/watch.py | 16 ++++++++++++---- firestore/tests/unit/test_watch.py | 6 +++--- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 4cf214a8afd2..1cb30126cb02 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -95,6 +95,10 @@ def remove(self, key): del self._dict[key] return self + def __iter__(self): + for k in self._dict: + yield k + def __len__(self): return len(self._dict) @@ -521,10 +525,12 @@ def _extract_changes(doc_map, changes, read_time): if name in doc_map: deletes.append(name) elif name in doc_map: - value.read_time = read_time + if read_time is not None: + value.read_time = read_time updates.append(value) else: - value.read_time = read_time + if read_time is not None: + value.read_time = read_time adds.append(value) _LOGGER.debug('deletes:{len(deletes)} adds:{len(adds)}') return (deletes, adds, updates) @@ -656,8 +662,10 @@ def _current_size(self): # XXX broken, no docMap or changeMap Returns the current count of all documents, including the changes from the current changeMap. """ - deletes, adds, _ = Watch._extract_changes(self.docMap, self.changeMap) - return self.docMap.size + len(adds) - len(deletes) + deletes, adds, _ = Watch._extract_changes( + self.doc_map, self.change_map, None + ) + return len(self.doc_map) + len(adds) - len(deletes) def _reset_docs(self): # XXX broken via formattedName """ diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index b2e93b799b10..ab56c8613423 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -309,7 +309,7 @@ class DummyRemove(object): inst.on_snapshot(proto) self.assertTrue(inst.change_map['fred'] is ChangeType.REMOVED) - def dont_test_on_snapshot_filter_update(self): # XXX _current_size broken + def test_on_snapshot_filter_update(self): inst = self._makeOne() proto = DummyProto() proto.target_change = '' @@ -323,9 +323,9 @@ class DummyFilter(object): proto.filter = DummyFilter() def reset(): - self._docs_reset = True + inst._docs_reset = True - proto._reset_docs = reset + inst._reset_docs = reset inst.on_snapshot(proto) self.assertTrue(inst._docs_reset) From 481ae83567b77e31e4ca1e72d917b1106cd373f0 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Tue, 21 Aug 2018 09:45:40 -0400 Subject: [PATCH 019/148] tests for various helper methods --- .../google/cloud/firestore_v1beta1/watch.py | 27 ++++++------ firestore/tests/unit/test_watch.py | 41 ++++++++++++++++++- 2 files changed, 55 insertions(+), 13 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 1cb30126cb02..577ce76212fd 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -491,17 +491,20 @@ def push(self, read_time, next_resume_token): # and sending them to the user. deletes, adds, updates = Watch._extract_changes( - self.doc_map, self.change_map, read_time) - updated_tree, updated_map, appliedChanges = \ - Watch._compute_snapshot( - self.doc_tree, self.doc_map, deletes, adds, updates) + self.doc_map, + self.change_map, + read_time, + ) - if not self.has_pushed or len(appliedChanges): - _LOGGER.debug( - 'Sending snapshot with {len(appliedChanges)} changes' - ' and {len(updated_tree)} documents') + updated_tree, updated_map, appliedChanges = Watch._compute_snapshot( + self.doc_tree, + self.doc_map, + deletes, + adds, + updates, + ) - _LOGGER.debug("updatedTree:{updated_tree}") + if not self.has_pushed or len(appliedChanges): self._snapshot_callback( updated_tree.keys(), appliedChanges, @@ -532,7 +535,7 @@ def _extract_changes(doc_map, changes, read_time): if read_time is not None: value.read_time = read_time adds.append(value) - _LOGGER.debug('deletes:{len(deletes)} adds:{len(adds)}') + return (deletes, adds, updates) @staticmethod @@ -649,7 +652,7 @@ def modify_doc(new_document, updated_tree, updated_map): return (updated_tree, updated_map, appliedChanges) def _affects_target(self, target_ids, current_id): - if target_ids is None or len(target_ids) == 0: + if target_ids is None: return True if current_id in target_ids: @@ -657,7 +660,7 @@ def _affects_target(self, target_ids, current_id): return False - def _current_size(self): # XXX broken, no docMap or changeMap + def _current_size(self): """ Returns the current count of all documents, including the changes from the current changeMap. diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index ab56c8613423..d1044362a3ca 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -1,3 +1,4 @@ +import datetime import unittest import mock from google.cloud.firestore_v1beta1.proto import firestore_pb2 @@ -8,6 +9,9 @@ def _makeOne(self): from google.cloud.firestore_v1beta1.watch import WatchDocTree return WatchDocTree() + def setUp(self): + self.snapshotted = None + def test_insert_and_keys(self): inst = self._makeOne() inst = inst.insert('b', 1) @@ -138,7 +142,7 @@ def _document_watch_comparator(self, doc1, doc2): return 0 def _snapshot_callback(self, docs, changes, read_time): - return True + self.snapshotted = (docs, changes, read_time) def test_ctor(self): inst = self._makeOne() @@ -344,6 +348,41 @@ def test_on_snapshot_unknown_listen_type(self): str(exc.exception) ) + def test_push_no_changes(self): + class DummyReadTime(object): + seconds = 1534858278 + inst = self._makeOne() + inst.push(DummyReadTime, 'token') + self.assertEqual( + self.snapshotted, + ([], [], datetime.datetime(2018, 8, 21, 9, 31, 18)), + ) + self.assertTrue(inst.has_pushed) + self.assertEqual(inst.resume_token, 'token') + + def test__current_size_empty(self): + inst = self._makeOne() + result = inst._current_size() + self.assertEqual(result, 0) + + def test__current_size_docmap_has_one(self): + inst = self._makeOne() + inst.doc_map['a'] = 1 + result = inst._current_size() + self.assertEqual(result, 1) + + def test__affects_target_target_id_None(self): + inst = self._makeOne() + self.assertTrue(inst._affects_target(None, [])) + + def test__affects_target_current_id_in_target_ids(self): + inst = self._makeOne() + self.assertTrue(inst._affects_target([1], 1)) + + def test__affects_target_current_id_not_in_target_ids(self): + inst = self._makeOne() + self.assertFalse(inst._affects_target([1], 2)) + class DummyFirestoreStub(object): def Listen(self): From a5c78a23e700fca570f79a8a385d49f0aac2e200 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Tue, 21 Aug 2018 12:54:55 -0700 Subject: [PATCH 020/148] Improve rpc_done and add early query support --- .../google/cloud/firestore_v1beta1/bidi.py | 7 +- .../google/cloud/firestore_v1beta1/query.py | 43 +++++----- .../google/cloud/firestore_v1beta1/watch.py | 83 ++++++++++++++----- 3 files changed, 89 insertions(+), 44 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/bidi.py b/firestore/google/cloud/firestore_v1beta1/bidi.py index e7629fc5df8e..53cfd7464c05 100644 --- a/firestore/google/cloud/firestore_v1beta1/bidi.py +++ b/firestore/google/cloud/firestore_v1beta1/bidi.py @@ -196,7 +196,6 @@ def open(self): request_generator = _RequestQueueGenerator( self._request_queue, initial_request=self._initial_request) - print('request generator created') call = self._start_rpc(iter(request_generator)) request_generator.call = call @@ -282,7 +281,7 @@ class ResumableBidiRpc(BidiRpc): def should_recover(exc): return ( isinstance(exc, grpc.RpcError) and - exc.code() == grpc.StatusCode.UNVAILABLE) + exc.code() == grpc.StatusCode.UNAVAILABLE) initial_request = example_pb2.StreamingRpcRequest( setting='example') @@ -429,7 +428,7 @@ class BackgroundConsumer(object): def should_recover(exc): return ( isinstance(exc, grpc.RpcError) and - exc.code() == grpc.StatusCode.UNVAILABLE) + exc.code() == grpc.StatusCode.UNAVAILABLE) initial_request = example_pb2.StreamingRpcRequest( setting='example') @@ -443,7 +442,7 @@ def on_response(response): print(response) consumer = BackgroundConsumer(rpc, on_response) - consumer.start() + consume.start() Note that error handling *must* be done by using the provided ``bidi_rpc``'s ``add_done_callback``. This helper will automatically exit diff --git a/firestore/google/cloud/firestore_v1beta1/query.py b/firestore/google/cloud/firestore_v1beta1/query.py index 2a88ad054678..229ae1afa8b1 100644 --- a/firestore/google/cloud/firestore_v1beta1/query.py +++ b/firestore/google/cloud/firestore_v1beta1/query.py @@ -29,7 +29,7 @@ from google.cloud.firestore_v1beta1 import document from google.cloud.firestore_v1beta1.gapic import enums from google.cloud.firestore_v1beta1.proto import query_pb2 - +from google.cloud.firestore_v1beta1.watch import Watch _EQ_OP = '==' _COMPARISON_OPERATORS = { @@ -601,33 +601,34 @@ def get(self, transaction=None): else: yield snapshot - # def on_snapshot(self, callback): - # """Monitor the documents in this collection that match this query. + def on_snapshot(self, callback): + """Monitor the documents in this collection that match this query. + + This starts a watch on this query using a background thread. The + provided callback is run on the snapshot of the documents. - # This starts a watch on this query using a background thread. The - # provided callback is run on the snapshot of the documents. + Args: + callback(~.firestore.query.QuerySnapshot): a callback to run when + a change occurs. - # Args: - # callback(~.firestore.query.QuerySnapshot): a callback to run when - # a change occurs. + Example: + from google.cloud import firestore - # Example: - # from google.cloud import firestore + db = firestore.Client() + query_ref = db.collection(u'users').where("user", "==", u'Ada') - # db = firestore.Client() - # query_ref = db.collection(u'users').where("user", "==", u'ada') + def on_snapshot(query_snapshot): + for doc in query_snapshot.documents: + print(u'{} => {}'.format(doc.id, doc.to_dict())) - # def on_snapshot(query_snapshot): - # for doc in query_snapshot.documents: - # print(u'{} => {}'.format(doc.id, doc.to_dict())) + # Watch this query + query_watch = query_ref.on_snapshot(on_snapshot) - # # Watch this query - # query_watch = query_ref.on_snapshot(on_snapshot) + # Terminate this watch + query_watch.unsubscribe() + """ + Watch.for_query(self, callback, document.DocumentSnapshot) - # # Terminate this watch - # query_watch.unsubscribe() - # """ - # raise NotImplemented def _enum_from_op_string(op_string): diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 1cb30126cb02..89c9ff46c960 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -187,11 +187,13 @@ def __init__(self, self._comparator = comparator self.DocumentSnapshot = DocumentSnapshotCls self._snapshot_callback = snapshot_callback - + self._closing = threading.Lock() + self._closed = False + def should_recover(exc): return ( isinstance(exc, grpc.RpcError) and - exc.code() == grpc.StatusCode.UNVAILABLE) + exc.code() == grpc.StatusCode.UNAVAILABLE) initial_request = firestore_pb2.ListenRequest( database=self._firestore._database_string, @@ -238,6 +240,41 @@ def should_recover(exc): self._consumer = BackgroundConsumer(self.rpc, self.on_snapshot) self._consumer.start() + @property + def is_active(self): + """bool: True if this manager is actively streaming. + + Note that ``False`` does not indicate this is complete shut down, + just that it stopped getting new messages. + """ + return self._consumer is not None and self._consumer.is_active + + def close(self, reason=None): + """Stop consuming messages and shutdown all helper threads. + + This method is idempotent. Additional calls will have no effect. + + Args: + reason (Any): The reason to close this. If None, this is considered + an "intentional" shutdown. + """ + with self._closing: + if self._closed: + return + + # Stop consuming messages. + if self.is_active: + _LOGGER.debug('Stopping consumer.') + self._consumer.stop() + self._consumer = None + + # TODO: Verify we don't have other helper threads that need to be + # shut down here. + + self._rpc = None + self._closed = True + _LOGGER.debug('Finished stopping manager.') + def _on_rpc_done(self, future): """Triggered whenever the underlying RPC terminates without recovery. @@ -249,11 +286,10 @@ def _on_rpc_done(self, future): with shutting everything down. This is to prevent blocking in the background consumer and preventing it from being ``joined()``. """ - # TODO: look at pushing this down into the background consumer _LOGGER.info( - 'RPC termination has signaled shutdown.') + 'RPC termination has signaled manager shutdown.') future = _maybe_wrap_exception(future) - thread = self.threading.Thread( + thread = threading.Thread( name=_RPC_ERROR_THREAD_NAME, target=self.close, kwargs={'reason': future}) @@ -289,19 +325,25 @@ def for_document(cls, document_ref, snapshot_callback, snapshot_callback, snapshot_class_instance) - # @classmethod - # def for_query(cls, query, snapshot_callback): - # return cls(query._client, - # { - # 'query': query.to_proto(), - # 'target_id': WATCH_TARGET_ID - # }, - # query.comparator(), - # snapshot_callback, - # snapshot_class_instance) + @classmethod + def for_query(cls, query, snapshot_callback, snapshot_class_instance): + query_target = firestore_pb2.Target.QueryTarget( + parent=query._parent.id, + structured_query=query._to_protobuf(), + ) + return cls(query, + query._client, + { + 'query': query_target, + 'target_id': WATCH_TARGET_ID + }, + document_watch_comparator, + snapshot_callback, + snapshot_class_instance) # @classmethod - # def for_collection(cls, collection_ref, snapshot_callback): + # def for_collection(cls, collection_ref, snapshot_callback, + # snapshot_class_instance): # return cls(collection_ref._client, # { # 'collection': collection_ref.to_proto(), @@ -379,9 +421,9 @@ def on_snapshot(self, proto): target_change = proto.target_change - if str(target_change): # XXX why if str - _LOGGER.debug('on_snapshot: target change') + if str(target_change): # XXX why if str - if it doesn't exist it will be empty (falsy). Otherwise always true. target_change_type = target_change.target_change_type + _LOGGER.debug('on_snapshot: target change: ' + str(target_change_type)) meth = target_changetype_dispatch.get(target_change_type) if meth is None: _LOGGER.info('on_snapshot: Unknown target change ' + @@ -394,7 +436,10 @@ def on_snapshot(self, proto): raise Exception('Unknown target change type: %s ' % str(target_change_type)) # XXX Exception? else: - meth(proto) + try: + meth(proto) + except Exception as exc2: + _LOGGER.debug("meth(proto) exc: " + str(exc2)) # XXX this is currently a no-op # affects_target = self._affects_target( From 4301130ad8ada1a807497310fcf2c085b65673de Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Tue, 21 Aug 2018 16:02:35 -0400 Subject: [PATCH 021/148] add more tests --- .../google/cloud/firestore_v1beta1/watch.py | 10 +-- firestore/tests/unit/test_watch.py | 71 +++++++++++++++++++ 2 files changed, 76 insertions(+), 5 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 577ce76212fd..107a2bbdaa20 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -559,10 +559,10 @@ def delete_doc(name, updated_tree, updated_map): """ assert name in updated_map, 'Document to delete does not exist' old_document = updated_map.get(name) - existing = updated_tree.find(old_document) + existing = updated_tree.find(name) old_index = existing.index # TODO: was existing.remove returning tree (presumably immuatable?) - updated_tree = updated_tree.remove(old_document) + updated_tree = updated_tree.remove(name) del updated_map[name] return (DocumentChange(ChangeType.REMOVED, old_document, @@ -625,7 +625,7 @@ def modify_doc(new_document, updated_tree, updated_map): for name in delete_changes: change, updated_tree, updated_map = delete_doc( name, updated_tree, updated_map) - if change: + if change: # XXX will always be True appliedChanges.append(change) # TODO: SORT @@ -635,7 +635,7 @@ def modify_doc(new_document, updated_tree, updated_map): _LOGGER.debug('in add_changes') change, updated_tree, updated_map = add_doc( snapshot, updated_tree, updated_map) - if change: + if change: # XXX will always be True appliedChanges.append(change) # TODO: SORT @@ -643,7 +643,7 @@ def modify_doc(new_document, updated_tree, updated_map): for snapshot in update_changes: change, updated_tree, updated_map = modify_doc( snapshot, updated_tree, updated_map) - if change: + if change: # XXX will always be True appliedChanges.append(change) assert len(updated_tree) == len(updated_map), \ diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index d1044362a3ca..dfc2d7c378fb 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -383,6 +383,77 @@ def test__affects_target_current_id_not_in_target_ids(self): inst = self._makeOne() self.assertFalse(inst._affects_target([1], 2)) + def test__extract_changes_doc_removed(self): + from google.cloud.firestore_v1beta1.watch import ChangeType + inst = self._makeOne() + changes = {'name':ChangeType.REMOVED} + doc_map = {'name':True} + results = inst._extract_changes(doc_map, changes, None) + self.assertEqual(results, (['name'], [], [])) + + def test__extract_changes_doc_updated(self): + inst = self._makeOne() + class Dummy(object): + pass + doc = Dummy() + snapshot = Dummy() + changes = {'name':snapshot} + doc_map = {'name':doc} + results = inst._extract_changes(doc_map, changes, 1) + self.assertEqual(results, ([], [], [snapshot])) + self.assertEqual(snapshot.read_time, 1) + + def test__extract_changes_doc_added(self): + inst = self._makeOne() + class Dummy(object): + pass + snapshot = Dummy() + changes = {'name':snapshot} + doc_map = {} + results = inst._extract_changes(doc_map, changes, 1) + self.assertEqual(results, ([], [snapshot], [])) + self.assertEqual(snapshot.read_time, 1) + + def test__compute_snapshot_doctree_and_docmap_disagree_about_length(self): + inst = self._makeOne() + doc_tree = {} + doc_map = {None:None} + self.assertRaises( + AssertionError, + inst._compute_snapshot, doc_tree, doc_map, None, None, None, + ) + + def test__compute_snapshot_operation_relative_ordering(self): + from google.cloud.firestore_v1beta1.watch import WatchDocTree + doc_tree = WatchDocTree() + class DummyDoc(object): + pass + deleted_doc = DummyDoc() + added_doc = DummyDoc() + updated_doc = DummyDoc() + doc_tree = doc_tree.insert('deleted', deleted_doc) + doc_tree = doc_tree.insert('added', added_doc) + doc_tree = doc_tree.insert('updated', updated_doc) + doc_map = { + 'deleted':deleted_doc, + 'added':added_doc, + 'updated':updated_doc, + } + added_snapshot = DummyDocumentSnapshot() + updated_snapshot = DummyDocumentSnapshot() + updated_snapshot.reference = updated_doc + delete_changes = ['deleted'] + add_changes = [added_snapshot] + update_changes = [updated_snapshot] + inst = self._makeOne() + updated_tree, updated_map, applied_changes = inst._compute_snapshot( + doc_tree, + doc_map, + delete_changes, + add_changes, + update_changes + ) + self.assertEqual(updated_map, None) class DummyFirestoreStub(object): def Listen(self): From 623e635601fa1dbdae0246efb864c1bf61c1f525 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Tue, 21 Aug 2018 16:23:40 -0400 Subject: [PATCH 022/148] add tests for close --- .../google/cloud/firestore_v1beta1/watch.py | 1 - firestore/tests/unit/test_watch.py | 37 +++++++++++++++++-- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 4851e775a09b..2ff8e500fafc 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -147,7 +147,6 @@ def document_watch_comparator(doc1, doc2): class Watch(object): - threading = threading # FBO unit tests BackgroundConsumer = BackgroundConsumer # FBO unit tests ResumableBidiRpc = ResumableBidiRpc # FBO unit tests MessageToDict = staticmethod(json_format.MessageToDict) # FBO unit tests diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index dfc2d7c378fb..cd7705c26483 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -149,14 +149,38 @@ def test_ctor(self): self.assertTrue(inst._consumer.started) self.assertTrue(inst.rpc.callbacks, [inst._on_rpc_done]) - def dont_test__on_rpc_done(self): # XXX fails + def test__on_rpc_done(self): inst = self._makeOne() threading = DummyThreading() - inst.threading = threading - inst._on_rpc_done(True) # no close method, fails + with mock.patch( + 'google.cloud.firestore_v1beta1.watch.threading', + threading + ): + inst._on_rpc_done(True) from google.cloud.firestore_v1beta1.watch import _RPC_ERROR_THREAD_NAME self.assertTrue(threading.threads[_RPC_ERROR_THREAD_NAME].started) + def test_close(self): + inst = self._makeOne() + inst.close() + self.assertEqual(inst._consumer, None) + self.assertEqual(inst._rpc, None) + self.assertTrue(inst._closed) + + def test_close_already_closed(self): + inst = self._makeOne() + inst._closed = True + old_consumer = inst._consumer + inst.close() + self.assertEqual(inst._consumer, old_consumer) + + def test_close_inactive(self): + inst = self._makeOne() + old_consumer = inst._consumer + old_consumer.is_active = False + inst.close() + self.assertEqual(old_consumer.stopped, False) + def test_unsubscribe(self): inst = self._makeOne() inst.unsubscribe() @@ -484,6 +508,7 @@ def __init__(self, **kw): class DummyBackgroundConsumer(object): started = False stopped = False + is_active = True def __init__(self, rpc, on_snapshot): self.rpc = rpc @@ -494,11 +519,17 @@ def start(self): def stop(self): self.stopped = True + self.is_active = False class DummyThread(object): started = False + def __init__(self, name, target, kwargs): + self.name = name + self.target = target + self.kwargs = kwargs + def start(self): self.started = True From 27de7bed8ea8d06b243f822a1a37926947553381 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Tue, 21 Aug 2018 16:26:31 -0400 Subject: [PATCH 023/148] not reraising in except broke tests --- .../google/cloud/firestore_v1beta1/watch.py | 1 + firestore/nox.py | 18 +++++++++--------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 2ff8e500fafc..7b9bf0a6e875 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -439,6 +439,7 @@ def on_snapshot(self, proto): meth(proto) except Exception as exc2: _LOGGER.debug("meth(proto) exc: " + str(exc2)) + raise # XXX this is currently a no-op # affects_target = self._affects_target( diff --git a/firestore/nox.py b/firestore/nox.py index 87aa2d1a8157..cb2616041e64 100644 --- a/firestore/nox.py +++ b/firestore/nox.py @@ -43,20 +43,20 @@ def default(session): session.run( 'py.test', '--quiet', - '--cov=google.cloud.firestore', - '--cov=google.cloud.firestore_v1beta1', - '--cov=tests.unit', - '--cov-append', - '--cov-config=.coveragerc', - '--cov-report=', - '--cov-fail-under=97', - os.path.join('tests', 'unit'), +# '--cov=google.cloud.firestore', +# '--cov=google.cloud.firestore_v1beta1', +# '--cov=tests.unit', +# '--cov-append', +# '--cov-config=.coveragerc', +# '--cov-report=', +# '--cov-fail-under=97', + os.path.join('tests', 'unit', 'test_watch.py'), *session.posargs ) @nox.session -@nox.parametrize('py', ['2.7', '3.5', '3.6', '3.7']) +@nox.parametrize('py', ['2.7', '3.7']) def unit(session, py): """Run the unit test suite.""" From c665d73c21b269b83ba318d8ff4eb544dac66a9a Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Tue, 21 Aug 2018 17:24:43 -0400 Subject: [PATCH 024/148] compute_snapshot_ordering test still fails but fails later than it used to --- .../google/cloud/firestore_v1beta1/watch.py | 25 +++++++++---------- firestore/tests/unit/test_watch.py | 15 +++++------ 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 7b9bf0a6e875..bf462e10b838 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -604,6 +604,7 @@ def delete_doc(name, updated_tree, updated_map): """ assert name in updated_map, 'Document to delete does not exist' old_document = updated_map.get(name) + # XXX probably should not expose IndexError when doc doesnt exist existing = updated_tree.find(name) old_index = existing.index # TODO: was existing.remove returning tree (presumably immuatable?) @@ -638,18 +639,18 @@ def modify_doc(new_document, updated_tree, updated_map): document map. Returns the DocumentChange event for successful modifications. """ - name = new_document.reference.formattedName - assert updated_map.has(name), 'Document to modify does not exist' - oldDocument = updated_map.get(name) - if oldDocument.updateTime != new_document.updateTime: - removeChange, updated_tree, updated_map = delete_doc( + name = new_document.reference._document_path + assert name in updated_map, 'Document to modify does not exist' + old_document = updated_map.get(name) + if old_document.update_time != new_document.update_time: + remove_change, updated_tree, updated_map = delete_doc( name, updated_tree, updated_map) - addChange, updated_tree, updated_map = add_doc( + add_change, updated_tree, updated_map = add_doc( new_document, updated_tree, updated_map) return (DocumentChange(ChangeType.MODIFIED, new_document, - removeChange.old_index, - addChange.new_index), + remove_change.old_index, + add_change.new_index), updated_tree, updated_map) return None @@ -670,8 +671,7 @@ def modify_doc(new_document, updated_tree, updated_map): for name in delete_changes: change, updated_tree, updated_map = delete_doc( name, updated_tree, updated_map) - if change: # XXX will always be True - appliedChanges.append(change) + appliedChanges.append(change) # TODO: SORT # add_changes.sort(self._comparator) @@ -680,15 +680,14 @@ def modify_doc(new_document, updated_tree, updated_map): _LOGGER.debug('in add_changes') change, updated_tree, updated_map = add_doc( snapshot, updated_tree, updated_map) - if change: # XXX will always be True - appliedChanges.append(change) + appliedChanges.append(change) # TODO: SORT # update_changes.sort(self._comparator) for snapshot in update_changes: change, updated_tree, updated_map = modify_doc( snapshot, updated_tree, updated_map) - if change: # XXX will always be True + if change is not None: appliedChanges.append(change) assert len(updated_tree) == len(updated_map), \ diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index cd7705c26483..fd808b50a801 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -454,19 +454,20 @@ class DummyDoc(object): pass deleted_doc = DummyDoc() added_doc = DummyDoc() + added_doc._document_path = '/added' updated_doc = DummyDoc() - doc_tree = doc_tree.insert('deleted', deleted_doc) - doc_tree = doc_tree.insert('added', added_doc) - doc_tree = doc_tree.insert('updated', updated_doc) + updated_doc._document_path = '/updated' + doc_tree = doc_tree.insert('/deleted', deleted_doc) + doc_tree = doc_tree.insert('/updated', updated_doc) doc_map = { - 'deleted':deleted_doc, - 'added':added_doc, - 'updated':updated_doc, + '/deleted':deleted_doc, + '/updated':updated_doc, } added_snapshot = DummyDocumentSnapshot() + added_snapshot.reference = added_doc updated_snapshot = DummyDocumentSnapshot() updated_snapshot.reference = updated_doc - delete_changes = ['deleted'] + delete_changes = ['/deleted'] add_changes = [added_snapshot] update_changes = [updated_snapshot] inst = self._makeOne() From 86628fdbb2add61fc94d9c9dc89e4497f1bb74b7 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Tue, 21 Aug 2018 17:25:29 -0400 Subject: [PATCH 025/148] remove incorrect comment --- firestore/google/cloud/firestore_v1beta1/watch.py | 1 - 1 file changed, 1 deletion(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index bf462e10b838..04cd1172379c 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -632,7 +632,6 @@ def add_doc(new_document, updated_tree, updated_map): new_index), updated_tree, updated_map) - # XXX modify_doc is broken via formattedName def modify_doc(new_document, updated_tree, updated_map): """ Applies a document modification to the document tree and the From 23eaad52d2e2468e0e98235ef48510c576588825 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Tue, 21 Aug 2018 15:19:59 -0700 Subject: [PATCH 026/148] parent is a fq path --- firestore/google/cloud/firestore_v1beta1/watch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 7b9bf0a6e875..ee54f5dce85b 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -188,7 +188,7 @@ def __init__(self, self._snapshot_callback = snapshot_callback self._closing = threading.Lock() self._closed = False - + def should_recover(exc): return ( isinstance(exc, grpc.RpcError) and @@ -327,7 +327,7 @@ def for_document(cls, document_ref, snapshot_callback, @classmethod def for_query(cls, query, snapshot_callback, snapshot_class_instance): query_target = firestore_pb2.Target.QueryTarget( - parent=query._parent.id, + parent=query._client._database_string, structured_query=query._to_protobuf(), ) return cls(query, From 0d9de3c2dbdb36f267a30c37c00728817fa058ec Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Tue, 21 Aug 2018 18:34:32 -0400 Subject: [PATCH 027/148] fix and add test for _reset_docs --- .../google/cloud/firestore_v1beta1/watch.py | 10 ++++---- firestore/tests/unit/test_watch.py | 23 ++++++++++++++++++- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index ec789f3eb044..80a4967c4744 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -74,6 +74,9 @@ def __init__(self): def keys(self): return list(self._dict.keys()) + def items(self): + return list(self._dict.items()) + def _copy(self): wdt = WatchDocTree() wdt._dict = self._dict.copy() @@ -713,7 +716,7 @@ def _current_size(self): ) return len(self.doc_map) + len(adds) - len(deletes) - def _reset_docs(self): # XXX broken via formattedName + def _reset_docs(self): """ Helper to clear the docs on RESET or filter mismatch. """ @@ -723,8 +726,7 @@ def _reset_docs(self): # XXX broken via formattedName # TODO: mark each document as deleted. If documents are not delete # they will be sent again by the server. - for snapshot in self.doc_tree: - document_name = snapshot.reference.formattedName - self.change_map[document_name] = ChangeType.REMOVED + for name, snapshot in self.doc_tree.items(): + self.change_map[name] = ChangeType.REMOVED self.current = False diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index fd808b50a801..4cb86df238f0 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -478,7 +478,28 @@ class DummyDoc(object): add_changes, update_changes ) - self.assertEqual(updated_map, None) + # assertion is incorrect below, but we don't get here yet; the tested + # code raises an exception before we get a result + self.assertEqual(updated_map, None) + + def test__reset_docs(self): + from google.cloud.firestore_v1beta1.watch import ChangeType + inst = self._makeOne() + inst.change_map = {None:None} + from google.cloud.firestore_v1beta1.watch import WatchDocTree + doc = DummyDocumentReference() + doc._document_path = '/doc' + doc_tree = WatchDocTree() + doc_tree = doc_tree.insert('/doc', doc) + doc_tree = doc_tree.insert('/doc', doc) + snapshot = DummyDocumentSnapshot() + snapshot.reference = doc + inst.doc_tree = doc_tree + inst._reset_docs() + self.assertEqual(inst.change_map, {'/doc':ChangeType.REMOVED}) + self.assertEqual(inst.resume_token, None) + self.assertFalse(inst.current) + class DummyFirestoreStub(object): def Listen(self): From 2a0723a73651021b30d7ec0645d495352fc48a08 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Wed, 22 Aug 2018 11:37:54 -0400 Subject: [PATCH 028/148] undo mistaken push --- firestore/nox.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/firestore/nox.py b/firestore/nox.py index cb2616041e64..efe32957a479 100644 --- a/firestore/nox.py +++ b/firestore/nox.py @@ -43,14 +43,14 @@ def default(session): session.run( 'py.test', '--quiet', -# '--cov=google.cloud.firestore', -# '--cov=google.cloud.firestore_v1beta1', -# '--cov=tests.unit', -# '--cov-append', -# '--cov-config=.coveragerc', -# '--cov-report=', -# '--cov-fail-under=97', - os.path.join('tests', 'unit', 'test_watch.py'), + '--cov=google.cloud.firestore', + '--cov=google.cloud.firestore_v1beta1', + '--cov=tests.unit', + '--cov-append', + '--cov-config=.coveragerc', + '--cov-report=', + '--cov-fail-under=97', + os.path.join('tests', 'unit'), *session.posargs ) From 889f3504488fdf1b1e6c246e815cf9079835066d Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Wed, 22 Aug 2018 11:51:13 -0400 Subject: [PATCH 029/148] appease the linter --- .../google/cloud/firestore_v1beta1/query.py | 1 - .../google/cloud/firestore_v1beta1/watch.py | 9 +++--- firestore/tests/unit/test_watch.py | 31 ++++++++++--------- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/query.py b/firestore/google/cloud/firestore_v1beta1/query.py index 229ae1afa8b1..1ca4ffe8e2a5 100644 --- a/firestore/google/cloud/firestore_v1beta1/query.py +++ b/firestore/google/cloud/firestore_v1beta1/query.py @@ -630,7 +630,6 @@ def on_snapshot(query_snapshot): Watch.for_query(self, callback, document.DocumentSnapshot) - def _enum_from_op_string(op_string): """Convert a string representation of a binary operator to an enum. diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 80a4967c4744..78300bf693c5 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -270,7 +270,7 @@ def close(self, reason=None): self._consumer.stop() self._consumer = None - # TODO: Verify we don't have other helper threads that need to be + # TODO: Verify we don't have other helper threads that need to be # shut down here. self._rpc = None @@ -423,9 +423,10 @@ def on_snapshot(self, proto): target_change = proto.target_change - if str(target_change): # XXX why if str - if it doesn't exist it will be empty (falsy). Otherwise always true. + if str(target_change): target_change_type = target_change.target_change_type - _LOGGER.debug('on_snapshot: target change: ' + str(target_change_type)) + _LOGGER.debug( + 'on_snapshot: target change: ' + str(target_change_type)) meth = target_changetype_dispatch.get(target_change_type) if meth is None: _LOGGER.info('on_snapshot: Unknown target change ' + @@ -454,7 +455,7 @@ def on_snapshot(self, proto): # # bidi rpc to do its thing. # pass - elif str(proto.document_change): # XXX why if str + elif str(proto.document_change): _LOGGER.debug('on_snapshot: document change') # No other target_ids can show up here, but we still need to see diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index 4cb86df238f0..8332e1ab3046 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -410,29 +410,33 @@ def test__affects_target_current_id_not_in_target_ids(self): def test__extract_changes_doc_removed(self): from google.cloud.firestore_v1beta1.watch import ChangeType inst = self._makeOne() - changes = {'name':ChangeType.REMOVED} - doc_map = {'name':True} + changes = {'name': ChangeType.REMOVED} + doc_map = {'name': True} results = inst._extract_changes(doc_map, changes, None) self.assertEqual(results, (['name'], [], [])) def test__extract_changes_doc_updated(self): inst = self._makeOne() + class Dummy(object): pass + doc = Dummy() snapshot = Dummy() - changes = {'name':snapshot} - doc_map = {'name':doc} + changes = {'name': snapshot} + doc_map = {'name': doc} results = inst._extract_changes(doc_map, changes, 1) self.assertEqual(results, ([], [], [snapshot])) self.assertEqual(snapshot.read_time, 1) - + def test__extract_changes_doc_added(self): inst = self._makeOne() + class Dummy(object): pass + snapshot = Dummy() - changes = {'name':snapshot} + changes = {'name': snapshot} doc_map = {} results = inst._extract_changes(doc_map, changes, 1) self.assertEqual(results, ([], [snapshot], [])) @@ -441,7 +445,7 @@ class Dummy(object): def test__compute_snapshot_doctree_and_docmap_disagree_about_length(self): inst = self._makeOne() doc_tree = {} - doc_map = {None:None} + doc_map = {None: None} self.assertRaises( AssertionError, inst._compute_snapshot, doc_tree, doc_map, None, None, None, @@ -450,8 +454,10 @@ def test__compute_snapshot_doctree_and_docmap_disagree_about_length(self): def test__compute_snapshot_operation_relative_ordering(self): from google.cloud.firestore_v1beta1.watch import WatchDocTree doc_tree = WatchDocTree() + class DummyDoc(object): pass + deleted_doc = DummyDoc() added_doc = DummyDoc() added_doc._document_path = '/added' @@ -459,10 +465,7 @@ class DummyDoc(object): updated_doc._document_path = '/updated' doc_tree = doc_tree.insert('/deleted', deleted_doc) doc_tree = doc_tree.insert('/updated', updated_doc) - doc_map = { - '/deleted':deleted_doc, - '/updated':updated_doc, - } + doc_map = {'/deleted': deleted_doc, '/updated': updated_doc} added_snapshot = DummyDocumentSnapshot() added_snapshot.reference = added_doc updated_snapshot = DummyDocumentSnapshot() @@ -480,12 +483,12 @@ class DummyDoc(object): ) # assertion is incorrect below, but we don't get here yet; the tested # code raises an exception before we get a result - self.assertEqual(updated_map, None) + self.assertEqual(updated_map, None) def test__reset_docs(self): from google.cloud.firestore_v1beta1.watch import ChangeType inst = self._makeOne() - inst.change_map = {None:None} + inst.change_map = {None: None} from google.cloud.firestore_v1beta1.watch import WatchDocTree doc = DummyDocumentReference() doc._document_path = '/doc' @@ -496,7 +499,7 @@ def test__reset_docs(self): snapshot.reference = doc inst.doc_tree = doc_tree inst._reset_docs() - self.assertEqual(inst.change_map, {'/doc':ChangeType.REMOVED}) + self.assertEqual(inst.change_map, {'/doc': ChangeType.REMOVED}) self.assertEqual(inst.resume_token, None) self.assertFalse(inst.current) From 698e51206d1b9c9dd422803171cbd6ab28e708d7 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Wed, 22 Aug 2018 11:54:31 -0400 Subject: [PATCH 030/148] idiom --- firestore/google/cloud/firestore_v1beta1/watch.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 78300bf693c5..0aa2184a38d5 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -702,11 +702,8 @@ def _affects_target(self, target_ids, current_id): if target_ids is None: return True - if current_id in target_ids: - return True - - return False - + return current_id in target_ids + def _current_size(self): """ Returns the current count of all documents, including the changes from From 3a701024a2e64e79007792b694b5758e2c9d8faa Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Wed, 22 Aug 2018 10:24:09 -0700 Subject: [PATCH 031/148] enable collection watches --- .../cloud/firestore_v1beta1/collection.py | 57 ++++++++++--------- .../google/cloud/firestore_v1beta1/query.py | 1 - .../google/cloud/firestore_v1beta1/watch.py | 21 ++----- 3 files changed, 35 insertions(+), 44 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/collection.py b/firestore/google/cloud/firestore_v1beta1/collection.py index 090da0cb0151..2fd3f09680ac 100644 --- a/firestore/google/cloud/firestore_v1beta1/collection.py +++ b/firestore/google/cloud/firestore_v1beta1/collection.py @@ -22,7 +22,8 @@ from google.cloud.firestore_v1beta1 import _helpers from google.cloud.firestore_v1beta1 import query as query_mod from google.cloud.firestore_v1beta1.proto import document_pb2 - +from google.cloud.firestore_v1beta1.watch import Watch +from google.cloud.firestore_v1beta1.document import DocumentSnapshot _AUTO_ID_CHARS = ( 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789') @@ -371,33 +372,33 @@ def get(self, transaction=None): query = query_mod.Query(self) return query.get(transaction=transaction) - # def on_snapshot(self, callback): - # """Monitor the documents in this collection. - # - # This starts a watch on this collection using a background thread. The - # provided callback is run on the snapshot of the documents. - # - # Args: - # callback(~.firestore.collection.CollectionSnapshot): a callback - # to run when a change occurs. - # - # Example: - # from google.cloud import firestore - # - # db = firestore.Client() - # collection_ref = db.collection(u'users') - # - # def on_snapshot(collection_snapshot): - # for doc in collection_snapshot.documents: - # print(u'{} => {}'.format(doc.id, doc.to_dict())) - # - # # Watch this collection - # collection_watch = collection_ref.on_snapshot(on_snapshot) - # - # # Terminate this watch - # collection_watch.unsubscribe() - # """ - # raise NotImplemented + def on_snapshot(self, callback): + """Monitor the documents in this collection. + + This starts a watch on this collection using a background thread. The + provided callback is run on the snapshot of the documents. + + Args: + callback(~.firestore.collection.CollectionSnapshot): a callback + to run when a change occurs. + + Example: + from google.cloud import firestore + + db = firestore.Client() + collection_ref = db.collection(u'users') + + def on_snapshot(collection_snapshot): + for doc in collection_snapshot.documents: + print(u'{} => {}'.format(doc.id, doc.to_dict())) + + # Watch this collection + collection_watch = collection_ref.on_snapshot(on_snapshot) + + # Terminate this watch + collection_watch.unsubscribe() + """ + Watch.for_query(query_mod.Query(self), callback, DocumentSnapshot) def _auto_id(): diff --git a/firestore/google/cloud/firestore_v1beta1/query.py b/firestore/google/cloud/firestore_v1beta1/query.py index 229ae1afa8b1..1ca4ffe8e2a5 100644 --- a/firestore/google/cloud/firestore_v1beta1/query.py +++ b/firestore/google/cloud/firestore_v1beta1/query.py @@ -630,7 +630,6 @@ def on_snapshot(query_snapshot): Watch.for_query(self, callback, document.DocumentSnapshot) - def _enum_from_op_string(op_string): """Convert a string representation of a binary operator to an enum. diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 80a4967c4744..e3988acde410 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -270,7 +270,7 @@ def close(self, reason=None): self._consumer.stop() self._consumer = None - # TODO: Verify we don't have other helper threads that need to be + # TODO: Verify we don't have other helper threads that need to be # shut down here. self._rpc = None @@ -343,18 +343,6 @@ def for_query(cls, query, snapshot_callback, snapshot_class_instance): snapshot_callback, snapshot_class_instance) - # @classmethod - # def for_collection(cls, collection_ref, snapshot_callback, - # snapshot_class_instance): - # return cls(collection_ref._client, - # { - # 'collection': collection_ref.to_proto(), - # 'target_id': WATCH_TARGET_ID - # }, - # document_watch_comparator, - # snapshot_callback, - # snapshot_class_instance) - def _on_snapshot_target_change_no_change(self, proto): _LOGGER.debug('on_snapshot: target change: NO_CHANGE') change = proto.target_change @@ -423,9 +411,12 @@ def on_snapshot(self, proto): target_change = proto.target_change - if str(target_change): # XXX why if str - if it doesn't exist it will be empty (falsy). Otherwise always true. + if str(target_change): + # XXX why if str - if it doesn't exist it will be empty (falsy). + # Otherwise this was always true. target_change_type = target_change.target_change_type - _LOGGER.debug('on_snapshot: target change: ' + str(target_change_type)) + _LOGGER.debug( + 'on_snapshot: target change: ' + str(target_change_type)) meth = target_changetype_dispatch.get(target_change_type) if meth is None: _LOGGER.info('on_snapshot: Unknown target change ' + From 2301a0e60db192c34d9859943b90d9d50c012b67 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Wed, 22 Aug 2018 18:10:35 -0400 Subject: [PATCH 032/148] undo spurious changes --- firestore/nox.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/firestore/nox.py b/firestore/nox.py index efe32957a479..87aa2d1a8157 100644 --- a/firestore/nox.py +++ b/firestore/nox.py @@ -56,7 +56,7 @@ def default(session): @nox.session -@nox.parametrize('py', ['2.7', '3.7']) +@nox.parametrize('py', ['2.7', '3.5', '3.6', '3.7']) def unit(session, py): """Run the unit test suite.""" From cca772ec918da1391610f7032bb7948274a6985f Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Wed, 22 Aug 2018 18:11:23 -0400 Subject: [PATCH 033/148] add unfinished test --- firestore/tests/system.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/firestore/tests/system.py b/firestore/tests/system.py index a39d98571563..5bdab39e416d 100644 --- a/firestore/tests/system.py +++ b/firestore/tests/system.py @@ -780,3 +780,32 @@ def on_response(*arg): sleep(1) if on_response.called_count != 1: raise AssertionError("Failed to get exactly one document change") + +def test_watch_collection(client, cleanup): + db = client + doc_ref = db.collection(u'users').document( + u'alovelace' + unique_resource_id()) + collection_ref = db.collection(u'users') + def on_snapshot(snapshot): + for doc in snapshot.documents: + print(u'{} => {}'.format(doc.id, doc.to_dict())) + + collection_ref.on_snapshot(on_snapshot) + + sleep(1) + + # Initial setting + doc_ref.set({ + u'first': u'Jane', + u'last': u'Doe', + u'born': 1900 + }) + + doc_ref.set({ + u'first': u'Ada', + u'last': u'Lovelace', + u'born': 1815 + }) + + # CM: had to stop here, this test is totally unfinished, trying to formalize + # https://gist.github.com/crwilcox/ce05f3857adc7a0ed86ffbd039b1a035 From 48fac6d84f6e98f09c1a43ae820c94cef7dea40c Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Thu, 23 Aug 2018 14:52:08 -0700 Subject: [PATCH 034/148] modify the way we compute document references to support query and collection watches --- .../cloud/firestore_v1beta1/collection.py | 4 ++-- .../cloud/firestore_v1beta1/document.py | 2 +- .../google/cloud/firestore_v1beta1/query.py | 2 +- .../google/cloud/firestore_v1beta1/watch.py | 19 ++++++++++++++----- 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/collection.py b/firestore/google/cloud/firestore_v1beta1/collection.py index 2fd3f09680ac..81229120a1e5 100644 --- a/firestore/google/cloud/firestore_v1beta1/collection.py +++ b/firestore/google/cloud/firestore_v1beta1/collection.py @@ -23,7 +23,7 @@ from google.cloud.firestore_v1beta1 import query as query_mod from google.cloud.firestore_v1beta1.proto import document_pb2 from google.cloud.firestore_v1beta1.watch import Watch -from google.cloud.firestore_v1beta1.document import DocumentSnapshot +from google.cloud.firestore_v1beta1 import document _AUTO_ID_CHARS = ( 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789') @@ -398,7 +398,7 @@ def on_snapshot(collection_snapshot): # Terminate this watch collection_watch.unsubscribe() """ - Watch.for_query(query_mod.Query(self), callback, DocumentSnapshot) + Watch.for_query(query_mod.Query(self), callback, document) def _auto_id(): diff --git a/firestore/google/cloud/firestore_v1beta1/document.py b/firestore/google/cloud/firestore_v1beta1/document.py index cd8c82f859d4..8e5a9a7bf110 100644 --- a/firestore/google/cloud/firestore_v1beta1/document.py +++ b/firestore/google/cloud/firestore_v1beta1/document.py @@ -471,7 +471,7 @@ def on_snapshot(document_snapshot): # Terminate this watch doc_watch.unsubscribe() """ - Watch.for_document(self, callback, DocumentSnapshot) + Watch.for_document(self, callback, __import__(__name__)) class DocumentSnapshot(object): diff --git a/firestore/google/cloud/firestore_v1beta1/query.py b/firestore/google/cloud/firestore_v1beta1/query.py index 1ca4ffe8e2a5..76712abc3c78 100644 --- a/firestore/google/cloud/firestore_v1beta1/query.py +++ b/firestore/google/cloud/firestore_v1beta1/query.py @@ -627,7 +627,7 @@ def on_snapshot(query_snapshot): # Terminate this watch query_watch.unsubscribe() """ - Watch.for_query(self, callback, document.DocumentSnapshot) + Watch.for_query(self, callback, document) def _enum_from_op_string(op_string): diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 9269014e5075..500711e59402 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -160,7 +160,7 @@ def __init__(self, target, comparator, snapshot_callback, - DocumentSnapshotCls, + document_module, BackgroundConsumer=None, # FBO unit testing ResumableBidiRpc=None, # FBO unit testing ): @@ -180,14 +180,15 @@ def __init__(self, snapshot was obtained. # TODO: Go had an err here and node.js provided size. # TODO: do we want to include either? - DocumentSnapshotCls: instance of the DocumentSnapshot class + document_module: instance of the Document module """ self._document_reference = document_reference self._firestore = firestore self._api = firestore._firestore_api self._targets = target self._comparator = comparator - self.DocumentSnapshot = DocumentSnapshotCls + self.DocumentSnapshot = document_module.DocumentSnapshot + self.DocumentReference = document_module.DocumentReference self._snapshot_callback = snapshot_callback self._closing = threading.Lock() self._closed = False @@ -410,7 +411,6 @@ def on_snapshot(self, proto): } target_change = proto.target_change - if str(target_change): target_change_type = target_change.target_change_type _LOGGER.debug( @@ -469,8 +469,17 @@ def on_snapshot(self, proto): data = self.MessageToDict(document) + # Create a snapshot. As Document and Query objects can be + # passed we need to get a Document Reference in a more manual + # fashion than self._document_reference + document_name = document.name + db_str = self._firestore._database_string + if document_name.startswith(db_str): + document_name = document_name[len(db_str):] + document_ref = self._firestore.document(document_name) + snapshot = self.DocumentSnapshot( - reference=self._document_reference, + reference=document_ref, data=data['fields'], exists=True, read_time=None, From 66b6071ffba1e8cf1356f6d18aa84983eaa4cf00 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Thu, 23 Aug 2018 21:41:37 -0700 Subject: [PATCH 035/148] add system tests for each variety of watch --- .../cloud/firestore_v1beta1/collection.py | 5 +- .../cloud/firestore_v1beta1/document.py | 2 +- .../google/cloud/firestore_v1beta1/query.py | 5 +- .../google/cloud/firestore_v1beta1/watch.py | 27 ++++-- firestore/tests/system.py | 95 ++++++++++++++++--- 5 files changed, 110 insertions(+), 24 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/collection.py b/firestore/google/cloud/firestore_v1beta1/collection.py index 81229120a1e5..1110858f4667 100644 --- a/firestore/google/cloud/firestore_v1beta1/collection.py +++ b/firestore/google/cloud/firestore_v1beta1/collection.py @@ -398,7 +398,10 @@ def on_snapshot(collection_snapshot): # Terminate this watch collection_watch.unsubscribe() """ - Watch.for_query(query_mod.Query(self), callback, document) + Watch.for_query(query_mod.Query(self), + callback, + document.DocumentSnapshot, + document.DocumentReference) def _auto_id(): diff --git a/firestore/google/cloud/firestore_v1beta1/document.py b/firestore/google/cloud/firestore_v1beta1/document.py index 8e5a9a7bf110..1cc105b0d828 100644 --- a/firestore/google/cloud/firestore_v1beta1/document.py +++ b/firestore/google/cloud/firestore_v1beta1/document.py @@ -471,7 +471,7 @@ def on_snapshot(document_snapshot): # Terminate this watch doc_watch.unsubscribe() """ - Watch.for_document(self, callback, __import__(__name__)) + Watch.for_document(self, callback, DocumentSnapshot, DocumentReference) class DocumentSnapshot(object): diff --git a/firestore/google/cloud/firestore_v1beta1/query.py b/firestore/google/cloud/firestore_v1beta1/query.py index 76712abc3c78..c39a5febea44 100644 --- a/firestore/google/cloud/firestore_v1beta1/query.py +++ b/firestore/google/cloud/firestore_v1beta1/query.py @@ -627,7 +627,10 @@ def on_snapshot(query_snapshot): # Terminate this watch query_watch.unsubscribe() """ - Watch.for_query(self, callback, document) + Watch.for_query(self, + callback, + document.DocumentSnapshot, + document.DocumentReference) def _enum_from_op_string(op_string): diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 500711e59402..b4eeee159bf5 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -160,7 +160,8 @@ def __init__(self, target, comparator, snapshot_callback, - document_module, + document_snapshot_cls, + document_reference_cls, BackgroundConsumer=None, # FBO unit testing ResumableBidiRpc=None, # FBO unit testing ): @@ -180,15 +181,16 @@ def __init__(self, snapshot was obtained. # TODO: Go had an err here and node.js provided size. # TODO: do we want to include either? - document_module: instance of the Document module + document_snapshot_cls: instance of DocumentSnapshot + document_reference_cls: instance of DocumentReference """ self._document_reference = document_reference self._firestore = firestore self._api = firestore._firestore_api self._targets = target self._comparator = comparator - self.DocumentSnapshot = document_module.DocumentSnapshot - self.DocumentReference = document_module.DocumentReference + self.DocumentSnapshot = document_snapshot_cls + self.DocumentReference = document_reference_cls self._snapshot_callback = snapshot_callback self._closing = threading.Lock() self._closed = False @@ -304,7 +306,7 @@ def unsubscribe(self): # XXX should this be aliased to close? @classmethod def for_document(cls, document_ref, snapshot_callback, - snapshot_class_instance): + snapshot_class_instance, reference_class_instance): """ Creates a watch snapshot listener for a document. snapshot_callback receives a DocumentChange object, but may also start to get @@ -313,8 +315,10 @@ def for_document(cls, document_ref, snapshot_callback, Args: document_ref: Reference to Document snapshot_callback: callback to be called on snapshot - snapshot_class_instance: instance of snapshot cls to make + snapshot_class_instance: instance of DocumentSnapshot to make snapshots with to pass to snapshot_callback + reference_class_instance: instance of DocumentReference to make + references """ return cls(document_ref, @@ -326,10 +330,12 @@ def for_document(cls, document_ref, snapshot_callback, }, document_watch_comparator, snapshot_callback, - snapshot_class_instance) + snapshot_class_instance, + reference_class_instance) @classmethod - def for_query(cls, query, snapshot_callback, snapshot_class_instance): + def for_query(cls, query, snapshot_callback, snapshot_class_instance, + reference_class_instance): query_target = firestore_pb2.Target.QueryTarget( parent=query._client._database_string, structured_query=query._to_protobuf(), @@ -342,7 +348,8 @@ def for_query(cls, query, snapshot_callback, snapshot_class_instance): }, document_watch_comparator, snapshot_callback, - snapshot_class_instance) + snapshot_class_instance, + reference_class_instance) def _on_snapshot_target_change_no_change(self, proto): _LOGGER.debug('on_snapshot: target change: NO_CHANGE') @@ -700,7 +707,7 @@ def _affects_target(self, target_ids, current_id): return True return current_id in target_ids - + def _current_size(self): """ Returns the current count of all documents, including the changes from diff --git a/firestore/tests/system.py b/firestore/tests/system.py index 5bdab39e416d..6edffb059283 100644 --- a/firestore/tests/system.py +++ b/firestore/tests/system.py @@ -763,12 +763,12 @@ def test_watch_document(client, cleanup): sleep(1) # Setup listener - def on_response(*arg): - on_response.called_count += 1 + def on_snapshot(docs, changes, read_time): + on_snapshot.called_count += 1 - on_response.called_count = 0 + on_snapshot.called_count = 0 - doc_ref.on_snapshot(on_response) + doc_ref.on_snapshot(on_snapshot) # Alter document doc_ref.set({ @@ -778,22 +778,74 @@ def on_response(*arg): }) sleep(1) - if on_response.called_count != 1: - raise AssertionError("Failed to get exactly one document change") + + for _ in range(10): + if on_snapshot.called_count == 1: + return + sleep(1) + + if on_snapshot.called_count != 1: + raise AssertionError( + "Failed to get exactly one document change: count: " + + str(on_snapshot.called_count)) + def test_watch_collection(client, cleanup): db = client doc_ref = db.collection(u'users').document( u'alovelace' + unique_resource_id()) collection_ref = db.collection(u'users') - def on_snapshot(snapshot): - for doc in snapshot.documents: - print(u'{} => {}'.format(doc.id, doc.to_dict())) + + # Initial setting + doc_ref.set({ + u'first': u'Jane', + u'last': u'Doe', + u'born': 1900 + }) + + # Setup listener + def on_snapshot(docs, changes, read_time): + on_snapshot.called_count += 1 + + on_snapshot.called_count = 0 + + # def on_snapshot(docs, changes, read_time): + # for doc in docs: + # print(u'{} => {}'.format(doc.id, doc.to_dict())) collection_ref.on_snapshot(on_snapshot) sleep(1) + doc_ref.set({ + u'first': u'Ada', + u'last': u'Lovelace', + u'born': 1815 + }) + + sleep(1) + + for _ in range(10): + if on_snapshot.called_count == 1: + return + sleep(1) + + if on_snapshot.called_count != 1: + raise AssertionError( + "Failed to get exactly one document change: count: " + + str(on_snapshot.called_count)) + + # CM: had to stop here, this test is totally unfinished, trying to + # formalize + # https://gist.github.com/crwilcox/ce05f3857adc7a0ed86ffbd039b1a035 + + +def test_watch_query(client, cleanup): + db = client + doc_ref = db.collection(u'users').document( + u'alovelace' + unique_resource_id()) + query_ref = db.collection(u'users').where("first", "==", u'Ada') + # Initial setting doc_ref.set({ u'first': u'Jane', @@ -801,11 +853,32 @@ def on_snapshot(snapshot): u'born': 1900 }) + sleep(1) + + # Setup listener + def on_snapshot(docs, changes, read_time): + on_snapshot.called_count += 1 + print("docs: " + docs) + print("changes: " + changes) + print("read_time: " + read_time) + + on_snapshot.called_count = 0 + + query_ref.on_snapshot(on_snapshot) + + # Alter document doc_ref.set({ u'first': u'Ada', u'last': u'Lovelace', u'born': 1815 }) - # CM: had to stop here, this test is totally unfinished, trying to formalize - # https://gist.github.com/crwilcox/ce05f3857adc7a0ed86ffbd039b1a035 + for _ in range(10): + if on_snapshot.called_count == 1: + return + sleep(1) + + if on_snapshot.called_count != 1: + raise AssertionError( + "Failed to get exactly one document change: count: " + + str(on_snapshot.called_count)) From 67a609fefddf41f82e390f80c58c71a1b8ac60b6 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Fri, 24 Aug 2018 10:42:52 -0400 Subject: [PATCH 036/148] fix most unit tests, 3 still fail --- firestore/tests/unit/test_watch.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index 8332e1ab3046..fb0708645530 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -106,8 +106,10 @@ def _makeOne( comparator=None, snapshot_callback=None, snapshot_class=None, + document_reference_class=None, ): from google.cloud.firestore_v1beta1.watch import Watch + from google.cloud.firestore_v1beta1.document import DocumentReference if document_reference is None: document_reference = DummyDocumentReference() if firestore is None: @@ -125,6 +127,8 @@ def _makeOne( snapshot_callback = self._snapshot_callback if snapshot_class is None: snapshot_class = DummyDocumentSnapshot + if document_reference_class is None: + document_reference_class = DocumentReference inst = Watch( document_reference, @@ -133,6 +137,7 @@ def _makeOne( comparator, snapshot_callback, snapshot_class, + document_reference_class, BackgroundConsumer=DummyBackgroundConsumer, ResumableBidiRpc=DummyRpc, ) From d598f323b22978d346bd50323bd8854c339dfe2d Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Fri, 24 Aug 2018 09:24:34 -0700 Subject: [PATCH 037/148] merge and apply --- firestore/tests/unit/test_watch.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index fb0708645530..a1bd38fb080d 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -106,10 +106,9 @@ def _makeOne( comparator=None, snapshot_callback=None, snapshot_class=None, - document_reference_class=None, + reference_class=None ): from google.cloud.firestore_v1beta1.watch import Watch - from google.cloud.firestore_v1beta1.document import DocumentReference if document_reference is None: document_reference = DummyDocumentReference() if firestore is None: @@ -127,9 +126,8 @@ def _makeOne( snapshot_callback = self._snapshot_callback if snapshot_class is None: snapshot_class = DummyDocumentSnapshot - if document_reference_class is None: - document_reference_class = DocumentReference - + if reference_class is None: + reference_class = DummyDocumentReference inst = Watch( document_reference, firestore, @@ -137,7 +135,7 @@ def _makeOne( comparator, snapshot_callback, snapshot_class, - document_reference_class, + reference_class, BackgroundConsumer=DummyBackgroundConsumer, ResumableBidiRpc=DummyRpc, ) @@ -196,6 +194,7 @@ def test_for_document(self): docref = DummyDocumentReference() snapshot_callback = self._snapshot_callback snapshot_class_instance = DummyDocumentSnapshot + document_reference_class_instance = DummyDocumentReference modulename = 'google.cloud.firestore_v1beta1.watch' with mock.patch( '%s.Watch.ResumableBidiRpc' % modulename, @@ -208,7 +207,8 @@ def test_for_document(self): inst = Watch.for_document( docref, snapshot_callback, - snapshot_class_instance + snapshot_class_instance, + document_reference_class_instance ) self.assertTrue(inst._consumer.started) self.assertTrue(inst.rpc.callbacks, [inst._on_rpc_done]) @@ -528,6 +528,7 @@ def __init__(self): class DummyFirestore(object): _firestore_api = DummyFirestoreClient() _database_string = '' + document = DummyDocumentReference class DummyDocumentSnapshot(object): From c6ae7253248bb4b398ce04ddc05023a43500d320 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Fri, 24 Aug 2018 09:43:06 -0700 Subject: [PATCH 038/148] expected time of test was not the same as read time, so false failure --- firestore/tests/unit/test_watch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index a1bd38fb080d..3e8d75c5c400 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -384,7 +384,7 @@ class DummyReadTime(object): inst.push(DummyReadTime, 'token') self.assertEqual( self.snapshotted, - ([], [], datetime.datetime(2018, 8, 21, 9, 31, 18)), + ([], [], datetime.datetime(2018, 8, 21, 6, 31, 18)), ) self.assertTrue(inst.has_pushed) self.assertEqual(inst.resume_token, 'token') From 87ccc8763a47f4d6d0048e23b8395bb11cf4c166 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Fri, 24 Aug 2018 10:11:09 -0700 Subject: [PATCH 039/148] tests passing --- firestore/tests/unit/test_watch.py | 52 +++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index 3e8d75c5c400..1644434916ab 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -461,7 +461,7 @@ def test__compute_snapshot_operation_relative_ordering(self): doc_tree = WatchDocTree() class DummyDoc(object): - pass + update_time = mock.sentinel deleted_doc = DummyDoc() added_doc = DummyDoc() @@ -471,9 +471,11 @@ class DummyDoc(object): doc_tree = doc_tree.insert('/deleted', deleted_doc) doc_tree = doc_tree.insert('/updated', updated_doc) doc_map = {'/deleted': deleted_doc, '/updated': updated_doc} - added_snapshot = DummyDocumentSnapshot() + added_snapshot = DummyDocumentSnapshot(added_doc, None, True, + None, None, None) added_snapshot.reference = added_doc - updated_snapshot = DummyDocumentSnapshot() + updated_snapshot = DummyDocumentSnapshot(updated_doc, None, True, + None, None, None) updated_snapshot.reference = updated_doc delete_changes = ['/deleted'] add_changes = [added_snapshot] @@ -486,9 +488,13 @@ class DummyDoc(object): add_changes, update_changes ) - # assertion is incorrect below, but we don't get here yet; the tested - # code raises an exception before we get a result - self.assertEqual(updated_map, None) + # TODO: + # Assertion is not verified correct below. Verify this test is good. + self.assertEqual(updated_map, + { + '/updated': updated_snapshot, + '/added': added_snapshot, + }) def test__reset_docs(self): from google.cloud.firestore_v1beta1.watch import ChangeType @@ -500,7 +506,7 @@ def test__reset_docs(self): doc_tree = WatchDocTree() doc_tree = doc_tree.insert('/doc', doc) doc_tree = doc_tree.insert('/doc', doc) - snapshot = DummyDocumentSnapshot() + snapshot = DummyDocumentSnapshot(doc, None, True, None, None, None) snapshot.reference = doc inst.doc_tree = doc_tree inst._reset_docs() @@ -520,20 +526,42 @@ def __init__(self): class DummyDocumentReference(object): - def __init__(self): - self._client = DummyFirestore() + def __init__(self, *document_path, **kw): + if 'client' not in kw: + self._client = DummyFirestore() + else: + self._client = kw['client'] + + self._path = document_path + self.__dict__.update(kw) + _document_path = '/' class DummyFirestore(object): _firestore_api = DummyFirestoreClient() _database_string = '' - document = DummyDocumentReference + + def document(self, *document_path): + if len(document_path) == 1: + path = document_path[0].split('/') + else: + path = document_path + + return DummyDocumentReference(*path, client=self) class DummyDocumentSnapshot(object): - def __init__(self, **kw): - self.__dict__.update(kw) + # def __init__(self, **kw): + # self.__dict__.update(kw) + def __init__(self, reference, data, exists, + read_time, create_time, update_time): + self.reference = reference + self.data = data + self.exists = exists + self.read_time = read_time + self.create_time = create_time + self.update_time = update_time class DummyBackgroundConsumer(object): From 725f4a40092b28269559af6b390be4079a41b30b Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Sat, 25 Aug 2018 13:10:54 -0400 Subject: [PATCH 040/148] make the datetime.datetime returned non-naive and assume it's in UTC --- firestore/google/cloud/firestore_v1beta1/watch.py | 4 +++- firestore/tests/unit/test_watch.py | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index b4eeee159bf5..be443f22f1c2 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -18,6 +18,8 @@ import datetime from enum import Enum +import pytz + from google.cloud.firestore_v1beta1.bidi import ResumableBidiRpc from google.cloud.firestore_v1beta1.bidi import BackgroundConsumer from google.cloud.firestore_v1beta1.proto import firestore_pb2 @@ -561,7 +563,7 @@ def push(self, read_time, next_resume_token): self._snapshot_callback( updated_tree.keys(), appliedChanges, - datetime.datetime.fromtimestamp(read_time.seconds) + datetime.datetime.fromtimestamp(read_time.seconds, pytz.utc) ) self.has_pushed = True diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index 1644434916ab..a1fa9c987a9c 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -378,13 +378,18 @@ def test_on_snapshot_unknown_listen_type(self): ) def test_push_no_changes(self): + import pytz class DummyReadTime(object): seconds = 1534858278 inst = self._makeOne() inst.push(DummyReadTime, 'token') self.assertEqual( self.snapshotted, - ([], [], datetime.datetime(2018, 8, 21, 6, 31, 18)), + ( + [], + [], + datetime.datetime.fromtimestamp(DummyReadTime.seconds, pytz.utc) + ), ) self.assertTrue(inst.has_pushed) self.assertEqual(inst.resume_token, 'token') From ee126081492b29821b42f61a89cc1c16a1e45ff4 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Sat, 25 Aug 2018 13:16:10 -0400 Subject: [PATCH 041/148] depends directly on pytz now --- firestore/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/firestore/setup.py b/firestore/setup.py index 81b63a520e50..bd3b98b175ab 100644 --- a/firestore/setup.py +++ b/firestore/setup.py @@ -31,6 +31,7 @@ dependencies = [ 'google-cloud-core<0.29dev,>=0.28.0', 'google-api-core[grpc]<2.0.0dev,>=0.1.1', + 'pytz', ] extras = { } From 6aac66414414d032673257d21fd8e78cb41f35ce Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Tue, 28 Aug 2018 15:32:05 -0400 Subject: [PATCH 042/148] 100pct statement coverage for watch and test_watch --- .../google/cloud/firestore_v1beta1/watch.py | 4 +- firestore/tests/unit/test_watch.py | 106 +++++++++++++++++- 2 files changed, 104 insertions(+), 6 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index be443f22f1c2..b91c679db065 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -197,7 +197,7 @@ def __init__(self, self._closing = threading.Lock() self._closed = False - def should_recover(exc): + def should_recover(exc): # pragma: NO COVER return ( isinstance(exc, grpc.RpcError) and exc.code() == grpc.StatusCode.UNAVAILABLE) @@ -662,7 +662,7 @@ def modify_doc(new_document, updated_tree, updated_map): add_change.new_index), updated_tree, updated_map) - return None + return None, updated_tree, updated_map # Process the sorted changes in the order that is expected by our # clients (removals, additions, and then modifications). We also need diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index a1fa9c987a9c..9dc7861c04cd 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -38,6 +38,12 @@ def test___len__(self): inst = inst.insert('a', 2) self.assertEqual(len(inst), 2) + def test___iter__(self): + inst = self._makeOne() + inst = inst.insert('b', 1) + inst = inst.insert('a', 2) + self.assertEqual(sorted(list(inst)), ['a', 'b']) + class TestDocumentChange(unittest.TestCase): def _makeOne(self, type, document, old_index, new_index): @@ -107,7 +113,7 @@ def _makeOne( snapshot_callback=None, snapshot_class=None, reference_class=None - ): + ): # pragma: NO COVER from google.cloud.firestore_v1beta1.watch import Watch if document_reference is None: document_reference = DummyDocumentReference() @@ -141,7 +147,7 @@ def _makeOne( ) return inst - def _document_watch_comparator(self, doc1, doc2): + def _document_watch_comparator(self, doc1, doc2): # pragma: NO COVER return 0 def _snapshot_callback(self, docs, changes, read_time): @@ -213,6 +219,36 @@ def test_for_document(self): self.assertTrue(inst._consumer.started) self.assertTrue(inst.rpc.callbacks, [inst._on_rpc_done]) + def test_for_query(self): + from google.cloud.firestore_v1beta1.watch import Watch + snapshot_callback = self._snapshot_callback + snapshot_class_instance = DummyDocumentSnapshot + document_reference_class_instance = DummyDocumentReference + modulename = 'google.cloud.firestore_v1beta1.watch' + pb2 = DummyPb2() + with mock.patch( + '%s.firestore_pb2' % modulename, + pb2, + ): + with mock.patch( + '%s.Watch.ResumableBidiRpc' % modulename, + DummyRpc, + ): + with mock.patch( + '%s.Watch.BackgroundConsumer' % modulename, + DummyBackgroundConsumer, + ): + query = DummyQuery() + inst = Watch.for_query( + query, + snapshot_callback, + snapshot_class_instance, + document_reference_class_instance + ) + self.assertTrue(inst._consumer.started) + self.assertTrue(inst.rpc.callbacks, [inst._on_rpc_done]) + self.assertEqual(inst._targets['query'], 'dummy query target') + def test_on_snapshot_target_no_change_no_target_ids_not_current(self): inst = self._makeOne() proto = DummyProto() @@ -254,6 +290,16 @@ def test_on_snapshot_target_remove(self): inst.on_snapshot(proto) self.assertEqual(str(exc.exception), 'Error 1: hi') + def test_on_snapshot_target_remove_nocause(self): + inst = self._makeOne() + proto = DummyProto() + target_change = proto.target_change + target_change.cause = None + target_change.target_change_type = firestore_pb2.TargetChange.REMOVE + with self.assertRaises(Exception) as exc: + inst.on_snapshot(proto) + self.assertEqual(str(exc.exception), 'Error 13: internal error') + def test_on_snapshot_target_reset(self): inst = self._makeOne() @@ -501,6 +547,36 @@ class DummyDoc(object): '/added': added_snapshot, }) + def test__compute_snapshot_modify_docs_updated_doc_no_timechange(self): + from google.cloud.firestore_v1beta1.watch import WatchDocTree + doc_tree = WatchDocTree() + + class DummyDoc(object): + pass + + updated_doc_v1 = DummyDoc() + updated_doc_v1.update_time = 1 + updated_doc_v1._document_path = '/updated' + updated_doc_v2 = DummyDoc() + updated_doc_v2.update_time = 1 + updated_doc_v2._document_path = '/updated' + doc_tree = doc_tree.insert('/updated', updated_doc_v1) + doc_map = {'/updated': updated_doc_v1} + updated_snapshot = DummyDocumentSnapshot(updated_doc_v2, None, True, + None, None, 1) + delete_changes = [] + add_changes = [] + update_changes = [updated_snapshot] + inst = self._makeOne() + updated_tree, updated_map, applied_changes = inst._compute_snapshot( + doc_tree, + doc_map, + delete_changes, + add_changes, + update_changes + ) + self.assertEqual(updated_map, doc_map) # no change + def test__reset_docs(self): from google.cloud.firestore_v1beta1.watch import ChangeType inst = self._makeOne() @@ -521,7 +597,7 @@ def test__reset_docs(self): class DummyFirestoreStub(object): - def Listen(self): + def Listen(self): # pragma: NO COVER pass @@ -542,12 +618,22 @@ def __init__(self, *document_path, **kw): _document_path = '/' +class DummyQuery(object): # pragma: NO COVER + def __init__(self, **kw): + if 'client' not in kw: + self._client = DummyFirestore() + else: + self._client = kw['client'] + + def _to_protobuf(self): + return '' + class DummyFirestore(object): _firestore_api = DummyFirestoreClient() _database_string = '' - def document(self, *document_path): + def document(self, *document_path): # pragma: NO COVER if len(document_path) == 1: path = document_path[0].split('/') else: @@ -642,3 +728,15 @@ class DummyProto(object): def __init__(self): self.target_change = DummyChange() self.document_change = DummyChange() + + +class DummyTarget(object): + def QueryTarget(self, **kw): + self.kw = kw + return 'dummy query target' + + +class DummyPb2(object): + Target = DummyTarget() + def ListenRequest(self, **kw): + pass From 97368c9105d2e615d30825322162af2140cab38c Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Tue, 28 Aug 2018 18:19:07 -0400 Subject: [PATCH 043/148] just cutnpaste this i guess --- firestore/tests/unit/test_bidi.py | 658 ++++++++++++++++++++++++++++++ 1 file changed, 658 insertions(+) create mode 100644 firestore/tests/unit/test_bidi.py diff --git a/firestore/tests/unit/test_bidi.py b/firestore/tests/unit/test_bidi.py new file mode 100644 index 000000000000..09a23b10405e --- /dev/null +++ b/firestore/tests/unit/test_bidi.py @@ -0,0 +1,658 @@ +# Copyright 2018, Google LLC All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import threading + +import grpc +import mock +import pytest +from six.moves import queue + +from google.api_core import exceptions +from google.cloud.firestore_v1beta1 import bidi + + +class Test_RequestQueueGenerator(object): + + def test_bounded_consume(self): + call = mock.create_autospec(grpc.Call, instance=True) + call.is_active.return_value = True + + def queue_generator(rpc): + yield mock.sentinel.A + yield queue.Empty() + yield mock.sentinel.B + rpc.is_active.return_value = False + yield mock.sentinel.C + + q = mock.create_autospec(queue.Queue, instance=True) + q.get.side_effect = queue_generator(call) + + generator = bidi._RequestQueueGenerator(q) + generator.call = call + + items = list(generator) + + assert items == [mock.sentinel.A, mock.sentinel.B] + + def test_yield_initial_and_exit(self): + q = mock.create_autospec(queue.Queue, instance=True) + q.get.side_effect = queue.Empty() + call = mock.create_autospec(grpc.Call, instance=True) + call.is_active.return_value = False + + generator = bidi._RequestQueueGenerator( + q, initial_request=mock.sentinel.A) + generator.call = call + + items = list(generator) + + assert items == [mock.sentinel.A] + + def test_yield_initial_callable_and_exit(self): + q = mock.create_autospec(queue.Queue, instance=True) + q.get.side_effect = queue.Empty() + call = mock.create_autospec(grpc.Call, instance=True) + call.is_active.return_value = False + + generator = bidi._RequestQueueGenerator( + q, initial_request=lambda: mock.sentinel.A) + generator.call = call + + items = list(generator) + + assert items == [mock.sentinel.A] + + def test_exit_when_inactive_with_item(self): + q = mock.create_autospec(queue.Queue, instance=True) + q.get.side_effect = [mock.sentinel.A, queue.Empty()] + call = mock.create_autospec(grpc.Call, instance=True) + call.is_active.return_value = False + + generator = bidi._RequestQueueGenerator(q) + generator.call = call + + items = list(generator) + + assert items == [] + # Make sure it put the item back. + q.put.assert_called_once_with(mock.sentinel.A) + + def test_exit_when_inactive_empty(self): + q = mock.create_autospec(queue.Queue, instance=True) + q.get.side_effect = queue.Empty() + call = mock.create_autospec(grpc.Call, instance=True) + call.is_active.return_value = False + + generator = bidi._RequestQueueGenerator(q) + generator.call = call + + items = list(generator) + + assert items == [] + + def test_exit_with_stop(self): + q = mock.create_autospec(queue.Queue, instance=True) + q.get.side_effect = [None, queue.Empty()] + call = mock.create_autospec(grpc.Call, instance=True) + call.is_active.return_value = True + + generator = bidi._RequestQueueGenerator(q) + generator.call = call + + items = list(generator) + + assert items == [] + + +class _CallAndFuture(grpc.Call, grpc.Future): + pass + + +def make_rpc(): + """Makes a mock RPC used to test Bidi classes.""" + call = mock.create_autospec(_CallAndFuture, instance=True) + rpc = mock.create_autospec(grpc.StreamStreamMultiCallable, instance=True) + + def rpc_side_effect(request): + call.is_active.return_value = True + call.request = request + return call + + rpc.side_effect = rpc_side_effect + + def cancel_side_effect(): + call.is_active.return_value = False + + call.cancel.side_effect = cancel_side_effect + + return rpc, call + + +class ClosedCall(object): + # NOTE: This is needed because defining `.next` on an **instance** + # rather than the **class** will not be iterable in Python 2. + # This is problematic since a `Mock` just sets members. + + def __init__(self, exception): + self.exception = exception + + def __next__(self): + raise self.exception + + next = __next__ # Python 2 + + def is_active(self): + return False + + +class TestBidiRpc(object): + def test_initial_state(self): + bidi_rpc = bidi.BidiRpc(None) + + assert bidi_rpc.is_active is False + + def test_done_callbacks(self): + bidi_rpc = bidi.BidiRpc(None) + callback = mock.Mock(spec=['__call__']) + + bidi_rpc.add_done_callback(callback) + bidi_rpc._on_call_done(mock.sentinel.future) + + callback.assert_called_once_with(mock.sentinel.future) + + def test_open(self): + rpc, call = make_rpc() + bidi_rpc = bidi.BidiRpc(rpc) + + bidi_rpc.open() + + assert bidi_rpc.call == call + assert bidi_rpc.is_active + call.add_done_callback.assert_called_once_with(bidi_rpc._on_call_done) + + def test_open_error_already_open(self): + rpc, _ = make_rpc() + bidi_rpc = bidi.BidiRpc(rpc) + + bidi_rpc.open() + + with pytest.raises(ValueError): + bidi_rpc.open() + + def test_close(self): + rpc, call = make_rpc() + bidi_rpc = bidi.BidiRpc(rpc) + bidi_rpc.open() + + bidi_rpc.close() + + call.cancel.assert_called_once() + assert bidi_rpc.call == call + assert bidi_rpc.is_active is False + # ensure the request queue was signaled to stop. + assert bidi_rpc.pending_requests == 1 + assert bidi_rpc._request_queue.get() is None + + def test_close_no_rpc(self): + bidi_rpc = bidi.BidiRpc(None) + bidi_rpc.close() + + def test_send(self): + rpc, call = make_rpc() + bidi_rpc = bidi.BidiRpc(rpc) + bidi_rpc.open() + + bidi_rpc.send(mock.sentinel.request) + + assert bidi_rpc.pending_requests == 1 + assert bidi_rpc._request_queue.get() is mock.sentinel.request + + def test_send_not_open(self): + rpc, call = make_rpc() + bidi_rpc = bidi.BidiRpc(rpc) + + with pytest.raises(ValueError): + bidi_rpc.send(mock.sentinel.request) + + def test_send_dead_rpc(self): + error = ValueError() + bidi_rpc = bidi.BidiRpc(None) + bidi_rpc.call = ClosedCall(error) + + with pytest.raises(ValueError) as exc_info: + bidi_rpc.send(mock.sentinel.request) + + assert exc_info.value == error + + def test_recv(self): + bidi_rpc = bidi.BidiRpc(None) + bidi_rpc.call = iter([mock.sentinel.response]) + + response = bidi_rpc.recv() + + assert response == mock.sentinel.response + + def test_recv_not_open(self): + rpc, call = make_rpc() + bidi_rpc = bidi.BidiRpc(rpc) + + with pytest.raises(ValueError): + bidi_rpc.recv() + + +class CallStub(object): + def __init__(self, values, active=True): + self.values = iter(values) + self._is_active = active + self.cancelled = False + + def __next__(self): + item = next(self.values) + if isinstance(item, Exception): + self._is_active = False + raise item + return item + + next = __next__ # Python 2 + + def is_active(self): + return self._is_active + + def add_done_callback(self, callback): + pass + + def cancel(self): + self.cancelled = True + + +class TestResumableBidiRpc(object): + def test_initial_state(self): + bidi_rpc = bidi.ResumableBidiRpc(None, lambda _: True) + + assert bidi_rpc.is_active is False + + def test_done_callbacks_recoverable(self): + start_rpc = mock.create_autospec( + grpc.StreamStreamMultiCallable, instance=True) + bidi_rpc = bidi.ResumableBidiRpc(start_rpc, lambda _: True) + callback = mock.Mock(spec=['__call__']) + + bidi_rpc.add_done_callback(callback) + bidi_rpc._on_call_done(mock.sentinel.future) + + callback.assert_not_called() + start_rpc.assert_called_once() + assert bidi_rpc.is_active + + def test_done_callbacks_non_recoverable(self): + bidi_rpc = bidi.ResumableBidiRpc(None, lambda _: False) + callback = mock.Mock(spec=['__call__']) + + bidi_rpc.add_done_callback(callback) + bidi_rpc._on_call_done(mock.sentinel.future) + + callback.assert_called_once_with(mock.sentinel.future) + + def test_send_recover(self): + error = ValueError() + call_1 = CallStub([error], active=False) + call_2 = CallStub([]) + start_rpc = mock.create_autospec( + grpc.StreamStreamMultiCallable, + instance=True, + side_effect=[call_1, call_2]) + should_recover = mock.Mock(spec=['__call__'], return_value=True) + bidi_rpc = bidi.ResumableBidiRpc(start_rpc, should_recover) + + bidi_rpc.open() + + bidi_rpc.send(mock.sentinel.request) + + assert bidi_rpc.pending_requests == 1 + assert bidi_rpc._request_queue.get() is mock.sentinel.request + + should_recover.assert_called_once_with(error) + assert bidi_rpc.call == call_2 + assert bidi_rpc.is_active is True + + def test_send_failure(self): + error = ValueError() + call = CallStub([error], active=False) + start_rpc = mock.create_autospec( + grpc.StreamStreamMultiCallable, + instance=True, + return_value=call) + should_recover = mock.Mock(spec=['__call__'], return_value=False) + bidi_rpc = bidi.ResumableBidiRpc(start_rpc, should_recover) + + bidi_rpc.open() + + with pytest.raises(ValueError) as exc_info: + bidi_rpc.send(mock.sentinel.request) + + assert exc_info.value == error + should_recover.assert_called_once_with(error) + assert bidi_rpc.call == call + assert bidi_rpc.is_active is False + assert call.cancelled is True + assert bidi_rpc.pending_requests == 1 + assert bidi_rpc._request_queue.get() is None + + def test_recv_recover(self): + error = ValueError() + call_1 = CallStub([1, error]) + call_2 = CallStub([2, 3]) + start_rpc = mock.create_autospec( + grpc.StreamStreamMultiCallable, + instance=True, + side_effect=[call_1, call_2]) + should_recover = mock.Mock(spec=['__call__'], return_value=True) + bidi_rpc = bidi.ResumableBidiRpc(start_rpc, should_recover) + + bidi_rpc.open() + + values = [] + for n in range(3): + values.append(bidi_rpc.recv()) + + assert values == [1, 2, 3] + should_recover.assert_called_once_with(error) + assert bidi_rpc.call == call_2 + assert bidi_rpc.is_active is True + + def test_recv_recover_race_condition(self): + # This test checks the race condition where two threads recv() and + # encounter an error and must re-open the stream. Only one thread + # should succeed in doing so. + error = ValueError() + call_1 = CallStub([error, error]) + call_2 = CallStub([1, 2]) + start_rpc = mock.create_autospec( + grpc.StreamStreamMultiCallable, + instance=True, + side_effect=[call_1, call_2]) + recovered_event = threading.Event() + + def second_thread_main(): + assert bidi_rpc.recv() == 2 + + second_thread = threading.Thread(target=second_thread_main) + + def should_recover(exception): + assert exception == error + if threading.current_thread() == second_thread: + recovered_event.wait() + return True + + bidi_rpc = bidi.ResumableBidiRpc(start_rpc, should_recover) + + bidi_rpc.open() + second_thread.start() + + assert bidi_rpc.recv() == 1 + recovered_event.set() + + assert bidi_rpc.call == call_2 + assert bidi_rpc.is_active is True + second_thread.join() + + def test_recv_failure(self): + error = ValueError() + call = CallStub([error]) + start_rpc = mock.create_autospec( + grpc.StreamStreamMultiCallable, + instance=True, + return_value=call) + should_recover = mock.Mock(spec=['__call__'], return_value=False) + bidi_rpc = bidi.ResumableBidiRpc(start_rpc, should_recover) + + bidi_rpc.open() + + with pytest.raises(ValueError) as exc_info: + bidi_rpc.recv() + + assert exc_info.value == error + should_recover.assert_called_once_with(error) + assert bidi_rpc.call == call + assert bidi_rpc.is_active is False + assert call.cancelled is True + + def test_reopen_failure_on_rpc_restart(self): + error1 = ValueError('1') + error2 = ValueError('2') + call = CallStub([error1]) + # Invoking start RPC a second time will trigger an error. + start_rpc = mock.create_autospec( + grpc.StreamStreamMultiCallable, + instance=True, + side_effect=[call, error2]) + should_recover = mock.Mock(spec=['__call__'], return_value=True) + callback = mock.Mock(spec=['__call__']) + + bidi_rpc = bidi.ResumableBidiRpc(start_rpc, should_recover) + bidi_rpc.add_done_callback(callback) + + bidi_rpc.open() + + with pytest.raises(ValueError) as exc_info: + bidi_rpc.recv() + + assert exc_info.value == error2 + should_recover.assert_called_once_with(error1) + assert bidi_rpc.call is None + assert bidi_rpc.is_active is False + callback.assert_called_once_with(error2) + + def test_finalize_idempotent(self): + error1 = ValueError('1') + error2 = ValueError('2') + callback = mock.Mock(spec=['__call__']) + should_recover = mock.Mock(spec=['__call__'], return_value=False) + + bidi_rpc = bidi.ResumableBidiRpc( + mock.sentinel.start_rpc, should_recover) + + bidi_rpc.add_done_callback(callback) + + bidi_rpc._on_call_done(error1) + bidi_rpc._on_call_done(error2) + + callback.assert_called_once_with(error1) + + +class TestBackgroundConsumer(object): + def test_consume_once_then_exit(self): + bidi_rpc = mock.create_autospec(bidi.BidiRpc, instance=True) + bidi_rpc.is_active = True + bidi_rpc.recv.side_effect = [mock.sentinel.response_1] + recved = threading.Event() + + def on_response(response): + assert response == mock.sentinel.response_1 + bidi_rpc.is_active = False + recved.set() + + consumer = bidi.BackgroundConsumer(bidi_rpc, on_response) + + consumer.start() + + recved.wait() + + bidi_rpc.recv.assert_called_once() + assert bidi_rpc.is_active is False + + consumer.stop() + + bidi_rpc.close.assert_called_once() + assert consumer.is_active is False + + def test_pause_resume_and_close(self): + # This test is relatively complex. It attempts to start the consumer, + # consume one item, pause the consumer, check the state of the world, + # then resume the consumer. Doing this in a deterministic fashion + # requires a bit more mocking and patching than usual. + + bidi_rpc = mock.create_autospec(bidi.BidiRpc, instance=True) + bidi_rpc.is_active = True + + def close_side_effect(): + bidi_rpc.is_active = False + + bidi_rpc.close.side_effect = close_side_effect + + # These are used to coordinate the two threads to ensure deterministic + # execution. + should_continue = threading.Event() + responses_and_events = { + mock.sentinel.response_1: threading.Event(), + mock.sentinel.response_2: threading.Event() + } + bidi_rpc.recv.side_effect = [ + mock.sentinel.response_1, mock.sentinel.response_2] + + recved_responses = [] + consumer = None + + def on_response(response): + if response == mock.sentinel.response_1: + consumer.pause() + + recved_responses.append(response) + responses_and_events[response].set() + should_continue.wait() + + consumer = bidi.BackgroundConsumer(bidi_rpc, on_response) + + consumer.start() + + # Wait for the first response to be recved. + responses_and_events[mock.sentinel.response_1].wait() + + # Ensure only one item has been recved and that the consumer is paused. + assert recved_responses == [mock.sentinel.response_1] + assert consumer.is_paused is True + assert consumer.is_active is True + + # Unpause the consumer, wait for the second item, then close the + # consumer. + should_continue.set() + consumer.resume() + + responses_and_events[mock.sentinel.response_2].wait() + + assert recved_responses == [ + mock.sentinel.response_1, mock.sentinel.response_2] + + consumer.stop() + + assert consumer.is_active is False + + def test_wake_on_error(self): + should_continue = threading.Event() + + bidi_rpc = mock.create_autospec(bidi.BidiRpc, instance=True) + bidi_rpc.is_active = True + bidi_rpc.add_done_callback.side_effect = ( + lambda _: should_continue.set()) + + consumer = bidi.BackgroundConsumer(bidi_rpc, mock.sentinel.on_response) + + # Start the consumer paused, which should immediately put it into wait + # state. + consumer.pause() + consumer.start() + + # Wait for add_done_callback to be called + should_continue.wait() + bidi_rpc.add_done_callback.assert_called_once_with( + consumer._on_call_done) + + # The consumer should now be blocked on waiting to be unpaused. + assert consumer.is_active + assert consumer.is_paused + + # Trigger the done callback, it should unpause the consumer and cause + # it to exit. + bidi_rpc.is_active = False + consumer._on_call_done(bidi_rpc) + + # It may take a few cycles for the thread to exit. + while consumer.is_active: + pass + + def test_consumer_expected_error(self, caplog): + caplog.set_level(logging.DEBUG) + + bidi_rpc = mock.create_autospec(bidi.BidiRpc, instance=True) + bidi_rpc.is_active = True + bidi_rpc.recv.side_effect = exceptions.ServiceUnavailable('Gone away') + + on_response = mock.Mock(spec=['__call__']) + + consumer = bidi.BackgroundConsumer(bidi_rpc, on_response) + + consumer.start() + + # Wait for the consumer's thread to exit. + while consumer.is_active: + pass + + on_response.assert_not_called() + bidi_rpc.recv.assert_called_once() + assert 'caught error' in caplog.text + + def test_consumer_unexpected_error(self, caplog): + caplog.set_level(logging.DEBUG) + + bidi_rpc = mock.create_autospec(bidi.BidiRpc, instance=True) + bidi_rpc.is_active = True + bidi_rpc.recv.side_effect = ValueError() + + on_response = mock.Mock(spec=['__call__']) + + consumer = bidi.BackgroundConsumer(bidi_rpc, on_response) + + consumer.start() + + # Wait for the consumer's thread to exit. + while consumer.is_active: + pass + + on_response.assert_not_called() + bidi_rpc.recv.assert_called_once() + assert 'caught unexpected exception' in caplog.text + + def test_double_stop(self, caplog): + caplog.set_level(logging.DEBUG) + bidi_rpc = mock.create_autospec(bidi.BidiRpc, instance=True) + bidi_rpc.is_active = True + on_response = mock.Mock(spec=['__call__']) + + def close_side_effect(): + bidi_rpc.is_active = False + + bidi_rpc.close.side_effect = close_side_effect + + consumer = bidi.BackgroundConsumer(bidi_rpc, on_response) + + consumer.start() + assert consumer.is_active is True + + consumer.stop() + assert consumer.is_active is False + + # calling stop twice should not result in an error. + consumer.stop() From 53d67452ed1641432ffe75b40e04826c69f54845 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Tue, 28 Aug 2018 18:29:08 -0400 Subject: [PATCH 044/148] coverage for collection and bidi modules --- firestore/tests/unit/test_bidi.py | 2 +- firestore/tests/unit/test_collection.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/firestore/tests/unit/test_bidi.py b/firestore/tests/unit/test_bidi.py index 09a23b10405e..80d8ecf48389 100644 --- a/firestore/tests/unit/test_bidi.py +++ b/firestore/tests/unit/test_bidi.py @@ -279,7 +279,7 @@ def cancel(self): class TestResumableBidiRpc(object): - def test_initial_state(self): + def test_initial_state(self): # pragma: NO COVER bidi_rpc = bidi.ResumableBidiRpc(None, lambda _: True) assert bidi_rpc.is_active is False diff --git a/firestore/tests/unit/test_collection.py b/firestore/tests/unit/test_collection.py index b5d348412ed5..de12059bb61a 100644 --- a/firestore/tests/unit/test_collection.py +++ b/firestore/tests/unit/test_collection.py @@ -415,6 +415,12 @@ def test_get_with_transaction(self, query_class): self.assertIs(get_response, query_instance.get.return_value) query_instance.get.assert_called_once_with(transaction=transaction) + @mock.patch('google.cloud.firestore_v1beta1.collection.Watch',autospec=True) + def test_on_snapshot(self, watch): + collection = self._make_one('collection') + collection.on_snapshot(None) + watch.for_query.assert_called_once() + class Test__auto_id(unittest.TestCase): From 1e73ab706e8c57170b537cada494b7d377bdfc24 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Tue, 28 Aug 2018 18:37:19 -0400 Subject: [PATCH 045/148] coverage for document and query methods added --- firestore/tests/unit/test_document.py | 9 +++++++++ firestore/tests/unit/test_query.py | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/firestore/tests/unit/test_document.py b/firestore/tests/unit/test_document.py index 401ae0b8b7ca..ba8d0f105515 100644 --- a/firestore/tests/unit/test_document.py +++ b/firestore/tests/unit/test_document.py @@ -556,6 +556,15 @@ def test_collections_wo_page_size(self): def test_collections_w_page_size(self): self._collections_helper(page_size=10) + @mock.patch('google.cloud.firestore_v1beta1.document.Watch', autospec=True) + def test_on_snapshot(self, watch): + client = mock.Mock( + _database_string='sprinklez', + spec=['_database_string']) + document = self._make_one('yellow', 'mellow', client=client) + document.on_snapshot(None) + watch.for_document.assert_called_once() + class TestDocumentSnapshot(unittest.TestCase): diff --git a/firestore/tests/unit/test_query.py b/firestore/tests/unit/test_query.py index 85f803c43fc3..e645408f537f 100644 --- a/firestore/tests/unit/test_query.py +++ b/firestore/tests/unit/test_query.py @@ -860,6 +860,12 @@ def test_get_empty_after_first_response(self): parent_path, query._to_protobuf(), transaction=None, metadata=client._rpc_metadata) + @mock.patch('google.cloud.firestore_v1beta1.query.Watch', autospec=True) + def test_on_snapshot(self, watch): + query = self._make_one(mock.sentinel.parent) + query.on_snapshot(None) + watch.for_query.assert_called_once() + class Test__enum_from_op_string(unittest.TestCase): From 6ade323c95e8042ba03387bb197d0687911ab496 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Wed, 29 Aug 2018 14:17:01 -0400 Subject: [PATCH 046/148] 100 percent branch coverage --- .../google/cloud/firestore_v1beta1/watch.py | 3 + firestore/tests/unit/test_watch.py | 116 +++++++++++++++++- 2 files changed, 114 insertions(+), 5 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index b91c679db065..152a51017198 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -107,6 +107,9 @@ def __iter__(self): def __len__(self): return len(self._dict) + def __contains__(self, k): + return k in self._dict + class ChangeType(Enum): ADDED = 0 diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index 9dc7861c04cd..452398a22821 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -9,9 +9,6 @@ def _makeOne(self): from google.cloud.firestore_v1beta1.watch import WatchDocTree return WatchDocTree() - def setUp(self): - self.snapshotted = None - def test_insert_and_keys(self): inst = self._makeOne() inst = inst.insert('b', 1) @@ -44,6 +41,12 @@ def test___iter__(self): inst = inst.insert('a', 2) self.assertEqual(sorted(list(inst)), ['a', 'b']) + def test___contains__(self): + inst = self._makeOne() + inst = inst.insert('b', 1) + self.assertTrue('b' in inst) + self.assertFalse('a' in inst) + class TestDocumentChange(unittest.TestCase): def _makeOne(self, type, document, old_index, new_index): @@ -147,6 +150,9 @@ def _makeOne( ) return inst + def setUp(self): + self.snapshotted = None + def _document_watch_comparator(self, doc1, doc2): # pragma: NO COVER return 0 @@ -372,6 +378,40 @@ class DummyDocument: inst.on_snapshot(proto) self.assertEqual(inst.change_map['fred'].data, None) + def test_on_snapshot_document_change_changed_docname_db_prefix(self): + # XXX This test asserts the current behavior, but I have no level + # of confidence that the change map should contain the + # db-prefixed document name instead of the bare document name. + from google.cloud.firestore_v1beta1.watch import WATCH_TARGET_ID + inst = self._makeOne() + + def message_to_dict(document): + return {'fields': None} + + inst.MessageToDict = message_to_dict + proto = DummyProto() + proto.target_change = '' + proto.document_change.target_ids = [WATCH_TARGET_ID] + + class DummyDocument: + name = 'abc://foo/fred' + create_time = None + update_time = None + + proto.document_change.document = DummyDocument() + inst._firestore._database_string = 'abc://foo/' + inst.on_snapshot(proto) + self.assertEqual(inst.change_map['abc://foo/fred'].data, None) + + def test_on_snapshot_document_change_neither_changed_nor_removed(self): + inst = self._makeOne() + proto = DummyProto() + proto.target_change = '' + proto.document_change.target_ids = [] + + inst.on_snapshot(proto) + self.assertTrue(not inst.change_map) + def test_on_snapshot_document_removed(self): from google.cloud.firestore_v1beta1.watch import ChangeType inst = self._makeOne() @@ -408,6 +448,23 @@ def reset(): inst.on_snapshot(proto) self.assertTrue(inst._docs_reset) + def test_on_snapshot_filter_update_no_size_change(self): + inst = self._makeOne() + proto = DummyProto() + proto.target_change = '' + proto.document_change = '' + proto.document_remove = None + proto.document_delete = None + + class DummyFilter(object): + count = 0 + + proto.filter = DummyFilter() + inst._docs_reset = False + + inst.on_snapshot(proto) + self.assertFalse(inst._docs_reset) + def test_on_snapshot_unknown_listen_type(self): inst = self._makeOne() proto = DummyProto() @@ -423,7 +480,7 @@ def test_on_snapshot_unknown_listen_type(self): str(exc.exception) ) - def test_push_no_changes(self): + def test_push_callback_called_no_changes(self): import pytz class DummyReadTime(object): seconds = 1534858278 @@ -440,6 +497,18 @@ class DummyReadTime(object): self.assertTrue(inst.has_pushed) self.assertEqual(inst.resume_token, 'token') + def test_push_already_pushed(self): + class DummyReadTime(object): + seconds = 1534858278 + inst = self._makeOne() + inst.has_pushed = True + inst.push(DummyReadTime, 'token') + self.assertEqual( + self.snapshotted, + None) + self.assertTrue(inst.has_pushed) + self.assertEqual(inst.resume_token, 'token') + def test__current_size_empty(self): inst = self._makeOne() result = inst._current_size() @@ -471,6 +540,14 @@ def test__extract_changes_doc_removed(self): results = inst._extract_changes(doc_map, changes, None) self.assertEqual(results, (['name'], [], [])) + def test__extract_changes_doc_removed_docname_not_in_docmap(self): + from google.cloud.firestore_v1beta1.watch import ChangeType + inst = self._makeOne() + changes = {'name': ChangeType.REMOVED} + doc_map = {} + results = inst._extract_changes(doc_map, changes, None) + self.assertEqual(results, ([], [], [])) + def test__extract_changes_doc_updated(self): inst = self._makeOne() @@ -485,6 +562,21 @@ class Dummy(object): self.assertEqual(results, ([], [], [snapshot])) self.assertEqual(snapshot.read_time, 1) + def test__extract_changes_doc_updated_read_time_is_None(self): + inst = self._makeOne() + + class Dummy(object): + pass + + doc = Dummy() + snapshot = Dummy() + snapshot.read_time = None + changes = {'name': snapshot} + doc_map = {'name': doc} + results = inst._extract_changes(doc_map, changes, None) + self.assertEqual(results, ([], [], [snapshot])) + self.assertEqual(snapshot.read_time, None) + def test__extract_changes_doc_added(self): inst = self._makeOne() @@ -498,6 +590,20 @@ class Dummy(object): self.assertEqual(results, ([], [snapshot], [])) self.assertEqual(snapshot.read_time, 1) + def test__extract_changes_doc_added_read_time_is_None(self): + inst = self._makeOne() + + class Dummy(object): + pass + + snapshot = Dummy() + snapshot.read_time = None + changes = {'name': snapshot} + doc_map = {} + results = inst._extract_changes(doc_map, changes, None) + self.assertEqual(results, ([], [snapshot], [])) + self.assertEqual(snapshot.read_time, None) + def test__compute_snapshot_doctree_and_docmap_disagree_about_length(self): inst = self._makeOne() doc_tree = {} @@ -631,7 +737,7 @@ def _to_protobuf(self): class DummyFirestore(object): _firestore_api = DummyFirestoreClient() - _database_string = '' + _database_string = 'abc://bar/' def document(self, *document_path): # pragma: NO COVER if len(document_path) == 1: From 2ee71febe0598a6b04482a08bd9df341ea352f4a Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Wed, 29 Aug 2018 14:21:23 -0400 Subject: [PATCH 047/148] appease linter --- firestore/tests/unit/test_collection.py | 3 ++- firestore/tests/unit/test_query.py | 2 +- firestore/tests/unit/test_watch.py | 10 ++++++++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/firestore/tests/unit/test_collection.py b/firestore/tests/unit/test_collection.py index de12059bb61a..ab4da4ccee8f 100644 --- a/firestore/tests/unit/test_collection.py +++ b/firestore/tests/unit/test_collection.py @@ -415,7 +415,8 @@ def test_get_with_transaction(self, query_class): self.assertIs(get_response, query_instance.get.return_value) query_instance.get.assert_called_once_with(transaction=transaction) - @mock.patch('google.cloud.firestore_v1beta1.collection.Watch',autospec=True) + @mock.patch('google.cloud.firestore_v1beta1.collection.Watch', + autospec=True) def test_on_snapshot(self, watch): collection = self._make_one('collection') collection.on_snapshot(None) diff --git a/firestore/tests/unit/test_query.py b/firestore/tests/unit/test_query.py index e645408f537f..4e4619841438 100644 --- a/firestore/tests/unit/test_query.py +++ b/firestore/tests/unit/test_query.py @@ -865,7 +865,7 @@ def test_on_snapshot(self, watch): query = self._make_one(mock.sentinel.parent) query.on_snapshot(None) watch.for_query.assert_called_once() - + class Test__enum_from_op_string(unittest.TestCase): diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index 452398a22821..10f970861572 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -116,7 +116,7 @@ def _makeOne( snapshot_callback=None, snapshot_class=None, reference_class=None - ): # pragma: NO COVER + ): # pragma: NO COVER from google.cloud.firestore_v1beta1.watch import Watch if document_reference is None: document_reference = DummyDocumentReference() @@ -482,8 +482,10 @@ def test_on_snapshot_unknown_listen_type(self): def test_push_callback_called_no_changes(self): import pytz + class DummyReadTime(object): seconds = 1534858278 + inst = self._makeOne() inst.push(DummyReadTime, 'token') self.assertEqual( @@ -491,7 +493,8 @@ class DummyReadTime(object): ( [], [], - datetime.datetime.fromtimestamp(DummyReadTime.seconds, pytz.utc) + datetime.datetime.fromtimestamp( + DummyReadTime.seconds, pytz.utc) ), ) self.assertTrue(inst.has_pushed) @@ -724,6 +727,7 @@ def __init__(self, *document_path, **kw): _document_path = '/' + class DummyQuery(object): # pragma: NO COVER def __init__(self, **kw): if 'client' not in kw: @@ -843,6 +847,8 @@ def QueryTarget(self, **kw): class DummyPb2(object): + Target = DummyTarget() + def ListenRequest(self, **kw): pass From 92e98d78a1c6206a57e5946a7e92c8e99d482b61 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Wed, 19 Sep 2018 12:14:02 -0700 Subject: [PATCH 048/148] should return object on snapshot watching. This is needed to unsubscribe --- firestore/google/cloud/firestore_v1beta1/collection.py | 2 +- firestore/google/cloud/firestore_v1beta1/document.py | 2 +- firestore/google/cloud/firestore_v1beta1/query.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/collection.py b/firestore/google/cloud/firestore_v1beta1/collection.py index 1110858f4667..8234bee6a21a 100644 --- a/firestore/google/cloud/firestore_v1beta1/collection.py +++ b/firestore/google/cloud/firestore_v1beta1/collection.py @@ -398,7 +398,7 @@ def on_snapshot(collection_snapshot): # Terminate this watch collection_watch.unsubscribe() """ - Watch.for_query(query_mod.Query(self), + return Watch.for_query(query_mod.Query(self), callback, document.DocumentSnapshot, document.DocumentReference) diff --git a/firestore/google/cloud/firestore_v1beta1/document.py b/firestore/google/cloud/firestore_v1beta1/document.py index 1cc105b0d828..004550f9542a 100644 --- a/firestore/google/cloud/firestore_v1beta1/document.py +++ b/firestore/google/cloud/firestore_v1beta1/document.py @@ -471,7 +471,7 @@ def on_snapshot(document_snapshot): # Terminate this watch doc_watch.unsubscribe() """ - Watch.for_document(self, callback, DocumentSnapshot, DocumentReference) + return Watch.for_document(self, callback, DocumentSnapshot, DocumentReference) class DocumentSnapshot(object): diff --git a/firestore/google/cloud/firestore_v1beta1/query.py b/firestore/google/cloud/firestore_v1beta1/query.py index c39a5febea44..27e282d9ffb4 100644 --- a/firestore/google/cloud/firestore_v1beta1/query.py +++ b/firestore/google/cloud/firestore_v1beta1/query.py @@ -627,7 +627,7 @@ def on_snapshot(query_snapshot): # Terminate this watch query_watch.unsubscribe() """ - Watch.for_query(self, + return Watch.for_query(self, callback, document.DocumentSnapshot, document.DocumentReference) From 213169e8706906f1ca6891981dda6784b16ac971 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Thu, 20 Sep 2018 11:13:21 -0700 Subject: [PATCH 049/148] Remove use of deprecated assert --- firestore/tests/unit/test__helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/firestore/tests/unit/test__helpers.py b/firestore/tests/unit/test__helpers.py index 18d80fa5ce8d..9ad07b9e5299 100644 --- a/firestore/tests/unit/test__helpers.py +++ b/firestore/tests/unit/test__helpers.py @@ -122,7 +122,7 @@ def test_invalid_chars_in_constructor(self): def test_component(self): field_path = self._make_one('a..b') - self.assertEquals(field_path.parts, ('a..b',)) + self.assertEqual(field_path.parts, ('a..b',)) def test_constructor_iterable(self): field_path = self._make_one('a', 'b', 'c') From abd5c975f81acd9d238bf04240be832d7c0f201e Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Thu, 20 Sep 2018 11:13:41 -0700 Subject: [PATCH 050/148] Fix bug in deletion of document from map (using wrong key) --- .../google/cloud/firestore_v1beta1/watch.py | 11 ++++---- firestore/tests/system.py | 25 ++++++++++++++----- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 152a51017198..bc2b237bc1a3 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -486,8 +486,10 @@ def on_snapshot(self, proto): # fashion than self._document_reference document_name = document.name db_str = self._firestore._database_string - if document_name.startswith(db_str): - document_name = document_name[len(db_str):] + db_str_documents = db_str + '/documents/' + if document_name.startswith(db_str_documents): + document_name = document_name[len(db_str_documents):] + document_ref = self._firestore.document(document_name) snapshot = self.DocumentSnapshot( @@ -547,7 +549,6 @@ def push(self, read_time, next_resume_token): """ # TODO: may need to lock here to avoid races on collecting snapshots # and sending them to the user. - deletes, adds, updates = Watch._extract_changes( self.doc_map, self.change_map, @@ -618,10 +619,10 @@ def delete_doc(name, updated_tree, updated_map): assert name in updated_map, 'Document to delete does not exist' old_document = updated_map.get(name) # XXX probably should not expose IndexError when doc doesnt exist - existing = updated_tree.find(name) + existing = updated_tree.find(old_document) old_index = existing.index # TODO: was existing.remove returning tree (presumably immuatable?) - updated_tree = updated_tree.remove(name) + updated_tree = updated_tree.remove(old_document) del updated_map[name] return (DocumentChange(ChangeType.REMOVED, old_document, diff --git a/firestore/tests/system.py b/firestore/tests/system.py index 6edffb059283..25d2f0958f0d 100644 --- a/firestore/tests/system.py +++ b/firestore/tests/system.py @@ -858,9 +858,10 @@ def test_watch_query(client, cleanup): # Setup listener def on_snapshot(docs, changes, read_time): on_snapshot.called_count += 1 - print("docs: " + docs) - print("changes: " + changes) - print("read_time: " + read_time) + + # A snapshot should return the same thing as if a query ran now. + query_ran = db.collection(u'users').where("first", "==", u'Ada').get() + assert len(docs) == len([i for i in query_ran]) on_snapshot.called_count = 0 @@ -875,10 +876,22 @@ def on_snapshot(docs, changes, read_time): for _ in range(10): if on_snapshot.called_count == 1: - return + break sleep(1) - if on_snapshot.called_count != 1: + # Alter document + doc_ref.set({ + u'first': u'Ada', + u'last': u'Lovelace', + u'born': 0 + }) + + for _ in range(10): + if on_snapshot.called_count == 2: + break + sleep(1) + + if on_snapshot.called_count != 2: raise AssertionError( - "Failed to get exactly one document change: count: " + + "Failed to get exactly two document changes: count: " + str(on_snapshot.called_count)) From 3ed821e7f8643ecd0da15a265cbca5a0086aa0f0 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Fri, 28 Sep 2018 16:38:55 -0700 Subject: [PATCH 051/148] startings of ordering --- .../google/cloud/firestore_v1beta1/order.py | 291 ++++++++++++++++++ firestore/tests/unit/test_order.py | 210 +++++++++++++ 2 files changed, 501 insertions(+) create mode 100644 firestore/google/cloud/firestore_v1beta1/order.py create mode 100644 firestore/tests/unit/test_order.py diff --git a/firestore/google/cloud/firestore_v1beta1/order.py b/firestore/google/cloud/firestore_v1beta1/order.py new file mode 100644 index 000000000000..4060673cf3f9 --- /dev/null +++ b/firestore/google/cloud/firestore_v1beta1/order.py @@ -0,0 +1,291 @@ +# Copyright 2017 Google LLC All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# package com.google.cloud.firestore; + +# import com.google.firestore.v1beta1.Value; +# import com.google.firestore.v1beta1.Value.ValueTypeCase; +# import com.google.protobuf.ByteString; +# import java.util.Comparator; +# import java.util.Iterator; +# import java.util.List; +# import java.util.Map.Entry; +# import java.util.SortedMap; +# import java.util.TreeMap; +# import javax.annotation.Nonnull; +from enum import Enum +from google.cloud.firestore_v1beta1._helpers import decode_value + +class TypeOrder(Enum): + # NOTE: This order is defined by the backend and cannot be changed. + NULL = 0 + BOOLEAN = 1 + NUMBER = 2 + TIMESTAMP = 3 + STRING = 4 + BLOB = 5 + REF = 6 + GEO_POINT = 7 + ARRAY = 8 + OBJECT = 9 + + def from_value(value): + v = value.WhichOneof('value_type') + + lut = { + 'null_value': NULL, + 'boolean_value': BOOLEAN, + 'integer_value': NUMBER, + 'double_value': NUMBER, + 'timestamp_value': TIMESTAMP, + 'string_value': STRING, + 'bytes_value': BLOB, + 'reference_value': REF, + 'geo_point_value': GEO_POINT, + 'array_value': ARRAY, + 'map_value': OBJECT, + } + + if v not in lut: + raise ArgumentException( + "Could not detect value type for " + value) + return lut[v] + + +class Order(object): + ''' + Order implements the ordering semantics of the backend. + ''' + def __init__(): + pass + + def compare(left, right): + ''' + Main comparison function for all Firestore types. + + @return -1 is left < right, 0 if left == right, otherwise 1 + ''' + + # First compare the types. + leftType = TypeOrder.from_value(left) + rightType = TypeOrder.from_value(right) + + if leftType != rightType: + if leftType < rightType: + return -1 + return 1 + + # TODO: may be able to use helpers.decode_value and do direct compares + # after converting to python types + value_type = value.WhichOneof('value_type') + + if value_type == 'null_value': + return 0 # nulls are all equal + elif value_type == 'boolean_value': + return _compareTo(decode_value(left), decode_value(right)) + elif value_type == 'integer_value': + return compare_numbers(left, right) + elif value_type == 'double_value': + return compare_numbers(left, right) + elif value_type == 'timestamp_value': + # NOTE: This conversion is "lossy", Python ``datetime.datetime`` + # has microsecond precision but ``timestamp_value`` has + # nanosecond precision. + #return _pb_timestamp_to_datetime(value.timestamp_value) + return compare_timestamps(left, right) + elif value_type == 'string_value': + #return value.string_value + return compare_strings(left, right) + elif value_type == 'bytes_value': + #return value.bytes_value + return compare_blobs(left, right) + elif value_type == 'reference_value': + #return reference_value_to_document(value.reference_value, client) + return compare_resource_paths(left, right) + elif value_type == 'geo_point_value': + #return GeoPoint( + # value.geo_point_value.latitude, + # value.geo_point_value.longitude) + return compare_geo_points(left, right) + elif value_type == 'array_value': + #return [decode_value(element, client) + # for element in value.array_value.values] + return compare_arrays(left, right) + elif value_type == 'map_value': + #return decode_dict(value.map_value.fields, client) + return compare_objects(left, right) + else: + raise ValueError('Unknown ``value_type``', value_type) + + +def compare_strings(left, right): + left_value = decode_value(left) + right_value = decode_value(right) + return _compareTo(left_value, right_value) + + +# private int compareBlobs(Value left, Value right) { +# ByteString leftBytes = left.getBytesValue(); +# ByteString rightBytes = right.getBytesValue(); + +# int size = Math.min(leftBytes.size(), rightBytes.size()); +# for (int i = 0; i < size; i++) { +# // Make sure the bytes are unsigned +# int thisByte = leftBytes.byteAt(i) & 0xff; +# int otherByte = rightBytes.byteAt(i) & 0xff; +# if (thisByte < otherByte) { +# return -1; +# } else if (thisByte > otherByte) { +# return 1; +# } +# // Byte values are equal, continue with comparison +# } +# return Integer.compare(leftBytes.size(), rightBytes.size()); +# } +def compare_blobs(left, right): + raise NotImplementedError() + + +def compare_timestamps(left, right): + left_value = left.timestamp_value + right_value = right.timestamp_value + + cmp = 0 + if left_value.seconds < right_value.seconds: + cmp = -1 + elif left_value.seconds == right_value.seconds: + cmp = 0 + else: + cmp = 0 + + if cmp != 0: + return cmp + else: + if left_value.nanos < right_value.nanos: + cmp = -1 + elif left_value.nanos == right_value.nanos: + cmp = 0 + else: + cmp = 1 + return cmp + + +def compare_geo_points(left, right): + left_value = decode_value(left) + right_value = decode_value(right) + cmp = 0 + if left_value.latitude < right_value.latitude: + cmp = -1 + elif left_value.latitude == right_value.latitude: + cmp = 0 + else: + cmp = 1 + + if cmp != 0: + return cmp + else: + if left.longitude < right.longitude: + cmp = -1 + elif left.longitude == right.longitude: + cmp = 0 + else: + cmp = 1 + return cmp + +# private int compareResourcePaths(Value left, Value right) { +# ResourcePath leftPath = ResourcePath.create(left.getReferenceValue()); +# ResourcePath rightPath = ResourcePath.create(right.getReferenceValue()); +# return leftPath.compareTo(rightPath); +# } +def compare_resource_paths(left, right): + raise NotImplementedError() + + +# private int compareArrays(Value left, Value right) { +# List leftValue = left.getArrayValue().getValuesList(); +# List rightValue = right.getArrayValue().getValuesList(); + +# int minLength = Math.min(leftValue.size(), rightValue.size()); +# for (int i = 0; i < minLength; i++) { +# int cmp = compare(leftValue.get(i), rightValue.get(i)); +# if (cmp != 0) { +# return cmp; +# } +# } +# return Integer.compare(leftValue.size(), rightValue.size()); +# } +def compare_arrays(left, right): + raise NotImplementedError() + + + +# private int compareObjects(Value left, Value right) { +# // This requires iterating over the keys in the object in order and doing a +# // deep comparison. +# SortedMap leftMap = new TreeMap<>(); +# leftMap.putAll(left.getMapValue().getFieldsMap()); +# SortedMap rightMap = new TreeMap<>(); +# rightMap.putAll(right.getMapValue().getFieldsMap()); + +# Iterator> leftIterator = leftMap.entrySet().iterator(); +# Iterator> rightIterator = rightMap.entrySet().iterator(); + +# while (leftIterator.hasNext() && rightIterator.hasNext()) { +# Entry leftEntry = leftIterator.next(); +# Entry rightEntry = rightIterator.next(); +# int keyCompare = leftEntry.getKey().compareTo(rightEntry.getKey()); +# if (keyCompare != 0) { +# return keyCompare; +# } +# int valueCompare = compare(leftEntry.getValue(), rightEntry.getValue()); +# if (valueCompare != 0) { +# return valueCompare; +# } +# } + +# // Only equal if both iterators are exhausted. +# return Boolean.compare(leftIterator.hasNext(), rightIterator.hasNext()); +# } +def compare_objects(left, right): + raise NotImplementedError() + +def compare_numbers(left, right): + left_value = decode_value(left) + right_value = decode_value(right) + return compare_doubles(left_value, right_value) + +def compare_doubles(left, right): + if math.isnan(left): + if math.isnan(right): + return 0 + return -1 + if math.isnan(right): + return 1 + + if left == -0.0: + left = 0 + if right == -0.0: + right = 0 + + return _compareTo(left, right) + + +def _compareTo(left, right): + if left < right: + return -1 + elif left == right: + return 0 + # left > right + return 1 diff --git a/firestore/tests/unit/test_order.py b/firestore/tests/unit/test_order.py new file mode 100644 index 000000000000..3f0cbd5348d2 --- /dev/null +++ b/firestore/tests/unit/test_order.py @@ -0,0 +1,210 @@ +# Copyright 2017 Google LLC All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import mock +from google.cloud.firestore_v1beta1._helpers import encode_value +from google.protobuf import timestamp_pb2 +from google.type import latlng_pb2 +import math + + +class TestOrder(unittest.TestCase): + @staticmethod + def _get_target_class(): + from google.cloud.firestore_v1beta1.order import Order + + return Order + + def _make_one(self, *args, **kwargs): + klass = self._get_target_class() + return klass(*args, **kwargs) + + def test_order(self): + + int_max_value = 10 ** 1000 + int_min_value = -10 ** 1000 + float_min_value = -10.0 ** 1000 + float_nan = float('nan') + + groups = [None] * 65 + + groups[0] = [nullValue()] + + groups[1] = [_booleanValue(False)] + groups[2] = [_booleanValue(True)] + + # numbers + groups[3] = [_doubleValue(float_nan), _doubleValue(float_nan)] + groups[4] = [_doubleValue(-math.inf)] + groups[5] = [_intValue(int_min_value - 1)] + groups[6] = [_intValue(int_min_value)] + groups[7] = [_doubleValue(-1.1)] + # Integers and Doubles order the same. + groups[8] = [_intValue(-1), _doubleValue(-1.0)] + groups[9] = [_doubleValue(-float_min_value)] + # zeros all compare the same. + groups[10] = [_intValue(0), _doubleValue(-0.0), + _doubleValue(0.0), _doubleValue(+0.0)] + groups[11] = [_doubleValue(float_min_value)] + groups[12] = [_intValue(1), _doubleValue(1.0)] + groups[13] = [_doubleValue(1.1)] + groups[14] = [_intValue(int_max_value)] + groups[15] = [_intValue(int_max_value + 1)] + groups[16] = [_doubleValue(math.inf)] + + groups[17] = [_timestampValue(123, 0)] + groups[18] = [_timestampValue(123, 123)] + groups[19] = [_timestampValue(345, 0)] + + # strings + groups[20] = [_stringValue("")] + groups[21] = [_stringValue("\u0000\ud7ff\ue000\uffff")] + groups[22] = [_stringValue("(╯°□°）╯︵ ┻━┻")] + groups[23] = [_stringValue("a")] + groups[24] = [_stringValue("abc def")] + # latin small letter e + combining acute accent + latin small letter b + groups[25] = [_stringValue("e\u0301b")] + groups[26] = [_stringValue("æ")] + # latin small letter e with acute accent + latin small letter a + groups[27] = [_stringValue("\u00e9a")] + + # blobs + groups[28] = [_blobValue(bytes())] + groups[29] = [_blobValue(bytes([0]))] + groups[30] = [_blobValue(bytes([0, 1, 2, 3, 4]))] + groups[31] = [_blobValue(bytes([0, 1, 2, 4, 3]))] + groups[32] = [_blobValue(bytes([127]))] + + # resource names + groups[33] = [ + _referenceValue("projects/p1/databases/d1/documents/c1/doc1")] + groups[34] = [ + _referenceValue("projects/p1/databases/d1/documents/c1/doc2")] + groups[35] = [ + _referenceValue( + "projects/p1/databases/d1/documents/c1/doc2/c2/doc1")] + groups[36] = [ + _referenceValue( + "projects/p1/databases/d1/documents/c1/doc2/c2/doc2")] + groups[37] = [ + _referenceValue("projects/p1/databases/d1/documents/c10/doc1")] + groups[38] = [ + _referenceValue("projects/p1/databases/d1/documents/c2/doc1")] + groups[39] = [ + _referenceValue("projects/p2/databases/d2/documents/c1/doc1")] + groups[40] = [ + _referenceValue("projects/p2/databases/d2/documents/c1-/doc1")] + groups[41] = [ + _referenceValue("projects/p2/databases/d3/documents/c1-/doc1")] + + # geo points + groups[42] = [_geoPointValue(-90, -180)] + groups[43] = [_geoPointValue(-90, 0)] + groups[44] = [_geoPointValue(-90, 180)] + groups[45] = [_geoPointValue(0, -180)] + groups[46] = [_geoPointValue(0, 0)] + groups[47] = [_geoPointValue(0, 180)] + groups[48] = [_geoPointValue(1, -180)] + groups[49] = [_geoPointValue(1, 0)] + groups[50] = [_geoPointValue(1, 180)] + groups[51] = [_geoPointValue(90, -180)] + groups[52] = [_geoPointValue(90, 0)] + groups[53] = [_geoPointValue(90, 180)] + + # arrays + groups[54] = [_arrayValue()] + groups[55] = [_arrayValue(_stringValue("bar"))] + groups[56] = [_arrayValue(_stringValue("foo"))] + groups[57] = [_arrayValue(_stringValue("foo"), _intValue(0))] + groups[58] = [_arrayValue(_stringValue("foo"), _intValue(1))] + groups[59] = [_arrayValue(_stringValue("foo"), _stringValue("0"))] + + # objects + groups[60] = [_objectValue({"bar": _intValue(0)})] + groups[61] = [_objectValue({ + "bar": _intValue(0), + "foo": _intValue(1) + })] + groups[62] = [_objectValue({"bar": _intValue(1)})] + groups[63] = [_objectValue({"bar": _intValue(2)})] + groups[64] = [_objectValue({"bar": _stringValue("0")})] + + target = self._make_one() + for left in groups: + for right in groups: + for i in groups[left]: + for j in groups[right]: + self.assertEqual( + _compare(left, right), + _compare( + target.compare( + groups[left][i], + groups[right][j]), 0), + "Order does not match for: groups[%d][%d] " + "and groups[%d][%d]".format(left, i, right, j) + ) + + +def _compare(left, right): + if left < right: + return -1 + elif left == right: + return 0 + return 1 + + +def _booleanValue(b): + return encode_value(b) + + +def _doubleValue(d): + return encode_value(d) + + +def _intValue(l): + return encode_value(l) + + +def _stringValue(s): + return encode_value(s) + + +def _referenceValue(r): + return encode_value(r) + + +def _blobValue(b): + return encode_value(b) + + +def nullValue(): + return encode_value(None) + + +def _timestampValue(seconds, nanos): + return timestamp_pb2.Timestamp(seconds=seconds, nanos=nanos) + + +def _geoPointValue(latitude, longitude): + return latlng_pb2.LatLng(latitude=latitude, + longitude=longitude) + + +def _arrayValue(values): + return encode_value(values) + + +def _objectValue(keysAndValues): + return encode_value(keysAndValues) From 2f3cbc71b698f1f503e95cd342d442e8ebe27de9 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Fri, 5 Oct 2018 16:14:18 -0700 Subject: [PATCH 052/148] update tests --- .../google/cloud/firestore_v1beta1/order.py | 419 ++++++++++-------- firestore/{nox.py => noxfile.py} | 0 firestore/tests/unit/test_order.py | 93 ++-- 3 files changed, 289 insertions(+), 223 deletions(-) rename firestore/{nox.py => noxfile.py} (100%) diff --git a/firestore/google/cloud/firestore_v1beta1/order.py b/firestore/google/cloud/firestore_v1beta1/order.py index 4060673cf3f9..274393a095a3 100644 --- a/firestore/google/cloud/firestore_v1beta1/order.py +++ b/firestore/google/cloud/firestore_v1beta1/order.py @@ -27,6 +27,7 @@ # import javax.annotation.Nonnull; from enum import Enum from google.cloud.firestore_v1beta1._helpers import decode_value +import math class TypeOrder(Enum): # NOTE: This order is defined by the backend and cannot be changed. @@ -45,17 +46,17 @@ def from_value(value): v = value.WhichOneof('value_type') lut = { - 'null_value': NULL, - 'boolean_value': BOOLEAN, - 'integer_value': NUMBER, - 'double_value': NUMBER, - 'timestamp_value': TIMESTAMP, - 'string_value': STRING, - 'bytes_value': BLOB, - 'reference_value': REF, - 'geo_point_value': GEO_POINT, - 'array_value': ARRAY, - 'map_value': OBJECT, + 'null_value': TypeOrder.NULL, + 'boolean_value': TypeOrder.BOOLEAN, + 'integer_value': TypeOrder.NUMBER, + 'double_value': TypeOrder.NUMBER, + 'timestamp_value': TypeOrder.TIMESTAMP, + 'string_value': TypeOrder.STRING, + 'bytes_value': TypeOrder.BLOB, + 'reference_value': TypeOrder.REF, + 'geo_point_value': TypeOrder.GEO_POINT, + 'array_value': TypeOrder.ARRAY, + 'map_value': TypeOrder.OBJECT, } if v not in lut: @@ -68,10 +69,10 @@ class Order(object): ''' Order implements the ordering semantics of the backend. ''' - def __init__(): + def __init__(self): pass - - def compare(left, right): + + def compare(self, left, right): ''' Main comparison function for all Firestore types. @@ -79,8 +80,8 @@ def compare(left, right): ''' # First compare the types. - leftType = TypeOrder.from_value(left) - rightType = TypeOrder.from_value(right) + leftType = TypeOrder.from_value(left).value + rightType = TypeOrder.from_value(right).value if leftType != rightType: if leftType < rightType: @@ -89,203 +90,245 @@ def compare(left, right): # TODO: may be able to use helpers.decode_value and do direct compares # after converting to python types - value_type = value.WhichOneof('value_type') + value_type = left.WhichOneof('value_type') if value_type == 'null_value': - return 0 # nulls are all equal + return 0 # nulls are all equal elif value_type == 'boolean_value': - return _compareTo(decode_value(left), decode_value(right)) + return self._compareTo(left.boolean_value, right.boolean_value) elif value_type == 'integer_value': - return compare_numbers(left, right) + return self.compare_numbers(left, right) elif value_type == 'double_value': - return compare_numbers(left, right) + return self.compare_numbers(left, right) elif value_type == 'timestamp_value': - # NOTE: This conversion is "lossy", Python ``datetime.datetime`` - # has microsecond precision but ``timestamp_value`` has - # nanosecond precision. - #return _pb_timestamp_to_datetime(value.timestamp_value) - return compare_timestamps(left, right) + return self.compare_timestamps(left, right) elif value_type == 'string_value': - #return value.string_value - return compare_strings(left, right) + return self._compareTo(left.string_value, right.string_value) elif value_type == 'bytes_value': - #return value.bytes_value - return compare_blobs(left, right) + return self.compare_blobs(left, right) elif value_type == 'reference_value': - #return reference_value_to_document(value.reference_value, client) - return compare_resource_paths(left, right) + return self.compare_resource_paths(left, right) elif value_type == 'geo_point_value': - #return GeoPoint( - # value.geo_point_value.latitude, - # value.geo_point_value.longitude) - return compare_geo_points(left, right) + return self.compare_geo_points(left, right) elif value_type == 'array_value': - #return [decode_value(element, client) - # for element in value.array_value.values] - return compare_arrays(left, right) + return self.compare_arrays(left, right) elif value_type == 'map_value': - #return decode_dict(value.map_value.fields, client) - return compare_objects(left, right) + return self.compare_objects(left, right) else: raise ValueError('Unknown ``value_type``', value_type) -def compare_strings(left, right): - left_value = decode_value(left) - right_value = decode_value(right) - return _compareTo(left_value, right_value) - - -# private int compareBlobs(Value left, Value right) { -# ByteString leftBytes = left.getBytesValue(); -# ByteString rightBytes = right.getBytesValue(); - -# int size = Math.min(leftBytes.size(), rightBytes.size()); -# for (int i = 0; i < size; i++) { -# // Make sure the bytes are unsigned -# int thisByte = leftBytes.byteAt(i) & 0xff; -# int otherByte = rightBytes.byteAt(i) & 0xff; -# if (thisByte < otherByte) { -# return -1; -# } else if (thisByte > otherByte) { -# return 1; -# } -# // Byte values are equal, continue with comparison -# } -# return Integer.compare(leftBytes.size(), rightBytes.size()); -# } -def compare_blobs(left, right): - raise NotImplementedError() - - -def compare_timestamps(left, right): - left_value = left.timestamp_value - right_value = right.timestamp_value - - cmp = 0 - if left_value.seconds < right_value.seconds: - cmp = -1 - elif left_value.seconds == right_value.seconds: - cmp = 0 - else: - cmp = 0 - - if cmp != 0: - return cmp - else: - if left_value.nanos < right_value.nanos: - cmp = -1 - elif left_value.nanos == right_value.nanos: - cmp = 0 - else: - cmp = 1 - return cmp + # private int compareBlobs(Value left, Value right) { + # ByteString leftBytes = left.getBytesValue(); + # ByteString rightBytes = right.getBytesValue(); + + # int size = Math.min(leftBytes.size(), rightBytes.size()); + # for (int i = 0; i < size; i++) { + # // Make sure the bytes are unsigned + # int thisByte = leftBytes.byteAt(i) & 0xff; + # int otherByte = rightBytes.byteAt(i) & 0xff; + # if (thisByte < otherByte) { + # return -1; + # } else if (thisByte > otherByte) { + # return 1; + # } + # // Byte values are equal, continue with comparison + # } + # return Integer.compare(leftBytes.size(), rightBytes.size()); + # } + @staticmethod + def compare_blobs(left, right): + left_bytes = left.bytes_value + right_bytes = right.bytes_value + + # TODO: verify this is okay. python can compare bytes so *shrugs* + return Order._compareTo(left_bytes, right_bytes) + @staticmethod + def compare_timestamps(left, right): + left = left.timestamp_value + right = right.timestamp_value -def compare_geo_points(left, right): - left_value = decode_value(left) - right_value = decode_value(right) - cmp = 0 - if left_value.latitude < right_value.latitude: - cmp = -1 - elif left_value.latitude == right_value.latitude: + seconds = Order._compareTo(left.seconds or 0, right.seconds or 0) + if seconds != 0: + return seconds + + return Order._compareTo(left.nanos or 0, right.nanos or 0) + + # cmp = 0 + # if left_value.seconds < right_value.seconds: + # cmp = -1 + # elif left_value.seconds == right_value.seconds: + # cmp = 0 + # else: + # cmp = 0 + + # if cmp != 0: + # return cmp + # else: + # if left_value.nanos < right_value.nanos: + # cmp = -1 + # elif left_value.nanos == right_value.nanos: + # cmp = 0 + # else: + # cmp = 1 + # return cmp + + @staticmethod + def compare_geo_points(left, right): + left_value = decode_value(left, None) + right_value = decode_value(right, None) cmp = 0 - else: - cmp = 1 - - if cmp != 0: - return cmp - else: - if left.longitude < right.longitude: + if left_value.latitude < right_value.latitude: cmp = -1 - elif left.longitude == right.longitude: + elif left_value.latitude == right_value.latitude: cmp = 0 else: cmp = 1 - return cmp - -# private int compareResourcePaths(Value left, Value right) { -# ResourcePath leftPath = ResourcePath.create(left.getReferenceValue()); -# ResourcePath rightPath = ResourcePath.create(right.getReferenceValue()); -# return leftPath.compareTo(rightPath); -# } -def compare_resource_paths(left, right): - raise NotImplementedError() - - -# private int compareArrays(Value left, Value right) { -# List leftValue = left.getArrayValue().getValuesList(); -# List rightValue = right.getArrayValue().getValuesList(); - -# int minLength = Math.min(leftValue.size(), rightValue.size()); -# for (int i = 0; i < minLength; i++) { -# int cmp = compare(leftValue.get(i), rightValue.get(i)); -# if (cmp != 0) { -# return cmp; -# } -# } -# return Integer.compare(leftValue.size(), rightValue.size()); -# } -def compare_arrays(left, right): - raise NotImplementedError() - - - -# private int compareObjects(Value left, Value right) { -# // This requires iterating over the keys in the object in order and doing a -# // deep comparison. -# SortedMap leftMap = new TreeMap<>(); -# leftMap.putAll(left.getMapValue().getFieldsMap()); -# SortedMap rightMap = new TreeMap<>(); -# rightMap.putAll(right.getMapValue().getFieldsMap()); - -# Iterator> leftIterator = leftMap.entrySet().iterator(); -# Iterator> rightIterator = rightMap.entrySet().iterator(); - -# while (leftIterator.hasNext() && rightIterator.hasNext()) { -# Entry leftEntry = leftIterator.next(); -# Entry rightEntry = rightIterator.next(); -# int keyCompare = leftEntry.getKey().compareTo(rightEntry.getKey()); -# if (keyCompare != 0) { -# return keyCompare; -# } -# int valueCompare = compare(leftEntry.getValue(), rightEntry.getValue()); -# if (valueCompare != 0) { -# return valueCompare; -# } -# } - -# // Only equal if both iterators are exhausted. -# return Boolean.compare(leftIterator.hasNext(), rightIterator.hasNext()); -# } -def compare_objects(left, right): - raise NotImplementedError() - -def compare_numbers(left, right): - left_value = decode_value(left) - right_value = decode_value(right) - return compare_doubles(left_value, right_value) - -def compare_doubles(left, right): - if math.isnan(left): - if math.isnan(right): - return 0 - return -1 - if math.isnan(right): - return 1 - if left == -0.0: - left = 0 - if right == -0.0: - right = 0 + if cmp != 0: + return cmp + else: + if left_value.longitude < right_value.longitude: + cmp = -1 + elif left_value.longitude == right_value.longitude: + cmp = 0 + else: + cmp = 1 + return cmp + + # private int compareResourcePaths(Value left, Value right) { + # ResourcePath leftPath = ResourcePath.create(left.getReferenceValue()); + # ResourcePath rightPath = ResourcePath.create(right.getReferenceValue()); + # return leftPath.compareTo(rightPath); + # } + @staticmethod + def compare_resource_paths(left, right): + """ + compareTo(other: Path): number { + const len = Math.min(left.segments.length, right.segments.length); + for (let i = 0; i < len; i++) { + if (left.segments[i] < right.segments[i]) { + return -1; + } + if (left.segments[i] > right.segments[i]) { + return 1; + } + } + if (left.segments.length < right.segments.length) { + return -1; + } + if (left.segments.length > right.segments.length) { + return 1; + } + return 0; + } + """ + left = left.reference_value + right = right.reference_value + + + left_segments = left.split('/') + right_segments = right.split('/') + shorter = min(len(left_segments), len(right_segments)) + # compare segments + for i in range(shorter): + if (left_segments[i] < right_segments[i]): + return -1 + + if (left_segments[i] > right_segments[i]): + return 1 + - return _compareTo(left, right) + left_length = len(left) + right_length = len(right) + if left_length < right_length: + return -1 + if left_length > right_length: + return 1 -def _compareTo(left, right): - if left < right: - return -1 - elif left == right: return 0 - # left > right - return 1 + #raise NotImplementedError() + + + # private int compareArrays(Value left, Value right) { + # List leftValue = left.getArrayValue().getValuesList(); + # List rightValue = right.getArrayValue().getValuesList(); + + # int minLength = Math.min(leftValue.size(), rightValue.size()); + # for (int i = 0; i < minLength; i++) { + # int cmp = compare(leftValue.get(i), rightValue.get(i)); + # if (cmp != 0) { + # return cmp; + # } + # } + # return Integer.compare(leftValue.size(), rightValue.size()); + # } + @staticmethod + def compare_arrays(left, right): + raise NotImplementedError() + + + + # private int compareObjects(Value left, Value right) { + # // This requires iterating over the keys in the object in order and doing a + # // deep comparison. + # SortedMap leftMap = new TreeMap<>(); + # leftMap.putAll(left.getMapValue().getFieldsMap()); + # SortedMap rightMap = new TreeMap<>(); + # rightMap.putAll(right.getMapValue().getFieldsMap()); + + # Iterator> leftIterator = leftMap.entrySet().iterator(); + # Iterator> rightIterator = rightMap.entrySet().iterator(); + + # while (leftIterator.hasNext() && rightIterator.hasNext()) { + # Entry leftEntry = leftIterator.next(); + # Entry rightEntry = rightIterator.next(); + # int keyCompare = leftEntry.getKey().compareTo(rightEntry.getKey()); + # if (keyCompare != 0) { + # return keyCompare; + # } + # int valueCompare = compare(leftEntry.getValue(), rightEntry.getValue()); + # if (valueCompare != 0) { + # return valueCompare; + # } + # } + + # // Only equal if both iterators are exhausted. + # return Boolean.compare(leftIterator.hasNext(), rightIterator.hasNext()); + # } + @staticmethod + def compare_objects(left, right): + raise NotImplementedError() + + @staticmethod + def compare_numbers(left, right): + left_value = decode_value(left, None) + right_value = decode_value(right, None) + return Order.compare_doubles(left_value, right_value) + + @staticmethod + def compare_doubles(left, right): + if math.isnan(left): + if math.isnan(right): + return 0 + return -1 + if math.isnan(right): + return 1 + + if left == -0.0: + left = 0 + if right == -0.0: + right = 0 + + return Order._compareTo(left, right) + + @staticmethod + def _compareTo(left, right): + if left < right: + return -1 + elif left == right: + return 0 + # left > right + return 1 diff --git a/firestore/nox.py b/firestore/noxfile.py similarity index 100% rename from firestore/nox.py rename to firestore/noxfile.py diff --git a/firestore/tests/unit/test_order.py b/firestore/tests/unit/test_order.py index 3f0cbd5348d2..5f16842763ad 100644 --- a/firestore/tests/unit/test_order.py +++ b/firestore/tests/unit/test_order.py @@ -14,11 +14,13 @@ import unittest import mock -from google.cloud.firestore_v1beta1._helpers import encode_value +from google.cloud.firestore_v1beta1._helpers import encode_value, GeoPoint from google.protobuf import timestamp_pb2 from google.type import latlng_pb2 import math - +from google.cloud.firestore_v1beta1.document import DocumentReference +from google.cloud.firestore_v1beta1.order import Order +from google.cloud.firestore_v1beta1.proto import document_pb2 class TestOrder(unittest.TestCase): @staticmethod @@ -33,9 +35,10 @@ def _make_one(self, *args, **kwargs): def test_order(self): - int_max_value = 10 ** 1000 - int_min_value = -10 ** 1000 - float_min_value = -10.0 ** 1000 + # Constants used to represent min/max values of storage types. + int_max_value = 2 ** 31 - 1 + int_min_value = -(2 ** 31) + float_min_value = 1.175494351 ** -38 float_nan = float('nan') groups = [None] * 65 @@ -71,14 +74,14 @@ def test_order(self): # strings groups[20] = [_stringValue("")] groups[21] = [_stringValue("\u0000\ud7ff\ue000\uffff")] - groups[22] = [_stringValue("(╯°□°）╯︵ ┻━┻")] + groups[22] = [_stringValue(u"(╯°□°）╯︵ ┻━┻")] groups[23] = [_stringValue("a")] groups[24] = [_stringValue("abc def")] # latin small letter e + combining acute accent + latin small letter b groups[25] = [_stringValue("e\u0301b")] - groups[26] = [_stringValue("æ")] + groups[26] = [_stringValue(u"æ")] # latin small letter e with acute accent + latin small letter a - groups[27] = [_stringValue("\u00e9a")] + groups[27] = [_stringValue(u"\u00e9a")] # blobs groups[28] = [_blobValue(bytes())] @@ -124,38 +127,58 @@ def test_order(self): groups[53] = [_geoPointValue(90, 180)] # arrays + # groups[54] = [_arrayValue()] + # groups[55] = [_arrayValue([_stringValue("bar"))] + # groups[56] = [_arrayValue(_stringValue("foo"))] + # groups[57] = [_arrayValue(_stringValue("foo"), _intValue(0))] + # groups[58] = [_arrayValue(_stringValue("foo"), _intValue(1))] + # groups[59] = [_arrayValue(_stringValue("foo"), _stringValue("0"))] groups[54] = [_arrayValue()] - groups[55] = [_arrayValue(_stringValue("bar"))] - groups[56] = [_arrayValue(_stringValue("foo"))] - groups[57] = [_arrayValue(_stringValue("foo"), _intValue(0))] - groups[58] = [_arrayValue(_stringValue("foo"), _intValue(1))] - groups[59] = [_arrayValue(_stringValue("foo"), _stringValue("0"))] + groups[55] = [_arrayValue(["bar"])] + groups[56] = [_arrayValue(["foo"])] + groups[57] = [_arrayValue(["foo", 0])] + groups[58] = [_arrayValue(["foo", 1])] + groups[59] = [_arrayValue(["foo", "0"])] # objects - groups[60] = [_objectValue({"bar": _intValue(0)})] + # groups[60] = [_objectValue({"bar": _intValue(0)})] + # groups[61] = [_objectValue({ + # "bar": _intValue(0), + # "foo": _intValue(1) + # })] + # groups[62] = [_objectValue({"bar": _intValue(1)})] + # groups[63] = [_objectValue({"bar": _intValue(2)})] + # groups[64] = [_objectValue({"bar": _stringValue("0")})] + + groups[60] = [_objectValue({"bar": 0})] groups[61] = [_objectValue({ - "bar": _intValue(0), - "foo": _intValue(1) + "bar":0, + "foo": 1 })] - groups[62] = [_objectValue({"bar": _intValue(1)})] - groups[63] = [_objectValue({"bar": _intValue(2)})] - groups[64] = [_objectValue({"bar": _stringValue("0")})] + groups[62] = [_objectValue({"bar": 1})] + groups[63] = [_objectValue({"bar": 2})] + groups[64] = [_objectValue({"bar": "0"})] target = self._make_one() - for left in groups: - for right in groups: - for i in groups[left]: - for j in groups[right]: + + for i in range(len(groups)): + for left in groups[i]: + for j in range(len(groups)): + for right in groups[j]: + expected = Order._compareTo(i,j) + self.assertEqual( - _compare(left, right), - _compare( - target.compare( - groups[left][i], - groups[right][j]), 0), - "Order does not match for: groups[%d][%d] " - "and groups[%d][%d]".format(left, i, right, j) + target.compare(left, right), expected, + "comparing L->R {} ({}) to {} ({})".format(i, left, j, right) ) + + expected = Order._compareTo(j, i); + self.assertEqual( + target.compare(right, left), expected, + #"comparing R->L {} to {}".format(right, left) + "comparing R->L {} ({}) to {} ({})".format(j, right, i, left) + ) def _compare(left, right): if left < right: @@ -182,7 +205,7 @@ def _stringValue(s): def _referenceValue(r): - return encode_value(r) + return document_pb2.Value(reference_value=r) def _blobValue(b): @@ -194,15 +217,15 @@ def nullValue(): def _timestampValue(seconds, nanos): - return timestamp_pb2.Timestamp(seconds=seconds, nanos=nanos) + return document_pb2.Value( + timestamp_value=timestamp_pb2.Timestamp(seconds=seconds, nanos=nanos)) def _geoPointValue(latitude, longitude): - return latlng_pb2.LatLng(latitude=latitude, - longitude=longitude) + return encode_value(GeoPoint(latitude,longitude)) -def _arrayValue(values): +def _arrayValue(values=[]): return encode_value(values) From 9290511fbe73c042d4b1438d6ad9cba5f56d8955 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Mon, 8 Oct 2018 13:06:28 -0700 Subject: [PATCH 053/148] complete implementation of order --- .../google/cloud/firestore_v1beta1/order.py | 76 ++++++++----------- 1 file changed, 32 insertions(+), 44 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/order.py b/firestore/google/cloud/firestore_v1beta1/order.py index 274393a095a3..a16ba9b7fb39 100644 --- a/firestore/google/cloud/firestore_v1beta1/order.py +++ b/firestore/google/cloud/firestore_v1beta1/order.py @@ -249,58 +249,46 @@ def compare_resource_paths(left, right): return 1 return 0 - #raise NotImplementedError() - # private int compareArrays(Value left, Value right) { - # List leftValue = left.getArrayValue().getValuesList(); - # List rightValue = right.getArrayValue().getValuesList(); - - # int minLength = Math.min(leftValue.size(), rightValue.size()); - # for (int i = 0; i < minLength; i++) { - # int cmp = compare(leftValue.get(i), rightValue.get(i)); - # if (cmp != 0) { - # return cmp; - # } - # } - # return Integer.compare(leftValue.size(), rightValue.size()); - # } @staticmethod def compare_arrays(left, right): - raise NotImplementedError() - - - - # private int compareObjects(Value left, Value right) { - # // This requires iterating over the keys in the object in order and doing a - # // deep comparison. - # SortedMap leftMap = new TreeMap<>(); - # leftMap.putAll(left.getMapValue().getFieldsMap()); - # SortedMap rightMap = new TreeMap<>(); - # rightMap.putAll(right.getMapValue().getFieldsMap()); + l_values = left.array_value.values#.keys() + r_values = right.array_value.values#.keys() + + length = min(len(l_values), len(r_values)) + for i in range(length): + cmp = Order().compare(l_values[i], r_values[i]) + if cmp != 0: + return cmp + + return Order._compareTo(len(l_values), len(r_values)) - # Iterator> leftIterator = leftMap.entrySet().iterator(); - # Iterator> rightIterator = rightMap.entrySet().iterator(); - # while (leftIterator.hasNext() && rightIterator.hasNext()) { - # Entry leftEntry = leftIterator.next(); - # Entry rightEntry = rightIterator.next(); - # int keyCompare = leftEntry.getKey().compareTo(rightEntry.getKey()); - # if (keyCompare != 0) { - # return keyCompare; - # } - # int valueCompare = compare(leftEntry.getValue(), rightEntry.getValue()); - # if (valueCompare != 0) { - # return valueCompare; - # } - # } - - # // Only equal if both iterators are exhausted. - # return Boolean.compare(leftIterator.hasNext(), rightIterator.hasNext()); - # } @staticmethod def compare_objects(left, right): - raise NotImplementedError() + left_fields = left.map_value.fields + right_fields = right.map_value.fields + + l_iter = left_fields.__iter__() + r_iter = right_fields.__iter__() + try: + while True: + left_key = l_iter.__next__() + right_key = r_iter.__next__() + + keyCompare = Order._compareTo(left_key, right_key) + if keyCompare != 0: + return keyCompare + + value_compare = Order().compare( + left_fields[left_key], right_fields[right_key]) + if value_compare != 0: + return value_compare + except StopIteration: + pass + + return Order._compareTo(len(left_fields), len(right_fields)) @staticmethod def compare_numbers(left, right): From f5734e9e851cb9c243cfb9b8ec5a548d18b58216 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Mon, 8 Oct 2018 13:08:01 -0700 Subject: [PATCH 054/148] remove commented code areas in order --- .../google/cloud/firestore_v1beta1/order.py | 75 +------------------ 1 file changed, 1 insertion(+), 74 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/order.py b/firestore/google/cloud/firestore_v1beta1/order.py index a16ba9b7fb39..d5a68e2ebf49 100644 --- a/firestore/google/cloud/firestore_v1beta1/order.py +++ b/firestore/google/cloud/firestore_v1beta1/order.py @@ -12,19 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - -# package com.google.cloud.firestore; - -# import com.google.firestore.v1beta1.Value; -# import com.google.firestore.v1beta1.Value.ValueTypeCase; -# import com.google.protobuf.ByteString; -# import java.util.Comparator; -# import java.util.Iterator; -# import java.util.List; -# import java.util.Map.Entry; -# import java.util.SortedMap; -# import java.util.TreeMap; -# import javax.annotation.Nonnull; from enum import Enum from google.cloud.firestore_v1beta1._helpers import decode_value import math @@ -118,24 +105,6 @@ def compare(self, left, right): raise ValueError('Unknown ``value_type``', value_type) - # private int compareBlobs(Value left, Value right) { - # ByteString leftBytes = left.getBytesValue(); - # ByteString rightBytes = right.getBytesValue(); - - # int size = Math.min(leftBytes.size(), rightBytes.size()); - # for (int i = 0; i < size; i++) { - # // Make sure the bytes are unsigned - # int thisByte = leftBytes.byteAt(i) & 0xff; - # int otherByte = rightBytes.byteAt(i) & 0xff; - # if (thisByte < otherByte) { - # return -1; - # } else if (thisByte > otherByte) { - # return 1; - # } - # // Byte values are equal, continue with comparison - # } - # return Integer.compare(leftBytes.size(), rightBytes.size()); - # } @staticmethod def compare_blobs(left, right): left_bytes = left.bytes_value @@ -155,24 +124,6 @@ def compare_timestamps(left, right): return Order._compareTo(left.nanos or 0, right.nanos or 0) - # cmp = 0 - # if left_value.seconds < right_value.seconds: - # cmp = -1 - # elif left_value.seconds == right_value.seconds: - # cmp = 0 - # else: - # cmp = 0 - - # if cmp != 0: - # return cmp - # else: - # if left_value.nanos < right_value.nanos: - # cmp = -1 - # elif left_value.nanos == right_value.nanos: - # cmp = 0 - # else: - # cmp = 1 - # return cmp @staticmethod def compare_geo_points(left, right): @@ -197,33 +148,9 @@ def compare_geo_points(left, right): cmp = 1 return cmp - # private int compareResourcePaths(Value left, Value right) { - # ResourcePath leftPath = ResourcePath.create(left.getReferenceValue()); - # ResourcePath rightPath = ResourcePath.create(right.getReferenceValue()); - # return leftPath.compareTo(rightPath); - # } + @staticmethod def compare_resource_paths(left, right): - """ - compareTo(other: Path): number { - const len = Math.min(left.segments.length, right.segments.length); - for (let i = 0; i < len; i++) { - if (left.segments[i] < right.segments[i]) { - return -1; - } - if (left.segments[i] > right.segments[i]) { - return 1; - } - } - if (left.segments.length < right.segments.length) { - return -1; - } - if (left.segments.length > right.segments.length) { - return 1; - } - return 0; - } - """ left = left.reference_value right = right.reference_value From 78a62a632912983a6832e88095c3abc4d50b1938 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Mon, 8 Oct 2018 13:09:21 -0700 Subject: [PATCH 055/148] refactor order --- .../google/cloud/firestore_v1beta1/order.py | 20 +- firestore/tests/unit/test_order.py | 185 ++++++++---------- 2 files changed, 92 insertions(+), 113 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/order.py b/firestore/google/cloud/firestore_v1beta1/order.py index d5a68e2ebf49..b1e1722daaa1 100644 --- a/firestore/google/cloud/firestore_v1beta1/order.py +++ b/firestore/google/cloud/firestore_v1beta1/order.py @@ -82,7 +82,7 @@ def compare(self, left, right): if value_type == 'null_value': return 0 # nulls are all equal elif value_type == 'boolean_value': - return self._compareTo(left.boolean_value, right.boolean_value) + return self._compare_to(left.boolean_value, right.boolean_value) elif value_type == 'integer_value': return self.compare_numbers(left, right) elif value_type == 'double_value': @@ -90,7 +90,7 @@ def compare(self, left, right): elif value_type == 'timestamp_value': return self.compare_timestamps(left, right) elif value_type == 'string_value': - return self._compareTo(left.string_value, right.string_value) + return self._compare_to(left.string_value, right.string_value) elif value_type == 'bytes_value': return self.compare_blobs(left, right) elif value_type == 'reference_value': @@ -111,18 +111,18 @@ def compare_blobs(left, right): right_bytes = right.bytes_value # TODO: verify this is okay. python can compare bytes so *shrugs* - return Order._compareTo(left_bytes, right_bytes) + return Order._compare_to(left_bytes, right_bytes) @staticmethod def compare_timestamps(left, right): left = left.timestamp_value right = right.timestamp_value - seconds = Order._compareTo(left.seconds or 0, right.seconds or 0) + seconds = Order._compare_to(left.seconds or 0, right.seconds or 0) if seconds != 0: return seconds - return Order._compareTo(left.nanos or 0, right.nanos or 0) + return Order._compare_to(left.nanos or 0, right.nanos or 0) @staticmethod @@ -189,7 +189,7 @@ def compare_arrays(left, right): if cmp != 0: return cmp - return Order._compareTo(len(l_values), len(r_values)) + return Order._compare_to(len(l_values), len(r_values)) @staticmethod @@ -204,7 +204,7 @@ def compare_objects(left, right): left_key = l_iter.__next__() right_key = r_iter.__next__() - keyCompare = Order._compareTo(left_key, right_key) + keyCompare = Order._compare_to(left_key, right_key) if keyCompare != 0: return keyCompare @@ -215,7 +215,7 @@ def compare_objects(left, right): except StopIteration: pass - return Order._compareTo(len(left_fields), len(right_fields)) + return Order._compare_to(len(left_fields), len(right_fields)) @staticmethod def compare_numbers(left, right): @@ -237,10 +237,10 @@ def compare_doubles(left, right): if right == -0.0: right = 0 - return Order._compareTo(left, right) + return Order._compare_to(left, right) @staticmethod - def _compareTo(left, right): + def _compare_to(left, right): if left < right: return -1 elif left == right: diff --git a/firestore/tests/unit/test_order.py b/firestore/tests/unit/test_order.py index 5f16842763ad..2a9e55765fe2 100644 --- a/firestore/tests/unit/test_order.py +++ b/firestore/tests/unit/test_order.py @@ -45,119 +45,104 @@ def test_order(self): groups[0] = [nullValue()] - groups[1] = [_booleanValue(False)] - groups[2] = [_booleanValue(True)] + groups[1] = [_boolean_value(False)] + groups[2] = [_boolean_value(True)] # numbers - groups[3] = [_doubleValue(float_nan), _doubleValue(float_nan)] - groups[4] = [_doubleValue(-math.inf)] - groups[5] = [_intValue(int_min_value - 1)] - groups[6] = [_intValue(int_min_value)] - groups[7] = [_doubleValue(-1.1)] + groups[3] = [_double_value(float_nan), _double_value(float_nan)] + groups[4] = [_double_value(-math.inf)] + groups[5] = [_int_value(int_min_value - 1)] + groups[6] = [_int_value(int_min_value)] + groups[7] = [_double_value(-1.1)] # Integers and Doubles order the same. - groups[8] = [_intValue(-1), _doubleValue(-1.0)] - groups[9] = [_doubleValue(-float_min_value)] + groups[8] = [_int_value(-1), _double_value(-1.0)] + groups[9] = [_double_value(-float_min_value)] # zeros all compare the same. - groups[10] = [_intValue(0), _doubleValue(-0.0), - _doubleValue(0.0), _doubleValue(+0.0)] - groups[11] = [_doubleValue(float_min_value)] - groups[12] = [_intValue(1), _doubleValue(1.0)] - groups[13] = [_doubleValue(1.1)] - groups[14] = [_intValue(int_max_value)] - groups[15] = [_intValue(int_max_value + 1)] - groups[16] = [_doubleValue(math.inf)] - - groups[17] = [_timestampValue(123, 0)] - groups[18] = [_timestampValue(123, 123)] - groups[19] = [_timestampValue(345, 0)] + groups[10] = [_int_value(0), _double_value(-0.0), + _double_value(0.0), _double_value(+0.0)] + groups[11] = [_double_value(float_min_value)] + groups[12] = [_int_value(1), _double_value(1.0)] + groups[13] = [_double_value(1.1)] + groups[14] = [_int_value(int_max_value)] + groups[15] = [_int_value(int_max_value + 1)] + groups[16] = [_double_value(math.inf)] + + groups[17] = [_timestamp_value(123, 0)] + groups[18] = [_timestamp_value(123, 123)] + groups[19] = [_timestamp_value(345, 0)] # strings - groups[20] = [_stringValue("")] - groups[21] = [_stringValue("\u0000\ud7ff\ue000\uffff")] - groups[22] = [_stringValue(u"(╯°□°）╯︵ ┻━┻")] - groups[23] = [_stringValue("a")] - groups[24] = [_stringValue("abc def")] + groups[20] = [_string_value("")] + groups[21] = [_string_value("\u0000\ud7ff\ue000\uffff")] + groups[22] = [_string_value(u"(╯°□°）╯︵ ┻━┻")] + groups[23] = [_string_value("a")] + groups[24] = [_string_value("abc def")] # latin small letter e + combining acute accent + latin small letter b - groups[25] = [_stringValue("e\u0301b")] - groups[26] = [_stringValue(u"æ")] + groups[25] = [_string_value("e\u0301b")] + groups[26] = [_string_value(u"æ")] # latin small letter e with acute accent + latin small letter a - groups[27] = [_stringValue(u"\u00e9a")] + groups[27] = [_string_value(u"\u00e9a")] # blobs - groups[28] = [_blobValue(bytes())] - groups[29] = [_blobValue(bytes([0]))] - groups[30] = [_blobValue(bytes([0, 1, 2, 3, 4]))] - groups[31] = [_blobValue(bytes([0, 1, 2, 4, 3]))] - groups[32] = [_blobValue(bytes([127]))] + groups[28] = [_blob_value(bytes())] + groups[29] = [_blob_value(bytes([0]))] + groups[30] = [_blob_value(bytes([0, 1, 2, 3, 4]))] + groups[31] = [_blob_value(bytes([0, 1, 2, 4, 3]))] + groups[32] = [_blob_value(bytes([127]))] # resource names groups[33] = [ - _referenceValue("projects/p1/databases/d1/documents/c1/doc1")] + _reference_value("projects/p1/databases/d1/documents/c1/doc1")] groups[34] = [ - _referenceValue("projects/p1/databases/d1/documents/c1/doc2")] + _reference_value("projects/p1/databases/d1/documents/c1/doc2")] groups[35] = [ - _referenceValue( + _reference_value( "projects/p1/databases/d1/documents/c1/doc2/c2/doc1")] groups[36] = [ - _referenceValue( + _reference_value( "projects/p1/databases/d1/documents/c1/doc2/c2/doc2")] groups[37] = [ - _referenceValue("projects/p1/databases/d1/documents/c10/doc1")] + _reference_value("projects/p1/databases/d1/documents/c10/doc1")] groups[38] = [ - _referenceValue("projects/p1/databases/d1/documents/c2/doc1")] + _reference_value("projects/p1/databases/d1/documents/c2/doc1")] groups[39] = [ - _referenceValue("projects/p2/databases/d2/documents/c1/doc1")] + _reference_value("projects/p2/databases/d2/documents/c1/doc1")] groups[40] = [ - _referenceValue("projects/p2/databases/d2/documents/c1-/doc1")] + _reference_value("projects/p2/databases/d2/documents/c1-/doc1")] groups[41] = [ - _referenceValue("projects/p2/databases/d3/documents/c1-/doc1")] + _reference_value("projects/p2/databases/d3/documents/c1-/doc1")] # geo points - groups[42] = [_geoPointValue(-90, -180)] - groups[43] = [_geoPointValue(-90, 0)] - groups[44] = [_geoPointValue(-90, 180)] - groups[45] = [_geoPointValue(0, -180)] - groups[46] = [_geoPointValue(0, 0)] - groups[47] = [_geoPointValue(0, 180)] - groups[48] = [_geoPointValue(1, -180)] - groups[49] = [_geoPointValue(1, 0)] - groups[50] = [_geoPointValue(1, 180)] - groups[51] = [_geoPointValue(90, -180)] - groups[52] = [_geoPointValue(90, 0)] - groups[53] = [_geoPointValue(90, 180)] + groups[42] = [_geoPoint_value(-90, -180)] + groups[43] = [_geoPoint_value(-90, 0)] + groups[44] = [_geoPoint_value(-90, 180)] + groups[45] = [_geoPoint_value(0, -180)] + groups[46] = [_geoPoint_value(0, 0)] + groups[47] = [_geoPoint_value(0, 180)] + groups[48] = [_geoPoint_value(1, -180)] + groups[49] = [_geoPoint_value(1, 0)] + groups[50] = [_geoPoint_value(1, 180)] + groups[51] = [_geoPoint_value(90, -180)] + groups[52] = [_geoPoint_value(90, 0)] + groups[53] = [_geoPoint_value(90, 180)] # arrays - # groups[54] = [_arrayValue()] - # groups[55] = [_arrayValue([_stringValue("bar"))] - # groups[56] = [_arrayValue(_stringValue("foo"))] - # groups[57] = [_arrayValue(_stringValue("foo"), _intValue(0))] - # groups[58] = [_arrayValue(_stringValue("foo"), _intValue(1))] - # groups[59] = [_arrayValue(_stringValue("foo"), _stringValue("0"))] - groups[54] = [_arrayValue()] - groups[55] = [_arrayValue(["bar"])] - groups[56] = [_arrayValue(["foo"])] - groups[57] = [_arrayValue(["foo", 0])] - groups[58] = [_arrayValue(["foo", 1])] - groups[59] = [_arrayValue(["foo", "0"])] + groups[54] = [_array_value()] + groups[55] = [_array_value(["bar"])] + groups[56] = [_array_value(["foo"])] + groups[57] = [_array_value(["foo", 0])] + groups[58] = [_array_value(["foo", 1])] + groups[59] = [_array_value(["foo", "0"])] # objects - # groups[60] = [_objectValue({"bar": _intValue(0)})] - # groups[61] = [_objectValue({ - # "bar": _intValue(0), - # "foo": _intValue(1) - # })] - # groups[62] = [_objectValue({"bar": _intValue(1)})] - # groups[63] = [_objectValue({"bar": _intValue(2)})] - # groups[64] = [_objectValue({"bar": _stringValue("0")})] - - groups[60] = [_objectValue({"bar": 0})] - groups[61] = [_objectValue({ + groups[60] = [_object_value({"bar": 0})] + groups[61] = [_object_value({ "bar":0, "foo": 1 })] - groups[62] = [_objectValue({"bar": 1})] - groups[63] = [_objectValue({"bar": 2})] - groups[64] = [_objectValue({"bar": "0"})] + groups[62] = [_object_value({"bar": 1})] + groups[63] = [_object_value({"bar": 2})] + groups[64] = [_object_value({"bar": "0"})] target = self._make_one() @@ -165,50 +150,44 @@ def test_order(self): for left in groups[i]: for j in range(len(groups)): for right in groups[j]: - expected = Order._compareTo(i,j) + expected = Order._compare_to(i,j) self.assertEqual( target.compare(left, right), expected, - "comparing L->R {} ({}) to {} ({})".format(i, left, j, right) + "comparing L->R {} ({}) to {} ({})".format( + i, left, j, right) ) - expected = Order._compareTo(j, i); + expected = Order._compare_to(j, i); self.assertEqual( target.compare(right, left), expected, - #"comparing R->L {} to {}".format(right, left) - "comparing R->L {} ({}) to {} ({})".format(j, right, i, left) + "comparing R->L {} ({}) to {} ({})".format( + j, right, i, left) ) -def _compare(left, right): - if left < right: - return -1 - elif left == right: - return 0 - return 1 - -def _booleanValue(b): +def _boolean_value(b): return encode_value(b) -def _doubleValue(d): +def _double_value(d): return encode_value(d) -def _intValue(l): +def _int_value(l): return encode_value(l) -def _stringValue(s): +def _string_value(s): return encode_value(s) -def _referenceValue(r): +def _reference_value(r): return document_pb2.Value(reference_value=r) -def _blobValue(b): +def _blob_value(b): return encode_value(b) @@ -216,18 +195,18 @@ def nullValue(): return encode_value(None) -def _timestampValue(seconds, nanos): +def _timestamp_value(seconds, nanos): return document_pb2.Value( timestamp_value=timestamp_pb2.Timestamp(seconds=seconds, nanos=nanos)) -def _geoPointValue(latitude, longitude): +def _geoPoint_value(latitude, longitude): return encode_value(GeoPoint(latitude,longitude)) -def _arrayValue(values=[]): +def _array_value(values=[]): return encode_value(values) -def _objectValue(keysAndValues): +def _object_value(keysAndValues): return encode_value(keysAndValues) From 925495c297dd6c9562e52eb7cce5a5ef21e27313 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Mon, 8 Oct 2018 14:53:48 -0700 Subject: [PATCH 056/148] refactor order compare_objects --- .../google/cloud/firestore_v1beta1/order.py | 27 +++++++------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/order.py b/firestore/google/cloud/firestore_v1beta1/order.py index b1e1722daaa1..a327639e5654 100644 --- a/firestore/google/cloud/firestore_v1beta1/order.py +++ b/firestore/google/cloud/firestore_v1beta1/order.py @@ -165,8 +165,6 @@ def compare_resource_paths(left, right): if (left_segments[i] > right_segments[i]): return 1 - - left_length = len(left) right_length = len(right) @@ -197,23 +195,16 @@ def compare_objects(left, right): left_fields = left.map_value.fields right_fields = right.map_value.fields - l_iter = left_fields.__iter__() - r_iter = right_fields.__iter__() - try: - while True: - left_key = l_iter.__next__() - right_key = r_iter.__next__() - - keyCompare = Order._compare_to(left_key, right_key) - if keyCompare != 0: - return keyCompare - - value_compare = Order().compare( - left_fields[left_key], right_fields[right_key]) - if value_compare != 0: + for left_key, right_key in zip(left_fields, right_fields): + keyCompare = Order._compare_to(left_key, right_key) + if keyCompare != 0: + return keyCompare + + value_compare = Order().compare( + left_fields[left_key], right_fields[right_key]) + if value_compare != 0: return value_compare - except StopIteration: - pass + return Order._compare_to(len(left_fields), len(right_fields)) From 4316fd1e607e4f00f78081012e818e68b7a234c2 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Mon, 8 Oct 2018 15:00:14 -0700 Subject: [PATCH 057/148] add system test for ordering (currently failing for non-ordering --- firestore/tests/system.py | 81 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/firestore/tests/system.py b/firestore/tests/system.py index 25d2f0958f0d..a3a7868f5c3d 100644 --- a/firestore/tests/system.py +++ b/firestore/tests/system.py @@ -895,3 +895,84 @@ def on_snapshot(docs, changes, read_time): raise AssertionError( "Failed to get exactly two document changes: count: " + str(on_snapshot.called_count)) + + +def test_watch_query_order(client, cleanup): + db = client + unique_id = unique_resource_id() + doc_ref = db.collection(u'users').document( + u'alovelace' + unique_id) + query_ref = db.collection(u'users').where( + "first", "==", u'Ada' + unique_id).order_by("last") + + + # Setup listener + def on_snapshot(docs, changes, read_time): + try: + on_snapshot.called_count += 1 + + # A snapshot should return the same thing as if a query ran now. + query_ran = query_ref.get() + query_ran_results = [i for i in query_ran] + assert len(docs) == len(query_ran_results) + print("doc length: {}".format(len(docs))) + print("changes length: {}".format(len(changes))) + print("readtime: {}".format(read_time)) + + # compare the order things are returned + for snapshot, query in zip(docs, query_ran_results): + assert snapshot.get('last')['stringValue'] == query.get( + 'last'), "expect the sort order to match" + + except Exception as e: + pytest.fail(e) + + on_snapshot.called_count = 0 + + # Initial setting + doc_ref.set({ + u'first': u'Jane', + u'last': u'Doe', + u'born': 1900 + }) + + sleep(1) + + query_ref.on_snapshot(on_snapshot) + + # Alter document + doc_ref.set({ + u'first': u'Ada' + unique_id, + u'last': u'Lovelace', + u'born': 1815 + }) + + for _ in range(10): + if on_snapshot.called_count == 1: + break + sleep(1) + + if on_snapshot.called_count != 1: + raise AssertionError( + "Initial set should have called on_snapshot 1 time: " + + str(on_snapshot.called_count)) + + # Create new document + doc_ref_2 = db.collection(u'users').document( + u'alovelace' + unique_resource_id()) + doc_ref_2.set({ + u'first': u'Ada' + unique_id, + u'last': u'ASecondLovelace', + u'born': 1815 + }) + + for _ in range(10): + if on_snapshot.called_count == 2: + break + sleep(1) + + + if on_snapshot.called_count != 2: + raise AssertionError( + "After new add on_snapshot should be called 2 times: " + + str(on_snapshot.called_count)) From 220ba99b4ee60cc46d340bfd49efffa0d83f947c Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Tue, 9 Oct 2018 21:32:09 -0700 Subject: [PATCH 058/148] add system test for query and verify order --- .../google/cloud/firestore_v1beta1/order.py | 33 +++++++------ .../google/cloud/firestore_v1beta1/query.py | 47 +++++++++++++++++++ .../google/cloud/firestore_v1beta1/watch.py | 39 ++++++++------- firestore/tests/system.py | 21 +++++---- 4 files changed, 96 insertions(+), 44 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/order.py b/firestore/google/cloud/firestore_v1beta1/order.py index a327639e5654..2ad62c96b906 100644 --- a/firestore/google/cloud/firestore_v1beta1/order.py +++ b/firestore/google/cloud/firestore_v1beta1/order.py @@ -16,6 +16,7 @@ from google.cloud.firestore_v1beta1._helpers import decode_value import math + class TypeOrder(Enum): # NOTE: This order is defined by the backend and cannot be changed. NULL = 0 @@ -56,16 +57,14 @@ class Order(object): ''' Order implements the ordering semantics of the backend. ''' - def __init__(self): - pass - def compare(self, left, right): + @classmethod + def compare(cls, left, right): ''' Main comparison function for all Firestore types. @return -1 is left < right, 0 if left == right, otherwise 1 ''' - # First compare the types. leftType = TypeOrder.from_value(left).value rightType = TypeOrder.from_value(right).value @@ -82,25 +81,25 @@ def compare(self, left, right): if value_type == 'null_value': return 0 # nulls are all equal elif value_type == 'boolean_value': - return self._compare_to(left.boolean_value, right.boolean_value) + return cls._compare_to(left.boolean_value, right.boolean_value) elif value_type == 'integer_value': - return self.compare_numbers(left, right) + return cls.compare_numbers(left, right) elif value_type == 'double_value': - return self.compare_numbers(left, right) + return cls.compare_numbers(left, right) elif value_type == 'timestamp_value': - return self.compare_timestamps(left, right) + return cls.compare_timestamps(left, right) elif value_type == 'string_value': - return self._compare_to(left.string_value, right.string_value) + return cls._compare_to(left.string_value, right.string_value) elif value_type == 'bytes_value': - return self.compare_blobs(left, right) + return cls.compare_blobs(left, right) elif value_type == 'reference_value': - return self.compare_resource_paths(left, right) + return cls.compare_resource_paths(left, right) elif value_type == 'geo_point_value': - return self.compare_geo_points(left, right) + return cls.compare_geo_points(left, right) elif value_type == 'array_value': - return self.compare_arrays(left, right) + return cls.compare_arrays(left, right) elif value_type == 'map_value': - return self.compare_objects(left, right) + return cls.compare_objects(left, right) else: raise ValueError('Unknown ``value_type``', value_type) @@ -110,7 +109,7 @@ def compare_blobs(left, right): left_bytes = left.bytes_value right_bytes = right.bytes_value - # TODO: verify this is okay. python can compare bytes so *shrugs* + # TODO: Should verify bytes comparisons in python work as expected return Order._compare_to(left_bytes, right_bytes) @staticmethod @@ -183,7 +182,7 @@ def compare_arrays(left, right): length = min(len(l_values), len(r_values)) for i in range(length): - cmp = Order().compare(l_values[i], r_values[i]) + cmp = Order.compare(l_values[i], r_values[i]) if cmp != 0: return cmp @@ -200,7 +199,7 @@ def compare_objects(left, right): if keyCompare != 0: return keyCompare - value_compare = Order().compare( + value_compare = Order.compare( left_fields[left_key], right_fields[right_key]) if value_compare != 0: return value_compare diff --git a/firestore/google/cloud/firestore_v1beta1/query.py b/firestore/google/cloud/firestore_v1beta1/query.py index 27e282d9ffb4..3febb1705a13 100644 --- a/firestore/google/cloud/firestore_v1beta1/query.py +++ b/firestore/google/cloud/firestore_v1beta1/query.py @@ -30,6 +30,7 @@ from google.cloud.firestore_v1beta1.gapic import enums from google.cloud.firestore_v1beta1.proto import query_pb2 from google.cloud.firestore_v1beta1.watch import Watch +from google.cloud.firestore_v1beta1.watch import Order _EQ_OP = '==' _COMPARISON_OPERATORS = { @@ -632,6 +633,52 @@ def on_snapshot(query_snapshot): document.DocumentSnapshot, document.DocumentReference) + def comparator(self, doc1, doc2): + _orders = self._orders + + # Add implicit sorting by name, using the last specified direction. + if len(_orders) == 0: + lastDirection = Query.ASCENDING + else: + if _orders[-1].direction == 1: + lastDirection = Query.ASCENDING + else: + lastDirection = Query.DESCENDING + + orderBys = list(_orders) + + order_pb = query_pb2.StructuredQuery.Order( + field=query_pb2.StructuredQuery.FieldReference( + field_path='id', + ), + direction=_enum_from_direction(lastDirection), + ) + orderBys.append(order_pb) + + for orderBy in orderBys: + if orderBy.field.field_path == 'id': + # If ordering by docuent id, compare resource paths. + comp = Order()._compare_to( + doc1.reference._path, doc2.reference._path) + else: + if orderBy.field.field_path not in doc1._data or \ + orderBy.field.field_path not in doc2._data: + raise Exception( + "Can only compare fields that exist in the " + "DocumentSnapshot. Please include the fields you are " + " ordering on in your select() call.") + v1 = doc1._data[orderBy.field.field_path] + v2 = doc2._data[orderBy.field.field_path] + encoded_v1 = _helpers.encode_value(v1) + encoded_v2 = _helpers.encode_value(v2) + comp = Order().compare(encoded_v1, encoded_v2) + + if (comp != 0): + # 1 == Ascending, -1 == Descending + return orderBy.direction * comp + + return 0 + def _enum_from_op_string(op_string): """Convert a string representation of a binary operator to an enum. diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index bc2b237bc1a3..4024ad030887 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -17,16 +17,20 @@ import threading import datetime from enum import Enum +import functools import pytz from google.cloud.firestore_v1beta1.bidi import ResumableBidiRpc from google.cloud.firestore_v1beta1.bidi import BackgroundConsumer +from google.cloud.firestore_v1beta1._helpers import encode_value from google.cloud.firestore_v1beta1.proto import firestore_pb2 +from google.cloud.firestore_v1beta1.order import Order from google.api_core import exceptions from google.protobuf import json_format + # from bidi import BidiRpc, ResumableBidiRpc import grpc @@ -344,14 +348,15 @@ def for_query(cls, query, snapshot_callback, snapshot_class_instance, query_target = firestore_pb2.Target.QueryTarget( parent=query._client._database_string, structured_query=query._to_protobuf(), - ) + ) + return cls(query, query._client, { 'query': query_target, 'target_id': WATCH_TARGET_ID }, - document_watch_comparator, + query.comparator, snapshot_callback, snapshot_class_instance, reference_class_instance) @@ -555,17 +560,21 @@ def push(self, read_time, next_resume_token): read_time, ) - updated_tree, updated_map, appliedChanges = Watch._compute_snapshot( + updated_tree, updated_map, appliedChanges = self._compute_snapshot( self.doc_tree, self.doc_map, deletes, adds, updates, - ) + ) if not self.has_pushed or len(appliedChanges): + # TODO: the tree should be ordered. Sort here for now. + key = functools.cmp_to_key(self._comparator) + keys = sorted(updated_tree.keys(), key=key) + self._snapshot_callback( - updated_tree.keys(), + keys, appliedChanges, datetime.datetime.fromtimestamp(read_time.seconds, pytz.utc) ) @@ -597,13 +606,11 @@ def _extract_changes(doc_map, changes, read_time): return (deletes, adds, updates) - @staticmethod - def _compute_snapshot(doc_tree, doc_map, delete_changes, add_changes, + def _compute_snapshot(self, doc_tree, doc_map, delete_changes, add_changes, update_changes): # TODO: ACTUALLY NEED TO CALCULATE # return {updated_tree, updated_map, appliedChanges}; # return doc_tree, doc_map, changes - updated_tree = doc_tree updated_map = doc_map @@ -674,20 +681,17 @@ def modify_doc(new_document, updated_tree, updated_map): # keep incrementing. appliedChanges = [] - # Deletes are sorted based on the order of the existing document. - - # TODO: SORT - # delete_changes.sort( - # lambda name1, name2: - # self._comparator(updated_map.get(name1), updated_map.get(name2))) + key = functools.cmp_to_key(self._comparator) + # Deletes are sorted based on the order of the existing document. + delete_changes = sorted(delete_changes, key=key) for name in delete_changes: change, updated_tree, updated_map = delete_doc( name, updated_tree, updated_map) appliedChanges.append(change) - # TODO: SORT - # add_changes.sort(self._comparator) + + add_changes = sorted(add_changes, key=key) _LOGGER.debug('walk over add_changes') for snapshot in add_changes: _LOGGER.debug('in add_changes') @@ -695,8 +699,7 @@ def modify_doc(new_document, updated_tree, updated_map): snapshot, updated_tree, updated_map) appliedChanges.append(change) - # TODO: SORT - # update_changes.sort(self._comparator) + update_changes = sorted(update_changes, key=key) for snapshot in update_changes: change, updated_tree, updated_map = modify_doc( snapshot, updated_tree, updated_map) diff --git a/firestore/tests/system.py b/firestore/tests/system.py index a3a7868f5c3d..3b4b79695725 100644 --- a/firestore/tests/system.py +++ b/firestore/tests/system.py @@ -909,25 +909,23 @@ def test_watch_query_order(client, cleanup): # Setup listener def on_snapshot(docs, changes, read_time): try: - on_snapshot.called_count += 1 - # A snapshot should return the same thing as if a query ran now. query_ran = query_ref.get() query_ran_results = [i for i in query_ran] assert len(docs) == len(query_ran_results) - print("doc length: {}".format(len(docs))) - print("changes length: {}".format(len(changes))) - print("readtime: {}".format(read_time)) # compare the order things are returned for snapshot, query in zip(docs, query_ran_results): - assert snapshot.get('last')['stringValue'] == query.get( - 'last'), "expect the sort order to match" + print("snapshot: " + snapshot.get('last')['stringValue'] + ", " + "query: " + query.get('last')) + assert snapshot.get('last')['stringValue'] == query.get( + 'last'), "expect the sort order to match" + on_snapshot.called_count += 1 except Exception as e: - pytest.fail(e) + on_snapshot.failed = e on_snapshot.called_count = 0 + on_snapshot.failed = None # Initial setting doc_ref.set({ @@ -959,7 +957,7 @@ def on_snapshot(docs, changes, read_time): # Create new document doc_ref_2 = db.collection(u'users').document( - u'alovelace' + unique_resource_id()) + u'asecondlovelace' + unique_id) doc_ref_2.set({ u'first': u'Ada' + unique_id, u'last': u'ASecondLovelace', @@ -976,3 +974,8 @@ def on_snapshot(docs, changes, read_time): raise AssertionError( "After new add on_snapshot should be called 2 times: " + str(on_snapshot.called_count)) + if on_snapshot.failed: + raise AssertionError( + "on_snapshot failed while trying to compare sort order: " + str( + on_snapshot.failed) + ) From 30972d909376ba77abddae014bc0e9f713b386f8 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Wed, 10 Oct 2018 10:06:13 -0700 Subject: [PATCH 059/148] Improve test for order --- firestore/tests/system.py | 77 ++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 41 deletions(-) diff --git a/firestore/tests/system.py b/firestore/tests/system.py index 3b4b79695725..25739e6f292a 100644 --- a/firestore/tests/system.py +++ b/firestore/tests/system.py @@ -900,8 +900,17 @@ def on_snapshot(docs, changes, read_time): def test_watch_query_order(client, cleanup): db = client unique_id = unique_resource_id() - doc_ref = db.collection(u'users').document( + doc_ref1 = db.collection(u'users').document( u'alovelace' + unique_id) + doc_ref2 = db.collection(u'users').document( + u'asecondlovelace' + unique_id) + doc_ref3 = db.collection(u'users').document( + u'athirdlovelace' + unique_id) + doc_ref4 = db.collection(u'users').document( + u'afourthlovelace' + unique_id) + doc_ref5 = db.collection(u'users').document( + u'afifthlovelace' + unique_id) + query_ref = db.collection(u'users').where( "first", "==", u'Ada' + unique_id).order_by("last") @@ -909,6 +918,8 @@ def test_watch_query_order(client, cleanup): # Setup listener def on_snapshot(docs, changes, read_time): try: + if len(docs) != 5: + return # A snapshot should return the same thing as if a query ran now. query_ran = query_ref.get() query_ran_results = [i for i in query_ran] @@ -916,66 +927,50 @@ def on_snapshot(docs, changes, read_time): # compare the order things are returned for snapshot, query in zip(docs, query_ran_results): - print("snapshot: " + snapshot.get('last')['stringValue'] + ", " + "query: " + query.get('last')) - assert snapshot.get('last')['stringValue'] == query.get( 'last'), "expect the sort order to match" on_snapshot.called_count += 1 + on_snapshot.last_doc_count = len(docs) except Exception as e: on_snapshot.failed = e on_snapshot.called_count = 0 + on_snapshot.last_doc_count = 0 on_snapshot.failed = None - - # Initial setting - doc_ref.set({ - u'first': u'Jane', - u'last': u'Doe', - u'born': 1900 - }) - - sleep(1) - query_ref.on_snapshot(on_snapshot) - # Alter document - doc_ref.set({ + doc_ref1.set({ u'first': u'Ada' + unique_id, u'last': u'Lovelace', u'born': 1815 }) - - for _ in range(10): - if on_snapshot.called_count == 1: - break - sleep(1) - - if on_snapshot.called_count != 1: - raise AssertionError( - "Initial set should have called on_snapshot 1 time: " + - str(on_snapshot.called_count)) - - # Create new document - doc_ref_2 = db.collection(u'users').document( - u'asecondlovelace' + unique_id) - doc_ref_2.set({ + doc_ref2.set({ + u'first': u'Ada' + unique_id, + u'last': u'SecondLovelace', + u'born': 1815 + }) + doc_ref3.set({ u'first': u'Ada' + unique_id, - u'last': u'ASecondLovelace', + u'last': u'ThirdLovelace', + u'born': 1815 + }) + doc_ref4.set({ + u'first': u'Ada' + unique_id, + u'last': u'FourthLovelace', + u'born': 1815 + }) + doc_ref5.set({ + u'first': u'Ada' + unique_id, + u'last': u'lovelace', u'born': 1815 }) for _ in range(10): - if on_snapshot.called_count == 2: + if on_snapshot.last_doc_count == 5: break sleep(1) - - if on_snapshot.called_count != 2: - raise AssertionError( - "After new add on_snapshot should be called 2 times: " + - str(on_snapshot.called_count)) - if on_snapshot.failed: + if on_snapshot.last_doc_count != 5: raise AssertionError( - "on_snapshot failed while trying to compare sort order: " + str( - on_snapshot.failed) - ) + "5 docs expected in snapshot method " + + str(on_snapshot.last_doc_count)) From d549145ab57aa6c00b30822c57e5110688b1bdf3 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Wed, 10 Oct 2018 10:06:51 -0700 Subject: [PATCH 060/148] Add comparator to kw creation of dummyquery, fix old test, move comparator to internal --- firestore/google/cloud/firestore_v1beta1/query.py | 9 +++++---- firestore/google/cloud/firestore_v1beta1/watch.py | 2 +- firestore/tests/unit/test_watch.py | 10 ++++++++-- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/query.py b/firestore/google/cloud/firestore_v1beta1/query.py index 3febb1705a13..d3b1d89e65b0 100644 --- a/firestore/google/cloud/firestore_v1beta1/query.py +++ b/firestore/google/cloud/firestore_v1beta1/query.py @@ -633,7 +633,7 @@ def on_snapshot(query_snapshot): document.DocumentSnapshot, document.DocumentReference) - def comparator(self, doc1, doc2): + def _comparator(self, doc1, doc2): _orders = self._orders # Add implicit sorting by name, using the last specified direction. @@ -664,9 +664,10 @@ def comparator(self, doc1, doc2): if orderBy.field.field_path not in doc1._data or \ orderBy.field.field_path not in doc2._data: raise Exception( - "Can only compare fields that exist in the " - "DocumentSnapshot. Please include the fields you are " - " ordering on in your select() call.") + "Can only compare fields that exist in the " + "DocumentSnapshot. Please include the fields you are " + "ordering on in your select() call." + ) v1 = doc1._data[orderBy.field.field_path] v2 = doc2._data[orderBy.field.field_path] encoded_v1 = _helpers.encode_value(v1) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 4024ad030887..ee79ffa92973 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -356,7 +356,7 @@ def for_query(cls, query, snapshot_callback, snapshot_class_instance, 'query': query_target, 'target_id': WATCH_TARGET_ID }, - query.comparator, + query._comparator, snapshot_callback, snapshot_class_instance, reference_class_instance) diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index 10f970861572..8bc33bd7a67f 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -628,8 +628,8 @@ class DummyDoc(object): added_doc._document_path = '/added' updated_doc = DummyDoc() updated_doc._document_path = '/updated' - doc_tree = doc_tree.insert('/deleted', deleted_doc) - doc_tree = doc_tree.insert('/updated', updated_doc) + doc_tree = doc_tree.insert(deleted_doc, None) + doc_tree = doc_tree.insert(updated_doc, None) doc_map = {'/deleted': deleted_doc, '/updated': updated_doc} added_snapshot = DummyDocumentSnapshot(added_doc, None, True, None, None, None) @@ -735,6 +735,12 @@ def __init__(self, **kw): else: self._client = kw['client'] + if 'comparator' not in kw: + # don't really do the comparison, just return 0 (equal) for all + self._comparator = lambda x,y: 1 + else: + self._comparator = kw['comparator'] + def _to_protobuf(self): return '' From 1abc70d338c5d4cf4a267c99d05c8a280b1af158 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Wed, 10 Oct 2018 12:50:14 -0700 Subject: [PATCH 061/148] flake8 fixes --- .../cloud/firestore_v1beta1/collection.py | 6 +- .../cloud/firestore_v1beta1/document.py | 3 +- .../google/cloud/firestore_v1beta1/order.py | 87 +++++++++---------- .../google/cloud/firestore_v1beta1/query.py | 10 +-- .../google/cloud/firestore_v1beta1/watch.py | 10 +-- firestore/tests/system.py | 24 ++--- firestore/tests/unit/test_order.py | 25 +++--- firestore/tests/unit/test_watch.py | 2 +- 8 files changed, 71 insertions(+), 96 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/collection.py b/firestore/google/cloud/firestore_v1beta1/collection.py index 8234bee6a21a..399766da7148 100644 --- a/firestore/google/cloud/firestore_v1beta1/collection.py +++ b/firestore/google/cloud/firestore_v1beta1/collection.py @@ -399,9 +399,9 @@ def on_snapshot(collection_snapshot): collection_watch.unsubscribe() """ return Watch.for_query(query_mod.Query(self), - callback, - document.DocumentSnapshot, - document.DocumentReference) + callback, + document.DocumentSnapshot, + document.DocumentReference) def _auto_id(): diff --git a/firestore/google/cloud/firestore_v1beta1/document.py b/firestore/google/cloud/firestore_v1beta1/document.py index 004550f9542a..f10df4302a28 100644 --- a/firestore/google/cloud/firestore_v1beta1/document.py +++ b/firestore/google/cloud/firestore_v1beta1/document.py @@ -471,7 +471,8 @@ def on_snapshot(document_snapshot): # Terminate this watch doc_watch.unsubscribe() """ - return Watch.for_document(self, callback, DocumentSnapshot, DocumentReference) + return Watch.for_document(self, callback, DocumentSnapshot, + DocumentReference) class DocumentSnapshot(object): diff --git a/firestore/google/cloud/firestore_v1beta1/order.py b/firestore/google/cloud/firestore_v1beta1/order.py index 2ad62c96b906..132416238474 100644 --- a/firestore/google/cloud/firestore_v1beta1/order.py +++ b/firestore/google/cloud/firestore_v1beta1/order.py @@ -18,51 +18,50 @@ class TypeOrder(Enum): - # NOTE: This order is defined by the backend and cannot be changed. - NULL = 0 - BOOLEAN = 1 - NUMBER = 2 - TIMESTAMP = 3 - STRING = 4 - BLOB = 5 - REF = 6 - GEO_POINT = 7 - ARRAY = 8 - OBJECT = 9 - - def from_value(value): - v = value.WhichOneof('value_type') - - lut = { - 'null_value': TypeOrder.NULL, - 'boolean_value': TypeOrder.BOOLEAN, - 'integer_value': TypeOrder.NUMBER, - 'double_value': TypeOrder.NUMBER, - 'timestamp_value': TypeOrder.TIMESTAMP, - 'string_value': TypeOrder.STRING, - 'bytes_value': TypeOrder.BLOB, - 'reference_value': TypeOrder.REF, - 'geo_point_value': TypeOrder.GEO_POINT, - 'array_value': TypeOrder.ARRAY, - 'map_value': TypeOrder.OBJECT, - } - - if v not in lut: - raise ArgumentException( - "Could not detect value type for " + value) - return lut[v] + # NOTE: This order is defined by the backend and cannot be changed. + NULL = 0 + BOOLEAN = 1 + NUMBER = 2 + TIMESTAMP = 3 + STRING = 4 + BLOB = 5 + REF = 6 + GEO_POINT = 7 + ARRAY = 8 + OBJECT = 9 + + def from_value(value): + v = value.WhichOneof('value_type') + + lut = { + 'null_value': TypeOrder.NULL, + 'boolean_value': TypeOrder.BOOLEAN, + 'integer_value': TypeOrder.NUMBER, + 'double_value': TypeOrder.NUMBER, + 'timestamp_value': TypeOrder.TIMESTAMP, + 'string_value': TypeOrder.STRING, + 'bytes_value': TypeOrder.BLOB, + 'reference_value': TypeOrder.REF, + 'geo_point_value': TypeOrder.GEO_POINT, + 'array_value': TypeOrder.ARRAY, + 'map_value': TypeOrder.OBJECT, + } + + if v not in lut: + raise ValueError( + "Could not detect value type for " + value) + return lut[v] class Order(object): ''' Order implements the ordering semantics of the backend. ''' - + @classmethod def compare(cls, left, right): ''' Main comparison function for all Firestore types. - @return -1 is left < right, 0 if left == right, otherwise 1 ''' # First compare the types. @@ -103,12 +102,11 @@ def compare(cls, left, right): else: raise ValueError('Unknown ``value_type``', value_type) - @staticmethod def compare_blobs(left, right): left_bytes = left.bytes_value right_bytes = right.bytes_value - + # TODO: Should verify bytes comparisons in python work as expected return Order._compare_to(left_bytes, right_bytes) @@ -120,9 +118,8 @@ def compare_timestamps(left, right): seconds = Order._compare_to(left.seconds or 0, right.seconds or 0) if seconds != 0: return seconds - - return Order._compare_to(left.nanos or 0, right.nanos or 0) + return Order._compare_to(left.nanos or 0, right.nanos or 0) @staticmethod def compare_geo_points(left, right): @@ -147,13 +144,11 @@ def compare_geo_points(left, right): cmp = 1 return cmp - @staticmethod def compare_resource_paths(left, right): left = left.reference_value right = right.reference_value - left_segments = left.split('/') right_segments = right.split('/') shorter = min(len(left_segments), len(right_segments)) @@ -161,7 +156,6 @@ def compare_resource_paths(left, right): for i in range(shorter): if (left_segments[i] < right_segments[i]): return -1 - if (left_segments[i] > right_segments[i]): return 1 @@ -174,20 +168,18 @@ def compare_resource_paths(left, right): return 0 - @staticmethod def compare_arrays(left, right): - l_values = left.array_value.values#.keys() - r_values = right.array_value.values#.keys() + l_values = left.array_value.values + r_values = right.array_value.values length = min(len(l_values), len(r_values)) for i in range(length): cmp = Order.compare(l_values[i], r_values[i]) if cmp != 0: return cmp - - return Order._compare_to(len(l_values), len(r_values)) + return Order._compare_to(len(l_values), len(r_values)) @staticmethod def compare_objects(left, right): @@ -204,7 +196,6 @@ def compare_objects(left, right): if value_compare != 0: return value_compare - return Order._compare_to(len(left_fields), len(right_fields)) @staticmethod diff --git a/firestore/google/cloud/firestore_v1beta1/query.py b/firestore/google/cloud/firestore_v1beta1/query.py index d3b1d89e65b0..85f9f7bbb264 100644 --- a/firestore/google/cloud/firestore_v1beta1/query.py +++ b/firestore/google/cloud/firestore_v1beta1/query.py @@ -29,8 +29,8 @@ from google.cloud.firestore_v1beta1 import document from google.cloud.firestore_v1beta1.gapic import enums from google.cloud.firestore_v1beta1.proto import query_pb2 +from google.cloud.firestore_v1beta1.order import Order from google.cloud.firestore_v1beta1.watch import Watch -from google.cloud.firestore_v1beta1.watch import Order _EQ_OP = '==' _COMPARISON_OPERATORS = { @@ -629,9 +629,9 @@ def on_snapshot(query_snapshot): query_watch.unsubscribe() """ return Watch.for_query(self, - callback, - document.DocumentSnapshot, - document.DocumentReference) + callback, + document.DocumentSnapshot, + document.DocumentReference) def _comparator(self, doc1, doc2): _orders = self._orders @@ -662,7 +662,7 @@ def _comparator(self, doc1, doc2): doc1.reference._path, doc2.reference._path) else: if orderBy.field.field_path not in doc1._data or \ - orderBy.field.field_path not in doc2._data: + orderBy.field.field_path not in doc2._data: raise Exception( "Can only compare fields that exist in the " "DocumentSnapshot. Please include the fields you are " diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index ee79ffa92973..01eebff57690 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -23,15 +23,10 @@ from google.cloud.firestore_v1beta1.bidi import ResumableBidiRpc from google.cloud.firestore_v1beta1.bidi import BackgroundConsumer -from google.cloud.firestore_v1beta1._helpers import encode_value from google.cloud.firestore_v1beta1.proto import firestore_pb2 -from google.cloud.firestore_v1beta1.order import Order from google.api_core import exceptions from google.protobuf import json_format - - -# from bidi import BidiRpc, ResumableBidiRpc import grpc """Python client for Google Cloud Firestore Watch.""" @@ -348,7 +343,7 @@ def for_query(cls, query, snapshot_callback, snapshot_class_instance, query_target = firestore_pb2.Target.QueryTarget( parent=query._client._database_string, structured_query=query._to_protobuf(), - ) + ) return cls(query, query._client, @@ -570,7 +565,7 @@ def push(self, read_time, next_resume_token): if not self.has_pushed or len(appliedChanges): # TODO: the tree should be ordered. Sort here for now. - key = functools.cmp_to_key(self._comparator) + key = functools.cmp_to_key(self._comparator) keys = sorted(updated_tree.keys(), key=key) self._snapshot_callback( @@ -690,7 +685,6 @@ def modify_doc(new_document, updated_tree, updated_map): name, updated_tree, updated_map) appliedChanges.append(change) - add_changes = sorted(add_changes, key=key) _LOGGER.debug('walk over add_changes') for snapshot in add_changes: diff --git a/firestore/tests/system.py b/firestore/tests/system.py index 25739e6f292a..f67cf4ec36bf 100644 --- a/firestore/tests/system.py +++ b/firestore/tests/system.py @@ -809,12 +809,9 @@ def on_snapshot(docs, changes, read_time): on_snapshot.called_count = 0 - # def on_snapshot(docs, changes, read_time): - # for doc in docs: - # print(u'{} => {}'.format(doc.id, doc.to_dict())) - collection_ref.on_snapshot(on_snapshot) + # delay here so initial on_snapshot occurs and isn't combined with set sleep(1) doc_ref.set({ @@ -823,22 +820,16 @@ def on_snapshot(docs, changes, read_time): u'born': 1815 }) - sleep(1) - for _ in range(10): if on_snapshot.called_count == 1: - return + break sleep(1) - if on_snapshot.called_count != 1: + if on_snapshot.called_count != 2: raise AssertionError( - "Failed to get exactly one document change: count: " + + "Expected 2 snapshots, initial, and change: " + str(on_snapshot.called_count)) - # CM: had to stop here, this test is totally unfinished, trying to - # formalize - # https://gist.github.com/crwilcox/ce05f3857adc7a0ed86ffbd039b1a035 - def test_watch_query(client, cleanup): db = client @@ -905,16 +896,15 @@ def test_watch_query_order(client, cleanup): doc_ref2 = db.collection(u'users').document( u'asecondlovelace' + unique_id) doc_ref3 = db.collection(u'users').document( - u'athirdlovelace' + unique_id) + u'athirdlovelace' + unique_id) doc_ref4 = db.collection(u'users').document( u'afourthlovelace' + unique_id) doc_ref5 = db.collection(u'users').document( - u'afifthlovelace' + unique_id) + u'afifthlovelace' + unique_id) query_ref = db.collection(u'users').where( "first", "==", u'Ada' + unique_id).order_by("last") - # Setup listener def on_snapshot(docs, changes, read_time): try: @@ -935,7 +925,7 @@ def on_snapshot(docs, changes, read_time): on_snapshot.failed = e on_snapshot.called_count = 0 - on_snapshot.last_doc_count = 0 + on_snapshot.last_doc_count = 0 on_snapshot.failed = None query_ref.on_snapshot(on_snapshot) diff --git a/firestore/tests/unit/test_order.py b/firestore/tests/unit/test_order.py index 2a9e55765fe2..4c0aafd7923f 100644 --- a/firestore/tests/unit/test_order.py +++ b/firestore/tests/unit/test_order.py @@ -13,15 +13,15 @@ # limitations under the License. import unittest -import mock -from google.cloud.firestore_v1beta1._helpers import encode_value, GeoPoint -from google.protobuf import timestamp_pb2 -from google.type import latlng_pb2 import math -from google.cloud.firestore_v1beta1.document import DocumentReference + +from google.cloud.firestore_v1beta1._helpers import encode_value, GeoPoint from google.cloud.firestore_v1beta1.order import Order from google.cloud.firestore_v1beta1.proto import document_pb2 +from google.protobuf import timestamp_pb2 + + class TestOrder(unittest.TestCase): @staticmethod def _get_target_class(): @@ -34,7 +34,6 @@ def _make_one(self, *args, **kwargs): return klass(*args, **kwargs) def test_order(self): - # Constants used to represent min/max values of storage types. int_max_value = 2 ** 31 - 1 int_min_value = -(2 ** 31) @@ -135,9 +134,9 @@ def test_order(self): groups[59] = [_array_value(["foo", "0"])] # objects - groups[60] = [_object_value({"bar": 0})] + groups[60] = [_object_value({"bar": 0})] groups[61] = [_object_value({ - "bar":0, + "bar": 0, "foo": 1 })] groups[62] = [_object_value({"bar": 1})] @@ -150,15 +149,15 @@ def test_order(self): for left in groups[i]: for j in range(len(groups)): for right in groups[j]: - expected = Order._compare_to(i,j) - + expected = Order._compare_to(i, j) + self.assertEqual( target.compare(left, right), expected, "comparing L->R {} ({}) to {} ({})".format( i, left, j, right) ) - - expected = Order._compare_to(j, i); + + expected = Order._compare_to(j, i) self.assertEqual( target.compare(right, left), expected, "comparing R->L {} ({}) to {} ({})".format( @@ -201,7 +200,7 @@ def _timestamp_value(seconds, nanos): def _geoPoint_value(latitude, longitude): - return encode_value(GeoPoint(latitude,longitude)) + return encode_value(GeoPoint(latitude, longitude)) def _array_value(values=[]): diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index 8bc33bd7a67f..29ed28119efa 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -737,7 +737,7 @@ def __init__(self, **kw): if 'comparator' not in kw: # don't really do the comparison, just return 0 (equal) for all - self._comparator = lambda x,y: 1 + self._comparator = lambda x, y: 1 else: self._comparator = kw['comparator'] From ab862d7b04da5c97d7682b9b37150a93e9d0e9bf Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Wed, 10 Oct 2018 15:33:27 -0700 Subject: [PATCH 062/148] coverage + tests --- .../cloud/firestore_v1beta1/_helpers.py | 2 +- .../google/cloud/firestore_v1beta1/order.py | 2 +- .../google/cloud/firestore_v1beta1/query.py | 2 +- firestore/tests/system.py | 11 ++-- firestore/tests/unit/test_order.py | 16 +++++ firestore/tests/unit/test_query.py | 62 +++++++++++++++++++ firestore/tests/unit/test_watch.py | 7 ++- 7 files changed, 92 insertions(+), 10 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/_helpers.py b/firestore/google/cloud/firestore_v1beta1/_helpers.py index 902942d895c6..a424649e8f65 100644 --- a/firestore/google/cloud/firestore_v1beta1/_helpers.py +++ b/firestore/google/cloud/firestore_v1beta1/_helpers.py @@ -745,7 +745,7 @@ def get_nested_value(field_path, data): nested_data = data for index, field_name in enumerate(field_names): - if isinstance(nested_data, collections.Mapping): + if isinstance(nested_data, collections.abc.Mapping): if field_name in nested_data: nested_data = nested_data[field_name] else: diff --git a/firestore/google/cloud/firestore_v1beta1/order.py b/firestore/google/cloud/firestore_v1beta1/order.py index 132416238474..b747e9bce7bc 100644 --- a/firestore/google/cloud/firestore_v1beta1/order.py +++ b/firestore/google/cloud/firestore_v1beta1/order.py @@ -100,7 +100,7 @@ def compare(cls, left, right): elif value_type == 'map_value': return cls.compare_objects(left, right) else: - raise ValueError('Unknown ``value_type``', value_type) + raise ValueError('Unknown ``value_type``', str(value_type)) @staticmethod def compare_blobs(left, right): diff --git a/firestore/google/cloud/firestore_v1beta1/query.py b/firestore/google/cloud/firestore_v1beta1/query.py index 85f9f7bbb264..2710e2e97026 100644 --- a/firestore/google/cloud/firestore_v1beta1/query.py +++ b/firestore/google/cloud/firestore_v1beta1/query.py @@ -663,7 +663,7 @@ def _comparator(self, doc1, doc2): else: if orderBy.field.field_path not in doc1._data or \ orderBy.field.field_path not in doc2._data: - raise Exception( + raise ValueError( "Can only compare fields that exist in the " "DocumentSnapshot. Please include the fields you are " "ordering on in your select() call." diff --git a/firestore/tests/system.py b/firestore/tests/system.py index f67cf4ec36bf..5609898770b2 100644 --- a/firestore/tests/system.py +++ b/firestore/tests/system.py @@ -806,8 +806,11 @@ def test_watch_collection(client, cleanup): # Setup listener def on_snapshot(docs, changes, read_time): on_snapshot.called_count += 1 + for doc in [doc for doc in docs if doc.id == doc_ref.id]: + on_snapshot.born = doc._data['born']['integerValue'] on_snapshot.called_count = 0 + on_snapshot.born = 0 collection_ref.on_snapshot(on_snapshot) @@ -821,14 +824,14 @@ def on_snapshot(docs, changes, read_time): }) for _ in range(10): - if on_snapshot.called_count == 1: + if on_snapshot.born == '1815': break sleep(1) - if on_snapshot.called_count != 2: + if on_snapshot.born != '1815': raise AssertionError( - "Expected 2 snapshots, initial, and change: " + - str(on_snapshot.called_count)) + "Expected the last document update to update born: " + + str(on_snapshot.born)) def test_watch_query(client, cleanup): diff --git a/firestore/tests/unit/test_order.py b/firestore/tests/unit/test_order.py index 4c0aafd7923f..b8490a8a4964 100644 --- a/firestore/tests/unit/test_order.py +++ b/firestore/tests/unit/test_order.py @@ -14,9 +14,13 @@ import unittest import math +import mock + from google.cloud.firestore_v1beta1._helpers import encode_value, GeoPoint from google.cloud.firestore_v1beta1.order import Order +from google.cloud.firestore_v1beta1.order import TypeOrder + from google.cloud.firestore_v1beta1.proto import document_pb2 from google.protobuf import timestamp_pb2 @@ -165,6 +169,18 @@ def test_order(self): ) + def test_failure_to_find_type(self): + target = self._make_one() + left = mock.Mock() + left.WhichOneOf.return_value = "imaginary-type" + right = mock.Mock() + # Patch from value to get to the deep compare. Since left is a bad type + # expect this to fail with value error. + with mock.patch.object(TypeOrder, 'from_value',) as to: + to.value = None + with self.assertRaisesRegex(ValueError, "'Unknown ``value_type``"): + target.compare(left, right) + def _boolean_value(b): return encode_value(b) diff --git a/firestore/tests/unit/test_query.py b/firestore/tests/unit/test_query.py index 4e4619841438..6cffafd8bc8f 100644 --- a/firestore/tests/unit/test_query.py +++ b/firestore/tests/unit/test_query.py @@ -866,6 +866,68 @@ def test_on_snapshot(self, watch): query.on_snapshot(None) watch.for_query.assert_called_once() + def test_comparator_no_ordering(self): + query = self._make_one(mock.sentinel.parent) + query._orders = [] + doc1 = mock.Mock() + doc1.reference._path = ('col', 'adocument1') + + doc2 = mock.Mock() + doc2.reference._path = ('col', 'adocument2') + + sort = query._comparator(doc1, doc2) + self.assertEqual(sort, -1) + + def test_comparator_no_ordering_same_id(self): + query = self._make_one(mock.sentinel.parent) + query._orders = [] + doc1 = mock.Mock() + doc1.reference._path = ('col', 'adocument1') + + doc2 = mock.Mock() + doc2.reference._path = ('col', 'adocument1') + + sort = query._comparator(doc1, doc2) + self.assertEqual(sort, 0) + + def test_comparator_ordering(self): + query = self._make_one(mock.sentinel.parent) + orderByMock = mock.Mock() + orderByMock.field.field_path = 'last' + orderByMock.direction = 1 # ascending + query._orders = [orderByMock] + + doc1 = mock.Mock() + doc1.reference._path = ('col', 'adocument1') + doc1._data = {'first': {'stringValue': 'Ada'}, + 'last': {'stringValue': 'secondlovelace'}} + doc2 = mock.Mock() + doc2.reference._path = ('col', 'adocument2') + doc2._data = {'first': {'stringValue': 'Ada'}, + 'last': {'stringValue': 'lovelace'}} + + sort = query._comparator(doc1, doc2) + self.assertEqual(sort, 1) + + def test_comparator_missing_order_by_field_in_data_raises(self): + query = self._make_one(mock.sentinel.parent) + orderByMock = mock.Mock() + orderByMock.field.field_path = 'last' + orderByMock.direction = 1 # ascending + query._orders = [orderByMock] + + doc1 = mock.Mock() + doc1.reference._path = ('col', 'adocument1') + doc1._data = {} + doc2 = mock.Mock() + doc2.reference._path = ('col', 'adocument2') + doc2._data = {'first': {'stringValue': 'Ada'}, + 'last': {'stringValue': 'lovelace'}} + + with self.assertRaisesRegex(ValueError, + "Can only compare fields "): + query._comparator(doc1, doc2) + class Test__enum_from_op_string(unittest.TestCase): diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index 29ed28119efa..ba5c306cbce8 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -394,14 +394,15 @@ def message_to_dict(document): proto.document_change.target_ids = [WATCH_TARGET_ID] class DummyDocument: - name = 'abc://foo/fred' + name = 'abc://foo/documents/fred' create_time = None update_time = None proto.document_change.document = DummyDocument() - inst._firestore._database_string = 'abc://foo/' + inst._firestore._database_string = 'abc://foo' inst.on_snapshot(proto) - self.assertEqual(inst.change_map['abc://foo/fred'].data, None) + self.assertEqual(inst.change_map['abc://foo/documents/fred'].data, + None) def test_on_snapshot_document_change_neither_changed_nor_removed(self): inst = self._makeOne() From be3584f01c467404fc19ce76d508d075498063f6 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Thu, 11 Oct 2018 10:39:02 -0700 Subject: [PATCH 063/148] Properly decode document snapshot data and fix tests --- firestore/google/cloud/firestore_v1beta1/watch.py | 8 ++++---- firestore/tests/system.py | 15 ++++++++++----- firestore/tests/unit/test_watch.py | 14 ++++---------- 3 files changed, 18 insertions(+), 19 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 01eebff57690..10c71cc24148 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -24,8 +24,9 @@ from google.cloud.firestore_v1beta1.bidi import ResumableBidiRpc from google.cloud.firestore_v1beta1.bidi import BackgroundConsumer from google.cloud.firestore_v1beta1.proto import firestore_pb2 +from google.cloud.firestore_v1beta1 import _helpers + from google.api_core import exceptions -from google.protobuf import json_format import grpc @@ -156,7 +157,6 @@ class Watch(object): BackgroundConsumer = BackgroundConsumer # FBO unit tests ResumableBidiRpc = ResumableBidiRpc # FBO unit tests - MessageToDict = staticmethod(json_format.MessageToDict) # FBO unit tests def __init__(self, document_reference, @@ -479,7 +479,7 @@ def on_snapshot(self, proto): # google.cloud.firestore_v1beta1.types.Document document = document_change.document - data = self.MessageToDict(document) + data = _helpers.decode_dict(document.fields, self._firestore) # Create a snapshot. As Document and Query objects can be # passed we need to get a Document Reference in a more manual @@ -494,7 +494,7 @@ def on_snapshot(self, proto): snapshot = self.DocumentSnapshot( reference=document_ref, - data=data['fields'], + data=data, exists=True, read_time=None, create_time=document.create_time, diff --git a/firestore/tests/system.py b/firestore/tests/system.py index 5609898770b2..00fdfb7a821c 100644 --- a/firestore/tests/system.py +++ b/firestore/tests/system.py @@ -807,7 +807,7 @@ def test_watch_collection(client, cleanup): def on_snapshot(docs, changes, read_time): on_snapshot.called_count += 1 for doc in [doc for doc in docs if doc.id == doc_ref.id]: - on_snapshot.born = doc._data['born']['integerValue'] + on_snapshot.born = doc.get('born') on_snapshot.called_count = 0 on_snapshot.born = 0 @@ -824,11 +824,11 @@ def on_snapshot(docs, changes, read_time): }) for _ in range(10): - if on_snapshot.born == '1815': + if on_snapshot.born == 1815: break sleep(1) - if on_snapshot.born != '1815': + if on_snapshot.born != 1815: raise AssertionError( "Expected the last document update to update born: " + str(on_snapshot.born)) @@ -920,8 +920,10 @@ def on_snapshot(docs, changes, read_time): # compare the order things are returned for snapshot, query in zip(docs, query_ran_results): - assert snapshot.get('last')['stringValue'] == query.get( - 'last'), "expect the sort order to match" + assert snapshot.get('last') == query.get( + 'last'), "expect the sort order to match, last" + assert snapshot.get('born') == query.get( + 'born'), "expect the sort order to match, born" on_snapshot.called_count += 1 on_snapshot.last_doc_count = len(docs) except Exception as e: @@ -963,6 +965,9 @@ def on_snapshot(docs, changes, read_time): break sleep(1) + if on_snapshot.failed: + raise on_snapshot.failed + if on_snapshot.last_doc_count != 5: raise AssertionError( "5 docs expected in snapshot method " + diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index ba5c306cbce8..7c49b64c0076 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -361,22 +361,19 @@ def test_on_snapshot_document_change_changed(self): from google.cloud.firestore_v1beta1.watch import WATCH_TARGET_ID inst = self._makeOne() - def message_to_dict(document): - return {'fields': None} - - inst.MessageToDict = message_to_dict proto = DummyProto() proto.target_change = '' proto.document_change.target_ids = [WATCH_TARGET_ID] class DummyDocument: name = 'fred' + fields = {} create_time = None update_time = None proto.document_change.document = DummyDocument() inst.on_snapshot(proto) - self.assertEqual(inst.change_map['fred'].data, None) + self.assertEqual(inst.change_map['fred'].data, {}) def test_on_snapshot_document_change_changed_docname_db_prefix(self): # XXX This test asserts the current behavior, but I have no level @@ -385,16 +382,13 @@ def test_on_snapshot_document_change_changed_docname_db_prefix(self): from google.cloud.firestore_v1beta1.watch import WATCH_TARGET_ID inst = self._makeOne() - def message_to_dict(document): - return {'fields': None} - - inst.MessageToDict = message_to_dict proto = DummyProto() proto.target_change = '' proto.document_change.target_ids = [WATCH_TARGET_ID] class DummyDocument: name = 'abc://foo/documents/fred' + fields = {} create_time = None update_time = None @@ -402,7 +396,7 @@ class DummyDocument: inst._firestore._database_string = 'abc://foo' inst.on_snapshot(proto) self.assertEqual(inst.change_map['abc://foo/documents/fred'].data, - None) + {}) def test_on_snapshot_document_change_neither_changed_nor_removed(self): inst = self._makeOne() From abdfe4033ab4002ddbbe4458be1c4ab902209415 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Thu, 11 Oct 2018 13:03:15 -0700 Subject: [PATCH 064/148] coverage --- .../google/cloud/firestore_v1beta1/Untitled-1 | 1 + .../google/cloud/firestore_v1beta1/order.py | 2 +- firestore/tests/unit/test_order.py | 17 ++++++++++++++++- firestore/tests/unit/test_query.py | 19 +++++++++++++++++++ 4 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 firestore/google/cloud/firestore_v1beta1/Untitled-1 diff --git a/firestore/google/cloud/firestore_v1beta1/Untitled-1 b/firestore/google/cloud/firestore_v1beta1/Untitled-1 new file mode 100644 index 000000000000..1af90026971e --- /dev/null +++ b/firestore/google/cloud/firestore_v1beta1/Untitled-1 @@ -0,0 +1 @@ +google/cloud/firestore_v1beta1/order.py 140 2 74 2 98% 51, 192, 50->51, 191->192 \ No newline at end of file diff --git a/firestore/google/cloud/firestore_v1beta1/order.py b/firestore/google/cloud/firestore_v1beta1/order.py index b747e9bce7bc..0c864114d05f 100644 --- a/firestore/google/cloud/firestore_v1beta1/order.py +++ b/firestore/google/cloud/firestore_v1beta1/order.py @@ -49,7 +49,7 @@ def from_value(value): if v not in lut: raise ValueError( - "Could not detect value type for " + value) + "Could not detect value type for " + v) return lut[v] diff --git a/firestore/tests/unit/test_order.py b/firestore/tests/unit/test_order.py index b8490a8a4964..c9ff1fd9186c 100644 --- a/firestore/tests/unit/test_order.py +++ b/firestore/tests/unit/test_order.py @@ -169,10 +169,18 @@ def test_order(self): ) + def test_typeorder_type_failure(self): + target = self._make_one() + left = mock.Mock() + left.WhichOneof.return_value = "imaginary-type" + + with self.assertRaisesRegex(ValueError, "Could not detect value"): + target.compare(left, mock.Mock()) + def test_failure_to_find_type(self): target = self._make_one() left = mock.Mock() - left.WhichOneOf.return_value = "imaginary-type" + left.WhichOneof.return_value = "imaginary-type" right = mock.Mock() # Patch from value to get to the deep compare. Since left is a bad type # expect this to fail with value error. @@ -181,6 +189,13 @@ def test_failure_to_find_type(self): with self.assertRaisesRegex(ValueError, "'Unknown ``value_type``"): target.compare(left, right) + def test_compare_objects_different_keys(self): + left = _object_value({"foo": 0}) + right = _object_value({"bar": 0}) + + target = self._make_one() + target.compare(left, right) + def _boolean_value(b): return encode_value(b) diff --git a/firestore/tests/unit/test_query.py b/firestore/tests/unit/test_query.py index 6cffafd8bc8f..16c3b4df2a5f 100644 --- a/firestore/tests/unit/test_query.py +++ b/firestore/tests/unit/test_query.py @@ -909,6 +909,25 @@ def test_comparator_ordering(self): sort = query._comparator(doc1, doc2) self.assertEqual(sort, 1) + def test_comparator_ordering_descending(self): + query = self._make_one(mock.sentinel.parent) + orderByMock = mock.Mock() + orderByMock.field.field_path = 'last' + orderByMock.direction = -1 # descending + query._orders = [orderByMock] + + doc1 = mock.Mock() + doc1.reference._path = ('col', 'adocument1') + doc1._data = {'first': {'stringValue': 'Ada'}, + 'last': {'stringValue': 'secondlovelace'}} + doc2 = mock.Mock() + doc2.reference._path = ('col', 'adocument2') + doc2._data = {'first': {'stringValue': 'Ada'}, + 'last': {'stringValue': 'lovelace'}} + + sort = query._comparator(doc1, doc2) + self.assertEqual(sort, -1) + def test_comparator_missing_order_by_field_in_data_raises(self): query = self._make_one(mock.sentinel.parent) orderByMock = mock.Mock() From 2af7db73698bd57cf6c090f702de642ebf3b0726 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Thu, 11 Oct 2018 13:43:50 -0700 Subject: [PATCH 065/148] noxfile from master --- firestore/noxfile.py | 36 +++++++----------------------------- 1 file changed, 7 insertions(+), 29 deletions(-) diff --git a/firestore/noxfile.py b/firestore/noxfile.py index 6b0dd65952ab..064f8044f182 100644 --- a/firestore/noxfile.py +++ b/firestore/noxfile.py @@ -26,7 +26,6 @@ ) -@nox.session def default(session): """Default unit test session. @@ -57,34 +56,20 @@ def default(session): ) -@nox.session -@nox.parametrize('py', ['2.7', '3.5', '3.6', '3.7']) -def unit(session, py): +@nox.session(python=['2.7', '3.5', '3.6', '3.7']) +def unit(session): """Run the unit test suite.""" - # Run unit tests against all supported versions of Python. - session.interpreter = 'python{}'.format(py) - - # Set the virtualenv dirname. - session.virtualenv_dirname = 'unit-' + py - default(session) -@nox.session -@nox.parametrize('py', ['2.7', '3.6']) -def system(session, py): +@nox.session(python=['2.7', '3.6']) +def system(session): """Run the system test suite.""" # Sanity check: Only run system tests if the environment variable is set. if not os.environ.get('FIRESTORE_APPLICATION_CREDENTIALS'): session.skip('Credentials must be set via environment variable.') - # Run the system tests against latest Python 2 and Python 3 only. - session.interpreter = 'python{}'.format(py) - - # Set the virtualenv dirname. - session.virtualenv_dirname = 'sys-' + py - # Use pre-release gRPC for system tests. session.install('--pre', 'grpcio') @@ -103,40 +88,33 @@ def system(session, py): ) -@nox.session +@nox.session(python='3.6') def lint(session): """Run linters. Returns a failure if the linters find linting errors or sufficiently serious code quality issues. """ - session.interpreter = 'python3.6' session.install('flake8', *LOCAL_DEPS) session.install('.') session.run('flake8', 'google', 'tests') -@nox.session +@nox.session(python='3.6') def lint_setup_py(session): """Verify that setup.py is valid (including RST check).""" - session.interpreter = 'python3.6' - - # Set the virtualenv dirname. - session.virtualenv_dirname = 'setup' - session.install('docutils', 'Pygments') session.run( 'python', 'setup.py', 'check', '--restructuredtext', '--strict') -@nox.session +@nox.session(python='3.6') def cover(session): """Run the final coverage report. This outputs the coverage report aggregating coverage from the unit test runs (not system test runs), and then erases coverage data. """ - session.interpreter = 'python3.6' session.chdir(os.path.dirname(__file__)) session.install('coverage', 'pytest-cov') session.run('coverage', 'report', '--show-missing', '--fail-under=100') From 4c52c5535fd56eda225ac1a0f9804965de05f4a8 Mon Sep 17 00:00:00 2001 From: Christopher Wilcox Date: Thu, 11 Oct 2018 14:03:13 -0700 Subject: [PATCH 066/148] Delete Untitled-1 --- firestore/google/cloud/firestore_v1beta1/Untitled-1 | 1 - 1 file changed, 1 deletion(-) delete mode 100644 firestore/google/cloud/firestore_v1beta1/Untitled-1 diff --git a/firestore/google/cloud/firestore_v1beta1/Untitled-1 b/firestore/google/cloud/firestore_v1beta1/Untitled-1 deleted file mode 100644 index 1af90026971e..000000000000 --- a/firestore/google/cloud/firestore_v1beta1/Untitled-1 +++ /dev/null @@ -1 +0,0 @@ -google/cloud/firestore_v1beta1/order.py 140 2 74 2 98% 51, 192, 50->51, 191->192 \ No newline at end of file From 17cd042419a89c37cb967877953d3c3a62cd4a8e Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Tue, 16 Oct 2018 12:24:13 -0700 Subject: [PATCH 067/148] modify to use bidi in api-core. one test has started to fail --- .../google/cloud/firestore_v1beta1/bidi.py | 559 --------------- .../google/cloud/firestore_v1beta1/watch.py | 4 +- firestore/tests/unit/test_bidi.py | 658 ------------------ 3 files changed, 2 insertions(+), 1219 deletions(-) delete mode 100644 firestore/google/cloud/firestore_v1beta1/bidi.py delete mode 100644 firestore/tests/unit/test_bidi.py diff --git a/firestore/google/cloud/firestore_v1beta1/bidi.py b/firestore/google/cloud/firestore_v1beta1/bidi.py deleted file mode 100644 index 53cfd7464c05..000000000000 --- a/firestore/google/cloud/firestore_v1beta1/bidi.py +++ /dev/null @@ -1,559 +0,0 @@ -# Copyright 2017, Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Bi-directional streaming RPC helpers.""" - -import logging -import threading - -from six.moves import queue - -from google.api_core import exceptions - -_LOGGER = logging.getLogger(__name__) -_BIDIRECTIONAL_CONSUMER_NAME = 'Thread-ConsumeBidirectionalStream' - - -class _RequestQueueGenerator(object): - """A helper for sending requests to a gRPC stream from a Queue. - - This generator takes requests off a given queue and yields them to gRPC. - - This helper is useful when you have an indeterminate, indefinite, or - otherwise open-ended set of requests to send through a request-streaming - (or bidirectional) RPC. - - The reason this is necessary is because gRPC takes an iterator as the - request for request-streaming RPCs. gRPC consumes this iterator in another - thread to allow it to block while generating requests for the stream. - However, if the generator blocks indefinitely gRPC will not be able to - clean up the thread as it'll be blocked on `next(iterator)` and not be able - to check the channel status to stop iterating. This helper mitigates that - by waiting on the queue with a timeout and checking the RPC state before - yielding. - - Finally, it allows for retrying without swapping queues because if it does - pull an item off the queue when the RPC is inactive, it'll immediately put - it back and then exit. This is necessary because yielding the item in this - case will cause gRPC to discard it. In practice, this means that the order - of messages is not guaranteed. If such a thing is necessary it would be - easy to use a priority queue. - - Example:: - - requests = request_queue_generator(q) - call = stub.StreamingRequest(iter(requests)) - requests.call = call - - for response in call: - print(response) - q.put(...) - - Note that it is possible to accomplish this behavior without "spinning" - (using a queue timeout). One possible way would be to use more threads to - multiplex the grpc end event with the queue, another possible way is to - use selectors and a custom event/queue object. Both of these approaches - are significant from an engineering perspective for small benefit - the - CPU consumed by spinning is pretty minuscule. - - Args: - queue (queue.Queue): The request queue. - period (float): The number of seconds to wait for items from the queue - before checking if the RPC is cancelled. In practice, this - determines the maximum amount of time the request consumption - thread will live after the RPC is cancelled. - initial_request (Union[protobuf.Message, - Callable[None, protobuf.Message]]): The initial request to - yield. This is done independently of the request queue to allow fo - easily restarting streams that require some initial configuration - request. - """ - def __init__(self, queue, period=1, initial_request=None): - self._queue = queue - self._period = period - self._initial_request = initial_request - self.call = None - - def _is_active(self): - # Note: there is a possibility that this starts *before* the call - # property is set. So we have to check if self.call is set before - # seeing if it's active. - if self.call is not None and not self.call.is_active(): - return False - else: - return True - - def __iter__(self): - if self._initial_request is not None: - if callable(self._initial_request): - yield self._initial_request() - else: - yield self._initial_request - - while True: - try: - item = self._queue.get(timeout=self._period) - except queue.Empty: - if not self._is_active(): - _LOGGER.debug( - 'Empty queue and inactive call, exiting request ' - 'generator.') - return - else: - # call is still active, keep waiting for queue items. - continue - - # The consumer explicitly sent "None", indicating that the request - # should end. - if item is None: - _LOGGER.debug('Cleanly exiting request generator.') - return - - if not self._is_active(): - # We have an item, but the call is closed. We should put the - # item back on the queue so that the next call can consume it. - self._queue.put(item) - _LOGGER.debug( - 'Inactive call, replacing item on queue and exiting ' - 'request generator.') - return - - yield item - - -class BidiRpc(object): - """A helper for consuming a bi-directional streaming RPC. - - This maps gRPC's built-in interface which uses a request iterator and a - response iterator into a socket-like :func:`send` and :func:`recv`. This - is a more useful pattern for long-running or asymmetric streams (streams - where there is not a direct correlation between the requests and - responses). - - Example:: - - initial_request = example_pb2.StreamingRpcRequest( - setting='example') - rpc = BidiRpc(stub.StreamingRpc, initial_request=initial_request) - - rpc.open() - - while rpc.is_active(): - print(rpc.recv()) - rpc.send(example_pb2.StreamingRpcRequest( - data='example')) - - This does *not* retry the stream on errors. See :class:`ResumableBidiRpc`. - - Args: - start_rpc (grpc.StreamStreamMultiCallable): The gRPC method used to - start the RPC. - initial_request (Union[protobuf.Message, - Callable[None, protobuf.Message]]): The initial request to - yield. This is useful if an initial request is needed to start the - stream. - """ - def __init__(self, start_rpc, initial_request=None): - self._start_rpc = start_rpc - self._initial_request = initial_request - self._request_queue = queue.Queue() - self._request_generator = None - self._is_active = False - self._callbacks = [] - self.call = None - - def add_done_callback(self, callback): - """Adds a callback that will be called when the RPC terminates. - - This occurs when the RPC errors or is successfully terminated. - - Args: - callback (Callable[[grpc.Future], None]): The callback to execute. - It will be provided with the same gRPC future as the underlying - stream which will also be a :class:`grpc.Call`. - """ - self._callbacks.append(callback) - - def _on_call_done(self, future): - for callback in self._callbacks: - callback(future) - - def open(self): - """Opens the stream.""" - if self.is_active: - raise ValueError('Can not open an already open stream.') - - request_generator = _RequestQueueGenerator( - self._request_queue, initial_request=self._initial_request) - call = self._start_rpc(iter(request_generator)) - - request_generator.call = call - - # TODO: api_core should expose the future interface for wrapped - # callables as well. - if hasattr(call, '_wrapped'): # pragma: NO COVER - call._wrapped.add_done_callback(self._on_call_done) - else: - call.add_done_callback(self._on_call_done) - - self._request_generator = request_generator - self.call = call - - def close(self): - """Closes the stream.""" - if self.call is None: - return - - self._request_queue.put(None) - self.call.cancel() - self._request_generator = None - # Don't set self.call to None. Keep it around so that send/recv can - # raise the error. - - def send(self, request): - """Queue a message to be sent on the stream. - - Send is non-blocking. - - If the underlying RPC has been closed, this will raise. - - Args: - request (protobuf.Message): The request to send. - """ - if self.call is None: - raise ValueError( - 'Can not send() on an RPC that has never been open()ed.') - - # Don't use self.is_active(), as ResumableBidiRpc will overload it - # to mean something semantically different. - if self.call.is_active(): - self._request_queue.put(request) - else: - # calling next should cause the call to raise. - next(self.call) - - def recv(self): - """Wait for a message to be returned from the stream. - - Recv is blocking. - - If the underlying RPC has been closed, this will raise. - - Returns: - protobuf.Message: The received message. - """ - if self.call is None: - raise ValueError( - 'Can not recv() on an RPC that has never been open()ed.') - - return next(self.call) - - @property - def is_active(self): - """bool: True if this stream is currently open and active.""" - return self.call is not None and self.call.is_active() - - @property - def pending_requests(self): - """int: Returns an estimate of the number of queued requests.""" - return self._request_queue.qsize() - - -class ResumableBidiRpc(BidiRpc): - """A :class:`BidiRpc` that can automatically resume the stream on errors. - - It uses the ``should_recover`` arg to determine if it should re-establish - the stream on error. - - Example:: - - def should_recover(exc): - return ( - isinstance(exc, grpc.RpcError) and - exc.code() == grpc.StatusCode.UNAVAILABLE) - - initial_request = example_pb2.StreamingRpcRequest( - setting='example') - - rpc = ResumeableBidiRpc( - stub.StreamingRpc, - initial_request=initial_request, - should_recover=should_recover) - - rpc.open() - - while rpc.is_active(): - print(rpc.recv()) - rpc.send(example_pb2.StreamingRpcRequest( - data='example')) - - Args: - start_rpc (grpc.StreamStreamMultiCallable): The gRPC method used to - start the RPC. - initial_request (Union[protobuf.Message, - Callable[None, protobuf.Message]]): The initial request to - yield. This is useful if an initial request is needed to start the - stream. - should_recover (Callable[[Exception], bool]): A function that returns - True if the stream should be recovered. This will be called - whenever an error is encountered on the stream. - """ - def __init__(self, start_rpc, should_recover, initial_request=None): - super(ResumableBidiRpc, self).__init__(start_rpc, initial_request) - self._should_recover = should_recover - self._operational_lock = threading.RLock() - self._finalized = False - self._finalize_lock = threading.Lock() - - def _finalize(self, result): - with self._finalize_lock: - if self._finalized: - return - - for callback in self._callbacks: - callback(result) - - self._finalized = True - - def _on_call_done(self, future): - # Unlike the base class, we only execute the callbacks on a terminal - # error, not for errors that we can recover from. Note that grpc's - # "future" here is also a grpc.RpcError. - if not self._should_recover(future): - self._finalize(future) - else: - _LOGGER.debug('Re-opening stream from gRPC callback.') - self._reopen() - - def _reopen(self): - with self._operational_lock: - # Another thread already managed to re-open this stream. - if self.call is not None and self.call.is_active(): - _LOGGER.debug('Stream was already re-established.') - return - - self.call = None - # Request generator should exit cleanly since the RPC its bound to - # has exited. - self.request_generator = None - - # Note: we do not currently do any sort of backoff here. The - # assumption is that re-establishing the stream under normal - # circumstances will happen in intervals greater than 60s. - # However, it is possible in a degenerative case that the server - # closes the stream rapidly which would lead to thrashing here, - # but hopefully in those cases the server would return a non- - # retryable error. - - try: - self.open() - # If re-opening or re-calling the method fails for any reason, - # consider it a terminal error and finalize the stream. - except Exception as exc: - self._finalize(exc) - raise - - _LOGGER.info('Re-established stream') - - def _recoverable(self, method, *args, **kwargs): - """Wraps a method to recover the stream and retry on error. - - If a retryable error occurs while making the call, then the stream will - be re-opened and the method will be retried. This happens indefinitely - so long as the error is a retryable one. If an error occurs while - re-opening the stream, then this method will raise immediately and - trigger finalization of this object. - - Args: - method (Callable[..., Any]): The method to call. - args: The args to pass to the method. - kwargs: The kwargs to pass to the method. - """ - while True: - try: - return method(*args, **kwargs) - - except Exception as exc: - _LOGGER.debug('Call to retryable %r caused %s.', method, exc) - if not self._should_recover(exc): - self.close() - _LOGGER.debug('Not retrying %r due to %s.', method, exc) - self._finalize(exc) - raise exc - - _LOGGER.debug('Re-opening stream from retryable %r.', method) - self._reopen() - - def send(self, request): - return self._recoverable( - super(ResumableBidiRpc, self).send, request) - - def recv(self): - return self._recoverable( - super(ResumableBidiRpc, self).recv) - - @property - def is_active(self): - """bool: True if this stream is currently open and active.""" - # Use the operational lock. It's entirely possible for something - # to check the active state *while* the RPC is being retried. - # Also, use finalized to track the actual terminal state here. - # This is because if the stream is re-established by the gRPC thread - # it's technically possible to check this between when gRPC marks the - # RPC as inactive and when gRPC executes our callback that re-opens - # the stream. - with self._operational_lock: - return self.call is not None and not self._finalized - - -class BackgroundConsumer(object): - """A bi-directional stream consumer that runs in a separate thread. - - This maps the consumption of a stream into a callback-based model. It also - provides :func:`pause` and :func:`resume` to allow for flow-control. - - Example:: - - def should_recover(exc): - return ( - isinstance(exc, grpc.RpcError) and - exc.code() == grpc.StatusCode.UNAVAILABLE) - - initial_request = example_pb2.StreamingRpcRequest( - setting='example') - - rpc = ResumeableBidiRpc( - stub.StreamingRpc, - initial_request=initial_request, - should_recover=should_recover) - - def on_response(response): - print(response) - - consumer = BackgroundConsumer(rpc, on_response) - consume.start() - - Note that error handling *must* be done by using the provided - ``bidi_rpc``'s ``add_done_callback``. This helper will automatically exit - whenever the RPC itself exits and will not provide any error details. - - Args: - bidi_rpc (BidiRpc): The RPC to consume. Should not have been - ``open()``ed yet. - on_response (Callable[[protobuf.Message], None]): The callback to - be called for every response on the stream. - """ - def __init__(self, bidi_rpc, on_response): - self._bidi_rpc = bidi_rpc - self._on_response = on_response - self._paused = False - self._wake = threading.Condition() - self._thread = None - self._operational_lock = threading.Lock() - - def _on_call_done(self, future): - # Resume the thread if it's paused, this prevents blocking forever - # when the RPC has terminated. - self.resume() - - def _thread_main(self): - try: - self._bidi_rpc.add_done_callback(self._on_call_done) - self._bidi_rpc.open() - - while self._bidi_rpc.is_active: - # Do not allow the paused status to change at all during this - # section. There is a condition where we could be resumed - # between checking if we are paused and calling wake.wait(), - # which means that we will miss the notification to wake up - # (oops!) and wait for a notification that will never come. - # Keeping the lock throughout avoids that. - # In the future, we could use `Condition.wait_for` if we drop - # Python 2.7. - with self._wake: - if self._paused: - _LOGGER.debug('paused, waiting for waking.') - self._wake.wait() - _LOGGER.debug('woken.') - - _LOGGER.debug('waiting for recv.') - response = self._bidi_rpc.recv() - _LOGGER.debug('recved response.') - self._on_response(response) - - except exceptions.GoogleAPICallError as exc: - _LOGGER.debug( - '%s caught error %s and will exit. Generally this is due to ' - 'the RPC itself being cancelled and the error will be ' - 'surfaced to the calling code.', - _BIDIRECTIONAL_CONSUMER_NAME, exc, exc_info=True) - - except Exception as exc: - _LOGGER.exception( - '%s caught unexpected exception %s and will exit.', - _BIDIRECTIONAL_CONSUMER_NAME, exc) - - else: - _LOGGER.error( - 'The bidirectional RPC unexpectedly exited. This is a truly ' - 'exceptional case. Please file a bug with your logs.') - - _LOGGER.info('%s exiting', _BIDIRECTIONAL_CONSUMER_NAME) - - def start(self): - """Start the background thread and begin consuming the thread.""" - with self._operational_lock: - thread = threading.Thread( - name=_BIDIRECTIONAL_CONSUMER_NAME, - target=self._thread_main) - thread.daemon = True - thread.start() - self._thread = thread - _LOGGER.debug('Started helper thread %s', thread.name) - - def stop(self): - """Stop consuming the stream and shutdown the background thread.""" - with self._operational_lock: - self._bidi_rpc.close() - - if self._thread is not None: - # Resume the thread to wake it up in case it is sleeping. - self.resume() - self._thread.join() - - self._thread = None - - @property - def is_active(self): - """bool: True if the background thread is active.""" - return self._thread is not None and self._thread.is_alive() - - def pause(self): - """Pauses the response stream. - - This does *not* pause the request stream. - """ - with self._wake: - self._paused = True - - def resume(self): - """Resumes the response stream.""" - with self._wake: - self._paused = False - self._wake.notifyAll() - - @property - def is_paused(self): - """bool: True if the response stream is paused.""" - return self._paused diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 10c71cc24148..b706dfc4c298 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -21,8 +21,8 @@ import pytz -from google.cloud.firestore_v1beta1.bidi import ResumableBidiRpc -from google.cloud.firestore_v1beta1.bidi import BackgroundConsumer +from google.api_core.bidi import ResumableBidiRpc +from google.api_core.bidi import BackgroundConsumer from google.cloud.firestore_v1beta1.proto import firestore_pb2 from google.cloud.firestore_v1beta1 import _helpers diff --git a/firestore/tests/unit/test_bidi.py b/firestore/tests/unit/test_bidi.py deleted file mode 100644 index 80d8ecf48389..000000000000 --- a/firestore/tests/unit/test_bidi.py +++ /dev/null @@ -1,658 +0,0 @@ -# Copyright 2018, Google LLC All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import threading - -import grpc -import mock -import pytest -from six.moves import queue - -from google.api_core import exceptions -from google.cloud.firestore_v1beta1 import bidi - - -class Test_RequestQueueGenerator(object): - - def test_bounded_consume(self): - call = mock.create_autospec(grpc.Call, instance=True) - call.is_active.return_value = True - - def queue_generator(rpc): - yield mock.sentinel.A - yield queue.Empty() - yield mock.sentinel.B - rpc.is_active.return_value = False - yield mock.sentinel.C - - q = mock.create_autospec(queue.Queue, instance=True) - q.get.side_effect = queue_generator(call) - - generator = bidi._RequestQueueGenerator(q) - generator.call = call - - items = list(generator) - - assert items == [mock.sentinel.A, mock.sentinel.B] - - def test_yield_initial_and_exit(self): - q = mock.create_autospec(queue.Queue, instance=True) - q.get.side_effect = queue.Empty() - call = mock.create_autospec(grpc.Call, instance=True) - call.is_active.return_value = False - - generator = bidi._RequestQueueGenerator( - q, initial_request=mock.sentinel.A) - generator.call = call - - items = list(generator) - - assert items == [mock.sentinel.A] - - def test_yield_initial_callable_and_exit(self): - q = mock.create_autospec(queue.Queue, instance=True) - q.get.side_effect = queue.Empty() - call = mock.create_autospec(grpc.Call, instance=True) - call.is_active.return_value = False - - generator = bidi._RequestQueueGenerator( - q, initial_request=lambda: mock.sentinel.A) - generator.call = call - - items = list(generator) - - assert items == [mock.sentinel.A] - - def test_exit_when_inactive_with_item(self): - q = mock.create_autospec(queue.Queue, instance=True) - q.get.side_effect = [mock.sentinel.A, queue.Empty()] - call = mock.create_autospec(grpc.Call, instance=True) - call.is_active.return_value = False - - generator = bidi._RequestQueueGenerator(q) - generator.call = call - - items = list(generator) - - assert items == [] - # Make sure it put the item back. - q.put.assert_called_once_with(mock.sentinel.A) - - def test_exit_when_inactive_empty(self): - q = mock.create_autospec(queue.Queue, instance=True) - q.get.side_effect = queue.Empty() - call = mock.create_autospec(grpc.Call, instance=True) - call.is_active.return_value = False - - generator = bidi._RequestQueueGenerator(q) - generator.call = call - - items = list(generator) - - assert items == [] - - def test_exit_with_stop(self): - q = mock.create_autospec(queue.Queue, instance=True) - q.get.side_effect = [None, queue.Empty()] - call = mock.create_autospec(grpc.Call, instance=True) - call.is_active.return_value = True - - generator = bidi._RequestQueueGenerator(q) - generator.call = call - - items = list(generator) - - assert items == [] - - -class _CallAndFuture(grpc.Call, grpc.Future): - pass - - -def make_rpc(): - """Makes a mock RPC used to test Bidi classes.""" - call = mock.create_autospec(_CallAndFuture, instance=True) - rpc = mock.create_autospec(grpc.StreamStreamMultiCallable, instance=True) - - def rpc_side_effect(request): - call.is_active.return_value = True - call.request = request - return call - - rpc.side_effect = rpc_side_effect - - def cancel_side_effect(): - call.is_active.return_value = False - - call.cancel.side_effect = cancel_side_effect - - return rpc, call - - -class ClosedCall(object): - # NOTE: This is needed because defining `.next` on an **instance** - # rather than the **class** will not be iterable in Python 2. - # This is problematic since a `Mock` just sets members. - - def __init__(self, exception): - self.exception = exception - - def __next__(self): - raise self.exception - - next = __next__ # Python 2 - - def is_active(self): - return False - - -class TestBidiRpc(object): - def test_initial_state(self): - bidi_rpc = bidi.BidiRpc(None) - - assert bidi_rpc.is_active is False - - def test_done_callbacks(self): - bidi_rpc = bidi.BidiRpc(None) - callback = mock.Mock(spec=['__call__']) - - bidi_rpc.add_done_callback(callback) - bidi_rpc._on_call_done(mock.sentinel.future) - - callback.assert_called_once_with(mock.sentinel.future) - - def test_open(self): - rpc, call = make_rpc() - bidi_rpc = bidi.BidiRpc(rpc) - - bidi_rpc.open() - - assert bidi_rpc.call == call - assert bidi_rpc.is_active - call.add_done_callback.assert_called_once_with(bidi_rpc._on_call_done) - - def test_open_error_already_open(self): - rpc, _ = make_rpc() - bidi_rpc = bidi.BidiRpc(rpc) - - bidi_rpc.open() - - with pytest.raises(ValueError): - bidi_rpc.open() - - def test_close(self): - rpc, call = make_rpc() - bidi_rpc = bidi.BidiRpc(rpc) - bidi_rpc.open() - - bidi_rpc.close() - - call.cancel.assert_called_once() - assert bidi_rpc.call == call - assert bidi_rpc.is_active is False - # ensure the request queue was signaled to stop. - assert bidi_rpc.pending_requests == 1 - assert bidi_rpc._request_queue.get() is None - - def test_close_no_rpc(self): - bidi_rpc = bidi.BidiRpc(None) - bidi_rpc.close() - - def test_send(self): - rpc, call = make_rpc() - bidi_rpc = bidi.BidiRpc(rpc) - bidi_rpc.open() - - bidi_rpc.send(mock.sentinel.request) - - assert bidi_rpc.pending_requests == 1 - assert bidi_rpc._request_queue.get() is mock.sentinel.request - - def test_send_not_open(self): - rpc, call = make_rpc() - bidi_rpc = bidi.BidiRpc(rpc) - - with pytest.raises(ValueError): - bidi_rpc.send(mock.sentinel.request) - - def test_send_dead_rpc(self): - error = ValueError() - bidi_rpc = bidi.BidiRpc(None) - bidi_rpc.call = ClosedCall(error) - - with pytest.raises(ValueError) as exc_info: - bidi_rpc.send(mock.sentinel.request) - - assert exc_info.value == error - - def test_recv(self): - bidi_rpc = bidi.BidiRpc(None) - bidi_rpc.call = iter([mock.sentinel.response]) - - response = bidi_rpc.recv() - - assert response == mock.sentinel.response - - def test_recv_not_open(self): - rpc, call = make_rpc() - bidi_rpc = bidi.BidiRpc(rpc) - - with pytest.raises(ValueError): - bidi_rpc.recv() - - -class CallStub(object): - def __init__(self, values, active=True): - self.values = iter(values) - self._is_active = active - self.cancelled = False - - def __next__(self): - item = next(self.values) - if isinstance(item, Exception): - self._is_active = False - raise item - return item - - next = __next__ # Python 2 - - def is_active(self): - return self._is_active - - def add_done_callback(self, callback): - pass - - def cancel(self): - self.cancelled = True - - -class TestResumableBidiRpc(object): - def test_initial_state(self): # pragma: NO COVER - bidi_rpc = bidi.ResumableBidiRpc(None, lambda _: True) - - assert bidi_rpc.is_active is False - - def test_done_callbacks_recoverable(self): - start_rpc = mock.create_autospec( - grpc.StreamStreamMultiCallable, instance=True) - bidi_rpc = bidi.ResumableBidiRpc(start_rpc, lambda _: True) - callback = mock.Mock(spec=['__call__']) - - bidi_rpc.add_done_callback(callback) - bidi_rpc._on_call_done(mock.sentinel.future) - - callback.assert_not_called() - start_rpc.assert_called_once() - assert bidi_rpc.is_active - - def test_done_callbacks_non_recoverable(self): - bidi_rpc = bidi.ResumableBidiRpc(None, lambda _: False) - callback = mock.Mock(spec=['__call__']) - - bidi_rpc.add_done_callback(callback) - bidi_rpc._on_call_done(mock.sentinel.future) - - callback.assert_called_once_with(mock.sentinel.future) - - def test_send_recover(self): - error = ValueError() - call_1 = CallStub([error], active=False) - call_2 = CallStub([]) - start_rpc = mock.create_autospec( - grpc.StreamStreamMultiCallable, - instance=True, - side_effect=[call_1, call_2]) - should_recover = mock.Mock(spec=['__call__'], return_value=True) - bidi_rpc = bidi.ResumableBidiRpc(start_rpc, should_recover) - - bidi_rpc.open() - - bidi_rpc.send(mock.sentinel.request) - - assert bidi_rpc.pending_requests == 1 - assert bidi_rpc._request_queue.get() is mock.sentinel.request - - should_recover.assert_called_once_with(error) - assert bidi_rpc.call == call_2 - assert bidi_rpc.is_active is True - - def test_send_failure(self): - error = ValueError() - call = CallStub([error], active=False) - start_rpc = mock.create_autospec( - grpc.StreamStreamMultiCallable, - instance=True, - return_value=call) - should_recover = mock.Mock(spec=['__call__'], return_value=False) - bidi_rpc = bidi.ResumableBidiRpc(start_rpc, should_recover) - - bidi_rpc.open() - - with pytest.raises(ValueError) as exc_info: - bidi_rpc.send(mock.sentinel.request) - - assert exc_info.value == error - should_recover.assert_called_once_with(error) - assert bidi_rpc.call == call - assert bidi_rpc.is_active is False - assert call.cancelled is True - assert bidi_rpc.pending_requests == 1 - assert bidi_rpc._request_queue.get() is None - - def test_recv_recover(self): - error = ValueError() - call_1 = CallStub([1, error]) - call_2 = CallStub([2, 3]) - start_rpc = mock.create_autospec( - grpc.StreamStreamMultiCallable, - instance=True, - side_effect=[call_1, call_2]) - should_recover = mock.Mock(spec=['__call__'], return_value=True) - bidi_rpc = bidi.ResumableBidiRpc(start_rpc, should_recover) - - bidi_rpc.open() - - values = [] - for n in range(3): - values.append(bidi_rpc.recv()) - - assert values == [1, 2, 3] - should_recover.assert_called_once_with(error) - assert bidi_rpc.call == call_2 - assert bidi_rpc.is_active is True - - def test_recv_recover_race_condition(self): - # This test checks the race condition where two threads recv() and - # encounter an error and must re-open the stream. Only one thread - # should succeed in doing so. - error = ValueError() - call_1 = CallStub([error, error]) - call_2 = CallStub([1, 2]) - start_rpc = mock.create_autospec( - grpc.StreamStreamMultiCallable, - instance=True, - side_effect=[call_1, call_2]) - recovered_event = threading.Event() - - def second_thread_main(): - assert bidi_rpc.recv() == 2 - - second_thread = threading.Thread(target=second_thread_main) - - def should_recover(exception): - assert exception == error - if threading.current_thread() == second_thread: - recovered_event.wait() - return True - - bidi_rpc = bidi.ResumableBidiRpc(start_rpc, should_recover) - - bidi_rpc.open() - second_thread.start() - - assert bidi_rpc.recv() == 1 - recovered_event.set() - - assert bidi_rpc.call == call_2 - assert bidi_rpc.is_active is True - second_thread.join() - - def test_recv_failure(self): - error = ValueError() - call = CallStub([error]) - start_rpc = mock.create_autospec( - grpc.StreamStreamMultiCallable, - instance=True, - return_value=call) - should_recover = mock.Mock(spec=['__call__'], return_value=False) - bidi_rpc = bidi.ResumableBidiRpc(start_rpc, should_recover) - - bidi_rpc.open() - - with pytest.raises(ValueError) as exc_info: - bidi_rpc.recv() - - assert exc_info.value == error - should_recover.assert_called_once_with(error) - assert bidi_rpc.call == call - assert bidi_rpc.is_active is False - assert call.cancelled is True - - def test_reopen_failure_on_rpc_restart(self): - error1 = ValueError('1') - error2 = ValueError('2') - call = CallStub([error1]) - # Invoking start RPC a second time will trigger an error. - start_rpc = mock.create_autospec( - grpc.StreamStreamMultiCallable, - instance=True, - side_effect=[call, error2]) - should_recover = mock.Mock(spec=['__call__'], return_value=True) - callback = mock.Mock(spec=['__call__']) - - bidi_rpc = bidi.ResumableBidiRpc(start_rpc, should_recover) - bidi_rpc.add_done_callback(callback) - - bidi_rpc.open() - - with pytest.raises(ValueError) as exc_info: - bidi_rpc.recv() - - assert exc_info.value == error2 - should_recover.assert_called_once_with(error1) - assert bidi_rpc.call is None - assert bidi_rpc.is_active is False - callback.assert_called_once_with(error2) - - def test_finalize_idempotent(self): - error1 = ValueError('1') - error2 = ValueError('2') - callback = mock.Mock(spec=['__call__']) - should_recover = mock.Mock(spec=['__call__'], return_value=False) - - bidi_rpc = bidi.ResumableBidiRpc( - mock.sentinel.start_rpc, should_recover) - - bidi_rpc.add_done_callback(callback) - - bidi_rpc._on_call_done(error1) - bidi_rpc._on_call_done(error2) - - callback.assert_called_once_with(error1) - - -class TestBackgroundConsumer(object): - def test_consume_once_then_exit(self): - bidi_rpc = mock.create_autospec(bidi.BidiRpc, instance=True) - bidi_rpc.is_active = True - bidi_rpc.recv.side_effect = [mock.sentinel.response_1] - recved = threading.Event() - - def on_response(response): - assert response == mock.sentinel.response_1 - bidi_rpc.is_active = False - recved.set() - - consumer = bidi.BackgroundConsumer(bidi_rpc, on_response) - - consumer.start() - - recved.wait() - - bidi_rpc.recv.assert_called_once() - assert bidi_rpc.is_active is False - - consumer.stop() - - bidi_rpc.close.assert_called_once() - assert consumer.is_active is False - - def test_pause_resume_and_close(self): - # This test is relatively complex. It attempts to start the consumer, - # consume one item, pause the consumer, check the state of the world, - # then resume the consumer. Doing this in a deterministic fashion - # requires a bit more mocking and patching than usual. - - bidi_rpc = mock.create_autospec(bidi.BidiRpc, instance=True) - bidi_rpc.is_active = True - - def close_side_effect(): - bidi_rpc.is_active = False - - bidi_rpc.close.side_effect = close_side_effect - - # These are used to coordinate the two threads to ensure deterministic - # execution. - should_continue = threading.Event() - responses_and_events = { - mock.sentinel.response_1: threading.Event(), - mock.sentinel.response_2: threading.Event() - } - bidi_rpc.recv.side_effect = [ - mock.sentinel.response_1, mock.sentinel.response_2] - - recved_responses = [] - consumer = None - - def on_response(response): - if response == mock.sentinel.response_1: - consumer.pause() - - recved_responses.append(response) - responses_and_events[response].set() - should_continue.wait() - - consumer = bidi.BackgroundConsumer(bidi_rpc, on_response) - - consumer.start() - - # Wait for the first response to be recved. - responses_and_events[mock.sentinel.response_1].wait() - - # Ensure only one item has been recved and that the consumer is paused. - assert recved_responses == [mock.sentinel.response_1] - assert consumer.is_paused is True - assert consumer.is_active is True - - # Unpause the consumer, wait for the second item, then close the - # consumer. - should_continue.set() - consumer.resume() - - responses_and_events[mock.sentinel.response_2].wait() - - assert recved_responses == [ - mock.sentinel.response_1, mock.sentinel.response_2] - - consumer.stop() - - assert consumer.is_active is False - - def test_wake_on_error(self): - should_continue = threading.Event() - - bidi_rpc = mock.create_autospec(bidi.BidiRpc, instance=True) - bidi_rpc.is_active = True - bidi_rpc.add_done_callback.side_effect = ( - lambda _: should_continue.set()) - - consumer = bidi.BackgroundConsumer(bidi_rpc, mock.sentinel.on_response) - - # Start the consumer paused, which should immediately put it into wait - # state. - consumer.pause() - consumer.start() - - # Wait for add_done_callback to be called - should_continue.wait() - bidi_rpc.add_done_callback.assert_called_once_with( - consumer._on_call_done) - - # The consumer should now be blocked on waiting to be unpaused. - assert consumer.is_active - assert consumer.is_paused - - # Trigger the done callback, it should unpause the consumer and cause - # it to exit. - bidi_rpc.is_active = False - consumer._on_call_done(bidi_rpc) - - # It may take a few cycles for the thread to exit. - while consumer.is_active: - pass - - def test_consumer_expected_error(self, caplog): - caplog.set_level(logging.DEBUG) - - bidi_rpc = mock.create_autospec(bidi.BidiRpc, instance=True) - bidi_rpc.is_active = True - bidi_rpc.recv.side_effect = exceptions.ServiceUnavailable('Gone away') - - on_response = mock.Mock(spec=['__call__']) - - consumer = bidi.BackgroundConsumer(bidi_rpc, on_response) - - consumer.start() - - # Wait for the consumer's thread to exit. - while consumer.is_active: - pass - - on_response.assert_not_called() - bidi_rpc.recv.assert_called_once() - assert 'caught error' in caplog.text - - def test_consumer_unexpected_error(self, caplog): - caplog.set_level(logging.DEBUG) - - bidi_rpc = mock.create_autospec(bidi.BidiRpc, instance=True) - bidi_rpc.is_active = True - bidi_rpc.recv.side_effect = ValueError() - - on_response = mock.Mock(spec=['__call__']) - - consumer = bidi.BackgroundConsumer(bidi_rpc, on_response) - - consumer.start() - - # Wait for the consumer's thread to exit. - while consumer.is_active: - pass - - on_response.assert_not_called() - bidi_rpc.recv.assert_called_once() - assert 'caught unexpected exception' in caplog.text - - def test_double_stop(self, caplog): - caplog.set_level(logging.DEBUG) - bidi_rpc = mock.create_autospec(bidi.BidiRpc, instance=True) - bidi_rpc.is_active = True - on_response = mock.Mock(spec=['__call__']) - - def close_side_effect(): - bidi_rpc.is_active = False - - bidi_rpc.close.side_effect = close_side_effect - - consumer = bidi.BackgroundConsumer(bidi_rpc, on_response) - - consumer.start() - assert consumer.is_active is True - - consumer.stop() - assert consumer.is_active is False - - # calling stop twice should not result in an error. - consumer.stop() From f1d079d1d72ad2f2f0f706a7d2b20a4a9be54084 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Thu, 18 Oct 2018 03:10:38 -0400 Subject: [PATCH 068/148] get tests passing (intermittently) on 2.7,3.5,3.6 --- .../cloud/firestore_v1beta1/_helpers.py | 8 +++- .../google/cloud/firestore_v1beta1/order.py | 1 + .../google/cloud/firestore_v1beta1/watch.py | 2 +- firestore/tests/unit/test_order.py | 37 +++++++++++-------- firestore/tests/unit/test_query.py | 4 +- 5 files changed, 32 insertions(+), 20 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/_helpers.py b/firestore/google/cloud/firestore_v1beta1/_helpers.py index 1474e33b851e..720e0111abd6 100644 --- a/firestore/google/cloud/firestore_v1beta1/_helpers.py +++ b/firestore/google/cloud/firestore_v1beta1/_helpers.py @@ -15,7 +15,11 @@ """Common helpers shared across Google Cloud Firestore modules.""" -import collections +try: + from collections import abc +except ImportError: # python 2.7 + import collections as abc + import datetime import re @@ -745,7 +749,7 @@ def get_nested_value(field_path, data): nested_data = data for index, field_name in enumerate(field_names): - if isinstance(nested_data, collections.abc.Mapping): + if isinstance(nested_data, abc.Mapping): if field_name in nested_data: nested_data = nested_data[field_name] else: diff --git a/firestore/google/cloud/firestore_v1beta1/order.py b/firestore/google/cloud/firestore_v1beta1/order.py index 0c864114d05f..81a3ecc9b0ad 100644 --- a/firestore/google/cloud/firestore_v1beta1/order.py +++ b/firestore/google/cloud/firestore_v1beta1/order.py @@ -30,6 +30,7 @@ class TypeOrder(Enum): ARRAY = 8 OBJECT = 9 + @staticmethod def from_value(value): v = value.WhichOneof('value_type') diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index b706dfc4c298..8e63808409f7 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -22,7 +22,7 @@ import pytz from google.api_core.bidi import ResumableBidiRpc -from google.api_core.bidi import BackgroundConsumer +from google.api_core.bidi import BackgroundConsumer from google.cloud.firestore_v1beta1.proto import firestore_pb2 from google.cloud.firestore_v1beta1 import _helpers diff --git a/firestore/tests/unit/test_order.py b/firestore/tests/unit/test_order.py index c9ff1fd9186c..af9488c7bcf0 100644 --- a/firestore/tests/unit/test_order.py +++ b/firestore/tests/unit/test_order.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # Copyright 2017 Google LLC All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,10 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import unittest -import math +import collections import mock - +import six +import sys +import unittest from google.cloud.firestore_v1beta1._helpers import encode_value, GeoPoint from google.cloud.firestore_v1beta1.order import Order @@ -43,6 +45,7 @@ def test_order(self): int_min_value = -(2 ** 31) float_min_value = 1.175494351 ** -38 float_nan = float('nan') + inf = float('inf') groups = [None] * 65 @@ -53,7 +56,7 @@ def test_order(self): # numbers groups[3] = [_double_value(float_nan), _double_value(float_nan)] - groups[4] = [_double_value(-math.inf)] + groups[4] = [_double_value(-inf)] groups[5] = [_int_value(int_min_value - 1)] groups[6] = [_int_value(int_min_value)] groups[7] = [_double_value(-1.1)] @@ -68,7 +71,7 @@ def test_order(self): groups[13] = [_double_value(1.1)] groups[14] = [_int_value(int_max_value)] groups[15] = [_int_value(int_max_value + 1)] - groups[16] = [_double_value(math.inf)] + groups[16] = [_double_value(inf)] groups[17] = [_timestamp_value(123, 0)] groups[18] = [_timestamp_value(123, 123)] @@ -77,21 +80,21 @@ def test_order(self): # strings groups[20] = [_string_value("")] groups[21] = [_string_value("\u0000\ud7ff\ue000\uffff")] - groups[22] = [_string_value(u"(╯°□°）╯︵ ┻━┻")] + groups[22] = [_string_value("(╯°□°）╯︵ ┻━┻")] groups[23] = [_string_value("a")] groups[24] = [_string_value("abc def")] # latin small letter e + combining acute accent + latin small letter b groups[25] = [_string_value("e\u0301b")] - groups[26] = [_string_value(u"æ")] + groups[26] = [_string_value("æ")] # latin small letter e with acute accent + latin small letter a - groups[27] = [_string_value(u"\u00e9a")] + groups[27] = [_string_value("\u00e9a")] # blobs - groups[28] = [_blob_value(bytes())] - groups[29] = [_blob_value(bytes([0]))] - groups[30] = [_blob_value(bytes([0, 1, 2, 3, 4]))] - groups[31] = [_blob_value(bytes([0, 1, 2, 4, 3]))] - groups[32] = [_blob_value(bytes([127]))] + groups[28] = [_blob_value(b'')] + groups[29] = [_blob_value(b'\x00')] + groups[30] = [_blob_value(b'\x00\x01\x02\x03\x04')] + groups[31] = [_blob_value(b'\x00\x01\x02\x04\x03')] + groups[32] = [_blob_value(b'\x7f')] # resource names groups[33] = [ @@ -174,7 +177,7 @@ def test_typeorder_type_failure(self): left = mock.Mock() left.WhichOneof.return_value = "imaginary-type" - with self.assertRaisesRegex(ValueError, "Could not detect value"): + with self.assertRaisesRegexp(ValueError, "Could not detect value"): target.compare(left, mock.Mock()) def test_failure_to_find_type(self): @@ -186,7 +189,9 @@ def test_failure_to_find_type(self): # expect this to fail with value error. with mock.patch.object(TypeOrder, 'from_value',) as to: to.value = None - with self.assertRaisesRegex(ValueError, "'Unknown ``value_type``"): + with self.assertRaisesRegexp( + ValueError, "'Unknown ``value_type``" + ): target.compare(left, right) def test_compare_objects_different_keys(self): @@ -210,6 +215,8 @@ def _int_value(l): def _string_value(s): + if not isinstance(s, six.text_type): + s = six.u(s) return encode_value(s) diff --git a/firestore/tests/unit/test_query.py b/firestore/tests/unit/test_query.py index 16c3b4df2a5f..98d3f7b4fdd8 100644 --- a/firestore/tests/unit/test_query.py +++ b/firestore/tests/unit/test_query.py @@ -943,8 +943,8 @@ def test_comparator_missing_order_by_field_in_data_raises(self): doc2._data = {'first': {'stringValue': 'Ada'}, 'last': {'stringValue': 'lovelace'}} - with self.assertRaisesRegex(ValueError, - "Can only compare fields "): + with self.assertRaisesRegexp(ValueError, + "Can only compare fields "): query._comparator(doc1, doc2) From e0cbda461250a7df52388fe4e0bfd420c5c47d0e Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Fri, 19 Oct 2018 00:23:51 -0400 Subject: [PATCH 069/148] fix failing test_order tests when left and right were dictionaries of differing sizes (groups\[61\] vs groups\[62\]) --- firestore/google/cloud/firestore_v1beta1/order.py | 4 +++- firestore/tests/unit/test_order.py | 2 -- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/order.py b/firestore/google/cloud/firestore_v1beta1/order.py index 81a3ecc9b0ad..de224f5ce039 100644 --- a/firestore/google/cloud/firestore_v1beta1/order.py +++ b/firestore/google/cloud/firestore_v1beta1/order.py @@ -187,7 +187,9 @@ def compare_objects(left, right): left_fields = left.map_value.fields right_fields = right.map_value.fields - for left_key, right_key in zip(left_fields, right_fields): + for left_key, right_key in zip( + sorted(left_fields), sorted(right_fields) + ): keyCompare = Order._compare_to(left_key, right_key) if keyCompare != 0: return keyCompare diff --git a/firestore/tests/unit/test_order.py b/firestore/tests/unit/test_order.py index af9488c7bcf0..9f1017b8807d 100644 --- a/firestore/tests/unit/test_order.py +++ b/firestore/tests/unit/test_order.py @@ -13,10 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import collections import mock import six -import sys import unittest from google.cloud.firestore_v1beta1._helpers import encode_value, GeoPoint From fb8142c8195e7add892426cb44d2f3bb54c0df1b Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Tue, 12 Jun 2018 13:33:58 -0700 Subject: [PATCH 070/148] groundwork for firestore --- .../cloud/firestore_v1beta1/__init__.py | 2 + .../cloud/firestore_v1beta1/collection.py | 6 ++ .../cloud/firestore_v1beta1/document.py | 5 ++ .../google/cloud/firestore_v1beta1/query.py | 13 +++++ .../google/cloud/firestore_v1beta1/watch.py | 58 +++++++++++++++++++ 5 files changed, 84 insertions(+) create mode 100644 firestore/google/cloud/firestore_v1beta1/watch.py diff --git a/firestore/google/cloud/firestore_v1beta1/__init__.py b/firestore/google/cloud/firestore_v1beta1/__init__.py index 1ae905bfdee1..d3bd90405f12 100644 --- a/firestore/google/cloud/firestore_v1beta1/__init__.py +++ b/firestore/google/cloud/firestore_v1beta1/__init__.py @@ -34,6 +34,7 @@ from google.cloud.firestore_v1beta1.query import Query from google.cloud.firestore_v1beta1.transaction import Transaction from google.cloud.firestore_v1beta1.transaction import transactional +from google.cloud.firestore_v1beta1.watch import Watch __all__ = [ @@ -53,6 +54,7 @@ 'Transaction', 'transactional', 'types', + 'Watch', 'WriteBatch', 'WriteOption', ] diff --git a/firestore/google/cloud/firestore_v1beta1/collection.py b/firestore/google/cloud/firestore_v1beta1/collection.py index 81e3dba448c3..c49c3e4080af 100644 --- a/firestore/google/cloud/firestore_v1beta1/collection.py +++ b/firestore/google/cloud/firestore_v1beta1/collection.py @@ -371,6 +371,12 @@ def get(self, transaction=None): query = query_mod.Query(self) return query.get(transaction=transaction) + def onSnapshot(options, callback): + ''' + given options and the callback, monitor this collection for changes + ''' + raise NotImplemented + def _auto_id(): """Generate a "random" automatically generated ID. diff --git a/firestore/google/cloud/firestore_v1beta1/document.py b/firestore/google/cloud/firestore_v1beta1/document.py index d9420470d62a..bae7533ee528 100644 --- a/firestore/google/cloud/firestore_v1beta1/document.py +++ b/firestore/google/cloud/firestore_v1beta1/document.py @@ -445,6 +445,11 @@ def collections(self, page_size=None): iterator.item_to_value = _item_to_collection_ref return iterator + def onSnapshot(options, callback): + ''' + given options and the callback, monitor this document for changes + ''' + raise NotImplemented class DocumentSnapshot(object): """A snapshot of document data in a Firestore database. diff --git a/firestore/google/cloud/firestore_v1beta1/query.py b/firestore/google/cloud/firestore_v1beta1/query.py index a4d0243a8724..909eb914e2ea 100644 --- a/firestore/google/cloud/firestore_v1beta1/query.py +++ b/firestore/google/cloud/firestore_v1beta1/query.py @@ -601,6 +601,19 @@ def get(self, transaction=None): else: yield snapshot + def onSnapshot(callback, options): + ''' + db.collection("cities").where("state", "==", "CA") + .onSnapshot(function(querySnapshot) { + var cities = []; + querySnapshot.forEach(function(doc) { + cities.push(doc.data().name); + }); + console.log("Current cities in CA: ", cities.join(", ")); + }); + ''' + raise NotImplemented + def _enum_from_op_string(op_string): """Convert a string representation of a binary operator to an enum. diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py new file mode 100644 index 000000000000..b79762c205d8 --- /dev/null +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -0,0 +1,58 @@ +# Copyright 2017 Google LLC All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Python client for Google Cloud Firestore Watch.""" + +WATCH_TARGET_ID = 0x5079 # "Py" + +class Watch(object): + pass + +''' +You can listen to a document with the onSnapshot() method. An initial call +using the callback you provide creates a document snapshot immediately with the +\current contents of the single document. Then, each time the contents change, +another call updates the document snapshot. + +db.collection("cities") + .onSnapshot + + +Internal: Count: 1, Average: 4.0 +Get Realtime Updates with Cloud Firestore +You can listen to a document with the onSnapshot() method. An initial call using +the callback you provide creates a document snapshot immediately with the +current contents of the single document. Then, each time the contents change, +another call updates the document snapshot. + +Note: Realtime listeners are not yet supported in the Python, Go, or PHP client +libraries. + +db.collection("cities").doc("SF") + .onSnapshot(function(doc) { + console.log("Current data: ", doc.data()); + }); +test.firestore.js + +Events for local changes +Local writes in your app will invoke snapshot listeners immediately. This is +because of an important feature called "latency compensation." When you perform +a write, your listeners will be notified with the new data before the data is +sent to the backend. + +Retrieved documents have a metadata.hasPendingWrites property that indicates +whether the document has local changes that haven't been written to the backend +yet. You can use this property to determine the source of events received by +your snapshot listener: +''' \ No newline at end of file From a5dbe62ae370b43788b273ec39c7e36a3b4e26a7 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Mon, 18 Jun 2018 12:57:40 -0700 Subject: [PATCH 071/148] Use chemelnucfin sample (pythonification of nodejs) as base --- .../google/cloud/firestore_v1beta1/watch.py | 406 +++++++++++++++++- 1 file changed, 400 insertions(+), 6 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index b79762c205d8..7c653ee29b5d 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -12,12 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Python client for Google Cloud Firestore Watch.""" - -WATCH_TARGET_ID = 0x5079 # "Py" +import logging -class Watch(object): - pass +"""Python client for Google Cloud Firestore Watch.""" ''' You can listen to a document with the onSnapshot() method. An initial call @@ -55,4 +52,401 @@ class Watch(object): whether the document has local changes that haven't been written to the backend yet. You can use this property to determine the source of events received by your snapshot listener: -''' \ No newline at end of file +''' + +_LOGGER = logging.getLogger(__name__) + +WATCH_TARGET_ID = 0xf0 + +GRPC_STATUS_CODE = { + 'OK': 0, + 'CANCELLED': 1, + 'UNKNOWN': 2, + 'INVALID_ARGUMENT': 3, + 'DEADLINE_EXCEEDED': 4, + 'NOT_FOUND': 5, + 'ALREADY_EXISTS': 6, + 'PERMISSION_DENIED': 7, + 'UNAUTHENTICATED': 16, + 'RESOURCE_EXHAUSTED': 8, + 'FAILED_PRECONDITION': 9, + 'ABORTED': 10, + 'OUT_OF_RANGE': 11, + 'UNIMPLEMENTED': 12, + 'INTERNAL': 13, + 'UNAVAILABLE': 14, + 'DATA_LOSS': 15, + 'DO_NOT_USE': -1 +} + + +def is_permanent_error(self, error): + try: + if (error.code == GRPC_STATUS_CODE['CANCELLED'] or + error.code == GRPC_STATUS_CODE['UNKNOWN'] or + error.code == GRPC_STATUS_CODE['DEADLINE_EXCEEDED'] or + error.code == GRPC_STATUS_CODE['RESOURCE_EXHAUSTED'] or + error.code == GRPC_STATUS_CODE['INTERNAL'] or + error.code == GRPC_STATUS_CODE['UNAVAILABLE'] or + error.code == GRPC_STATUS_CODE['UNAUTHENTICATED'] + ): + return False + else: + return True + except AttributeError: + _LOGGER.error("Unable to determine error code") + return False + + +class Watch(object): + def __init__(self, firestore, target, comparator): + self._firestore = firestore + self._api = firestore.api + self._targets = target + self._comparator = comparator + self._backoff = ExponentialBackOff() + + @classmethod + def for_document(cls, document_ref): + return cls(document_ref.firestore, + {documents: {documents: [document_ref.formatted_name],}, + target_id: WATCH_TARGET_ID, + }, + DOCUMENT_WATCH_COMPARATOR) + + @classmethod + def for_query(cls, query): + return cls(query.firestore, + {query: query.to_proto(), + target_id: WATCH_TARGET_ID + } + query.comparator()) + + def on_snapshot(self, on_next, on_error): + doc_tree = rbtree(self.comparator) + doc_map = {} + change_map = {} + + current = False + has_pushed = False + is_active = True + + REMOVED = {} + + request = { database: self._firestore.formatted_name, + add_target: self._targets + } + + stream = through.obj() + + current_stream = None + + def reset_docs(): + log() + change_map.clear() + del resume_token + for snapshot in doc_tree: + change_map.set(snapshot.ref.formatted_name, REMOVED) + current = False + + def close_stream(err): + if current_stream is not None: + current_stream.unpipe(stream) + current_stream.end() + current_stream = None + stream.end() + + if is_active: + is_active = False + _LOGGER.error('Invoking on_error: ', err) + on_error(err) + + def maybe_reopen_stream(err): + if is_active and not is_permanent_error(err): + _LOGGER.error('Stream ended, re-opening after retryable error: ', err) + request.add_target.resume_token = resume_token + change_map.clear() + + if is_resource_exhausted_error(err): + self._backoff.reset_to_max() + reset_stream() + else: + _LOGGER.error('Stream ended, sending error: ', err) + close_stream(err) + + def reset_stream(): + _LOGGER.info('Opening new stream') + if current_stream: + current_stream.unpipe(stream) + current_stream.end() + current_stream = None + init_stream() + + def init_stream(): + self._backoff.back_off_and_wait() + if not is_active: + _LOGGER.info('Not initializing inactive stream') + return + + backend_stream = self._firestore.read_write_stream( + self._api.Firestore._listen.bind(self._api.Firestore), + request, + ) + + + if not is_active: + _LOGGER.info('Closing inactive stream') + backend_stream.end() + _LOGGER.info('Opened new stream') + current_stream = backend_stream + + def on_error(err): + maybe_reopen_stream(err) + + current_stream.on('error')(on_error) + + def on_end(): + err = Error('Stream ended unexpectedly') + err.code = GRPC_STATUS_CODE['UNKNOWN'] + maybe_reopen_stream(err) + + current_stream.on('end')(on_end) + current_stream.pipe(stream) + current_stream.resume() + + current_stream.catch(close_stream) + + def affects_target(target_ids, current_id): + for target_id in target_ids: + if target_id == current_id: + return True + return False + + def extract_changes(doc_map, changes, read_time): + deletes = [] + adds = [] + updates = [] + + for value, name in changes: + if value == REMOVED: + if doc_map.has(name): + deletes.append(name) + elif doc_map.has(name): + value.read_time = read_time + upates.append(value.build()) + else: + value.read_time = read_time + adds.append(value.build()) + return deletes, adds, updates + + def compute_snapshot(doc_tree, doc_map, changes): + if len(doc_tree) != doc_map: + raise ValueError('The document tree and document map should' + 'have the same number of entries.') + updated_tree = doc_tree + updated_map = doc_map + + def delete_doc(name): + """ raises KeyError if name not in updated_map""" + old_document = updated_map.pop(name) # Raises KeyError + existing = updated_tree.find(old_document) + old_index = existing.index + updated_tree = existing.remove() + return DocumentChange('removed', + old_document, + old_index, + -1) + + def add_doc(new_document): + name = new_document.ref.formatted_name + if name in updated_map: + raise ValueError('Document to add already exists') + updated_tree = updated_tree.insert(new_document, null) + new_index = updated_tree.find(new_document).index + updated_map[name] = new_document + return DocumentChange('added', + new_document, + -1, + new_index) + + def modify_doc(new_document): + name = new_document.ref.formattedName + if not name in updated_map: + raise ValueError('Document to modify does not exsit') + old_document = updated_map[name] + if old_document.update_time != new_document.update_time): + remove_change = delete_doc(name) + add_change = add_doc(new_document) + return DocumentChange('modified', + new_document, + remove_change.old_index, + add_change.new_index) + return None + + applied_changes = [] + + def compartor_sort(name1, name2): + return self._comparator(updated_map[name1], updated_map[name2]) + changes.deletes.sort(comparator_sort) + + + for name in changes.deletes: + changes.delete_doc(name) + if change: + applied_changes.push(change) + + changes.adds.sort(self._compartor) + + for snapshot in changes.adds: + change = add_doc(snapshot) + if change: + applied_changes.push(change) + + changes.updates.sort(self._compartor) + + for snapshot in changes.updates: + change = modify_doc(snapshot) + if change: + applied_changes.push(change) + + if not len(updated_tree) == len(updated_map): + raise RuntimeError('The update document tree and document ' + 'map should have the same number of ' + 'entries') + + + return {updated_tree, updated_map, applied_changes) + + def push(read_time, next_resume_token): + changes = extract_changes(doc_map, change_map, read_time) + diff = compute_snapshot(doc_tree, doc_map, changes) + + if not has_pushed or len(diff.applied_changes) > 0: + _LOGGER.info('Sending snapshot with %d changes and %d documents' + % (len(diff.applied_changes), len(updated_tree))) + + next(read_time, diff.updatedTree.keys, diff.applied_changes) + + doc_tree = diff.updated_tree + doc_map = diff.updated_map + change_map.clear() + resume_token = next_resume_token + + def current_size(): + changes = extract_changes(doc_map, change_map): + return doc_map.size + len(changes.adds) - len(changes.deletes) + + init_stream() + + def proto(): + if proto.target_change: + _LOGGER.log('Processing target change') + change = proto.target_change + no_target_ids = not target_ids + if change.target_change_type == 'NO_CHANGE': + if no_target_ids and change.read_time and current: + push(DocumentSnapshot.to_ISO_time(change.read_time) + change.resume_token) + elif change.target_change_type == 'ADD': + if WATCH_TARGET_ID != change.target_ids[0]: + raise ValueError('Unexpected target ID sent by server') + elif change.target_change_type == 'REMOVE': + code = 13 + message = 'internal error' + if change.cause: + code = change.cause.code + message = change.cause.message + close_stream(Error('Error ' + code + ': ' + message)) + elif change.target_change_type == 'RESET': + reset_docs() + elif change.target_change_type == 'CURRENT': + current = true + else: + close_stream(Error('Unknown target change type: ' + str(change))) + + stream.on('data', proto) # ?? + + if change.resume_token and affects_target(change.target_ids, WATCH_TARGET_ID): + self._backoff.reset() + + elif proto.document_change: + _LOGGER.info('Processing change event') + + target_ids = proto.document_change.target_ids + removed_target_ids = proto.document_change.removed_target_ids + + changed = False + + removed = False + for target_id in target_ids: + if target_id == WATCH_TARGET_ID: + changed = True + + for target_id in removed_target_ids: + if removed_target_ids == WATCH_TARGET_ID: + removed = True + + document = proto.document_change.document + name = document.name + + if changed: + _LOGGER.info('Received document change') + snapshot = DocumentSnapshot.Builder() + snapshot.ref = DocumentReference(self._firestore, + ResourcePath.from_slash_separated_string(name)) + snapshot.fields_proto = document.fields + snapshot.create_time = DocumentSnapshot.to_ISO_time(document.create_time) + snapshot.update_time = DocumentSnapshot.to_ISO_time(document.update_time) + change_map[name] = snapshot + elif removed: + _LOGGER.info('Received document remove') + change_map[name] = REMOVED + elif proto.document_delete + _LOGGER.info('Processing remove event') + name = proto.document_delete.document + change_map[name] = REMOVED + elif proto.document_remove: + _LOGGER.info('Processing remove event') + name = proto.document_remove.document + change_map[name] = REMOVED + elif proto.filter: + _LOGGER.info('Processing filter update') + if proto.filter.count != current_size(): + reset_docs() + reset_stream() + else: + close_stream(Error('Unknown listen response type: ' + str(proto))) + + def on_end(): + _LOGGER.info('Processing stream end') + if current_stream: + current_stream.end() + + on('end', on_end) + + def initialize(): + return {} + + def end_stream(): + _LOGGER.info('Ending stream') + is_active = False + on_next = initialize + on_error = initialize + stream.end() + + return end_stream + + + + + + + + + + + + + + + \ No newline at end of file From 8a5bd5f9a0482012bda73ee0429df5007a28f9d0 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Mon, 18 Jun 2018 13:10:26 -0700 Subject: [PATCH 072/148] syntactic and style fixes --- .../google/cloud/firestore_v1beta1/watch.py | 131 +++++++----------- 1 file changed, 52 insertions(+), 79 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 7c653ee29b5d..62b33a88bf8b 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -13,50 +13,14 @@ # limitations under the License. import logging +from google.firestore.v1beta1 import DocumentChange -"""Python client for Google Cloud Firestore Watch.""" - -''' -You can listen to a document with the onSnapshot() method. An initial call -using the callback you provide creates a document snapshot immediately with the -\current contents of the single document. Then, each time the contents change, -another call updates the document snapshot. - -db.collection("cities") - .onSnapshot - - -Internal: Count: 1, Average: 4.0 -Get Realtime Updates with Cloud Firestore -You can listen to a document with the onSnapshot() method. An initial call using -the callback you provide creates a document snapshot immediately with the -current contents of the single document. Then, each time the contents change, -another call updates the document snapshot. - -Note: Realtime listeners are not yet supported in the Python, Go, or PHP client -libraries. -db.collection("cities").doc("SF") - .onSnapshot(function(doc) { - console.log("Current data: ", doc.data()); - }); -test.firestore.js - -Events for local changes -Local writes in your app will invoke snapshot listeners immediately. This is -because of an important feature called "latency compensation." When you perform -a write, your listeners will be notified with the new data before the data is -sent to the backend. - -Retrieved documents have a metadata.hasPendingWrites property that indicates -whether the document has local changes that haven't been written to the backend -yet. You can use this property to determine the source of events received by -your snapshot listener: -''' +"""Python client for Google Cloud Firestore Watch.""" _LOGGER = logging.getLogger(__name__) -WATCH_TARGET_ID = 0xf0 +WATCH_TARGET_ID = 0x5079 # "Py" GRPC_STATUS_CODE = { 'OK': 0, @@ -83,13 +47,12 @@ def is_permanent_error(self, error): try: if (error.code == GRPC_STATUS_CODE['CANCELLED'] or - error.code == GRPC_STATUS_CODE['UNKNOWN'] or - error.code == GRPC_STATUS_CODE['DEADLINE_EXCEEDED'] or - error.code == GRPC_STATUS_CODE['RESOURCE_EXHAUSTED'] or - error.code == GRPC_STATUS_CODE['INTERNAL'] or - error.code == GRPC_STATUS_CODE['UNAVAILABLE'] or - error.code == GRPC_STATUS_CODE['UNAUTHENTICATED'] - ): + error.code == GRPC_STATUS_CODE['UNKNOWN'] or + error.code == GRPC_STATUS_CODE['DEADLINE_EXCEEDED'] or + error.code == GRPC_STATUS_CODE['RESOURCE_EXHAUSTED'] or + error.code == GRPC_STATUS_CODE['INTERNAL'] or + error.code == GRPC_STATUS_CODE['UNAVAILABLE'] or + error.code == GRPC_STATUS_CODE['UNAUTHENTICATED']): return False else: return True @@ -98,6 +61,11 @@ def is_permanent_error(self, error): return False +def document_watch_comparator(doc1, doc2): + assert doc1 == doc2, 'Document watches only support one document.' + return 0 + + class Watch(object): def __init__(self, firestore, target, comparator): self._firestore = firestore @@ -109,17 +77,19 @@ def __init__(self, firestore, target, comparator): @classmethod def for_document(cls, document_ref): return cls(document_ref.firestore, - {documents: {documents: [document_ref.formatted_name],}, - target_id: WATCH_TARGET_ID, + { + documents: {documents: [document_ref.formatted_name]}, + target_id: WATCH_TARGET_ID }, - DOCUMENT_WATCH_COMPARATOR) - + document_watch_comparator) + @classmethod def for_query(cls, query): return cls(query.firestore, - {query: query.to_proto(), - target_id: WATCH_TARGET_ID - } + { + query: query.to_proto(), + target_id: WATCH_TARGET_ID + }, query.comparator()) def on_snapshot(self, on_next, on_error): @@ -132,10 +102,9 @@ def on_snapshot(self, on_next, on_error): is_active = True REMOVED = {} - - request = { database: self._firestore.formatted_name, - add_target: self._targets - } + + request = {database: self._firestore.formatted_name, + add_target: self._targets} stream = through.obj() @@ -163,7 +132,8 @@ def close_stream(err): def maybe_reopen_stream(err): if is_active and not is_permanent_error(err): - _LOGGER.error('Stream ended, re-opening after retryable error: ', err) + _LOGGER.error( + 'Stream ended, re-opening after retryable error: ', err) request.add_target.resume_token = resume_token change_map.clear() @@ -193,18 +163,17 @@ def init_stream(): request, ) - if not is_active: _LOGGER.info('Closing inactive stream') backend_stream.end() _LOGGER.info('Opened new stream') current_stream = backend_stream - + def on_error(err): maybe_reopen_stream(err) current_stream.on('error')(on_error) - + def on_end(): err = Error('Stream ended unexpectedly') err.code = GRPC_STATUS_CODE['UNKNOWN'] @@ -215,7 +184,7 @@ def on_end(): current_stream.resume() current_stream.catch(close_stream) - + def affects_target(target_ids, current_id): for target_id in target_ids: if target_id == current_id: @@ -243,7 +212,7 @@ def compute_snapshot(doc_tree, doc_map, changes): if len(doc_tree) != doc_map: raise ValueError('The document tree and document map should' 'have the same number of entries.') - updated_tree = doc_tree + updated_tree = doc_tree updated_map = doc_map def delete_doc(name): @@ -271,10 +240,10 @@ def add_doc(new_document): def modify_doc(new_document): name = new_document.ref.formattedName - if not name in updated_map: + if name not in updated_map: raise ValueError('Document to modify does not exsit') old_document = updated_map[name] - if old_document.update_time != new_document.update_time): + if old_document.update_time != new_document.update_time: remove_change = delete_doc(name) add_change = add_doc(new_document) return DocumentChange('modified', @@ -289,7 +258,6 @@ def compartor_sort(name1, name2): return self._comparator(updated_map[name1], updated_map[name2]) changes.deletes.sort(comparator_sort) - for name in changes.deletes: changes.delete_doc(name) if change: @@ -314,16 +282,16 @@ def compartor_sort(name1, name2): 'map should have the same number of ' 'entries') - - return {updated_tree, updated_map, applied_changes) + return {updated_tree, updated_map, applied_changes} def push(read_time, next_resume_token): changes = extract_changes(doc_map, change_map, read_time) diff = compute_snapshot(doc_tree, doc_map, changes) if not has_pushed or len(diff.applied_changes) > 0: - _LOGGER.info('Sending snapshot with %d changes and %d documents' - % (len(diff.applied_changes), len(updated_tree))) + _LOGGER.info( + 'Sending snapshot with %d changes and %d documents' + % (len(diff.applied_changes), len(updated_tree))) next(read_time, diff.updatedTree.keys, diff.applied_changes) @@ -333,7 +301,7 @@ def push(read_time, next_resume_token): resume_token = next_resume_token def current_size(): - changes = extract_changes(doc_map, change_map): + changes = extract_changes(doc_map, change_map) return doc_map.size + len(changes.adds) - len(changes.deletes) init_stream() @@ -345,7 +313,7 @@ def proto(): no_target_ids = not target_ids if change.target_change_type == 'NO_CHANGE': if no_target_ids and change.read_time and current: - push(DocumentSnapshot.to_ISO_time(change.read_time) + push(DocumentSnapshot.to_ISO_time(change.read_time), change.resume_token) elif change.target_change_type == 'ADD': if WATCH_TARGET_ID != change.target_ids[0]: @@ -362,11 +330,13 @@ def proto(): elif change.target_change_type == 'CURRENT': current = true else: - close_stream(Error('Unknown target change type: ' + str(change))) + close_stream( + Error('Unknown target change type: ' + str(change))) stream.on('data', proto) # ?? - if change.resume_token and affects_target(change.target_ids, WATCH_TARGET_ID): + if change.resume_token and \ + affects_target(change.target_ids, WATCH_TARGET_ID): self._backoff.reset() elif proto.document_change: @@ -392,16 +362,19 @@ def proto(): if changed: _LOGGER.info('Received document change') snapshot = DocumentSnapshot.Builder() - snapshot.ref = DocumentReference(self._firestore, - ResourcePath.from_slash_separated_string(name)) + snapshot.ref = DocumentReference( + self._firestore, + ResourcePath.from_slash_separated_string(name)) snapshot.fields_proto = document.fields - snapshot.create_time = DocumentSnapshot.to_ISO_time(document.create_time) - snapshot.update_time = DocumentSnapshot.to_ISO_time(document.update_time) + snapshot.create_time = DocumentSnapshot.to_ISO_time( + document.create_time) + snapshot.update_time = DocumentSnapshot.to_ISO_time( + document.update_time) change_map[name] = snapshot elif removed: _LOGGER.info('Received document remove') change_map[name] = REMOVED - elif proto.document_delete + elif proto.document_delete: _LOGGER.info('Processing remove event') name = proto.document_delete.document change_map[name] = REMOVED From e296b22d103428b6fde3264faf30da416e45e33c Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Wed, 27 Jun 2018 14:39:35 -0700 Subject: [PATCH 073/148] hold work --- firestore/google/cloud/firestore.py | 2 + .../google/cloud/firestore_v1beta1/bidi.py | 559 +++++++++++++ .../cloud/firestore_v1beta1/document.py | 17 +- .../google/cloud/firestore_v1beta1/watch.py | 747 ++++++++++-------- firestore/tests/system.py | 678 ++++++++++++++++ 5 files changed, 1666 insertions(+), 337 deletions(-) create mode 100644 firestore/google/cloud/firestore_v1beta1/bidi.py diff --git a/firestore/google/cloud/firestore.py b/firestore/google/cloud/firestore.py index b7bec0c3adf5..a03ae65ea798 100644 --- a/firestore/google/cloud/firestore.py +++ b/firestore/google/cloud/firestore.py @@ -31,6 +31,7 @@ from google.cloud.firestore_v1beta1 import Transaction from google.cloud.firestore_v1beta1 import transactional from google.cloud.firestore_v1beta1 import types +from google.cloud.firestore_v1beta1 import Watch from google.cloud.firestore_v1beta1 import WriteBatch from google.cloud.firestore_v1beta1 import WriteOption @@ -52,6 +53,7 @@ 'Transaction', 'transactional', 'types', + 'Watch', 'WriteBatch', 'WriteOption', ] diff --git a/firestore/google/cloud/firestore_v1beta1/bidi.py b/firestore/google/cloud/firestore_v1beta1/bidi.py new file mode 100644 index 000000000000..00877e70058e --- /dev/null +++ b/firestore/google/cloud/firestore_v1beta1/bidi.py @@ -0,0 +1,559 @@ +# Copyright 2017, Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Bi-directional streaming RPC helpers.""" + +import logging +import threading + +from six.moves import queue + +from google.api_core import exceptions + +_LOGGER = logging.getLogger(__name__) +_BIDIRECTIONAL_CONSUMER_NAME = 'Thread-ConsumeBidirectionalStream' + + +class _RequestQueueGenerator(object): + """A helper for sending requests to a gRPC stream from a Queue. + + This generator takes requests off a given queue and yields them to gRPC. + + This helper is useful when you have an indeterminate, indefinite, or + otherwise open-ended set of requests to send through a request-streaming + (or bidirectional) RPC. + + The reason this is necessary is because gRPC takes an iterator as the + request for request-streaming RPCs. gRPC consumes this iterator in another + thread to allow it to block while generating requests for the stream. + However, if the generator blocks indefinitely gRPC will not be able to + clean up the thread as it'll be blocked on `next(iterator)` and not be able + to check the channel status to stop iterating. This helper mitigates that + by waiting on the queue with a timeout and checking the RPC state before + yielding. + + Finally, it allows for retrying without swapping queues because if it does + pull an item off the queue when the RPC is inactive, it'll immediately put + it back and then exit. This is necessary because yielding the item in this + case will cause gRPC to discard it. In practice, this means that the order + of messages is not guaranteed. If such a thing is necessary it would be + easy to use a priority queue. + + Example:: + + requests = request_queue_generator(q) + call = stub.StreamingRequest(iter(requests)) + requests.call = call + + for response in call: + print(response) + q.put(...) + + Note that it is possible to accomplish this behavior without "spinning" + (using a queue timeout). One possible way would be to use more threads to + multiplex the grpc end event with the queue, another possible way is to + use selectors and a custom event/queue object. Both of these approaches + are significant from an engineering perspective for small benefit - the + CPU consumed by spinning is pretty minuscule. + + Args: + queue (queue.Queue): The request queue. + period (float): The number of seconds to wait for items from the queue + before checking if the RPC is cancelled. In practice, this + determines the maximum amount of time the request consumption + thread will live after the RPC is cancelled. + initial_request (Union[protobuf.Message, + Callable[None, protobuf.Message]]): The initial request to + yield. This is done independently of the request queue to allow fo + easily restarting streams that require some initial configuration + request. + """ + def __init__(self, queue, period=1, initial_request=None): + self._queue = queue + self._period = period + self._initial_request = initial_request + self.call = None + + def _is_active(self): + # Note: there is a possibility that this starts *before* the call + # property is set. So we have to check if self.call is set before + # seeing if it's active. + if self.call is not None and not self.call.is_active(): + return False + else: + return True + + def __iter__(self): + if self._initial_request is not None: + if callable(self._initial_request): + yield self._initial_request() + else: + yield self._initial_request + + while True: + try: + item = self._queue.get(timeout=self._period) + except queue.Empty: + if not self._is_active(): + _LOGGER.debug( + 'Empty queue and inactive call, exiting request ' + 'generator.') + return + else: + # call is still active, keep waiting for queue items. + continue + + # The consumer explicitly sent "None", indicating that the request + # should end. + if item is None: + _LOGGER.debug('Cleanly exiting request generator.') + return + + if not self._is_active(): + # We have an item, but the call is closed. We should put the + # item back on the queue so that the next call can consume it. + self._queue.put(item) + _LOGGER.debug( + 'Inactive call, replacing item on queue and exiting ' + 'request generator.') + return + + yield item + + +class BidiRpc(object): + """A helper for consuming a bi-directional streaming RPC. + + This maps gRPC's built-in interface which uses a request iterator and a + response iterator into a socket-like :func:`send` and :func:`recv`. This + is a more useful pattern for long-running or asymmetric streams (streams + where there is not a direct correlation between the requests and + responses). + + Example:: + + initial_request = example_pb2.StreamingRpcRequest( + setting='example') + rpc = BidiRpc(stub.StreamingRpc, initial_request=initial_request) + + rpc.open() + + while rpc.is_active(): + print(rpc.recv()) + rpc.send(example_pb2.StreamingRpcRequest( + data='example')) + + This does *not* retry the stream on errors. See :class:`ResumableBidiRpc`. + + Args: + start_rpc (grpc.StreamStreamMultiCallable): The gRPC method used to + start the RPC. + initial_request (Union[protobuf.Message, + Callable[None, protobuf.Message]]): The initial request to + yield. This is useful if an initial request is needed to start the + stream. + """ + def __init__(self, start_rpc, initial_request=None): + self._start_rpc = start_rpc + self._initial_request = initial_request + self._request_queue = queue.Queue() + self._request_generator = None + self._is_active = False + self._callbacks = [] + self.call = None + + def add_done_callback(self, callback): + """Adds a callback that will be called when the RPC terminates. + + This occurs when the RPC errors or is successfully terminated. + + Args: + callback (Callable[[grpc.Future], None]): The callback to execute. + It will be provided with the same gRPC future as the underlying + stream which will also be a :class:`grpc.Call`. + """ + self._callbacks.append(callback) + + def _on_call_done(self, future): + for callback in self._callbacks: + callback(future) + + def open(self): + """Opens the stream.""" + if self.is_active: + raise ValueError('Can not open an already open stream.') + + request_generator = _RequestQueueGenerator( + self._request_queue, initial_request=self._initial_request) + call = self._start_rpc(iter(request_generator)) + + request_generator.call = call + + # TODO: api_core should expose the future interface for wrapped + # callables as well. + if hasattr(call, '_wrapped'): # pragma: NO COVER + call._wrapped.add_done_callback(self._on_call_done) + else: + call.add_done_callback(self._on_call_done) + + self._request_generator = request_generator + self.call = call + + def close(self): + """Closes the stream.""" + if self.call is None: + return + + self._request_queue.put(None) + self.call.cancel() + self._request_generator = None + # Don't set self.call to None. Keep it around so that send/recv can + # raise the error. + + def send(self, request): + """Queue a message to be sent on the stream. + + Send is non-blocking. + + If the underlying RPC has been closed, this will raise. + + Args: + request (protobuf.Message): The request to send. + """ + if self.call is None: + raise ValueError( + 'Can not send() on an RPC that has never been open()ed.') + + # Don't use self.is_active(), as ResumableBidiRpc will overload it + # to mean something semantically different. + if self.call.is_active(): + self._request_queue.put(request) + else: + # calling next should cause the call to raise. + next(self.call) + + def recv(self): + """Wait for a message to be returned from the stream. + + Recv is blocking. + + If the underlying RPC has been closed, this will raise. + + Returns: + protobuf.Message: The received message. + """ + if self.call is None: + raise ValueError( + 'Can not recv() on an RPC that has never been open()ed.') + + return next(self.call) + + @property + def is_active(self): + """bool: True if this stream is currently open and active.""" + return self.call is not None and self.call.is_active() + + @property + def pending_requests(self): + """int: Returns an estimate of the number of queued requests.""" + return self._request_queue.qsize() + + +class ResumableBidiRpc(BidiRpc): + """A :class:`BidiRpc` that can automatically resume the stream on errors. + + It uses the ``should_recover`` arg to determine if it should re-establish + the stream on error. + + Example:: + + def should_recover(exc): + return ( + isinstance(exc, grpc.RpcError) and + exc.code() == grpc.StatusCode.UNVAILABLE) + + initial_request = example_pb2.StreamingRpcRequest( + setting='example') + + rpc = ResumeableBidiRpc( + stub.StreamingRpc, + initial_request=initial_request, + should_recover=should_recover) + + rpc.open() + + while rpc.is_active(): + print(rpc.recv()) + rpc.send(example_pb2.StreamingRpcRequest( + data='example')) + + Args: + start_rpc (grpc.StreamStreamMultiCallable): The gRPC method used to + start the RPC. + initial_request (Union[protobuf.Message, + Callable[None, protobuf.Message]]): The initial request to + yield. This is useful if an initial request is needed to start the + stream. + should_recover (Callable[[Exception], bool]): A function that returns + True if the stream should be recovered. This will be called + whenever an error is encountered on the stream. + """ + def __init__(self, start_rpc, should_recover, initial_request=None): + super(ResumableBidiRpc, self).__init__(start_rpc, initial_request) + self._should_recover = should_recover + self._operational_lock = threading.RLock() + self._finalized = False + self._finalize_lock = threading.Lock() + + def _finalize(self, result): + with self._finalize_lock: + if self._finalized: + return + + for callback in self._callbacks: + callback(result) + + self._finalized = True + + def _on_call_done(self, future): + # Unlike the base class, we only execute the callbacks on a terminal + # error, not for errors that we can recover from. Note that grpc's + # "future" here is also a grpc.RpcError. + if not self._should_recover(future): + self._finalize(future) + else: + _LOGGER.debug('Re-opening stream from gRPC callback.') + self._reopen() + + def _reopen(self): + with self._operational_lock: + # Another thread already managed to re-open this stream. + if self.call is not None and self.call.is_active(): + _LOGGER.debug('Stream was already re-established.') + return + + self.call = None + # Request generator should exit cleanly since the RPC its bound to + # has exited. + self.request_generator = None + + # Note: we do not currently do any sort of backoff here. The + # assumption is that re-establishing the stream under normal + # circumstances will happen in intervals greater than 60s. + # However, it is possible in a degenerative case that the server + # closes the stream rapidly which would lead to thrashing here, + # but hopefully in those cases the server would return a non- + # retryable error. + + try: + self.open() + # If re-opening or re-calling the method fails for any reason, + # consider it a terminal error and finalize the stream. + except Exception as exc: + self._finalize(exc) + raise + + _LOGGER.info('Re-established stream') + + def _recoverable(self, method, *args, **kwargs): + """Wraps a method to recover the stream and retry on error. + + If a retryable error occurs while making the call, then the stream will + be re-opened and the method will be retried. This happens indefinitely + so long as the error is a retryable one. If an error occurs while + re-opening the stream, then this method will raise immediately and + trigger finalization of this object. + + Args: + method (Callable[..., Any]): The method to call. + args: The args to pass to the method. + kwargs: The kwargs to pass to the method. + """ + while True: + try: + return method(*args, **kwargs) + + except Exception as exc: + _LOGGER.debug('Call to retryable %r caused %s.', method, exc) + if not self._should_recover(exc): + self.close() + _LOGGER.debug('Not retrying %r due to %s.', method, exc) + self._finalize(exc) + raise exc + + _LOGGER.debug('Re-opening stream from retryable %r.', method) + self._reopen() + + def send(self, request): + return self._recoverable( + super(ResumableBidiRpc, self).send, request) + + def recv(self): + return self._recoverable( + super(ResumableBidiRpc, self).recv) + + @property + def is_active(self): + """bool: True if this stream is currently open and active.""" + # Use the operational lock. It's entirely possible for something + # to check the active state *while* the RPC is being retried. + # Also, use finalized to track the actual terminal state here. + # This is because if the stream is re-established by the gRPC thread + # it's technically possible to check this between when gRPC marks the + # RPC as inactive and when gRPC executes our callback that re-opens + # the stream. + with self._operational_lock: + return self.call is not None and not self._finalized + + +class BackgroundConsumer(object): + """A bi-directional stream consumer that runs in a separate thread. + + This maps the consumption of a stream into a callback-based model. It also + provides :func:`pause` and :func:`resume` to allow for flow-control. + + Example:: + + def should_recover(exc): + return ( + isinstance(exc, grpc.RpcError) and + exc.code() == grpc.StatusCode.UNVAILABLE) + + initial_request = example_pb2.StreamingRpcRequest( + setting='example') + + rpc = ResumeableBidiRpc( + stub.StreamingRpc, + initial_request=initial_request, + should_recover=should_recover) + + def on_response(response): + print(response) + + consumer = BackgroundConsumer(rpc, on_response) + consume.start() + + Note that error handling *must* be done by using the provided + ``bidi_rpc``'s ``add_done_callback``. This helper will automatically exit + whenever the RPC itself exits and will not provide any error details. + + Args: + bidi_rpc (BidiRpc): The RPC to consume. Should not have been + ``open()``ed yet. + on_response (Callable[[protobuf.Message], None]): The callback to + be called for every response on the stream. + """ + def __init__(self, bidi_rpc, on_response): + self._bidi_rpc = bidi_rpc + self._on_response = on_response + self._paused = False + self._wake = threading.Condition() + self._thread = None + self._operational_lock = threading.Lock() + + def _on_call_done(self, future): + # Resume the thread if it's paused, this prevents blocking forever + # when the RPC has terminated. + self.resume() + + def _thread_main(self): + try: + self._bidi_rpc.add_done_callback(self._on_call_done) + self._bidi_rpc.open() + + while self._bidi_rpc.is_active: + # Do not allow the paused status to change at all during this + # section. There is a condition where we could be resumed + # between checking if we are paused and calling wake.wait(), + # which means that we will miss the notification to wake up + # (oops!) and wait for a notification that will never come. + # Keeping the lock throughout avoids that. + # In the future, we could use `Condition.wait_for` if we drop + # Python 2.7. + with self._wake: + if self._paused: + _LOGGER.debug('paused, waiting for waking.') + self._wake.wait() + _LOGGER.debug('woken.') + + _LOGGER.debug('waiting for recv.') + response = self._bidi_rpc.recv() + _LOGGER.debug('recved response.') + self._on_response(response) + + except exceptions.GoogleAPICallError as exc: + _LOGGER.debug( + '%s caught error %s and will exit. Generally this is due to ' + 'the RPC itself being cancelled and the error will be ' + 'surfaced to the calling code.', + _BIDIRECTIONAL_CONSUMER_NAME, exc, exc_info=True) + + except Exception as exc: + _LOGGER.exception( + '%s caught unexpected exception %s and will exit.', + _BIDIRECTIONAL_CONSUMER_NAME, exc) + + else: + _LOGGER.error( + 'The bidirectional RPC unexpectedly exited. This is a truly ' + 'exceptional case. Please file a bug with your logs.') + + _LOGGER.info('%s exiting', _BIDIRECTIONAL_CONSUMER_NAME) + + def start(self): + """Start the background thread and begin consuming the thread.""" + with self._operational_lock: + thread = threading.Thread( + name=_BIDIRECTIONAL_CONSUMER_NAME, + target=self._thread_main) + thread.daemon = True + thread.start() + self._thread = thread + _LOGGER.debug('Started helper thread %s', thread.name) + + def stop(self): + """Stop consuming the stream and shutdown the background thread.""" + with self._operational_lock: + self._bidi_rpc.close() + + if self._thread is not None: + # Resume the thread to wake it up in case it is sleeping. + self.resume() + self._thread.join() + + self._thread = None + + @property + def is_active(self): + """bool: True if the background thread is active.""" + return self._thread is not None and self._thread.is_alive() + + def pause(self): + """Pauses the response stream. + + This does *not* pause the request stream. + """ + with self._wake: + self._paused = True + + def resume(self): + """Resumes the response stream.""" + with self._wake: + self._paused = False + self._wake.notifyAll() + + @property + def is_paused(self): + """bool: True if the response stream is paused.""" + return self._paused diff --git a/firestore/google/cloud/firestore_v1beta1/document.py b/firestore/google/cloud/firestore_v1beta1/document.py index bae7533ee528..8a32bbcf0de5 100644 --- a/firestore/google/cloud/firestore_v1beta1/document.py +++ b/firestore/google/cloud/firestore_v1beta1/document.py @@ -19,7 +19,8 @@ import six from google.cloud.firestore_v1beta1 import _helpers - +from google.cloud.firestore_v1beta1.watch import Watch +from google.cloud.firestore_v1beta1.proto.firestore_pb2 import Target class DocumentReference(object): """A reference to a document in a Firestore database. @@ -445,11 +446,21 @@ def collections(self, page_size=None): iterator.item_to_value = _item_to_collection_ref return iterator - def onSnapshot(options, callback): + def on_snapshot(self, options, callback): ''' given options and the callback, monitor this document for changes ''' - raise NotImplemented + #google.firestore.v1beta1.Target.DocumentsTarget + documentsTarget = Target.DocumentsTarget( + documents=[self._document_path]) + + Watch( + self._client, + Target( + documents=documentsTarget + ), + None) + class DocumentSnapshot(object): """A snapshot of document data in a Firestore database. diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 62b33a88bf8b..cf374c984cae 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -13,8 +13,19 @@ # limitations under the License. import logging -from google.firestore.v1beta1 import DocumentChange +#from google.cloud.firestore_v1beta1 import DocumentReference, DocumentSnapshot + +#from google.cloud.firestore_v1beta1.document import DocumentReference +#from google.cloud.firestore_v1beta1.document import DocumentSnapshot +#import google.cloud.firestore_v1beta1.client as client +from google.cloud.firestore_v1beta1.bidi import BidiRpc, ResumableBidiRpc +from google.cloud.firestore_v1beta1.proto import firestore_pb2 + +#from bidi import BidiRpc, ResumableBidiRpc +import time +import random +import grpc """Python client for Google Cloud Firestore Watch.""" @@ -66,20 +77,98 @@ def document_watch_comparator(doc1, doc2): return 0 +class ExponentialBackOff(object): + _INITIAL_SLEEP = 1.0 + """float: Initial "max" for sleep interval.""" + _MAX_SLEEP = 30.0 + """float: Eventual "max" sleep time.""" + _MULTIPLIER = 2.0 + """float: Multiplier for exponential backoff.""" + + def __init__(self, initial_sleep=_INITIAL_SLEEP, max_sleep=_MAX_SLEEP, + multiplier=_MULTIPLIER): + self.initial_sleep = self.current_sleep = initial_sleep + self.max_sleep = max_sleep + self.multipler = multiplier + + def back_off(self): + self.current_sleep = self._sleep(self.current_sleep, + self.max_sleep, + self.multipler) + + def reset_to_max(self): + self.current_sleep = self.max_sleep + + def reset(self): + self.current_sleep = self._INITIAL_SLEEP + + def _sleep(self, current_sleep, max_sleep=_MAX_SLEEP, + multiplier=_MULTIPLIER): + """Sleep and produce a new sleep time. + + .. _Exponential Backoff And Jitter: https://www.awsarchitectureblog.com/\ + 2015/03/backoff.html + + Select a duration between zero and ``current_sleep``. It might seem + counterintuitive to have so much jitter, but + `Exponential Backoff And Jitter`_ argues that "full jitter" is + the best strategy. + + Args: + current_sleep (float): The current "max" for sleep interval. + max_sleep (Optional[float]): Eventual "max" sleep time + multiplier (Optional[float]): Multiplier for exponential backoff. + + Returns: + float: Newly doubled ``current_sleep`` or ``max_sleep`` (whichever + is smaller) + """ + actual_sleep = random.uniform(0.0, self.current_sleep) + time.sleep(actual_sleep) + return min(self.multiplier * self.current_sleep, self.max_sleep) + class Watch(object): - def __init__(self, firestore, target, comparator): + def __init__(self, + firestore, #: client.Client, + target, + comparator): + self._firestore = firestore - self._api = firestore.api + self._api = firestore._firestore_api self._targets = target self._comparator = comparator self._backoff = ExponentialBackOff() + def should_recover(exc): + return ( + isinstance(exc, grpc.RpcError) and + exc.code() == grpc.StatusCode.UNVAILABLE) + + initial_request = firestore_pb2.ListenRequest( + #database=firestore.database_root_path, + add_target=target + # database, add_taret, remove_target, labels + ) + + rpc = ResumableBidiRpc( + # self._api.firestore_stub.Listen, + #firestore_pb2.BetaFirestoreStub.Listen, + self._api.firestore_stub.Listen, + initial_request=initial_request, + should_recover=should_recover) + + rpc.open() + + while rpc.is_active: + print(rpc.recv()) + @classmethod def for_document(cls, document_ref): return cls(document_ref.firestore, { - documents: {documents: [document_ref.formatted_name]}, - target_id: WATCH_TARGET_ID + 'documents': { + 'documents': [document_ref._document_path]}, + 'target_id': WATCH_TARGET_ID }, document_watch_comparator) @@ -87,339 +176,329 @@ def for_document(cls, document_ref): def for_query(cls, query): return cls(query.firestore, { - query: query.to_proto(), - target_id: WATCH_TARGET_ID + 'query': query.to_proto(), + 'target_id': WATCH_TARGET_ID }, query.comparator()) - def on_snapshot(self, on_next, on_error): - doc_tree = rbtree(self.comparator) - doc_map = {} - change_map = {} - - current = False - has_pushed = False - is_active = True - - REMOVED = {} - - request = {database: self._firestore.formatted_name, - add_target: self._targets} - - stream = through.obj() - - current_stream = None - - def reset_docs(): - log() - change_map.clear() - del resume_token - for snapshot in doc_tree: - change_map.set(snapshot.ref.formatted_name, REMOVED) - current = False - - def close_stream(err): - if current_stream is not None: - current_stream.unpipe(stream) - current_stream.end() - current_stream = None - stream.end() - - if is_active: - is_active = False - _LOGGER.error('Invoking on_error: ', err) - on_error(err) - - def maybe_reopen_stream(err): - if is_active and not is_permanent_error(err): - _LOGGER.error( - 'Stream ended, re-opening after retryable error: ', err) - request.add_target.resume_token = resume_token - change_map.clear() - - if is_resource_exhausted_error(err): - self._backoff.reset_to_max() - reset_stream() - else: - _LOGGER.error('Stream ended, sending error: ', err) - close_stream(err) - - def reset_stream(): - _LOGGER.info('Opening new stream') - if current_stream: - current_stream.unpipe(stream) - current_stream.end() - current_stream = None - init_stream() - - def init_stream(): - self._backoff.back_off_and_wait() - if not is_active: - _LOGGER.info('Not initializing inactive stream') - return - - backend_stream = self._firestore.read_write_stream( - self._api.Firestore._listen.bind(self._api.Firestore), - request, - ) - - if not is_active: - _LOGGER.info('Closing inactive stream') - backend_stream.end() - _LOGGER.info('Opened new stream') - current_stream = backend_stream - - def on_error(err): - maybe_reopen_stream(err) - - current_stream.on('error')(on_error) - - def on_end(): - err = Error('Stream ended unexpectedly') - err.code = GRPC_STATUS_CODE['UNKNOWN'] - maybe_reopen_stream(err) - - current_stream.on('end')(on_end) - current_stream.pipe(stream) - current_stream.resume() - - current_stream.catch(close_stream) - - def affects_target(target_ids, current_id): - for target_id in target_ids: - if target_id == current_id: - return True - return False - - def extract_changes(doc_map, changes, read_time): - deletes = [] - adds = [] - updates = [] - - for value, name in changes: - if value == REMOVED: - if doc_map.has(name): - deletes.append(name) - elif doc_map.has(name): - value.read_time = read_time - upates.append(value.build()) - else: - value.read_time = read_time - adds.append(value.build()) - return deletes, adds, updates - - def compute_snapshot(doc_tree, doc_map, changes): - if len(doc_tree) != doc_map: - raise ValueError('The document tree and document map should' - 'have the same number of entries.') - updated_tree = doc_tree - updated_map = doc_map - - def delete_doc(name): - """ raises KeyError if name not in updated_map""" - old_document = updated_map.pop(name) # Raises KeyError - existing = updated_tree.find(old_document) - old_index = existing.index - updated_tree = existing.remove() - return DocumentChange('removed', - old_document, - old_index, - -1) - - def add_doc(new_document): - name = new_document.ref.formatted_name - if name in updated_map: - raise ValueError('Document to add already exists') - updated_tree = updated_tree.insert(new_document, null) - new_index = updated_tree.find(new_document).index - updated_map[name] = new_document - return DocumentChange('added', - new_document, - -1, - new_index) - - def modify_doc(new_document): - name = new_document.ref.formattedName - if name not in updated_map: - raise ValueError('Document to modify does not exsit') - old_document = updated_map[name] - if old_document.update_time != new_document.update_time: - remove_change = delete_doc(name) - add_change = add_doc(new_document) - return DocumentChange('modified', - new_document, - remove_change.old_index, - add_change.new_index) - return None - - applied_changes = [] - - def compartor_sort(name1, name2): - return self._comparator(updated_map[name1], updated_map[name2]) - changes.deletes.sort(comparator_sort) - - for name in changes.deletes: - changes.delete_doc(name) - if change: - applied_changes.push(change) - - changes.adds.sort(self._compartor) - - for snapshot in changes.adds: - change = add_doc(snapshot) - if change: - applied_changes.push(change) - - changes.updates.sort(self._compartor) - - for snapshot in changes.updates: - change = modify_doc(snapshot) - if change: - applied_changes.push(change) - - if not len(updated_tree) == len(updated_map): - raise RuntimeError('The update document tree and document ' - 'map should have the same number of ' - 'entries') - - return {updated_tree, updated_map, applied_changes} - - def push(read_time, next_resume_token): - changes = extract_changes(doc_map, change_map, read_time) - diff = compute_snapshot(doc_tree, doc_map, changes) - - if not has_pushed or len(diff.applied_changes) > 0: - _LOGGER.info( - 'Sending snapshot with %d changes and %d documents' - % (len(diff.applied_changes), len(updated_tree))) - - next(read_time, diff.updatedTree.keys, diff.applied_changes) - - doc_tree = diff.updated_tree - doc_map = diff.updated_map - change_map.clear() - resume_token = next_resume_token - - def current_size(): - changes = extract_changes(doc_map, change_map) - return doc_map.size + len(changes.adds) - len(changes.deletes) - - init_stream() - - def proto(): - if proto.target_change: - _LOGGER.log('Processing target change') - change = proto.target_change - no_target_ids = not target_ids - if change.target_change_type == 'NO_CHANGE': - if no_target_ids and change.read_time and current: - push(DocumentSnapshot.to_ISO_time(change.read_time), - change.resume_token) - elif change.target_change_type == 'ADD': - if WATCH_TARGET_ID != change.target_ids[0]: - raise ValueError('Unexpected target ID sent by server') - elif change.target_change_type == 'REMOVE': - code = 13 - message = 'internal error' - if change.cause: - code = change.cause.code - message = change.cause.message - close_stream(Error('Error ' + code + ': ' + message)) - elif change.target_change_type == 'RESET': - reset_docs() - elif change.target_change_type == 'CURRENT': - current = true - else: - close_stream( - Error('Unknown target change type: ' + str(change))) - - stream.on('data', proto) # ?? - - if change.resume_token and \ - affects_target(change.target_ids, WATCH_TARGET_ID): - self._backoff.reset() - - elif proto.document_change: - _LOGGER.info('Processing change event') - - target_ids = proto.document_change.target_ids - removed_target_ids = proto.document_change.removed_target_ids - - changed = False - - removed = False - for target_id in target_ids: - if target_id == WATCH_TARGET_ID: - changed = True - - for target_id in removed_target_ids: - if removed_target_ids == WATCH_TARGET_ID: - removed = True - - document = proto.document_change.document - name = document.name - - if changed: - _LOGGER.info('Received document change') - snapshot = DocumentSnapshot.Builder() - snapshot.ref = DocumentReference( - self._firestore, - ResourcePath.from_slash_separated_string(name)) - snapshot.fields_proto = document.fields - snapshot.create_time = DocumentSnapshot.to_ISO_time( - document.create_time) - snapshot.update_time = DocumentSnapshot.to_ISO_time( - document.update_time) - change_map[name] = snapshot - elif removed: - _LOGGER.info('Received document remove') - change_map[name] = REMOVED - elif proto.document_delete: - _LOGGER.info('Processing remove event') - name = proto.document_delete.document - change_map[name] = REMOVED - elif proto.document_remove: - _LOGGER.info('Processing remove event') - name = proto.document_remove.document - change_map[name] = REMOVED - elif proto.filter: - _LOGGER.info('Processing filter update') - if proto.filter.count != current_size(): - reset_docs() - reset_stream() - else: - close_stream(Error('Unknown listen response type: ' + str(proto))) - - def on_end(): - _LOGGER.info('Processing stream end') - if current_stream: - current_stream.end() - - on('end', on_end) - - def initialize(): - return {} - - def end_stream(): - _LOGGER.info('Ending stream') - is_active = False - on_next = initialize - on_error = initialize - stream.end() - - return end_stream - - - + # def on_snapshot(self, on_next, on_error): + # doc_dict = {} + # doc_map = {} + # change_map = {} + + # current = False + # has_pushed = False + # is_active = True + + # REMOVED = {} + + # request = {'database': self._firestore.formatted_name, + # 'add_target': self._targets} + + # stream = through.obj() # TODO: fix through (node holdover) + + # current_stream = None + + # def reset_docs(): + # log() + # change_map.clear() + # del resume_token + # for snapshot in doc_dict: + # change_map.set(snapshot.ref.formatted_name, REMOVED) + # current = False + + # def close_stream(err): + # if current_stream is not None: + # current_stream.unpipe(stream) + # current_stream.end() + # current_stream = None + # stream.end() + + # if is_active: + # is_active = False + # _LOGGER.error('Invoking on_error: ', err) + # on_error(err) + + # def maybe_reopen_stream(err): + # if is_active and not is_permanent_error(err): + # _LOGGER.error( + # 'Stream ended, re-opening after retryable error: ', err) + # request.add_target.resume_token = resume_token + # change_map.clear() + + # if is_resource_exhausted_error(err): + # self._backoff.reset_to_max() + # reset_stream() + # else: + # _LOGGER.error('Stream ended, sending error: ', err) + # close_stream(err) + + # def reset_stream(): + # _LOGGER.info('Opening new stream') + # if current_stream: + # current_stream.unpipe(stream) + # current_stream.end() + # current_stream = None + # init_stream() + + # def init_stream(): + # self._backoff.back_off() + # if not is_active: + # _LOGGER.info('Not initializing inactive stream') + # return + + # backend_stream = self._firestore.read_write_stream( + # self._api.Firestore._listen.bind(self._api.Firestore), + # request, + # ) + + # if not is_active: + # _LOGGER.info('Closing inactive stream') + # backend_stream.end() + # _LOGGER.info('Opened new stream') + # current_stream = backend_stream + + # def on_error(err): + # maybe_reopen_stream(err) + + # current_stream.on('error')(on_error) + + # def on_end(): + # err = Exception('Stream ended unexpectedly') + # err.code = GRPC_STATUS_CODE['UNKNOWN'] + # maybe_reopen_stream(err) + + # current_stream.on('end')(on_end) + # current_stream.pipe(stream) + # current_stream.resume() + + # current_stream.catch(close_stream) + + # def affects_target(target_ids, current_id): + # for target_id in target_ids: + # if target_id == current_id: + # return True + # return False + + # def extract_changes(doc_map, changes, read_time): + # deletes = [] + # adds = [] + # updates = [] + + # for value, name in changes: + # if value == REMOVED: + # if doc_map.has(name): + # deletes.append(name) + # elif doc_map.has(name): + # value.read_time = read_time + # updates.append(value.build()) + # else: + # value.read_time = read_time + # adds.append(value.build()) + # return deletes, adds, updates + + # def compute_snapshot(doc_dict, doc_map, changes): + # if len(doc_dict) != doc_map: + # raise ValueError('The document tree and document map should' + # 'have the same number of entries.') + # updated_dict = doc_dict + # updated_map = doc_map + + # def delete_doc(name): + # """ raises KeyError if name not in updated_map""" + # old_document = updated_map.pop(name) # Raises KeyError + # existing = updated_dict.find(old_document) + # old_index = existing.index + # updated_dict = existing.remove() + # return DocumentChange('removed', + # old_document, + # old_index, + # -1) + + # def add_doc(new_document): + # name = new_document.ref.formatted_name + # if name in updated_map: + # raise ValueError('Document to add already exists') + # updated_dict = updated_dict.insert(new_document, null) + # new_index = updated_dict.find(new_document).index + # updated_map[name] = new_document + # return DocumentChange('added', + # new_document, + # -1, + # new_index) + + # def modify_doc(new_document): + # name = new_document.ref.formattedName + # if name not in updated_map: + # raise ValueError('Document to modify does not exsit') + # old_document = updated_map[name] + # if old_document.update_time != new_document.update_time: + # remove_change = delete_doc(name) + # add_change = add_doc(new_document) + # return DocumentChange('modified', + # new_document, + # remove_change.old_index, + # add_change.new_index) + # return None + + # applied_changes = [] + + # def comparator_sort(name1, name2): + # return self._comparator(updated_map[name1], updated_map[name2]) + + # changes.deletes.sort(comparator_sort) + + # for name in changes.deletes: + # changes.delete_doc(name) + # if change: + # applied_changes.push(change) + + # changes.adds.sort(self._compartor) + + # for snapshot in changes.adds: + # change = add_doc(snapshot) + # if change: + # applied_changes.push(change) + + # changes.updates.sort(self._compartor) + + # for snapshot in changes.updates: + # change = modify_doc(snapshot) + # if change: + # applied_changes.push(change) + + # if not len(updated_dict) == len(updated_map): + # raise RuntimeError('The update document tree and document ' + # 'map should have the same number of ' + # 'entries') + + # return {updated_dict, updated_map, applied_changes} + + # def push(read_time, next_resume_token): + # changes = extract_changes(doc_map, change_map, read_time) + # diff = compute_snapshot(doc_dict, doc_map, changes) + + # if not has_pushed or len(diff.applied_changes) > 0: + # _LOGGER.info( + # 'Sending snapshot with %d changes and %d documents' + # % (len(diff.applied_changes), len(updated_dict))) + + # next(read_time, diff.updatedTree.keys, diff.applied_changes) + + # doc_dict = diff.updated_dict + # doc_map = diff.updated_map + # change_map.clear() + # resume_token = next_resume_token + + # def current_size(): + # changes = extract_changes(doc_map, change_map) + # return doc_map.size + len(changes.adds) - len(changes.deletes) + + # init_stream() + + # def proto(): + # if proto.target_change: + # _LOGGER.log('Processing target change') + # change = proto.target_change + # no_target_ids = not target_ids + # if change.target_change_type == 'NO_CHANGE': + # if no_target_ids and change.read_time and current: + # push(DocumentSnapshot.to_ISO_time(change.read_time), + # change.resume_token) + # elif change.target_change_type == 'ADD': + # if WATCH_TARGET_ID != change.target_ids[0]: + # raise ValueError('Unexpected target ID sent by server') + # elif change.target_change_type == 'REMOVE': + # code = 13 + # message = 'internal error' + # if change.cause: + # code = change.cause.code + # message = change.cause.message + # close_stream(Error('Error ' + code + ': ' + message)) + # elif change.target_change_type == 'RESET': + # reset_docs() + # elif change.target_change_type == 'CURRENT': + # current = true + # else: + # close_stream( + # Exception('Unknown target change type: ' + str(change))) + + # stream.on('data', proto) # ?? + + # if change.resume_token and \ + # affects_target(change.target_ids, WATCH_TARGET_ID): + # self._backoff.reset() + + # elif proto.document_change: + # _LOGGER.info('Processing change event') + + # target_ids = proto.document_change.target_ids + # removed_target_ids = proto.document_change.removed_target_ids + + # changed = False + + # removed = False + # for target_id in target_ids: + # if target_id == WATCH_TARGET_ID: + # changed = True + + # for target_id in removed_target_ids: + # if removed_target_ids == WATCH_TARGET_ID: + # removed = True + + # document = proto.document_change.document + # name = document.name + + # if changed: + # _LOGGER.info('Received document change') + # snapshot = DocumentSnapshot.Builder() + # snapshot.ref = DocumentReference( + # self._firestore, + # ResourcePath.from_slash_separated_string(name)) + # snapshot.fields_proto = document.fields + # snapshot.create_time = DocumentSnapshot.to_ISO_time( + # document.create_time) + # snapshot.update_time = DocumentSnapshot.to_ISO_time( + # document.update_time) + # change_map[name] = snapshot + # elif removed: + # _LOGGER.info('Received document remove') + # change_map[name] = REMOVED + # elif proto.document_delete: + # _LOGGER.info('Processing remove event') + # name = proto.document_delete.document + # change_map[name] = REMOVED + # elif proto.document_remove: + # _LOGGER.info('Processing remove event') + # name = proto.document_remove.document + # change_map[name] = REMOVED + # elif proto.filter: + # _LOGGER.info('Processing filter update') + # if proto.filter.count != current_size(): + # reset_docs() + # reset_stream() + # else: + # close_stream(Error('Unknown listen response type: ' + str(proto))) + + # def on_end(): + # _LOGGER.info('Processing stream end') + # if current_stream: + # current_stream.end() + + # on('end', on_end) + + # def initialize(): + # return {} + + # def end_stream(): + # _LOGGER.info('Ending stream') + # is_active = False + # on_next = initialize + # on_error = initialize + # stream.end() + + # return end_stream - - - - - - - - \ No newline at end of file diff --git a/firestore/tests/system.py b/firestore/tests/system.py index 53b529a91966..5f93b54e751e 100644 --- a/firestore/tests/system.py +++ b/firestore/tests/system.py @@ -794,3 +794,681 @@ def test_batch(client, cleanup): assert snapshot2.update_time == write_result2.update_time assert not document3.get().exists +def test_watch_document(client, cleanup): + # Add a new document + db = client + doc_ref = db.collection(u'users').document(u'alovelace') + doc_ref.set({ + u'first': u'Ada', + u'last': u'Lovelace', + u'born': 1815 + }) + doc_ref.on_snapshot(None, None) + + +# def test_create_document(client, cleanup): +# now = datetime.datetime.utcnow().replace(tzinfo=UTC) +# document_id = 'shun' + unique_resource_id('-') +# document = client.document('collek', document_id) +# # Add to clean-up before API request (in case ``create()`` fails). +# cleanup(document) + +# data = { +# 'now': firestore.SERVER_TIMESTAMP, +# 'eenta-ger': 11, +# 'bites': b'\xe2\x98\x83 \xe2\x9b\xb5', +# 'also': { +# 'nestednow': firestore.SERVER_TIMESTAMP, +# 'quarter': 0.25, +# }, +# } +# write_result = document.create(data) +# updated = _pb_timestamp_to_datetime(write_result.update_time) +# delta = updated - now +# # Allow a bit of clock skew, but make sure timestamps are close. +# assert -300.0 < delta.total_seconds() < 300.0 + +# with pytest.raises(AlreadyExists): +# document.create(data) + +# # Verify the server times. +# snapshot = document.get() +# stored_data = snapshot.to_dict() +# server_now = stored_data['now'] + +# delta = updated - server_now +# # NOTE: We could check the ``transform_results`` from the write result +# # for the document transform, but this value gets dropped. Instead +# # we make sure the timestamps are close. +# assert 0.0 <= delta.total_seconds() < 5.0 +# expected_data = { +# 'now': server_now, +# 'eenta-ger': data['eenta-ger'], +# 'bites': data['bites'], +# 'also': { +# 'nestednow': server_now, +# 'quarter': data['also']['quarter'], +# }, +# } +# assert stored_data == expected_data + + +# def test_cannot_use_foreign_key(client, cleanup): +# document_id = 'cannot' + unique_resource_id('-') +# document = client.document('foreign-key', document_id) +# # Add to clean-up before API request (in case ``create()`` fails). +# cleanup(document) + +# other_client = firestore.Client( +# project='other-prahj', +# credentials=client._credentials, +# database='dee-bee') +# assert other_client._database_string != client._database_string +# fake_doc = other_client.document('foo', 'bar') +# with pytest.raises(InvalidArgument): +# document.create({'ref': fake_doc}) + + +# def assert_timestamp_less(timestamp_pb1, timestamp_pb2): +# dt_val1 = _pb_timestamp_to_datetime(timestamp_pb1) +# dt_val2 = _pb_timestamp_to_datetime(timestamp_pb2) +# assert dt_val1 < dt_val2 + + +# def test_no_document(client, cleanup): +# document_id = 'no_document' + unique_resource_id('-') +# document = client.document('abcde', document_id) +# snapshot = document.get() +# assert snapshot.to_dict() is None + + +# def test_document_set(client, cleanup): +# document_id = 'for-set' + unique_resource_id('-') +# document = client.document('i-did-it', document_id) +# # Add to clean-up before API request (in case ``set()`` fails). +# cleanup(document) + +# # 0. Make sure the document doesn't exist yet +# snapshot = document.get() +# assert snapshot.to_dict() is None + +# # 1. Use ``create()`` to create the document. +# data1 = {'foo': 88} +# write_result1 = document.create(data1) +# snapshot1 = document.get() +# assert snapshot1.to_dict() == data1 +# # Make sure the update is what created the document. +# assert snapshot1.create_time == snapshot1.update_time +# assert snapshot1.update_time == write_result1.update_time + +# # 2. Call ``set()`` again to overwrite. +# data2 = {'bar': None} +# write_result2 = document.set(data2) +# snapshot2 = document.get() +# assert snapshot2.to_dict() == data2 +# # Make sure the create time hasn't changed. +# assert snapshot2.create_time == snapshot1.create_time +# assert snapshot2.update_time == write_result2.update_time + + +# def test_document_integer_field(client, cleanup): +# document_id = 'for-set' + unique_resource_id('-') +# document = client.document('i-did-it', document_id) +# # Add to clean-up before API request (in case ``set()`` fails). +# cleanup(document) + +# data1 = { +# '1a': { +# '2b': '3c', +# 'ab': '5e'}, +# '6f': { +# '7g': '8h', +# 'cd': '0j'} +# } +# document.create(data1) + +# data2 = {'1a.ab': '4d', '6f.7g': '9h'} +# option2 = client.write_option(exists=True) +# document.update(data2, option=option2) +# snapshot = document.get() +# expected = { +# '1a': { +# '2b': '3c', +# 'ab': '4d'}, +# '6f': { +# '7g': '9h', +# 'cd': '0j'} +# } +# assert snapshot.to_dict() == expected + + +# def test_document_set_merge(client, cleanup): +# document_id = 'for-set' + unique_resource_id('-') +# document = client.document('i-did-it', document_id) +# # Add to clean-up before API request (in case ``set()`` fails). +# cleanup(document) + +# # 0. Make sure the document doesn't exist yet +# snapshot = document.get() +# assert not snapshot.exists + +# # 1. Use ``create()`` to create the document. +# data1 = {'name': 'Sam', +# 'address': {'city': 'SF', +# 'state': 'CA'}} +# write_result1 = document.create(data1) +# snapshot1 = document.get() +# assert snapshot1.to_dict() == data1 +# # Make sure the update is what created the document. +# assert snapshot1.create_time == snapshot1.update_time +# assert snapshot1.update_time == write_result1.update_time + +# # 2. Call ``set()`` to merge +# data2 = {'address': {'city': 'LA'}} +# write_result2 = document.set(data2, merge=True) +# snapshot2 = document.get() +# assert snapshot2.to_dict() == {'name': 'Sam', +# 'address': {'city': 'LA', +# 'state': 'CA'}} +# # Make sure the create time hasn't changed. +# assert snapshot2.create_time == snapshot1.create_time +# assert snapshot2.update_time == write_result2.update_time + + +# def test_update_document(client, cleanup): +# document_id = 'for-update' + unique_resource_id('-') +# document = client.document('made', document_id) +# # Add to clean-up before API request (in case ``create()`` fails). +# cleanup(document) + +# # 0. Try to update before the document exists. +# with pytest.raises(NotFound) as exc_info: +# document.update({'not': 'there'}) +# assert exc_info.value.message.startswith(MISSING_DOCUMENT) +# assert document_id in exc_info.value.message + +# # 1. Try to update before the document exists (now with an option). +# option1 = client.write_option(exists=True) +# with pytest.raises(NotFound) as exc_info: +# document.update({'still': 'not-there'}, option=option1) +# assert exc_info.value.message.startswith(MISSING_DOCUMENT) +# assert document_id in exc_info.value.message + +# # 2. Update and create the document (with an option). +# data = { +# 'foo': { +# 'bar': 'baz', +# }, +# 'scoop': { +# 'barn': 981, +# }, +# 'other': True, +# } +# option2 = client.write_option(exists=False) +# write_result2 = document.update(data, option=option2) + +# # 3. Send an update without a field path (no option). +# field_updates3 = {'foo': {'quux': 800}} +# write_result3 = document.update(field_updates3) +# assert_timestamp_less(write_result2.update_time, write_result3.update_time) +# snapshot3 = document.get() +# expected3 = { +# 'foo': field_updates3['foo'], +# 'scoop': data['scoop'], +# 'other': data['other'], +# } +# assert snapshot3.to_dict() == expected3 + +# # 4. Send an update **with** a field path and a delete and a valid +# # "last timestamp" option. +# field_updates4 = { +# 'scoop.silo': None, +# 'other': firestore.DELETE_FIELD, +# } +# option4 = client.write_option(last_update_time=snapshot3.update_time) +# write_result4 = document.update(field_updates4, option=option4) +# assert_timestamp_less(write_result3.update_time, write_result4.update_time) +# snapshot4 = document.get() +# expected4 = { +# 'foo': field_updates3['foo'], +# 'scoop': { +# 'barn': data['scoop']['barn'], +# 'silo': field_updates4['scoop.silo'], +# }, +# } +# assert snapshot4.to_dict() == expected4 + +# # 5. Call ``update()`` with invalid (in the past) "last timestamp" option. +# assert_timestamp_less(option4._last_update_time, snapshot4.update_time) +# with pytest.raises(FailedPrecondition) as exc_info: +# document.update({'bad': 'time-past'}, option=option4) + +# # 6. Call ``update()`` with invalid (in future) "last timestamp" option. +# timestamp_pb = timestamp_pb2.Timestamp( +# seconds=snapshot4.update_time.nanos + 3600, +# nanos=snapshot4.update_time.nanos, +# ) +# option6 = client.write_option(last_update_time=timestamp_pb) +# with pytest.raises(FailedPrecondition) as exc_info: +# document.update({'bad': 'time-future'}, option=option6) + + +# def check_snapshot(snapshot, document, data, write_result): +# assert snapshot.reference is document +# assert snapshot.to_dict() == data +# assert snapshot.exists +# assert snapshot.create_time == write_result.update_time +# assert snapshot.update_time == write_result.update_time + + +# def test_document_get(client, cleanup): +# now = datetime.datetime.utcnow().replace(tzinfo=UTC) +# document_id = 'for-get' + unique_resource_id('-') +# document = client.document('created', document_id) +# # Add to clean-up before API request (in case ``create()`` fails). +# cleanup(document) + +# # First make sure it doesn't exist. +# assert not document.get().exists + +# ref_doc = client.document('top', 'middle1', 'middle2', 'bottom') +# data = { +# 'turtle': 'power', +# 'cheese': 19.5, +# 'fire': 199099299, +# 'referee': ref_doc, +# 'gio': firestore.GeoPoint(45.5, 90.0), +# 'deep': [ +# u'some', +# b'\xde\xad\xbe\xef', +# ], +# 'map': { +# 'ice': True, +# 'water': None, +# 'vapor': { +# 'deeper': now, +# }, +# }, +# } +# write_result = document.create(data) +# snapshot = document.get() +# check_snapshot(snapshot, document, data, write_result) +# assert_timestamp_less(snapshot.create_time, snapshot.read_time) + + +# def test_document_delete(client, cleanup): +# document_id = 'deleted' + unique_resource_id('-') +# document = client.document('here-to-be', document_id) +# # Add to clean-up before API request (in case ``create()`` fails). +# cleanup(document) +# document.create({'not': 'much'}) + +# # 1. Call ``delete()`` with invalid (in the past) "last timestamp" option. +# snapshot1 = document.get() +# timestamp_pb = timestamp_pb2.Timestamp( +# seconds=snapshot1.update_time.nanos - 3600, +# nanos=snapshot1.update_time.nanos, +# ) +# option1 = client.write_option(last_update_time=timestamp_pb) +# with pytest.raises(FailedPrecondition): +# document.delete(option=option1) + +# # 2. Call ``delete()`` with invalid (in future) "last timestamp" option. +# timestamp_pb = timestamp_pb2.Timestamp( +# seconds=snapshot1.update_time.nanos + 3600, +# nanos=snapshot1.update_time.nanos, +# ) +# option2 = client.write_option(last_update_time=timestamp_pb) +# with pytest.raises(FailedPrecondition): +# document.delete(option=option2) + +# # 3. Actually ``delete()`` the document. +# delete_time3 = document.delete() + +# # 4. ``delete()`` again, even though we know the document is gone. +# delete_time4 = document.delete() +# assert_timestamp_less(delete_time3, delete_time4) + + +# def test_collection_add(client, cleanup): +# collection1 = client.collection('collek') +# collection2 = client.collection('collek', 'shun', 'child') +# explicit_doc_id = 'hula' + unique_resource_id('-') + +# # Auto-ID at top-level. +# data1 = {'foo': 'bar'} +# update_time1, document_ref1 = collection1.add(data1) +# cleanup(document_ref1) +# snapshot1 = document_ref1.get() +# assert snapshot1.to_dict() == data1 +# assert snapshot1.create_time == update_time1 +# assert snapshot1.update_time == update_time1 +# assert RANDOM_ID_REGEX.match(document_ref1.id) + +# # Explicit ID at top-level. +# data2 = {'baz': 999} +# update_time2, document_ref2 = collection1.add( +# data2, document_id=explicit_doc_id) +# cleanup(document_ref2) +# snapshot2 = document_ref2.get() +# assert snapshot2.to_dict() == data2 +# assert snapshot2.create_time == update_time2 +# assert snapshot2.update_time == update_time2 +# assert document_ref2.id == explicit_doc_id + +# # Auto-ID for nested collection. +# data3 = {'quux': b'\x00\x01\x02\x03'} +# update_time3, document_ref3 = collection2.add(data3) +# cleanup(document_ref3) +# snapshot3 = document_ref3.get() +# assert snapshot3.to_dict() == data3 +# assert snapshot3.create_time == update_time3 +# assert snapshot3.update_time == update_time3 +# assert RANDOM_ID_REGEX.match(document_ref3.id) + +# # Explicit for nested collection. +# data4 = {'kazaam': None, 'bad': False} +# update_time4, document_ref4 = collection2.add( +# data4, document_id=explicit_doc_id) +# cleanup(document_ref4) +# snapshot4 = document_ref4.get() +# assert snapshot4.to_dict() == data4 +# assert snapshot4.create_time == update_time4 +# assert snapshot4.update_time == update_time4 +# assert document_ref4.id == explicit_doc_id + + +# def test_query_get(client, cleanup): +# sub_collection = 'child' + unique_resource_id('-') +# collection = client.collection('collek', 'shun', sub_collection) + +# stored = {} +# num_vals = 5 +# allowed_vals = six.moves.xrange(num_vals) +# for a_val in allowed_vals: +# for b_val in allowed_vals: +# document_data = { +# 'a': a_val, +# 'b': b_val, +# 'stats': { +# 'sum': a_val + b_val, +# 'product': a_val * b_val, +# }, +# } +# _, doc_ref = collection.add(document_data) +# # Add to clean-up. +# cleanup(doc_ref) +# stored[doc_ref.id] = document_data + +# # 0. Limit to snapshots where ``a==1``. +# query0 = collection.where('a', '==', 1) +# values0 = { +# snapshot.id: snapshot.to_dict() +# for snapshot in query0.get() +# } +# assert len(values0) == num_vals +# for key, value in six.iteritems(values0): +# assert stored[key] == value +# assert value['a'] == 1 + +# # 1. Order by ``b``. +# query1 = collection.order_by('b', direction=query0.DESCENDING) +# values1 = [ +# (snapshot.id, snapshot.to_dict()) +# for snapshot in query1.get() +# ] +# assert len(values1) == len(stored) +# b_vals1 = [] +# for key, value in values1: +# assert stored[key] == value +# b_vals1.append(value['b']) +# # Make sure the ``b``-values are in DESCENDING order. +# assert sorted(b_vals1, reverse=True) == b_vals1 + +# # 2. Limit to snapshots where ``stats.sum > 1`` (a field path). +# query2 = collection.where('stats.sum', '>', 4) +# values2 = { +# snapshot.id: snapshot.to_dict() +# for snapshot in query2.get() +# } +# assert len(values2) == 10 +# ab_pairs2 = set() +# for key, value in six.iteritems(values2): +# assert stored[key] == value +# ab_pairs2.add((value['a'], value['b'])) + +# expected_ab_pairs = set([ +# (a_val, b_val) +# for a_val in allowed_vals +# for b_val in allowed_vals +# if a_val + b_val > 4 +# ]) +# assert expected_ab_pairs == ab_pairs2 + +# # 3. Use a start and end cursor. +# query3 = collection.start_at({'a': num_vals - 2}) +# query3 = query3.order_by('a') +# query3 = query3.end_before({'a': num_vals - 1}) +# values3 = [ +# (snapshot.id, snapshot.to_dict()) +# for snapshot in query3.get() +# ] +# assert len(values3) == num_vals +# for key, value in values3: +# assert stored[key] == value +# assert value['a'] == num_vals - 2 +# b_vals1.append(value['b']) + +# # 4. Send a query with no results. +# query4 = collection.where('b', '==', num_vals + 100) +# values4 = list(query4.get()) +# assert len(values4) == 0 + +# # 5. Select a subset of fields. +# query5 = collection.where('b', '<=', 1) +# query5 = query5.select(['a', 'stats.product']) +# values5 = { +# snapshot.id: snapshot.to_dict() +# for snapshot in query5.get() +# } +# assert len(values5) == num_vals * 2 # a ANY, b in (0, 1) +# for key, value in six.iteritems(values5): +# expected = { +# 'a': stored[key]['a'], +# 'stats': { +# 'product': stored[key]['stats']['product'], +# }, +# } +# assert expected == value + +# # 6. Add multiple filters via ``where()``. +# query6 = collection.where('stats.product', '>', 5) +# query6 = query6.where('stats.product', '<', 10) +# values6 = { +# snapshot.id: snapshot.to_dict() +# for snapshot in query6.get() +# } + +# matching_pairs = [ +# (a_val, b_val) +# for a_val in allowed_vals +# for b_val in allowed_vals +# if 5 < a_val * b_val < 10 +# ] +# assert len(values6) == len(matching_pairs) +# for key, value in six.iteritems(values6): +# assert stored[key] == value +# pair = (value['a'], value['b']) +# assert pair in matching_pairs + +# # 7. Skip the first three results, when ``b==2`` +# query7 = collection.where('b', '==', 2) +# offset = 3 +# query7 = query7.offset(offset) +# values7 = { +# snapshot.id: snapshot.to_dict() +# for snapshot in query7.get() +# } +# # NOTE: We don't check the ``a``-values, since that would require +# # an ``order_by('a')``, which combined with the ``b == 2`` +# # filter would necessitate an index. +# assert len(values7) == num_vals - offset +# for key, value in six.iteritems(values7): +# assert stored[key] == value +# assert value['b'] == 2 + + +# def test_query_unary(client, cleanup): +# collection_name = 'unary' + unique_resource_id('-') +# collection = client.collection(collection_name) +# field_name = 'foo' + +# _, document0 = collection.add({field_name: None}) +# # Add to clean-up. +# cleanup(document0) + +# nan_val = float('nan') +# _, document1 = collection.add({field_name: nan_val}) +# # Add to clean-up. +# cleanup(document1) + +# # 0. Query for null. +# query0 = collection.where(field_name, '==', None) +# values0 = list(query0.get()) +# assert len(values0) == 1 +# snapshot0 = values0[0] +# assert snapshot0.reference._path == document0._path +# assert snapshot0.to_dict() == {field_name: None} + +# # 1. Query for a NAN. +# query1 = collection.where(field_name, '==', nan_val) +# values1 = list(query1.get()) +# assert len(values1) == 1 +# snapshot1 = values1[0] +# assert snapshot1.reference._path == document1._path +# data1 = snapshot1.to_dict() +# assert len(data1) == 1 +# assert math.isnan(data1[field_name]) + + +# def test_get_all(client, cleanup): +# collection_name = 'get-all' + unique_resource_id('-') + +# document1 = client.document(collection_name, 'a') +# document2 = client.document(collection_name, 'b') +# document3 = client.document(collection_name, 'c') +# # Add to clean-up before API requests (in case ``create()`` fails). +# cleanup(document1) +# cleanup(document3) + +# data1 = { +# 'a': { +# 'b': 2, +# 'c': 3, +# }, +# 'd': 4, +# 'e': 0, +# } +# write_result1 = document1.create(data1) +# data3 = { +# 'a': { +# 'b': 5, +# 'c': 6, +# }, +# 'd': 7, +# 'e': 100, +# } +# write_result3 = document3.create(data3) + +# # 0. Get 3 unique documents, one of which is missing. +# snapshots = list(client.get_all( +# [document1, document2, document3])) + +# assert snapshots[0].exists +# assert snapshots[1].exists +# assert not snapshots[2].exists +# snapshots = [snapshot for snapshot in snapshots if snapshot.exists] +# id_attr = operator.attrgetter('id') +# snapshots.sort(key=id_attr) + +# snapshot1, snapshot3 = snapshots +# check_snapshot(snapshot1, document1, data1, write_result1) +# check_snapshot(snapshot3, document3, data3, write_result3) + +# # 1. Get 2 colliding documents. +# document1_also = client.document(collection_name, 'a') +# snapshots = list(client.get_all([document1, document1_also])) + +# assert len(snapshots) == 1 +# assert document1 is not document1_also +# check_snapshot(snapshots[0], document1_also, data1, write_result1) + +# # 2. Use ``field_paths`` / projection in ``get_all()``. +# snapshots = list(client.get_all( +# [document1, document3], field_paths=['a.b', 'd'])) + +# assert len(snapshots) == 2 +# snapshots.sort(key=id_attr) + +# snapshot1, snapshot3 = snapshots +# restricted1 = { +# 'a': {'b': data1['a']['b']}, +# 'd': data1['d'], +# } +# check_snapshot(snapshot1, document1, restricted1, write_result1) +# restricted3 = { +# 'a': {'b': data3['a']['b']}, +# 'd': data3['d'], +# } +# check_snapshot(snapshot3, document3, restricted3, write_result3) + + +# def test_batch(client, cleanup): +# collection_name = 'batch' + unique_resource_id('-') + +# document1 = client.document(collection_name, 'abc') +# document2 = client.document(collection_name, 'mno') +# document3 = client.document(collection_name, 'xyz') +# # Add to clean-up before API request (in case ``create()`` fails). +# cleanup(document1) +# cleanup(document2) +# cleanup(document3) + +# data2 = { +# 'some': { +# 'deep': 'stuff', +# 'and': 'here', +# }, +# 'water': 100.0, +# } +# document2.create(data2) +# document3.create({'other': 19}) + +# batch = client.batch() +# data1 = {'all': True} +# batch.create(document1, data1) +# new_value = 'there' +# batch.update(document2, {'some.and': new_value}) +# batch.delete(document3) +# write_results = batch.commit() + +# assert len(write_results) == 3 + +# write_result1 = write_results[0] +# write_result2 = write_results[1] +# write_result3 = write_results[2] +# assert not write_result3.HasField('update_time') + +# snapshot1 = document1.get() +# assert snapshot1.to_dict() == data1 +# assert snapshot1.create_time == write_result1.update_time +# assert snapshot1.update_time == write_result1.update_time + +# snapshot2 = document2.get() +# assert snapshot2.to_dict() != data2 +# data2['some']['and'] = new_value +# assert snapshot2.to_dict() == data2 +# assert_timestamp_less(snapshot2.create_time, write_result2.update_time) +# assert snapshot2.update_time == write_result2.update_time + +# assert not document3.get().exists From 9499c8977318e4171513c30192e9f7481f4ffa13 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Wed, 27 Jun 2018 15:48:35 -0700 Subject: [PATCH 074/148] use helper for_document --- .../google/cloud/firestore_v1beta1/document.py | 16 +++++++--------- .../google/cloud/firestore_v1beta1/watch.py | 8 ++++---- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/document.py b/firestore/google/cloud/firestore_v1beta1/document.py index 8a32bbcf0de5..749e44b8f61e 100644 --- a/firestore/google/cloud/firestore_v1beta1/document.py +++ b/firestore/google/cloud/firestore_v1beta1/document.py @@ -451,15 +451,13 @@ def on_snapshot(self, options, callback): given options and the callback, monitor this document for changes ''' #google.firestore.v1beta1.Target.DocumentsTarget - documentsTarget = Target.DocumentsTarget( - documents=[self._document_path]) - - Watch( - self._client, - Target( - documents=documentsTarget - ), - None) + # documentsTarget = Target.DocumentsTarget( + # documents=[self._document_path]) + Watch.for_document(self) + # Watch( + # self._client, + # Target(documents=documentsTarget), + # None) class DocumentSnapshot(object): diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index cf374c984cae..60835260eb21 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -145,9 +145,9 @@ def should_recover(exc): exc.code() == grpc.StatusCode.UNVAILABLE) initial_request = firestore_pb2.ListenRequest( - #database=firestore.database_root_path, + database=firestore._database_string, add_target=target - # database, add_taret, remove_target, labels + # database, add_target, remove_target, labels ) rpc = ResumableBidiRpc( @@ -164,7 +164,7 @@ def should_recover(exc): @classmethod def for_document(cls, document_ref): - return cls(document_ref.firestore, + return cls(document_ref._client, { 'documents': { 'documents': [document_ref._document_path]}, @@ -174,7 +174,7 @@ def for_document(cls, document_ref): @classmethod def for_query(cls, query): - return cls(query.firestore, + return cls(query._client, { 'query': query.to_proto(), 'target_id': WATCH_TARGET_ID From e9855fd2c0ac45c2cb6434cdaf6f99f541b6e132 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Tue, 3 Jul 2018 15:31:20 -0700 Subject: [PATCH 075/148] Staging changes to firestore for watch. still incomplete but returning document changes --- .../google/cloud/firestore_v1beta1/bidi.py | 3 +- .../cloud/firestore_v1beta1/document.py | 11 +- .../google/cloud/firestore_v1beta1/watch.py | 584 +++++++----------- firestore/tests/system.py | 32 +- 4 files changed, 267 insertions(+), 363 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/bidi.py b/firestore/google/cloud/firestore_v1beta1/bidi.py index 00877e70058e..e7629fc5df8e 100644 --- a/firestore/google/cloud/firestore_v1beta1/bidi.py +++ b/firestore/google/cloud/firestore_v1beta1/bidi.py @@ -196,6 +196,7 @@ def open(self): request_generator = _RequestQueueGenerator( self._request_queue, initial_request=self._initial_request) + print('request generator created') call = self._start_rpc(iter(request_generator)) request_generator.call = call @@ -442,7 +443,7 @@ def on_response(response): print(response) consumer = BackgroundConsumer(rpc, on_response) - consume.start() + consumer.start() Note that error handling *must* be done by using the provided ``bidi_rpc``'s ``add_done_callback``. This helper will automatically exit diff --git a/firestore/google/cloud/firestore_v1beta1/document.py b/firestore/google/cloud/firestore_v1beta1/document.py index 749e44b8f61e..ad37b450c254 100644 --- a/firestore/google/cloud/firestore_v1beta1/document.py +++ b/firestore/google/cloud/firestore_v1beta1/document.py @@ -446,18 +446,11 @@ def collections(self, page_size=None): iterator.item_to_value = _item_to_collection_ref return iterator - def on_snapshot(self, options, callback): + def on_snapshot(self, callback): ''' given options and the callback, monitor this document for changes ''' - #google.firestore.v1beta1.Target.DocumentsTarget - # documentsTarget = Target.DocumentsTarget( - # documents=[self._document_path]) - Watch.for_document(self) - # Watch( - # self._client, - # Target(documents=documentsTarget), - # None) + Watch.for_document(self, callback) class DocumentSnapshot(object): diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 60835260eb21..acf154971dea 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -13,16 +13,15 @@ # limitations under the License. import logging +import threading -#from google.cloud.firestore_v1beta1 import DocumentReference, DocumentSnapshot - -#from google.cloud.firestore_v1beta1.document import DocumentReference -#from google.cloud.firestore_v1beta1.document import DocumentSnapshot #import google.cloud.firestore_v1beta1.client as client -from google.cloud.firestore_v1beta1.bidi import BidiRpc, ResumableBidiRpc +from google.cloud.firestore_v1beta1.bidi import BidiRpc, ResumableBidiRpc, BackgroundConsumer from google.cloud.firestore_v1beta1.proto import firestore_pb2 +from google.api_core import exceptions + -#from bidi import BidiRpc, ResumableBidiRpc +# from bidi import BidiRpc, ResumableBidiRpc import time import random import grpc @@ -53,6 +52,21 @@ 'DATA_LOSS': 15, 'DO_NOT_USE': -1 } +_RPC_ERROR_THREAD_NAME = 'Thread-OnRpcTerminated' +_RETRYABLE_STREAM_ERRORS = ( + exceptions.DeadlineExceeded, + exceptions.ServiceUnavailable, + exceptions.InternalServerError, + exceptions.Unknown, + exceptions.GatewayTimeout +) + + +def _maybe_wrap_exception(exception): + """Wraps a gRPC exception class, if needed.""" + if isinstance(exception, grpc.RpcError): + return exceptions.from_grpc_error(exception) + return exception def is_permanent_error(self, error): @@ -127,12 +141,24 @@ def _sleep(self, current_sleep, max_sleep=_MAX_SLEEP, time.sleep(actual_sleep) return min(self.multiplier * self.current_sleep, self.max_sleep) + class Watch(object): def __init__(self, - firestore, #: client.Client, + document_reference, + firestore, target, - comparator): - + comparator, + on_response): + """ + Args: + firestore: + target: ß + comparator: + on_response: Callback method that reveives a + `google.cloud.firestore_v1beta1.types.ListenResponse` object to + be acted on. + """ + self._document_reference = document_reference self._firestore = firestore self._api = firestore._firestore_api self._targets = target @@ -145,360 +171,218 @@ def should_recover(exc): exc.code() == grpc.StatusCode.UNVAILABLE) initial_request = firestore_pb2.ListenRequest( - database=firestore._database_string, - add_target=target + database=self._firestore._database_string, + add_target=self._targets # database, add_target, remove_target, labels ) rpc = ResumableBidiRpc( - # self._api.firestore_stub.Listen, - #firestore_pb2.BetaFirestoreStub.Listen, self._api.firestore_stub.Listen, initial_request=initial_request, should_recover=should_recover) - rpc.open() + rpc.add_done_callback(self._on_rpc_done) + + def consumer_callback(response): + processed_response = self.process_response(response) + if processed_response: + _LOGGER.debug("running provided callback") + on_response(processed_response) + + self._consumer = BackgroundConsumer(rpc, consumer_callback) + self._consumer.start() + + def _on_rpc_done(self, future): + """Triggered whenever the underlying RPC terminates without recovery. - while rpc.is_active: - print(rpc.recv()) + This is typically triggered from one of two threads: the background + consumer thread (when calling ``recv()`` produces a non-recoverable + error) or the grpc management thread (when cancelling the RPC). + + This method is *non-blocking*. It will start another thread to deal + with shutting everything down. This is to prevent blocking in the + background consumer and preventing it from being ``joined()``. + """ + # TODO: look at pushing this down into the background consumer + _LOGGER.info( + 'RPC termination has signaled shutdown.') + future = _maybe_wrap_exception(future) + thread = threading.Thread( + name=_RPC_ERROR_THREAD_NAME, + target=self.close, + kwargs={'reason': future}) + thread.daemon = True + thread.start() @classmethod - def for_document(cls, document_ref): - return cls(document_ref._client, + def for_document(cls, document_ref, on_response): + """ + Creates a watch snapshot listener for a document. on_response receives + a DocumentChange object, but may also start to get targetChange and such + soon + """ + + + return cls(document_ref, + document_ref._client, { 'documents': { 'documents': [document_ref._document_path]}, 'target_id': WATCH_TARGET_ID }, - document_watch_comparator) - - @classmethod - def for_query(cls, query): - return cls(query._client, - { - 'query': query.to_proto(), - 'target_id': WATCH_TARGET_ID - }, - query.comparator()) - - - # def on_snapshot(self, on_next, on_error): - # doc_dict = {} - # doc_map = {} - # change_map = {} - - # current = False - # has_pushed = False - # is_active = True - - # REMOVED = {} - - # request = {'database': self._firestore.formatted_name, - # 'add_target': self._targets} - - # stream = through.obj() # TODO: fix through (node holdover) - - # current_stream = None - - # def reset_docs(): - # log() - # change_map.clear() - # del resume_token - # for snapshot in doc_dict: - # change_map.set(snapshot.ref.formatted_name, REMOVED) - # current = False - - # def close_stream(err): - # if current_stream is not None: - # current_stream.unpipe(stream) - # current_stream.end() - # current_stream = None - # stream.end() - - # if is_active: - # is_active = False - # _LOGGER.error('Invoking on_error: ', err) - # on_error(err) - - # def maybe_reopen_stream(err): - # if is_active and not is_permanent_error(err): - # _LOGGER.error( - # 'Stream ended, re-opening after retryable error: ', err) - # request.add_target.resume_token = resume_token - # change_map.clear() - - # if is_resource_exhausted_error(err): - # self._backoff.reset_to_max() - # reset_stream() - # else: - # _LOGGER.error('Stream ended, sending error: ', err) - # close_stream(err) - - # def reset_stream(): - # _LOGGER.info('Opening new stream') - # if current_stream: - # current_stream.unpipe(stream) - # current_stream.end() - # current_stream = None - # init_stream() - - # def init_stream(): - # self._backoff.back_off() - # if not is_active: - # _LOGGER.info('Not initializing inactive stream') - # return - - # backend_stream = self._firestore.read_write_stream( - # self._api.Firestore._listen.bind(self._api.Firestore), - # request, - # ) - - # if not is_active: - # _LOGGER.info('Closing inactive stream') - # backend_stream.end() - # _LOGGER.info('Opened new stream') - # current_stream = backend_stream - - # def on_error(err): - # maybe_reopen_stream(err) - - # current_stream.on('error')(on_error) - - # def on_end(): - # err = Exception('Stream ended unexpectedly') - # err.code = GRPC_STATUS_CODE['UNKNOWN'] - # maybe_reopen_stream(err) - - # current_stream.on('end')(on_end) - # current_stream.pipe(stream) - # current_stream.resume() - - # current_stream.catch(close_stream) - - # def affects_target(target_ids, current_id): - # for target_id in target_ids: - # if target_id == current_id: - # return True - # return False - - # def extract_changes(doc_map, changes, read_time): - # deletes = [] - # adds = [] - # updates = [] - - # for value, name in changes: - # if value == REMOVED: - # if doc_map.has(name): - # deletes.append(name) - # elif doc_map.has(name): - # value.read_time = read_time - # updates.append(value.build()) - # else: - # value.read_time = read_time - # adds.append(value.build()) - # return deletes, adds, updates - - # def compute_snapshot(doc_dict, doc_map, changes): - # if len(doc_dict) != doc_map: - # raise ValueError('The document tree and document map should' - # 'have the same number of entries.') - # updated_dict = doc_dict - # updated_map = doc_map - - # def delete_doc(name): - # """ raises KeyError if name not in updated_map""" - # old_document = updated_map.pop(name) # Raises KeyError - # existing = updated_dict.find(old_document) - # old_index = existing.index - # updated_dict = existing.remove() - # return DocumentChange('removed', - # old_document, - # old_index, - # -1) - - # def add_doc(new_document): - # name = new_document.ref.formatted_name - # if name in updated_map: - # raise ValueError('Document to add already exists') - # updated_dict = updated_dict.insert(new_document, null) - # new_index = updated_dict.find(new_document).index - # updated_map[name] = new_document - # return DocumentChange('added', - # new_document, - # -1, - # new_index) - - # def modify_doc(new_document): - # name = new_document.ref.formattedName - # if name not in updated_map: - # raise ValueError('Document to modify does not exsit') - # old_document = updated_map[name] - # if old_document.update_time != new_document.update_time: - # remove_change = delete_doc(name) - # add_change = add_doc(new_document) - # return DocumentChange('modified', - # new_document, - # remove_change.old_index, - # add_change.new_index) - # return None - - # applied_changes = [] - - # def comparator_sort(name1, name2): - # return self._comparator(updated_map[name1], updated_map[name2]) - - # changes.deletes.sort(comparator_sort) - - # for name in changes.deletes: - # changes.delete_doc(name) - # if change: - # applied_changes.push(change) - - # changes.adds.sort(self._compartor) - - # for snapshot in changes.adds: - # change = add_doc(snapshot) - # if change: - # applied_changes.push(change) - - # changes.updates.sort(self._compartor) - - # for snapshot in changes.updates: - # change = modify_doc(snapshot) - # if change: - # applied_changes.push(change) - - # if not len(updated_dict) == len(updated_map): - # raise RuntimeError('The update document tree and document ' - # 'map should have the same number of ' - # 'entries') - - # return {updated_dict, updated_map, applied_changes} - - # def push(read_time, next_resume_token): - # changes = extract_changes(doc_map, change_map, read_time) - # diff = compute_snapshot(doc_dict, doc_map, changes) - - # if not has_pushed or len(diff.applied_changes) > 0: - # _LOGGER.info( - # 'Sending snapshot with %d changes and %d documents' - # % (len(diff.applied_changes), len(updated_dict))) - - # next(read_time, diff.updatedTree.keys, diff.applied_changes) - - # doc_dict = diff.updated_dict - # doc_map = diff.updated_map - # change_map.clear() - # resume_token = next_resume_token - - # def current_size(): - # changes = extract_changes(doc_map, change_map) - # return doc_map.size + len(changes.adds) - len(changes.deletes) - - # init_stream() - - # def proto(): - # if proto.target_change: - # _LOGGER.log('Processing target change') - # change = proto.target_change - # no_target_ids = not target_ids - # if change.target_change_type == 'NO_CHANGE': - # if no_target_ids and change.read_time and current: - # push(DocumentSnapshot.to_ISO_time(change.read_time), - # change.resume_token) - # elif change.target_change_type == 'ADD': - # if WATCH_TARGET_ID != change.target_ids[0]: - # raise ValueError('Unexpected target ID sent by server') - # elif change.target_change_type == 'REMOVE': - # code = 13 - # message = 'internal error' - # if change.cause: - # code = change.cause.code - # message = change.cause.message - # close_stream(Error('Error ' + code + ': ' + message)) - # elif change.target_change_type == 'RESET': - # reset_docs() - # elif change.target_change_type == 'CURRENT': - # current = true - # else: - # close_stream( - # Exception('Unknown target change type: ' + str(change))) - - # stream.on('data', proto) # ?? - - # if change.resume_token and \ - # affects_target(change.target_ids, WATCH_TARGET_ID): - # self._backoff.reset() - - # elif proto.document_change: - # _LOGGER.info('Processing change event') - - # target_ids = proto.document_change.target_ids - # removed_target_ids = proto.document_change.removed_target_ids - - # changed = False - - # removed = False - # for target_id in target_ids: - # if target_id == WATCH_TARGET_ID: - # changed = True - - # for target_id in removed_target_ids: - # if removed_target_ids == WATCH_TARGET_ID: - # removed = True - - # document = proto.document_change.document - # name = document.name - - # if changed: - # _LOGGER.info('Received document change') - # snapshot = DocumentSnapshot.Builder() - # snapshot.ref = DocumentReference( - # self._firestore, - # ResourcePath.from_slash_separated_string(name)) - # snapshot.fields_proto = document.fields - # snapshot.create_time = DocumentSnapshot.to_ISO_time( - # document.create_time) - # snapshot.update_time = DocumentSnapshot.to_ISO_time( - # document.update_time) - # change_map[name] = snapshot - # elif removed: - # _LOGGER.info('Received document remove') - # change_map[name] = REMOVED - # elif proto.document_delete: - # _LOGGER.info('Processing remove event') - # name = proto.document_delete.document - # change_map[name] = REMOVED - # elif proto.document_remove: - # _LOGGER.info('Processing remove event') - # name = proto.document_remove.document - # change_map[name] = REMOVED - # elif proto.filter: - # _LOGGER.info('Processing filter update') - # if proto.filter.count != current_size(): - # reset_docs() - # reset_stream() - # else: - # close_stream(Error('Unknown listen response type: ' + str(proto))) - - # def on_end(): - # _LOGGER.info('Processing stream end') - # if current_stream: - # current_stream.end() - - # on('end', on_end) - - # def initialize(): - # return {} - - # def end_stream(): - # _LOGGER.info('Ending stream') - # is_active = False - # on_next = initialize - # on_error = initialize - # stream.end() - - # return end_stream - - - + document_watch_comparator, + on_response) + + # @classmethod + # def for_query(cls, query, on_response): + # return cls(query._client, + # { + # 'query': query.to_proto(), + # 'target_id': WATCH_TARGET_ID + # }, + # query.comparator(), + # on_response) + + def process_response(self, proto): + """ + Args: + listen_response(`google.cloud.firestore_v1beta1.types.ListenResponse`): + Callback method that reveives a object to + """ + _LOGGER.debug('process_response') + TargetChange = firestore_pb2.TargetChange + + # TODO FIGURE OUT CONDITIONAL OF THIS + _LOGGER.debug(f"STATE: document_change: {proto.document_change} target_change: {proto.target_change} target_change_type: {proto.target_change.target_change_type}") + if str(proto.document_change): + _LOGGER.debug("Document Change") + if str(proto.target_change): + _LOGGER.debug("Target Change") + + if str(proto.target_change): + _LOGGER.info('process_response: Processing target change') + change = proto.target_change + + notarget_ids = change.target_ids is None or len(change.target_ids) + if change.target_change_type == TargetChange.NO_CHANGE: + _LOGGER.info("process_response: " + "Processing target change NO_CHANGE") + # if notarget_ids and change.read_time and current) { + # // This means everything is up-to-date, so emit the current set of + # // docs as a snapshot, if there were changes. + # push( + # DocumentSnapshot.toISOTime(change.readTime), + # change.resumeToken + # ); + # } + elif change.target_change_type == TargetChange.ADD: + _LOGGER.info('process_response: Processing target change ADD') + assert WATCH_TARGET_ID == change.target_ids[0], 'Unexpected target ID sent by server' + + elif change.target_change_type == TargetChange.REMOVE: + _LOGGER.info("process_response: " + "Processing target change REMOVE") + # let code = 13; + # let message = 'internal error'; + # if (change.cause) { + # code = change.cause.code; + # message = change.cause.message; + # } + # // @todo: Surface a .code property on the exception. + # closeStream(new Error('Error ' + code + ': ' + message)); + elif change.target_change_type == TargetChange.RESET: + _LOGGER.info("process_response: " + "Processing target change RESET") + # // Whatever changes have happened so far no longer matter. + # resetDocs(); + elif change.target_change_type == TargetChange.CURRENT: + _LOGGER.info("process_response: " + "Processing target change CURRENT") + # current = True + else: + _LOGGER.info('process_response: Processing target change ELSE') + _LOGGER.info('process_response: Unknown target change ' + + str(change.target_change_type)) + + # closeStream( + # new Error('Unknown target change type: ' + JSON.stringify(change)) + + # if ( + # change.resumeToken and affectsTarget(change.target_ids, WATCH_TARGET_ID) + # ) { + # this._backoff.reset(); + # } + + elif str(proto.document_change): + _LOGGER.debug('Watch.onSnapshot Processing document_change event') + + # No other target_ids can show up here, but we still need to see if the + # targetId was in the added list or removed list. + target_ids = proto.document_change.target_ids or [] + removed_target_ids = proto.document_change.removed_target_ids or [] + changed = False + removed = False + + for target in target_ids: + if target == WATCH_TARGET_ID: + changed = True + + for target in removed_target_ids: + if target == WATCH_TARGET_ID: + removed = True + + document = proto.document_change.document + # name = document.name + + if changed: + _LOGGER.debug('Received document change') + + # reference = DocumentReference( + # self._firestore, + # ResourcePath.fromSlashSeparatedString(name)) + reference = self._document_reference + #create_time = DocumentSnapshot.toISOTime(document.create_time) + #update_time = DocumentSnapshot.toISOTime(document.update_time) + #read_time = DocumentSnapshot.toISOTime(document.read_time) + create_time = document.create_time + update_time = document.update_time + + #read_time = document.read_time + + # TODO: other clients seem to return snapshots + # snapshot = DocumentSnapshot( + # reference, + # document.fields, # DATA? + # exists=True, + # read_time=read_time, + # create_time=create_time, + # update_time=update_time) + # #changeMap.set(name, snapshot); + # return snapshot + return document + + elif removed: + _LOGGER.debug('Watch.onSnapshot Received document remove') + # changeMap.set(name, REMOVED); + + + # Document Delete or Document Remove? + elif (proto.document_delete or proto.document_remove): + _LOGGER.debug('Watch.onSnapshot Processing remove event') + # const name = (proto.document_delete || proto.document_remove).document + # changeMap.set(name, REMOVED); + + elif (proto.filter): + _LOGGER.debug('Watch.onSnapshot Processing filter update') + # if (proto.filter.count !== currentSize()) { + # // We need to remove all the current results. + # resetDocs(); + # // The filter didn't match, so re-issue the query. + # resetStream(); + + else: + _LOGGER.debug("UNKNOWN TYPE. UHOH") + # closeStream( + # new Error('Unknown listen response type: ' + JSON.stringify(proto)) + # ) + \ No newline at end of file diff --git a/firestore/tests/system.py b/firestore/tests/system.py index 5f93b54e751e..393c5a983935 100644 --- a/firestore/tests/system.py +++ b/firestore/tests/system.py @@ -32,6 +32,7 @@ from google.cloud import firestore from test_utils.system import unique_resource_id +from time import sleep FIRESTORE_CREDS = os.environ.get('FIRESTORE_APPLICATION_CREDENTIALS') FIRESTORE_PROJECT = os.environ.get('GCLOUD_PROJECT') @@ -795,15 +796,40 @@ def test_batch(client, cleanup): assert not document3.get().exists def test_watch_document(client, cleanup): - # Add a new document db = client - doc_ref = db.collection(u'users').document(u'alovelace') + doc_ref = db.collection(u'users').document( + u'alovelace' + unique_resource_id()) + + # Initial setting + doc_ref.set({ + u'first': u'Jane', + u'last': u'Doe', + u'born': 1900 + }) + + sleep(1) + + # Setup listener + def on_response(response): + on_response.called_count += 1 + print(f'Response: {response}') + print(type(response)) + + on_response.called_count = 0 + + doc_ref.on_snapshot(on_response) + + # Alter document doc_ref.set({ u'first': u'Ada', u'last': u'Lovelace', u'born': 1815 }) - doc_ref.on_snapshot(None, None) + + sleep(1) + if on_response.called_count != 1: + raise AssertionError("Failed to get exactly one document change") + # def test_create_document(client, cleanup): From 82a4a064cdf1f076b6f55bf1816173adcf54a6a0 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Mon, 9 Jul 2018 17:16:09 -0700 Subject: [PATCH 076/148] returning watch result now --- .../cloud/firestore_v1beta1/document.py | 6 +- .../google/cloud/firestore_v1beta1/watch.py | 221 ++++++++++-------- 2 files changed, 127 insertions(+), 100 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/document.py b/firestore/google/cloud/firestore_v1beta1/document.py index ad37b450c254..d47925acf11a 100644 --- a/firestore/google/cloud/firestore_v1beta1/document.py +++ b/firestore/google/cloud/firestore_v1beta1/document.py @@ -20,7 +20,7 @@ from google.cloud.firestore_v1beta1 import _helpers from google.cloud.firestore_v1beta1.watch import Watch -from google.cloud.firestore_v1beta1.proto.firestore_pb2 import Target + class DocumentReference(object): """A reference to a document in a Firestore database. @@ -450,8 +450,8 @@ def on_snapshot(self, callback): ''' given options and the callback, monitor this document for changes ''' - Watch.for_document(self, callback) - + Watch.for_document(self, callback, DocumentSnapshot) + class DocumentSnapshot(object): """A snapshot of document data in a Firestore database. diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index acf154971dea..d2faaecdf8bb 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -14,11 +14,13 @@ import logging import threading +from enum import Enum -#import google.cloud.firestore_v1beta1.client as client -from google.cloud.firestore_v1beta1.bidi import BidiRpc, ResumableBidiRpc, BackgroundConsumer +from google.cloud.firestore_v1beta1.bidi import ResumableBidiRpc +from google.cloud.firestore_v1beta1.bidi import BackgroundConsumer from google.cloud.firestore_v1beta1.proto import firestore_pb2 from google.api_core import exceptions +from google.protobuf import json_format # from bidi import BidiRpc, ResumableBidiRpc @@ -142,21 +144,36 @@ def _sleep(self, current_sleep, max_sleep=_MAX_SLEEP, return min(self.multiplier * self.current_sleep, self.max_sleep) +class WatchChangeType(Enum): + ADDED = 0 + MODIFIED = 1 + REMOVED = 2 + + +class WatchResult(object): + def __init__(self, snapshot, name, change_type): + self.snapshot = snapshot + self.name = name + self.change_type = change_type + + class Watch(object): - def __init__(self, + def __init__(self, document_reference, firestore, target, comparator, - on_response): + on_snapshot, + DocumentSnapshotCls): """ Args: firestore: - target: ß + target: comparator: - on_response: Callback method that reveives a - `google.cloud.firestore_v1beta1.types.ListenResponse` object to - be acted on. + on_snapshot: Callback method that receives two arguments, + list(snapshots) and + list(tuple(document_id, change_type)) + DocumentSnapshotCls: instance of the DocumentSnapshot class """ self._document_reference = document_reference self._firestore = firestore @@ -164,6 +181,7 @@ def __init__(self, self._targets = target self._comparator = comparator self._backoff = ExponentialBackOff() + self.DocumentSnapshot = DocumentSnapshotCls def should_recover(exc): return ( @@ -173,23 +191,22 @@ def should_recover(exc): initial_request = firestore_pb2.ListenRequest( database=self._firestore._database_string, add_target=self._targets - # database, add_target, remove_target, labels ) - rpc = ResumableBidiRpc( + self.rpc = ResumableBidiRpc( self._api.firestore_stub.Listen, initial_request=initial_request, should_recover=should_recover) - rpc.add_done_callback(self._on_rpc_done) + self.rpc.add_done_callback(self._on_rpc_done) def consumer_callback(response): processed_response = self.process_response(response) if processed_response: _LOGGER.debug("running provided callback") - on_response(processed_response) + on_snapshot(processed_response) - self._consumer = BackgroundConsumer(rpc, consumer_callback) + self._consumer = BackgroundConsumer(self.rpc, consumer_callback) self._consumer.start() def _on_rpc_done(self, future): @@ -215,14 +232,19 @@ def _on_rpc_done(self, future): thread.start() @classmethod - def for_document(cls, document_ref, on_response): + def for_document(cls, document_ref, on_snapshot, snapshot_class_instance): """ - Creates a watch snapshot listener for a document. on_response receives + Creates a watch snapshot listener for a document. on_snapshot receives a DocumentChange object, but may also start to get targetChange and such soon - """ + Args: + document_ref: Reference to Document + on_snapshot: callback to be called on snapshot + snapshot_class_instance: instance of snapshot cls to make snapshots with to + pass to on_snapshot + """ return cls(document_ref, document_ref._client, { @@ -231,90 +253,93 @@ def for_document(cls, document_ref, on_response): 'target_id': WATCH_TARGET_ID }, document_watch_comparator, - on_response) + on_snapshot, + snapshot_class_instance) # @classmethod - # def for_query(cls, query, on_response): + # def for_query(cls, query, on_snapshot): # return cls(query._client, # { # 'query': query.to_proto(), # 'target_id': WATCH_TARGET_ID # }, # query.comparator(), - # on_response) - + # on_snapshot) + def process_response(self, proto): """ Args: listen_response(`google.cloud.firestore_v1beta1.types.ListenResponse`): - Callback method that reveives a object to + Callback method that receives a object to """ - _LOGGER.debug('process_response') TargetChange = firestore_pb2.TargetChange - - # TODO FIGURE OUT CONDITIONAL OF THIS - _LOGGER.debug(f"STATE: document_change: {proto.document_change} target_change: {proto.target_change} target_change_type: {proto.target_change.target_change_type}") - if str(proto.document_change): - _LOGGER.debug("Document Change") - if str(proto.target_change): - _LOGGER.debug("Target Change") if str(proto.target_change): - _LOGGER.info('process_response: Processing target change') - change = proto.target_change + _LOGGER.debug('process_response: Processing target change') + + change = proto.target_change # google.cloud.firestore_v1beta1.types.TargetChange notarget_ids = change.target_ids is None or len(change.target_ids) if change.target_change_type == TargetChange.NO_CHANGE: - _LOGGER.info("process_response: " - "Processing target change NO_CHANGE") - # if notarget_ids and change.read_time and current) { - # // This means everything is up-to-date, so emit the current set of - # // docs as a snapshot, if there were changes. - # push( - # DocumentSnapshot.toISOTime(change.readTime), - # change.resumeToken - # ); - # } + _LOGGER.debug("process_response: target change: NO_CHANGE") + if notarget_ids and change.read_time: # and current: # current is used to reflect if the local copy of tree is accurate? + # This means everything is up-to-date, so emit the current set of + # docs as a snapshot, if there were changes. + # push( + # DocumentSnapshot.toISOTime(change.readTime), + # change.resumeToken + # ); + # } + # For now, we can do nothing here since there isn't anything to do + # eventually it seems it makes sens to record this as a snapshot? + # TODO : node calls the callback with no change? + pass elif change.target_change_type == TargetChange.ADD: - _LOGGER.info('process_response: Processing target change ADD') + _LOGGER.debug("process_response: target change: ADD") assert WATCH_TARGET_ID == change.target_ids[0], 'Unexpected target ID sent by server' + # TODO : do anything here? + return WatchResult( + None, + self._document_reference.id, + WatchChangeType.ADDED) elif change.target_change_type == TargetChange.REMOVE: - _LOGGER.info("process_response: " - "Processing target change REMOVE") - # let code = 13; - # let message = 'internal error'; - # if (change.cause) { - # code = change.cause.code; - # message = change.cause.message; - # } - # // @todo: Surface a .code property on the exception. - # closeStream(new Error('Error ' + code + ': ' + message)); + _LOGGER.debug("process_response: target change: REMOVE") + + code = 13 + message = 'internal error' + if change.cause: + code = change.cause.code + message = change.cause.message + + # TODO: Surface a .code property on the exception. + raise Exception('Error ' + code + ': ' + message) elif change.target_change_type == TargetChange.RESET: - _LOGGER.info("process_response: " - "Processing target change RESET") + _LOGGER.debug("process_response: target change: RESET") + # // Whatever changes have happened so far no longer matter. - # resetDocs(); + # resetDocs(); # TODO + # TODO : do something here? elif change.target_change_type == TargetChange.CURRENT: - _LOGGER.info("process_response: " - "Processing target change CURRENT") - # current = True + _LOGGER.debug("process_response: target change: CURRENT") + + # current = True # TODO + # TODO: do something here? else: - _LOGGER.info('process_response: Processing target change ELSE') _LOGGER.info('process_response: Unknown target change ' + str(change.target_change_type)) # closeStream( - # new Error('Unknown target change type: ' + JSON.stringify(change)) + # new Error('Unknown target change type: ' + JSON.stringify(change)) + # TODO : make this exit the inner function and stop processing? + raise Exception('Unknown target change type: ' + str(change)) - # if ( - # change.resumeToken and affectsTarget(change.target_ids, WATCH_TARGET_ID) - # ) { - # this._backoff.reset(); - # } + if change.resume_token and self._affects_target(change.target_ids, + WATCH_TARGET_ID): + self._backoff.reset() elif str(proto.document_change): - _LOGGER.debug('Watch.onSnapshot Processing document_change event') + _LOGGER.debug('process_response: Processing document change') # No other target_ids can show up here, but we still need to see if the # targetId was in the added list or removed list. @@ -331,47 +356,40 @@ def process_response(self, proto): if target == WATCH_TARGET_ID: removed = True - document = proto.document_change.document - # name = document.name - if changed: _LOGGER.debug('Received document change') - # reference = DocumentReference( - # self._firestore, - # ResourcePath.fromSlashSeparatedString(name)) - reference = self._document_reference - #create_time = DocumentSnapshot.toISOTime(document.create_time) - #update_time = DocumentSnapshot.toISOTime(document.update_time) - #read_time = DocumentSnapshot.toISOTime(document.read_time) - create_time = document.create_time - update_time = document.update_time - - #read_time = document.read_time - - # TODO: other clients seem to return snapshots - # snapshot = DocumentSnapshot( - # reference, - # document.fields, # DATA? - # exists=True, - # read_time=read_time, - # create_time=create_time, - # update_time=update_time) - # #changeMap.set(name, snapshot); - # return snapshot - return document - + # google.cloud.firestore_v1beta1.types.DocumentChange + document_change = proto.document_change + # google.cloud.firestore_v1beta1.types.Document + document = document_change.document + + data = json_format.MessageToDict(document) + + snapshot = self.DocumentSnapshot( + reference=self._document_reference, + data=data['fields'], + exists=True, + read_time=None, + create_time=document.create_time, + update_time=document.update_time) + + return WatchResult(snapshot, + self._document_reference.id, + WatchChangeType.MODIFIED) + elif removed: _LOGGER.debug('Watch.onSnapshot Received document remove') # changeMap.set(name, REMOVED); - # Document Delete or Document Remove? elif (proto.document_delete or proto.document_remove): _LOGGER.debug('Watch.onSnapshot Processing remove event') # const name = (proto.document_delete || proto.document_remove).document # changeMap.set(name, REMOVED); - + return WatchResult(None, + self._document_reference.id, + WatchChangeType.REMOVED) elif (proto.filter): _LOGGER.debug('Watch.onSnapshot Processing filter update') # if (proto.filter.count !== currentSize()) { @@ -379,10 +397,19 @@ def process_response(self, proto): # resetDocs(); # // The filter didn't match, so re-issue the query. # resetStream(); - + else: _LOGGER.debug("UNKNOWN TYPE. UHOH") # closeStream( # new Error('Unknown listen response type: ' + JSON.stringify(proto)) # ) - \ No newline at end of file + + def _affects_target(self, target_ids, current_id): + if target_ids is None or len(target_ids) == 0: + return True + + for target_id in target_ids: + if target_id == current_id: + return True + + return False From a83ebde1653807ad60a0ed90819b49215806e568 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Tue, 10 Jul 2018 16:33:22 -0700 Subject: [PATCH 077/148] broken currently, but nearing handling of multiple documents in collection --- .../google/cloud/firestore_v1beta1/watch.py | 479 ++++++++++++++---- 1 file changed, 380 insertions(+), 99 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index d2faaecdf8bb..a266c255a2a6 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -13,7 +13,9 @@ # limitations under the License. import logging +import collections import threading +import datetime from enum import Enum from google.cloud.firestore_v1beta1.bidi import ResumableBidiRpc @@ -63,6 +65,62 @@ exceptions.GatewayTimeout ) +DocTreeEntry = collections.namedtuple('DocTreeEntry', ['value', 'index']) + + +class WatchDocTree(object): + def __init__(self): + self._dict = {} + self._index = 0 + + def keys(self): + return list(self._dict.keys()) + + def insert(self, key, value): + self._dict[key] = DocTreeEntry(value, self._index) + self._index += 1 + return self + + def find(self, key): + return self._dict[key] + + def remove(self, key): + del self._dict[key] + return self + + def __len__(self): + return len(self._dict) + + +class ChangeType(Enum): + ADDED = 0 + MODIFIED = 1 + REMOVED = 2 + + +class DocumentChange(object): + def __init__(self, type, document, old_index, new_index): + """DocumentChange + + Args: + type (ChangeType): + document (document.DocumentSnapshot): + old_index (int): + new_index (int): + """ + # TODO: spec indicated an isEqual param also + self.type = type + self.document = document + self.old_index = old_index + self.new_index = new_index + + +class WatchResult(object): + def __init__(self, snapshot, name, change_type): + self.snapshot = snapshot + self.name = name + self.change_type = change_type + def _maybe_wrap_exception(exception): """Wraps a gRPC exception class, if needed.""" @@ -122,8 +180,8 @@ def _sleep(self, current_sleep, max_sleep=_MAX_SLEEP, multiplier=_MULTIPLIER): """Sleep and produce a new sleep time. - .. _Exponential Backoff And Jitter: https://www.awsarchitectureblog.com/\ - 2015/03/backoff.html + .. _Exponential Backoff And Jitter: + https://www.awsarchitectureblog.com/2015/03/backoff.html Select a duration between zero and ``current_sleep``. It might seem counterintuitive to have so much jitter, but @@ -144,35 +202,30 @@ def _sleep(self, current_sleep, max_sleep=_MAX_SLEEP, return min(self.multiplier * self.current_sleep, self.max_sleep) -class WatchChangeType(Enum): - ADDED = 0 - MODIFIED = 1 - REMOVED = 2 - - -class WatchResult(object): - def __init__(self, snapshot, name, change_type): - self.snapshot = snapshot - self.name = name - self.change_type = change_type - - class Watch(object): def __init__(self, document_reference, firestore, target, comparator, - on_snapshot, + snapshot_callback, DocumentSnapshotCls): """ Args: firestore: target: comparator: - on_snapshot: Callback method that receives two arguments, - list(snapshots) and - list(tuple(document_id, change_type)) + snapshot_callback: Callback method to process snapshots. + Args: + docs (List(DocumentSnapshot)): A callback that returns the + ordered list of documents stored in this snapshot. + changes (List(str)): A callback that returns the list of + changed documents since the last snapshot delivered for + this watch. + read_time (string): The ISO 8601 time at which this + snapshot was obtained. + # TODO: Go had an err here and node.js provided size. + # TODO: do we want to include either? DocumentSnapshotCls: instance of the DocumentSnapshot class """ self._document_reference = document_reference @@ -180,8 +233,8 @@ def __init__(self, self._api = firestore._firestore_api self._targets = target self._comparator = comparator - self._backoff = ExponentialBackOff() self.DocumentSnapshot = DocumentSnapshotCls + self._snapshot_callback = snapshot_callback def should_recover(exc): return ( @@ -200,13 +253,32 @@ def should_recover(exc): self.rpc.add_done_callback(self._on_rpc_done) - def consumer_callback(response): - processed_response = self.process_response(response) - if processed_response: - _LOGGER.debug("running provided callback") - on_snapshot(processed_response) + # Initialize state for on_snapshot + # The sorted tree of QueryDocumentSnapshots as sent in the last + # snapshot. We only look at the keys. + # TODO: using ordered dict right now but not great maybe + self.doc_tree = WatchDocTree() # TODO: rbtree(this._comparator) + + # A map of document names to QueryDocumentSnapshots for the last sent + # snapshot. + self.doc_map = {} + + # The accumulates map of document changes (keyed by document name) for + # the current snapshot. + self.change_map = {} - self._consumer = BackgroundConsumer(self.rpc, consumer_callback) + # The current state of the query results. + self.current = False + + # We need this to track whether we've pushed an initial set of changes, + # since we should push those even when there are no changes, if there + # aren't docs. + self.has_pushed = False + + # The server assigns and updates the resume token. + self.resume_token = None + + self._consumer = BackgroundConsumer(self.rpc, self.on_snapshot) self._consumer.start() def _on_rpc_done(self, future): @@ -232,17 +304,18 @@ def _on_rpc_done(self, future): thread.start() @classmethod - def for_document(cls, document_ref, on_snapshot, snapshot_class_instance): + def for_document(cls, document_ref, snapshot_callback, + snapshot_class_instance): """ - Creates a watch snapshot listener for a document. on_snapshot receives - a DocumentChange object, but may also start to get targetChange and such - soon + Creates a watch snapshot listener for a document. snapshot_callback + receives a DocumentChange object, but may also start to get + targetChange and such soon Args: document_ref: Reference to Document - on_snapshot: callback to be called on snapshot - snapshot_class_instance: instance of snapshot cls to make snapshots with to - pass to on_snapshot + snapshot_callback: callback to be called on snapshot + snapshot_class_instance: instance of snapshot cls to make + snapshots with to pass to snapshot_callback """ return cls(document_ref, @@ -253,21 +326,25 @@ def for_document(cls, document_ref, on_snapshot, snapshot_class_instance): 'target_id': WATCH_TARGET_ID }, document_watch_comparator, - on_snapshot, + snapshot_callback, snapshot_class_instance) # @classmethod - # def for_query(cls, query, on_snapshot): + # def for_query(cls, query, snapshot_callback): # return cls(query._client, # { # 'query': query.to_proto(), # 'target_id': WATCH_TARGET_ID # }, # query.comparator(), - # on_snapshot) + # snapshot_callback) - def process_response(self, proto): + def on_snapshot(self, proto): """ + Called everytime there is a response from listen. Collect changes + and 'push' the changes in a batch to the customer when we receive + 'current' from the listen response. + Args: listen_response(`google.cloud.firestore_v1beta1.types.ListenResponse`): Callback method that receives a object to @@ -275,36 +352,34 @@ def process_response(self, proto): TargetChange = firestore_pb2.TargetChange if str(proto.target_change): - _LOGGER.debug('process_response: Processing target change') + _LOGGER.debug('on_snapshot: target change') - change = proto.target_change # google.cloud.firestore_v1beta1.types.TargetChange + # google.cloud.firestore_v1beta1.types.TargetChange + change = proto.target_change - notarget_ids = change.target_ids is None or len(change.target_ids) + no_target_ids = change.target_ids is None or \ + len(change.target_ids) == 0 if change.target_change_type == TargetChange.NO_CHANGE: - _LOGGER.debug("process_response: target change: NO_CHANGE") - if notarget_ids and change.read_time: # and current: # current is used to reflect if the local copy of tree is accurate? - # This means everything is up-to-date, so emit the current set of - # docs as a snapshot, if there were changes. - # push( - # DocumentSnapshot.toISOTime(change.readTime), - # change.resumeToken - # ); - # } - # For now, we can do nothing here since there isn't anything to do - # eventually it seems it makes sens to record this as a snapshot? - # TODO : node calls the callback with no change? - pass + _LOGGER.debug('on_snapshot: target change: NO_CHANGE') + if no_target_ids and change.read_time and self.current: + # TargetChange.CURRENT followed by TargetChange.NO_CHANGE + # signals a consistent state. Invoke the onSnapshot + # callback as specified by the user. + self.push(change.read_time, change.resume_token) elif change.target_change_type == TargetChange.ADD: - _LOGGER.debug("process_response: target change: ADD") - assert WATCH_TARGET_ID == change.target_ids[0], 'Unexpected target ID sent by server' - # TODO : do anything here? - - return WatchResult( - None, - self._document_reference.id, - WatchChangeType.ADDED) + _LOGGER.debug("on_snapshot: target change: ADD") + assert WATCH_TARGET_ID == change.target_ids[0], \ + 'Unexpected target ID sent by server' + # TODO : do anything here? Node didn't so I think this isn't + # the right thing to do + # wr = WatchResult( + # None, + # self._document_reference.id, + # ChangeType.ADDED) + # self._snapshot_callback(wr) + elif change.target_change_type == TargetChange.REMOVE: - _LOGGER.debug("process_response: target change: REMOVE") + _LOGGER.debug("on_snapshot: target change: REMOVE") code = 13 message = 'internal error' @@ -315,34 +390,34 @@ def process_response(self, proto): # TODO: Surface a .code property on the exception. raise Exception('Error ' + code + ': ' + message) elif change.target_change_type == TargetChange.RESET: - _LOGGER.debug("process_response: target change: RESET") - - # // Whatever changes have happened so far no longer matter. - # resetDocs(); # TODO - # TODO : do something here? + # Whatever changes have happened so far no longer matter. + _LOGGER.debug("on_snapshot: target change: RESET") + self._reset_docs() elif change.target_change_type == TargetChange.CURRENT: - _LOGGER.debug("process_response: target change: CURRENT") - - # current = True # TODO - # TODO: do something here? + _LOGGER.debug("on_snapshot: target change: CURRENT") + self.current = True else: - _LOGGER.info('process_response: Unknown target change ' + + _LOGGER.info('on_snapshot: Unknown target change ' + str(change.target_change_type)) + self._consumer.stop() # closeStream( - # new Error('Unknown target change type: ' + JSON.stringify(change)) + # new Error('Unknown target change type: ' + + # JSON.stringify(change)) # TODO : make this exit the inner function and stop processing? raise Exception('Unknown target change type: ' + str(change)) if change.resume_token and self._affects_target(change.target_ids, WATCH_TARGET_ID): - self._backoff.reset() + # TODO: they node version resets backoff here. We allow + # bidi rpc to do its thing. + pass elif str(proto.document_change): - _LOGGER.debug('process_response: Processing document change') + _LOGGER.debug('on_snapshot: document change') - # No other target_ids can show up here, but we still need to see if the - # targetId was in the added list or removed list. + # No other target_ids can show up here, but we still need to see + # if the targetId was in the added list or removed list. target_ids = proto.document_change.target_ids or [] removed_target_ids = proto.document_change.removed_target_ids or [] changed = False @@ -357,7 +432,7 @@ def process_response(self, proto): removed = True if changed: - _LOGGER.debug('Received document change') + _LOGGER.debug('on_snapshot: document change: CHANGED') # google.cloud.firestore_v1beta1.types.DocumentChange document_change = proto.document_change @@ -374,35 +449,215 @@ def process_response(self, proto): create_time=document.create_time, update_time=document.update_time) - return WatchResult(snapshot, - self._document_reference.id, - WatchChangeType.MODIFIED) + self.change_map[document.name] = snapshot + # TODO: ensure we call this later, on current returend. + # wr = WatchResult(snapshot, + # self._document_reference.id, + # ChangeType.MODIFIED) + # self._snapshot_callback(wr) elif removed: - _LOGGER.debug('Watch.onSnapshot Received document remove') - # changeMap.set(name, REMOVED); + _LOGGER.debug('on_snapshot: document change: REMOVED') + self.change_map[document.name] = ChangeType.REMOVED - # Document Delete or Document Remove? elif (proto.document_delete or proto.document_remove): - _LOGGER.debug('Watch.onSnapshot Processing remove event') - # const name = (proto.document_delete || proto.document_remove).document - # changeMap.set(name, REMOVED); - return WatchResult(None, - self._document_reference.id, - WatchChangeType.REMOVED) + _LOGGER.debug('on_snapshot: document change: DELETE/REMOVE') + name = (proto.document_delete or proto.document_remove).document + self.change_map[name] = ChangeType.REMOVED + # wr = WatchResult(None, + # self._document_reference.id, + # ChangeType.REMOVED) + # self._snapshot_callback(wr) + elif (proto.filter): - _LOGGER.debug('Watch.onSnapshot Processing filter update') - # if (proto.filter.count !== currentSize()) { - # // We need to remove all the current results. - # resetDocs(); - # // The filter didn't match, so re-issue the query. - # resetStream(); + _LOGGER.debug('on_snapshot: filter update') + if proto.filter.count != self._current_size(): + # We need to remove all the current results. + self._reset_docs() + # The filter didn't match, so re-issue the query. + # TODO: reset stream method? + # self._reset_stream(); else: _LOGGER.debug("UNKNOWN TYPE. UHOH") - # closeStream( - # new Error('Unknown listen response type: ' + JSON.stringify(proto)) - # ) + self._consumer.stop() + raise Exception( + 'Unknown listen response type: ' + proto) + # TODO: can we stop but raise an error? + # closeStream( + # new Error('Unknown listen response type: ' + + # JSON.stringify(proto)) + # ) + + def push(self, read_time, next_resume_token): + """ + Assembles a new snapshot from the current set of changes and invokes + the user's callback. Clears the current changes on completion. + """ + # TODO: may need to lock here to avoid races on collecting snapshots + # and sending them to the user. + + deletes, adds, updates = Watch._extract_changes( + self.doc_map, self.change_map, read_time) + updated_tree, updated_map, appliedChanges = \ + Watch._compute_snapshot( + self.doc_tree, self.doc_map, deletes, adds, updates) +# _LOGGER.debug(f"""push +# self.doc_map {self.doc_map} +# self.change_map {self.change_map} +# read_time {read_time} +# deletes {deletes} +# adds {adds} +# updates {updates} +# updated_tree {updated_tree} +# """) + if not self.has_pushed or len(appliedChanges): + _LOGGER.debug( + f'Sending snapshot with {len(appliedChanges)} changes' + f' and {len(updated_tree)} documents') + + _LOGGER.debug(f"updatedTree:{updated_tree}") + self._snapshot_callback( + updated_tree.keys(), + appliedChanges, + datetime.datetime.fromtimestamp(read_time.seconds) + ) + self.has_pushed = True + + self.doc_tree = updated_tree + self.doc_map = updated_map + self.change_map.clear() + self.resume_token = next_resume_token + + def _extract_changes(doc_map, changes, read_time): + deletes = [] + adds = [] + updates = [] + + for name, value in changes.items(): + if value == ChangeType.REMOVED: + if name in doc_map: + deletes.append(name) + elif name in doc_map: + value.read_time = read_time + updates.append(value) + else: + value.read_time = read_time + adds.append(value) + _LOGGER.debug(f'deletes:{len(deletes)} adds:{len(adds)}') + return (deletes, adds, updates) + + def _compute_snapshot(doc_tree, doc_map, delete_changes, add_changes, + update_changes): + # TODO: ACTUALLY NEED TO CALCULATE + # return {updated_tree, updated_map, appliedChanges}; + # return doc_tree, doc_map, changes + + updated_tree = doc_tree + updated_map = doc_map + + assert len(doc_tree) == len(doc_map), \ + 'The document tree and document map should have the same ' + \ + 'number of entries.' + + def delete_doc(name, updated_tree, updated_map): + """ + Applies a document delete to the document tree and document map. + Returns the corresponding DocumentChange event. + """ + assert name in updated_map(name), 'Document to delete does not exist' + old_document = updated_map.get(name) + existing = updated_tree.find(old_document) + old_index = existing.index + # TODO: was existing.remove returning tree (presumably immuatable?) + updated_tree = updated_tree.remove(old_document) + updated_map.delete(name) + return (DocumentChange(ChangeType.REMOVED, + old_document, + old_index, + -1), + updated_tree, updated_map) + + def add_doc(new_document, updated_tree, updated_map): + """ + Applies a document add to the document tree and the document map. + Returns the corresponding DocumentChange event. + """ + name = new_document.reference._document_path + assert name not in updated_map, 'Document to add already exists' + updated_tree = updated_tree.insert(new_document, None) + new_index = updated_tree.find(new_document).index + updated_map[name] = new_document + return (DocumentChange(ChangeType.ADDED, + new_document, + -1, + new_index), + updated_tree, updated_map) + + def modify_doc(new_document, updated_tree, updated_map): + """ + Applies a document modification to the document tree and the + document map. + Returns the DocumentChange event for successful modifications. + """ + name = new_document.ref.formattedName + assert updated_map.has(name), 'Document to modify does not exist' + oldDocument = updated_map.get(name) + if oldDocument.updateTime != new_document.updateTime: + removeChange, updated_tree, updated_map = delete_doc( + name, updated_tree, updated_map) + addChange, updated_tree, updated_map = add_doc( + new_document, updated_tree, updated_map) + return (DocumentChange(ChangeType.MODIFIED, + new_document, + removeChange.old_index, + addChange.new_index), + updated_tree, updated_map) + + return None + + # Process the sorted changes in the order that is expected by our + # clients (removals, additions, and then modifications). We also need + # to sort the individual changes to assure that old_index/new_index + # keep incrementing. + appliedChanges = [] + + # Deletes are sorted based on the order of the existing document. + + # TODO: SORT + # delete_changes.sort( + # lambda name1, name2: + # self._comparator(updated_map.get(name1), updated_map.get(name2))) + + for name in delete_changes: + change, updated_tree, updated_map = delete_doc( + name, updated_tree, updated_map) + if change: + appliedChanges.append(change) + + # TODO: SORT + # add_changes.sort(self._comparator) + _LOGGER.debug('walk over add_changes') + for snapshot in add_changes: + _LOGGER.debug('in add_changes') + change, updated_tree, updated_map = add_doc( + snapshot, updated_tree, updated_map) + if change: + appliedChanges.append(change) + + # TODO: SORT + # update_changes.sort(self._comparator) + for snapshot in update_changes: + change, updated_tree, updated_map = modify_doc( + snapshot, updated_tree, updated_map) + if change: + appliedChanges.append(change) + + assert len(updated_tree) == len(updated_map), \ + 'The update document ' + \ + 'tree and document map should have the same number of entries.' + _LOGGER.debug(f"tree:{updated_tree}, map:{updated_map}, applied:{appliedChanges}") + return (updated_tree, updated_map, appliedChanges) def _affects_target(self, target_ids, current_id): if target_ids is None or len(target_ids) == 0: @@ -413,3 +668,29 @@ def _affects_target(self, target_ids, current_id): return True return False + + def _current_size(self): + """ + Returns the current count of all documents, including the changes from + the current changeMap. + """ + deletes, adds, _ = Watch._extract_changes(self.docMap, self.changeMap) + return self.docMap.size + len(adds) - len(deletes) + + def _reset_docs(self): + """ + Helper to clear the docs on RESET or filter mismatch. + """ + _LOGGER.debug("resetting documents") + self.change_map.clear() + self.resume_token = None + + # TODO: mark each document as deleted. If documents are not delete + # they will be sent again by the server. + # docTree.forEach(snapshot => { + # // Mark each document as deleted. If documents are not deleted, + # // they + # // will be send again by the server. + # changeMap.set(snapshot.ref.formattedName, REMOVED); + + self.current = False From 1d1be5349b7db579f44de4c714b2cad548f1cf83 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Tue, 10 Jul 2018 17:18:54 -0700 Subject: [PATCH 078/148] small fixes. seems mostly working --- firestore/google/cloud/firestore_v1beta1/watch.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index a266c255a2a6..bfb9b8d848db 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -76,7 +76,15 @@ def __init__(self): def keys(self): return list(self._dict.keys()) + def _copy(self): + wdt = WatchDocTree() + wdt._dict = self._dict.copy() + wdt._index = self._index + self = wdt + return self + def insert(self, key, value): + self = self._copy() self._dict[key] = DocTreeEntry(value, self._index) self._index += 1 return self @@ -85,6 +93,7 @@ def find(self, key): return self._dict[key] def remove(self, key): + self = self._copy() del self._dict[key] return self @@ -565,13 +574,13 @@ def delete_doc(name, updated_tree, updated_map): Applies a document delete to the document tree and document map. Returns the corresponding DocumentChange event. """ - assert name in updated_map(name), 'Document to delete does not exist' + assert name in updated_map, 'Document to delete does not exist' old_document = updated_map.get(name) existing = updated_tree.find(old_document) old_index = existing.index # TODO: was existing.remove returning tree (presumably immuatable?) updated_tree = updated_tree.remove(old_document) - updated_map.delete(name) + del updated_map[name] return (DocumentChange(ChangeType.REMOVED, old_document, old_index, From 80c6bea62f4d92efd7f405bdc3cc3e7ddfabb05b Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Wed, 11 Jul 2018 09:07:47 -0700 Subject: [PATCH 079/148] fix variable name, remove things from doc map on remove --- firestore/google/cloud/firestore_v1beta1/watch.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index bfb9b8d848db..15ef7e8b0a90 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -609,7 +609,7 @@ def modify_doc(new_document, updated_tree, updated_map): document map. Returns the DocumentChange event for successful modifications. """ - name = new_document.ref.formattedName + name = new_document.reference.formattedName assert updated_map.has(name), 'Document to modify does not exist' oldDocument = updated_map.get(name) if oldDocument.updateTime != new_document.updateTime: @@ -696,10 +696,8 @@ def _reset_docs(self): # TODO: mark each document as deleted. If documents are not delete # they will be sent again by the server. - # docTree.forEach(snapshot => { - # // Mark each document as deleted. If documents are not deleted, - # // they - # // will be send again by the server. - # changeMap.set(snapshot.ref.formattedName, REMOVED); + for snapshot in self.doc_tree: + document_name = snapshot.reference.formattedName + self.change_map[document_name] = ChangeType.REMOVED self.current = False From 799fac9232b22a5101fa99f35b6ab367f63cb250 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Tue, 14 Aug 2018 10:47:43 -0700 Subject: [PATCH 080/148] improve doc strings and comment out yet to be done methods --- .../cloud/firestore_v1beta1/collection.py | 31 +++++++++++++--- .../cloud/firestore_v1beta1/document.py | 30 +++++++++++++-- .../google/cloud/firestore_v1beta1/query.py | 37 +++++++++++++------ .../google/cloud/firestore_v1beta1/watch.py | 27 +++++++++----- firestore/tests/system.py | 2 + 5 files changed, 97 insertions(+), 30 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/collection.py b/firestore/google/cloud/firestore_v1beta1/collection.py index c49c3e4080af..ed5c06baa10d 100644 --- a/firestore/google/cloud/firestore_v1beta1/collection.py +++ b/firestore/google/cloud/firestore_v1beta1/collection.py @@ -371,11 +371,32 @@ def get(self, transaction=None): query = query_mod.Query(self) return query.get(transaction=transaction) - def onSnapshot(options, callback): - ''' - given options and the callback, monitor this collection for changes - ''' - raise NotImplemented + # def on_snapshot(self, callback): + # """Monitor the documents in this collection. + # + # This starts a watch on this collection using a background thread. The + # provided callback is run on the snapshot of the documents. + # + # Args: + # callback(CollectionSnapshot): a callback to run when a change occurs + # + # Example: + # from google.cloud import firestore + # + # db = firestore.Client() + # collection_ref = db.collection(u'users') + # + # def on_snapshot(collection_snapshot): + # for doc in collection_snapshot.docs: + # print(u'{} => {}'.format(doc.id, doc.to_dict())) + # + # # Watch this collection + # collection_watch = collection_ref.on_snapshot(on_snapshot) + # + # # Terminate this watch + # collection_watch.unsubscribe() + # """ + # raise NotImplemented def _auto_id(): diff --git a/firestore/google/cloud/firestore_v1beta1/document.py b/firestore/google/cloud/firestore_v1beta1/document.py index d47925acf11a..a7b2f061849d 100644 --- a/firestore/google/cloud/firestore_v1beta1/document.py +++ b/firestore/google/cloud/firestore_v1beta1/document.py @@ -447,9 +447,33 @@ def collections(self, page_size=None): return iterator def on_snapshot(self, callback): - ''' - given options and the callback, monitor this document for changes - ''' + """Watch this document. + + This starts a watch on this document using a background thread. The + provided callback is run on the snapshot. + + Args: + callback(DocumentSnapshot): a callback to run when a change occurs + + Example: + from google.cloud import firestore + + db = firestore.Client() + collection_ref = db.collection(u'users') + + def on_snapshot(document_snapshot): + doc = document_snapshot + print(u'{} => {}'.format(doc.id, doc.to_dict())) + + doc_ref = db.collection(u'users').document( + u'alovelace' + unique_resource_id()) + + # Watch this document + doc_watch = doc_ref.on_snapshot(on_snapshot) + + # Terminate this watch + doc_watch.unsubscribe() + """ Watch.for_document(self, callback, DocumentSnapshot) diff --git a/firestore/google/cloud/firestore_v1beta1/query.py b/firestore/google/cloud/firestore_v1beta1/query.py index 909eb914e2ea..05a6ba4e44ca 100644 --- a/firestore/google/cloud/firestore_v1beta1/query.py +++ b/firestore/google/cloud/firestore_v1beta1/query.py @@ -601,19 +601,32 @@ def get(self, transaction=None): else: yield snapshot - def onSnapshot(callback, options): - ''' - db.collection("cities").where("state", "==", "CA") - .onSnapshot(function(querySnapshot) { - var cities = []; - querySnapshot.forEach(function(doc) { - cities.push(doc.data().name); - }); - console.log("Current cities in CA: ", cities.join(", ")); - }); - ''' - raise NotImplemented + # def on_snapshot(self, callback): + # """Monitor the documents in this collection that match this query. + # This starts a watch on this query using a background thread. The + # provided callback is run on the snapshot of the documents. + + # Args: + # callback(QuerySnapshot): a callback to run when a change occurs + + # Example: + # from google.cloud import firestore + + # db = firestore.Client() + # query_ref = db.collection(u'users').where("user", "==", u'ada') + + # def on_snapshot(query_snapshot): + # for doc in query_snapshot.docs: + # print(u'{} => {}'.format(doc.id, doc.to_dict())) + + # # Watch this query + # query_watch = query_ref.on_snapshot(on_snapshot) + + # # Terminate this watch + # query_watch.unsubscribe() + # """ + # raise NotImplemented def _enum_from_op_string(op_string): """Convert a string representation of a binary operator to an enum. diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 15ef7e8b0a90..6d8d201aa71e 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -312,6 +312,9 @@ def _on_rpc_done(self, future): thread.daemon = True thread.start() + def unsubscribe(self): + self.rpc.close() + @classmethod def for_document(cls, document_ref, snapshot_callback, snapshot_class_instance): @@ -346,7 +349,19 @@ def for_document(cls, document_ref, snapshot_callback, # 'target_id': WATCH_TARGET_ID # }, # query.comparator(), - # snapshot_callback) + # snapshot_callback, + # snapshot_class_instance) + + # @classmethod + # def for_collection(cls, collection_ref, snapshot_callback): + # return cls(collection_ref._client, + # { + # 'collection': collection_ref.to_proto(), + # 'target_id': WATCH_TARGET_ID + # }, + # document_watch_comparator, + # snapshot_callback, + # snapshot_class_instance) def on_snapshot(self, proto): """ @@ -511,15 +526,7 @@ def push(self, read_time, next_resume_token): updated_tree, updated_map, appliedChanges = \ Watch._compute_snapshot( self.doc_tree, self.doc_map, deletes, adds, updates) -# _LOGGER.debug(f"""push -# self.doc_map {self.doc_map} -# self.change_map {self.change_map} -# read_time {read_time} -# deletes {deletes} -# adds {adds} -# updates {updates} -# updated_tree {updated_tree} -# """) + if not self.has_pushed or len(appliedChanges): _LOGGER.debug( f'Sending snapshot with {len(appliedChanges)} changes' diff --git a/firestore/tests/system.py b/firestore/tests/system.py index 393c5a983935..995f5fc004b1 100644 --- a/firestore/tests/system.py +++ b/firestore/tests/system.py @@ -795,6 +795,8 @@ def test_batch(client, cleanup): assert snapshot2.update_time == write_result2.update_time assert not document3.get().exists + + def test_watch_document(client, cleanup): db = client doc_ref = db.collection(u'users').document( From 77eea6a75176e7cc589c54aeca83508515179b70 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Tue, 14 Aug 2018 13:58:14 -0400 Subject: [PATCH 081/148] remove fstrings for 2.7 compat --- firestore/google/cloud/firestore_v1beta1/watch.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 6d8d201aa71e..57332a858982 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -529,10 +529,10 @@ def push(self, read_time, next_resume_token): if not self.has_pushed or len(appliedChanges): _LOGGER.debug( - f'Sending snapshot with {len(appliedChanges)} changes' - f' and {len(updated_tree)} documents') + 'Sending snapshot with {len(appliedChanges)} changes' + ' and {len(updated_tree)} documents') - _LOGGER.debug(f"updatedTree:{updated_tree}") + _LOGGER.debug("updatedTree:{updated_tree}") self._snapshot_callback( updated_tree.keys(), appliedChanges, @@ -560,7 +560,7 @@ def _extract_changes(doc_map, changes, read_time): else: value.read_time = read_time adds.append(value) - _LOGGER.debug(f'deletes:{len(deletes)} adds:{len(adds)}') + _LOGGER.debug('deletes:{len(deletes)} adds:{len(adds)}') return (deletes, adds, updates) def _compute_snapshot(doc_tree, doc_map, delete_changes, add_changes, @@ -672,7 +672,7 @@ def modify_doc(new_document, updated_tree, updated_map): assert len(updated_tree) == len(updated_map), \ 'The update document ' + \ 'tree and document map should have the same number of entries.' - _LOGGER.debug(f"tree:{updated_tree}, map:{updated_map}, applied:{appliedChanges}") + _LOGGER.debug("tree:{updated_tree}, map:{updated_map}, applied:{appliedChanges}") return (updated_tree, updated_map, appliedChanges) def _affects_target(self, target_ids, current_id): From 0db6a9dad895b041d20cd08ee162949ca405a999 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Tue, 14 Aug 2018 11:02:26 -0700 Subject: [PATCH 082/148] be more specific on snapshot type --- firestore/google/cloud/firestore_v1beta1/collection.py | 5 +++-- firestore/google/cloud/firestore_v1beta1/document.py | 3 ++- firestore/google/cloud/firestore_v1beta1/query.py | 5 +++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/collection.py b/firestore/google/cloud/firestore_v1beta1/collection.py index ed5c06baa10d..090da0cb0151 100644 --- a/firestore/google/cloud/firestore_v1beta1/collection.py +++ b/firestore/google/cloud/firestore_v1beta1/collection.py @@ -378,7 +378,8 @@ def get(self, transaction=None): # provided callback is run on the snapshot of the documents. # # Args: - # callback(CollectionSnapshot): a callback to run when a change occurs + # callback(~.firestore.collection.CollectionSnapshot): a callback + # to run when a change occurs. # # Example: # from google.cloud import firestore @@ -387,7 +388,7 @@ def get(self, transaction=None): # collection_ref = db.collection(u'users') # # def on_snapshot(collection_snapshot): - # for doc in collection_snapshot.docs: + # for doc in collection_snapshot.documents: # print(u'{} => {}'.format(doc.id, doc.to_dict())) # # # Watch this collection diff --git a/firestore/google/cloud/firestore_v1beta1/document.py b/firestore/google/cloud/firestore_v1beta1/document.py index a7b2f061849d..b68aa93e93aa 100644 --- a/firestore/google/cloud/firestore_v1beta1/document.py +++ b/firestore/google/cloud/firestore_v1beta1/document.py @@ -453,7 +453,8 @@ def on_snapshot(self, callback): provided callback is run on the snapshot. Args: - callback(DocumentSnapshot): a callback to run when a change occurs + callback(~.firestore.document.DocumentSnapshot):a callback to run + when a change occurs Example: from google.cloud import firestore diff --git a/firestore/google/cloud/firestore_v1beta1/query.py b/firestore/google/cloud/firestore_v1beta1/query.py index 05a6ba4e44ca..88299039c448 100644 --- a/firestore/google/cloud/firestore_v1beta1/query.py +++ b/firestore/google/cloud/firestore_v1beta1/query.py @@ -608,7 +608,8 @@ def get(self, transaction=None): # provided callback is run on the snapshot of the documents. # Args: - # callback(QuerySnapshot): a callback to run when a change occurs + # callback(~.firestore.query.QuerySnapshot): a callback to run when + # a change occurs. # Example: # from google.cloud import firestore @@ -617,7 +618,7 @@ def get(self, transaction=None): # query_ref = db.collection(u'users').where("user", "==", u'ada') # def on_snapshot(query_snapshot): - # for doc in query_snapshot.docs: + # for doc in query_snapshot.documents: # print(u'{} => {}'.format(doc.id, doc.to_dict())) # # Watch this query From 374775b47ec175cb149ada28a2dff78852933ecf Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Mon, 20 Aug 2018 11:01:41 -0400 Subject: [PATCH 083/148] unit tests for watch module --- .../google/cloud/firestore_v1beta1/query.py | 1 + .../google/cloud/firestore_v1beta1/watch.py | 235 ++++------ firestore/tests/system.py | 2 +- firestore/tests/unit/test_watch.py | 439 ++++++++++++++++++ 4 files changed, 542 insertions(+), 135 deletions(-) create mode 100644 firestore/tests/unit/test_watch.py diff --git a/firestore/google/cloud/firestore_v1beta1/query.py b/firestore/google/cloud/firestore_v1beta1/query.py index 88299039c448..2a88ad054678 100644 --- a/firestore/google/cloud/firestore_v1beta1/query.py +++ b/firestore/google/cloud/firestore_v1beta1/query.py @@ -629,6 +629,7 @@ def get(self, transaction=None): # """ # raise NotImplemented + def _enum_from_op_string(op_string): """Convert a string representation of a binary operator to an enum. diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 57332a858982..a44742b75111 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -26,8 +26,6 @@ # from bidi import BidiRpc, ResumableBidiRpc -import time -import random import grpc """Python client for Google Cloud Firestore Watch.""" @@ -138,87 +136,28 @@ def _maybe_wrap_exception(exception): return exception -def is_permanent_error(self, error): - try: - if (error.code == GRPC_STATUS_CODE['CANCELLED'] or - error.code == GRPC_STATUS_CODE['UNKNOWN'] or - error.code == GRPC_STATUS_CODE['DEADLINE_EXCEEDED'] or - error.code == GRPC_STATUS_CODE['RESOURCE_EXHAUSTED'] or - error.code == GRPC_STATUS_CODE['INTERNAL'] or - error.code == GRPC_STATUS_CODE['UNAVAILABLE'] or - error.code == GRPC_STATUS_CODE['UNAUTHENTICATED']): - return False - else: - return True - except AttributeError: - _LOGGER.error("Unable to determine error code") - return False - - def document_watch_comparator(doc1, doc2): assert doc1 == doc2, 'Document watches only support one document.' return 0 -class ExponentialBackOff(object): - _INITIAL_SLEEP = 1.0 - """float: Initial "max" for sleep interval.""" - _MAX_SLEEP = 30.0 - """float: Eventual "max" sleep time.""" - _MULTIPLIER = 2.0 - """float: Multiplier for exponential backoff.""" - - def __init__(self, initial_sleep=_INITIAL_SLEEP, max_sleep=_MAX_SLEEP, - multiplier=_MULTIPLIER): - self.initial_sleep = self.current_sleep = initial_sleep - self.max_sleep = max_sleep - self.multipler = multiplier - - def back_off(self): - self.current_sleep = self._sleep(self.current_sleep, - self.max_sleep, - self.multipler) - - def reset_to_max(self): - self.current_sleep = self.max_sleep - - def reset(self): - self.current_sleep = self._INITIAL_SLEEP - - def _sleep(self, current_sleep, max_sleep=_MAX_SLEEP, - multiplier=_MULTIPLIER): - """Sleep and produce a new sleep time. - - .. _Exponential Backoff And Jitter: - https://www.awsarchitectureblog.com/2015/03/backoff.html - - Select a duration between zero and ``current_sleep``. It might seem - counterintuitive to have so much jitter, but - `Exponential Backoff And Jitter`_ argues that "full jitter" is - the best strategy. - - Args: - current_sleep (float): The current "max" for sleep interval. - max_sleep (Optional[float]): Eventual "max" sleep time - multiplier (Optional[float]): Multiplier for exponential backoff. - - Returns: - float: Newly doubled ``current_sleep`` or ``max_sleep`` (whichever - is smaller) - """ - actual_sleep = random.uniform(0.0, self.current_sleep) - time.sleep(actual_sleep) - return min(self.multiplier * self.current_sleep, self.max_sleep) +class Watch(object): + threading = threading # FBO unit tests + BackgroundConsumer = BackgroundConsumer # FBO unit tests + ResumableBidiRpc = ResumableBidiRpc # FBO unit tests + MessageToDict = json_format.MessageToDict # FBO unit tests -class Watch(object): def __init__(self, document_reference, firestore, target, comparator, snapshot_callback, - DocumentSnapshotCls): + DocumentSnapshotCls, + BackgroundConsumer=None, # FBO unit testing + ResumableBidiRpc=None, # FBO unit testing + ): """ Args: firestore: @@ -234,7 +173,7 @@ def __init__(self, read_time (string): The ISO 8601 time at which this snapshot was obtained. # TODO: Go had an err here and node.js provided size. - # TODO: do we want to include either? + # TODO: do we want to include either? DocumentSnapshotCls: instance of the DocumentSnapshot class """ self._document_reference = document_reference @@ -255,6 +194,9 @@ def should_recover(exc): add_target=self._targets ) + if ResumableBidiRpc is None: + ResumableBidiRpc = self.ResumableBidiRpc # FBO unit tests + self.rpc = ResumableBidiRpc( self._api.firestore_stub.Listen, initial_request=initial_request, @@ -278,7 +220,7 @@ def should_recover(exc): # The current state of the query results. self.current = False - + # We need this to track whether we've pushed an initial set of changes, # since we should push those even when there are no changes, if there # aren't docs. @@ -286,6 +228,8 @@ def should_recover(exc): # The server assigns and updates the resume token. self.resume_token = None + if BackgroundConsumer is None: # FBO unit tests + BackgroundConsumer = self.BackgroundConsumer self._consumer = BackgroundConsumer(self.rpc, self.on_snapshot) self._consumer.start() @@ -305,14 +249,14 @@ def _on_rpc_done(self, future): _LOGGER.info( 'RPC termination has signaled shutdown.') future = _maybe_wrap_exception(future) - thread = threading.Thread( + thread = self.threading.Thread( name=_RPC_ERROR_THREAD_NAME, target=self.close, kwargs={'reason': future}) thread.daemon = True thread.start() - def unsubscribe(self): + def unsubscribe(self): # XXX should this be aliased to close? self.rpc.close() @classmethod @@ -326,7 +270,7 @@ def for_document(cls, document_ref, snapshot_callback, Args: document_ref: Reference to Document snapshot_callback: callback to be called on snapshot - snapshot_class_instance: instance of snapshot cls to make + snapshot_class_instance: instance of snapshot cls to make snapshots with to pass to snapshot_callback """ @@ -363,6 +307,52 @@ def for_document(cls, document_ref, snapshot_callback, # snapshot_callback, # snapshot_class_instance) + def _on_snapshot_target_change_no_change(self, proto): + _LOGGER.debug('on_snapshot: target change: NO_CHANGE') + change = proto.target_change + + no_target_ids = (change.target_ids is None or + len(change.target_ids) == 0) + if no_target_ids and change.read_time and self.current: + # TargetChange.CURRENT followed by TargetChange.NO_CHANGE + # signals a consistent state. Invoke the onSnapshot + # callback as specified by the user. + self.push(change.read_time, change.resume_token) + + def _on_snapshot_target_change_add(self, proto): + _LOGGER.debug("on_snapshot: target change: ADD") + assert WATCH_TARGET_ID == proto.target_change.target_ids[0], \ + 'Unexpected target ID sent by server' + # TODO : do anything here? Node didn't so I think this isn't + # the right thing to do + # wr = WatchResult( + # None, + # self._document_reference.id, + # ChangeType.ADDED) + # self._snapshot_callback(wr) + + def _on_snapshot_target_change_remove(self, proto): + _LOGGER.debug("on_snapshot: target change: REMOVE") + change = proto.target_change + + code = 13 + message = 'internal error' + if change.cause: + code = change.cause.code + message = change.cause.message + + # TODO: Surface a .code property on the exception. + raise Exception('Error %s: %s' % (code, message)) # XXX Exception? + + def _on_snapshot_target_change_reset(self, proto): + # Whatever changes have happened so far no longer matter. + _LOGGER.debug("on_snapshot: target change: RESET") + self._reset_docs() + + def _on_snapshot_target_change_current(self, proto): + _LOGGER.debug("on_snapshot: target change: CURRENT") + self.current = True + def on_snapshot(self, proto): """ Called everytime there is a response from listen. Collect changes @@ -375,72 +365,47 @@ def on_snapshot(self, proto): """ TargetChange = firestore_pb2.TargetChange - if str(proto.target_change): - _LOGGER.debug('on_snapshot: target change') - - # google.cloud.firestore_v1beta1.types.TargetChange - change = proto.target_change - - no_target_ids = change.target_ids is None or \ - len(change.target_ids) == 0 - if change.target_change_type == TargetChange.NO_CHANGE: - _LOGGER.debug('on_snapshot: target change: NO_CHANGE') - if no_target_ids and change.read_time and self.current: - # TargetChange.CURRENT followed by TargetChange.NO_CHANGE - # signals a consistent state. Invoke the onSnapshot - # callback as specified by the user. - self.push(change.read_time, change.resume_token) - elif change.target_change_type == TargetChange.ADD: - _LOGGER.debug("on_snapshot: target change: ADD") - assert WATCH_TARGET_ID == change.target_ids[0], \ - 'Unexpected target ID sent by server' - # TODO : do anything here? Node didn't so I think this isn't - # the right thing to do - # wr = WatchResult( - # None, - # self._document_reference.id, - # ChangeType.ADDED) - # self._snapshot_callback(wr) + target_changetype_dispatch = { + TargetChange.NO_CHANGE: self._on_snapshot_target_change_no_change, + TargetChange.ADD: self._on_snapshot_target_change_add, + TargetChange.REMOVE: self._on_snapshot_target_change_remove, + TargetChange.RESET: self._on_snapshot_target_change_reset, + TargetChange.CURRENT: self._on_snapshot_target_change_current, + } - elif change.target_change_type == TargetChange.REMOVE: - _LOGGER.debug("on_snapshot: target change: REMOVE") + target_change = proto.target_change - code = 13 - message = 'internal error' - if change.cause: - code = change.cause.code - message = change.cause.message - - # TODO: Surface a .code property on the exception. - raise Exception('Error ' + code + ': ' + message) - elif change.target_change_type == TargetChange.RESET: - # Whatever changes have happened so far no longer matter. - _LOGGER.debug("on_snapshot: target change: RESET") - self._reset_docs() - elif change.target_change_type == TargetChange.CURRENT: - _LOGGER.debug("on_snapshot: target change: CURRENT") - self.current = True - else: + if str(target_change): # XXX why if str + _LOGGER.debug('on_snapshot: target change') + target_change_type = target_change.target_change_type + meth = target_changetype_dispatch.get(target_change_type) + if meth is None: _LOGGER.info('on_snapshot: Unknown target change ' + - str(change.target_change_type)) - + str(target_change_type)) self._consumer.stop() # closeStream( # new Error('Unknown target change type: ' + # JSON.stringify(change)) # TODO : make this exit the inner function and stop processing? - raise Exception('Unknown target change type: ' + str(change)) + raise Exception('Unknown target change type: %s ' % + str(target_change_type)) # XXX Exception? + else: + meth(proto) + + # XXX this is currently a no-op + # affects_target = self._affects_target( + # target_change.target_ids, WATCH_TARGET_ID + # ) - if change.resume_token and self._affects_target(change.target_ids, - WATCH_TARGET_ID): - # TODO: they node version resets backoff here. We allow - # bidi rpc to do its thing. - pass + # if target_change.resume_token and affects_target: + # # TODO: they node version resets backoff here. We allow + # # bidi rpc to do its thing. + # pass - elif str(proto.document_change): + elif str(proto.document_change): # XXX why if str _LOGGER.debug('on_snapshot: document change') - # No other target_ids can show up here, but we still need to see + # No other target_ids can show up here, but we still need to see # if the targetId was in the added list or removed list. target_ids = proto.document_change.target_ids or [] removed_target_ids = proto.document_change.removed_target_ids or [] @@ -463,7 +428,7 @@ def on_snapshot(self, proto): # google.cloud.firestore_v1beta1.types.Document document = document_change.document - data = json_format.MessageToDict(document) + data = self.MessageToDict(document) snapshot = self.DocumentSnapshot( reference=self._document_reference, @@ -482,6 +447,7 @@ def on_snapshot(self, proto): elif removed: _LOGGER.debug('on_snapshot: document change: REMOVED') + document = proto.document_change.document self.change_map[document.name] = ChangeType.REMOVED elif (proto.document_delete or proto.document_remove): @@ -506,7 +472,8 @@ def on_snapshot(self, proto): _LOGGER.debug("UNKNOWN TYPE. UHOH") self._consumer.stop() raise Exception( - 'Unknown listen response type: ' + proto) + 'Unknown listen response type: %s' % proto + ) # XXX Exception? # TODO: can we stop but raise an error? # closeStream( # new Error('Unknown listen response type: ' + @@ -610,6 +577,7 @@ def add_doc(new_document, updated_tree, updated_map): new_index), updated_tree, updated_map) + # XXX modify_doc is broken via formattedName def modify_doc(new_document, updated_tree, updated_map): """ Applies a document modification to the document tree and the @@ -672,7 +640,6 @@ def modify_doc(new_document, updated_tree, updated_map): assert len(updated_tree) == len(updated_map), \ 'The update document ' + \ 'tree and document map should have the same number of entries.' - _LOGGER.debug("tree:{updated_tree}, map:{updated_map}, applied:{appliedChanges}") return (updated_tree, updated_map, appliedChanges) def _affects_target(self, target_ids, current_id): @@ -685,7 +652,7 @@ def _affects_target(self, target_ids, current_id): return False - def _current_size(self): + def _current_size(self): # XXX broken, no docMap or changeMap """ Returns the current count of all documents, including the changes from the current changeMap. @@ -693,7 +660,7 @@ def _current_size(self): deletes, adds, _ = Watch._extract_changes(self.docMap, self.changeMap) return self.docMap.size + len(adds) - len(deletes) - def _reset_docs(self): + def _reset_docs(self): # XXX broken via formattedName """ Helper to clear the docs on RESET or filter mismatch. """ diff --git a/firestore/tests/system.py b/firestore/tests/system.py index 995f5fc004b1..858489c081f1 100644 --- a/firestore/tests/system.py +++ b/firestore/tests/system.py @@ -814,7 +814,7 @@ def test_watch_document(client, cleanup): # Setup listener def on_response(response): on_response.called_count += 1 - print(f'Response: {response}') + print('Response: %s' % response) print(type(response)) on_response.called_count = 0 diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py new file mode 100644 index 000000000000..b2e93b799b10 --- /dev/null +++ b/firestore/tests/unit/test_watch.py @@ -0,0 +1,439 @@ +import unittest +import mock +from google.cloud.firestore_v1beta1.proto import firestore_pb2 + + +class TestWatchDocTree(unittest.TestCase): + def _makeOne(self): + from google.cloud.firestore_v1beta1.watch import WatchDocTree + return WatchDocTree() + + def test_insert_and_keys(self): + inst = self._makeOne() + inst = inst.insert('b', 1) + inst = inst.insert('a', 2) + self.assertEqual(sorted(inst.keys()), ['a', 'b']) + + def test_remove_and_keys(self): + inst = self._makeOne() + inst = inst.insert('b', 1) + inst = inst.insert('a', 2) + inst = inst.remove('a') + self.assertEqual(sorted(inst.keys()), ['b']) + + def test_insert_and_find(self): + inst = self._makeOne() + inst = inst.insert('b', 1) + inst = inst.insert('a', 2) + val = inst.find('a') + self.assertEqual(val.value, 2) + + def test___len__(self): + inst = self._makeOne() + inst = inst.insert('b', 1) + inst = inst.insert('a', 2) + self.assertEqual(len(inst), 2) + + +class TestDocumentChange(unittest.TestCase): + def _makeOne(self, type, document, old_index, new_index): + from google.cloud.firestore_v1beta1.watch import DocumentChange + return DocumentChange(type, document, old_index, new_index) + + def test_ctor(self): + inst = self._makeOne('type', 'document', 'old_index', 'new_index') + self.assertEqual(inst.type, 'type') + self.assertEqual(inst.document, 'document') + self.assertEqual(inst.old_index, 'old_index') + self.assertEqual(inst.new_index, 'new_index') + + +class TestWatchResult(unittest.TestCase): + def _makeOne(self, snapshot, name, change_type): + from google.cloud.firestore_v1beta1.watch import WatchResult + return WatchResult(snapshot, name, change_type) + + def test_ctor(self): + inst = self._makeOne('snapshot', 'name', 'change_type') + self.assertEqual(inst.snapshot, 'snapshot') + self.assertEqual(inst.name, 'name') + self.assertEqual(inst.change_type, 'change_type') + + +class Test_maybe_wrap_exception(unittest.TestCase): + def _callFUT(self, exc): + from google.cloud.firestore_v1beta1.watch import _maybe_wrap_exception + return _maybe_wrap_exception(exc) + + def test_is_grpc_error(self): + import grpc + from google.api_core.exceptions import GoogleAPICallError + exc = grpc.RpcError() + result = self._callFUT(exc) + self.assertEqual(result.__class__, GoogleAPICallError) + + def test_is_not_grpc_error(self): + exc = ValueError() + result = self._callFUT(exc) + self.assertEqual(result.__class__, ValueError) + + +class Test_document_watch_comparator(unittest.TestCase): + def _callFUT(self, doc1, doc2): + from google.cloud.firestore_v1beta1.watch import ( + document_watch_comparator, + ) + return document_watch_comparator(doc1, doc2) + + def test_same_doc(self): + result = self._callFUT(1, 1) + self.assertEqual(result, 0) + + def test_diff_doc(self): + self.assertRaises(AssertionError, self._callFUT, 1, 2) + + +class TestWatch(unittest.TestCase): + def _makeOne( + self, + document_reference=None, + firestore=None, + target=None, + comparator=None, + snapshot_callback=None, + snapshot_class=None, + ): + from google.cloud.firestore_v1beta1.watch import Watch + if document_reference is None: + document_reference = DummyDocumentReference() + if firestore is None: + firestore = DummyFirestore() + if target is None: + WATCH_TARGET_ID = 0x5079 # "Py" + target = { + 'documents': { + 'documents': ['/']}, + 'target_id': WATCH_TARGET_ID + } + if comparator is None: + comparator = self._document_watch_comparator + if snapshot_callback is None: + snapshot_callback = self._snapshot_callback + if snapshot_class is None: + snapshot_class = DummyDocumentSnapshot + + inst = Watch( + document_reference, + firestore, + target, + comparator, + snapshot_callback, + snapshot_class, + BackgroundConsumer=DummyBackgroundConsumer, + ResumableBidiRpc=DummyRpc, + ) + return inst + + def _document_watch_comparator(self, doc1, doc2): + return 0 + + def _snapshot_callback(self, docs, changes, read_time): + return True + + def test_ctor(self): + inst = self._makeOne() + self.assertTrue(inst._consumer.started) + self.assertTrue(inst.rpc.callbacks, [inst._on_rpc_done]) + + def dont_test__on_rpc_done(self): # XXX fails + inst = self._makeOne() + threading = DummyThreading() + inst.threading = threading + inst._on_rpc_done(True) # no close method, fails + from google.cloud.firestore_v1beta1.watch import _RPC_ERROR_THREAD_NAME + self.assertTrue(threading.threads[_RPC_ERROR_THREAD_NAME].started) + + def test_unsubscribe(self): + inst = self._makeOne() + inst.unsubscribe() + self.assertTrue(inst.rpc.closed) + + def test_for_document(self): + from google.cloud.firestore_v1beta1.watch import Watch + docref = DummyDocumentReference() + snapshot_callback = self._snapshot_callback + snapshot_class_instance = DummyDocumentSnapshot + modulename = 'google.cloud.firestore_v1beta1.watch' + with mock.patch( + '%s.Watch.ResumableBidiRpc' % modulename, + DummyRpc, + ): + with mock.patch( + '%s.Watch.BackgroundConsumer' % modulename, + DummyBackgroundConsumer, + ): + inst = Watch.for_document( + docref, + snapshot_callback, + snapshot_class_instance + ) + self.assertTrue(inst._consumer.started) + self.assertTrue(inst.rpc.callbacks, [inst._on_rpc_done]) + + def test_on_snapshot_target_no_change_no_target_ids_not_current(self): + inst = self._makeOne() + proto = DummyProto() + inst.on_snapshot(proto) # nothing to assert, no mutations, no rtnval + + def test_on_snapshot_target_no_change_no_target_ids_current(self): + inst = self._makeOne() + proto = DummyProto() + proto.target_change.read_time = 1 + inst.current = True + + def push(read_time, next_resume_token): + inst._read_time = read_time + inst._next_resume_token = next_resume_token + + inst.push = push + inst.on_snapshot(proto) + self.assertEqual(inst._read_time, 1) + self.assertEqual(inst._next_resume_token, None) + + def test_on_snapshot_target_add(self): + inst = self._makeOne() + proto = DummyProto() + proto.target_change.target_change_type = firestore_pb2.TargetChange.ADD + proto.target_change.target_ids = [1] # not "Py" + with self.assertRaises(Exception) as exc: + inst.on_snapshot(proto) + self.assertEqual( + str(exc.exception), + 'Unexpected target ID sent by server' + ) + + def test_on_snapshot_target_remove(self): + inst = self._makeOne() + proto = DummyProto() + target_change = proto.target_change + target_change.target_change_type = firestore_pb2.TargetChange.REMOVE + with self.assertRaises(Exception) as exc: + inst.on_snapshot(proto) + self.assertEqual(str(exc.exception), 'Error 1: hi') + + def test_on_snapshot_target_reset(self): + inst = self._makeOne() + + def reset(): + inst._docs_reset = True + + inst._reset_docs = reset + proto = DummyProto() + target_change = proto.target_change + target_change.target_change_type = firestore_pb2.TargetChange.RESET + inst.on_snapshot(proto) + self.assertTrue(inst._docs_reset) + + def test_on_snapshot_target_current(self): + inst = self._makeOne() + inst.current = False + proto = DummyProto() + target_change = proto.target_change + target_change.target_change_type = firestore_pb2.TargetChange.CURRENT + inst.on_snapshot(proto) + self.assertTrue(inst.current) + + def test_on_snapshot_target_unknown(self): + inst = self._makeOne() + proto = DummyProto() + proto.target_change.target_change_type = 'unknown' + with self.assertRaises(Exception) as exc: + inst.on_snapshot(proto) + self.assertTrue(inst._consumer.stopped) + self.assertEqual( + str(exc.exception), + 'Unknown target change type: unknown ' + ) + + def test_on_snapshot_document_change_removed(self): + from google.cloud.firestore_v1beta1.watch import ( + WATCH_TARGET_ID, + ChangeType, + ) + inst = self._makeOne() + proto = DummyProto() + proto.target_change = '' + proto.document_change.removed_target_ids = [WATCH_TARGET_ID] + + class DummyDocument: + name = 'fred' + + proto.document_change.document = DummyDocument() + inst.on_snapshot(proto) + self.assertTrue(inst.change_map['fred'] is ChangeType.REMOVED) + + def test_on_snapshot_document_change_changed(self): + from google.cloud.firestore_v1beta1.watch import WATCH_TARGET_ID + inst = self._makeOne() + + def message_to_dict(document): + return {'fields': None} + + inst.MessageToDict = message_to_dict + proto = DummyProto() + proto.target_change = '' + proto.document_change.target_ids = [WATCH_TARGET_ID] + + class DummyDocument: + name = 'fred' + create_time = None + update_time = None + + proto.document_change.document = DummyDocument() + inst.on_snapshot(proto) + self.assertEqual(inst.change_map['fred'].data, None) + + def test_on_snapshot_document_removed(self): + from google.cloud.firestore_v1beta1.watch import ChangeType + inst = self._makeOne() + proto = DummyProto() + proto.target_change = '' + proto.document_change = '' + + class DummyRemove(object): + document = 'fred' + + remove = DummyRemove() + proto.document_remove = remove + proto.document_delete = None + inst.on_snapshot(proto) + self.assertTrue(inst.change_map['fred'] is ChangeType.REMOVED) + + def dont_test_on_snapshot_filter_update(self): # XXX _current_size broken + inst = self._makeOne() + proto = DummyProto() + proto.target_change = '' + proto.document_change = '' + proto.document_remove = None + proto.document_delete = None + + class DummyFilter(object): + count = 999 + + proto.filter = DummyFilter() + + def reset(): + self._docs_reset = True + + proto._reset_docs = reset + inst.on_snapshot(proto) + self.assertTrue(inst._docs_reset) + + def test_on_snapshot_unknown_listen_type(self): + inst = self._makeOne() + proto = DummyProto() + proto.target_change = '' + proto.document_change = '' + proto.document_remove = None + proto.document_delete = None + proto.filter = '' + with self.assertRaises(Exception) as exc: + inst.on_snapshot(proto) + self.assertTrue( + str(exc.exception).startswith('Unknown listen response type'), + str(exc.exception) + ) + + +class DummyFirestoreStub(object): + def Listen(self): + pass + + +class DummyFirestoreClient(object): + def __init__(self): + self.firestore_stub = DummyFirestoreStub() + + +class DummyDocumentReference(object): + def __init__(self): + self._client = DummyFirestore() + _document_path = '/' + + +class DummyFirestore(object): + _firestore_api = DummyFirestoreClient() + _database_string = '' + + +class DummyDocumentSnapshot(object): + def __init__(self, **kw): + self.__dict__.update(kw) + + +class DummyBackgroundConsumer(object): + started = False + stopped = False + + def __init__(self, rpc, on_snapshot): + self.rpc = rpc + self.on_snapshot = on_snapshot + + def start(self): + self.started = True + + def stop(self): + self.stopped = True + + +class DummyThread(object): + started = False + + def start(self): + self.started = True + + +class DummyThreading(object): + def __init__(self): + self.threads = {} + + def Thread(self, name, target, kwargs): + thread = DummyThread(name, target, kwargs) + self.threads[name] = thread + return thread + + +class DummyRpc(object): + def __init__(self, listen, initial_request, should_recover): + self.listen = listen + self.initial_request = initial_request + self.should_recover = should_recover + self.closed = False + self.callbacks = [] + + def add_done_callback(self, callback): + self.callbacks.append(callback) + + def close(self): + self.closed = True + + +class DummyCause(object): + code = 1 + message = 'hi' + + +class DummyChange(object): + def __init__(self): + self.target_ids = [] + self.removed_target_ids = [] + self.read_time = 0 + self.target_change_type = firestore_pb2.TargetChange.NO_CHANGE + self.resume_token = None + self.cause = DummyCause() + + +class DummyProto(object): + def __init__(self): + self.target_change = DummyChange() + self.document_change = DummyChange() From 36e431b4840e6cc53602b659a571d803bf8605a4 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Mon, 20 Aug 2018 14:19:31 -0400 Subject: [PATCH 084/148] these must be staticmethods; improve performance in _affects_target --- firestore/google/cloud/firestore_v1beta1/watch.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index a44742b75111..369fb9e40cea 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -146,7 +146,7 @@ class Watch(object): threading = threading # FBO unit tests BackgroundConsumer = BackgroundConsumer # FBO unit tests ResumableBidiRpc = ResumableBidiRpc # FBO unit tests - MessageToDict = json_format.MessageToDict # FBO unit tests + MessageToDict = staticmethod(json_format.MessageToDict) # FBO unit tests def __init__(self, document_reference, @@ -512,6 +512,7 @@ def push(self, read_time, next_resume_token): self.change_map.clear() self.resume_token = next_resume_token + @staticmethod def _extract_changes(doc_map, changes, read_time): deletes = [] adds = [] @@ -530,6 +531,7 @@ def _extract_changes(doc_map, changes, read_time): _LOGGER.debug('deletes:{len(deletes)} adds:{len(adds)}') return (deletes, adds, updates) + @staticmethod def _compute_snapshot(doc_tree, doc_map, delete_changes, add_changes, update_changes): # TODO: ACTUALLY NEED TO CALCULATE @@ -646,9 +648,8 @@ def _affects_target(self, target_ids, current_id): if target_ids is None or len(target_ids) == 0: return True - for target_id in target_ids: - if target_id == current_id: - return True + if current_id in target_ids: + return True return False From 4f08ac33d3d2a8ddbe08b5420f20cf04dbbc3bea Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Mon, 20 Aug 2018 14:46:15 -0400 Subject: [PATCH 085/148] make system test for watch pass --- firestore/tests/system.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/firestore/tests/system.py b/firestore/tests/system.py index 858489c081f1..48f8a24a269a 100644 --- a/firestore/tests/system.py +++ b/firestore/tests/system.py @@ -812,10 +812,8 @@ def test_watch_document(client, cleanup): sleep(1) # Setup listener - def on_response(response): + def on_response(*arg): on_response.called_count += 1 - print('Response: %s' % response) - print(type(response)) on_response.called_count = 0 From d0bffa9b8e7a91beba2225edb1e6f79deeada176 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Mon, 20 Aug 2018 15:12:37 -0400 Subject: [PATCH 086/148] containment check instead of iteration --- firestore/google/cloud/firestore_v1beta1/watch.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 369fb9e40cea..4cf214a8afd2 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -412,13 +412,11 @@ def on_snapshot(self, proto): changed = False removed = False - for target in target_ids: - if target == WATCH_TARGET_ID: - changed = True + if WATCH_TARGET_ID in target_ids: + changed = True - for target in removed_target_ids: - if target == WATCH_TARGET_ID: - removed = True + if WATCH_TARGET_ID in removed_target_ids: + removed = True if changed: _LOGGER.debug('on_snapshot: document change: CHANGED') From 3de5310f6065a25515acf13d5ad3f4160b4c5d51 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Mon, 20 Aug 2018 15:55:10 -0400 Subject: [PATCH 087/148] fix filter update test --- .../google/cloud/firestore_v1beta1/watch.py | 16 ++++++++++++---- firestore/tests/unit/test_watch.py | 6 +++--- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 4cf214a8afd2..1cb30126cb02 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -95,6 +95,10 @@ def remove(self, key): del self._dict[key] return self + def __iter__(self): + for k in self._dict: + yield k + def __len__(self): return len(self._dict) @@ -521,10 +525,12 @@ def _extract_changes(doc_map, changes, read_time): if name in doc_map: deletes.append(name) elif name in doc_map: - value.read_time = read_time + if read_time is not None: + value.read_time = read_time updates.append(value) else: - value.read_time = read_time + if read_time is not None: + value.read_time = read_time adds.append(value) _LOGGER.debug('deletes:{len(deletes)} adds:{len(adds)}') return (deletes, adds, updates) @@ -656,8 +662,10 @@ def _current_size(self): # XXX broken, no docMap or changeMap Returns the current count of all documents, including the changes from the current changeMap. """ - deletes, adds, _ = Watch._extract_changes(self.docMap, self.changeMap) - return self.docMap.size + len(adds) - len(deletes) + deletes, adds, _ = Watch._extract_changes( + self.doc_map, self.change_map, None + ) + return len(self.doc_map) + len(adds) - len(deletes) def _reset_docs(self): # XXX broken via formattedName """ diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index b2e93b799b10..ab56c8613423 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -309,7 +309,7 @@ class DummyRemove(object): inst.on_snapshot(proto) self.assertTrue(inst.change_map['fred'] is ChangeType.REMOVED) - def dont_test_on_snapshot_filter_update(self): # XXX _current_size broken + def test_on_snapshot_filter_update(self): inst = self._makeOne() proto = DummyProto() proto.target_change = '' @@ -323,9 +323,9 @@ class DummyFilter(object): proto.filter = DummyFilter() def reset(): - self._docs_reset = True + inst._docs_reset = True - proto._reset_docs = reset + inst._reset_docs = reset inst.on_snapshot(proto) self.assertTrue(inst._docs_reset) From 40e8b0b185c85582ea34c8906c275c7188cf00a7 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Tue, 21 Aug 2018 09:45:40 -0400 Subject: [PATCH 088/148] tests for various helper methods --- .../google/cloud/firestore_v1beta1/watch.py | 27 ++++++------ firestore/tests/unit/test_watch.py | 41 ++++++++++++++++++- 2 files changed, 55 insertions(+), 13 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 1cb30126cb02..577ce76212fd 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -491,17 +491,20 @@ def push(self, read_time, next_resume_token): # and sending them to the user. deletes, adds, updates = Watch._extract_changes( - self.doc_map, self.change_map, read_time) - updated_tree, updated_map, appliedChanges = \ - Watch._compute_snapshot( - self.doc_tree, self.doc_map, deletes, adds, updates) + self.doc_map, + self.change_map, + read_time, + ) - if not self.has_pushed or len(appliedChanges): - _LOGGER.debug( - 'Sending snapshot with {len(appliedChanges)} changes' - ' and {len(updated_tree)} documents') + updated_tree, updated_map, appliedChanges = Watch._compute_snapshot( + self.doc_tree, + self.doc_map, + deletes, + adds, + updates, + ) - _LOGGER.debug("updatedTree:{updated_tree}") + if not self.has_pushed or len(appliedChanges): self._snapshot_callback( updated_tree.keys(), appliedChanges, @@ -532,7 +535,7 @@ def _extract_changes(doc_map, changes, read_time): if read_time is not None: value.read_time = read_time adds.append(value) - _LOGGER.debug('deletes:{len(deletes)} adds:{len(adds)}') + return (deletes, adds, updates) @staticmethod @@ -649,7 +652,7 @@ def modify_doc(new_document, updated_tree, updated_map): return (updated_tree, updated_map, appliedChanges) def _affects_target(self, target_ids, current_id): - if target_ids is None or len(target_ids) == 0: + if target_ids is None: return True if current_id in target_ids: @@ -657,7 +660,7 @@ def _affects_target(self, target_ids, current_id): return False - def _current_size(self): # XXX broken, no docMap or changeMap + def _current_size(self): """ Returns the current count of all documents, including the changes from the current changeMap. diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index ab56c8613423..d1044362a3ca 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -1,3 +1,4 @@ +import datetime import unittest import mock from google.cloud.firestore_v1beta1.proto import firestore_pb2 @@ -8,6 +9,9 @@ def _makeOne(self): from google.cloud.firestore_v1beta1.watch import WatchDocTree return WatchDocTree() + def setUp(self): + self.snapshotted = None + def test_insert_and_keys(self): inst = self._makeOne() inst = inst.insert('b', 1) @@ -138,7 +142,7 @@ def _document_watch_comparator(self, doc1, doc2): return 0 def _snapshot_callback(self, docs, changes, read_time): - return True + self.snapshotted = (docs, changes, read_time) def test_ctor(self): inst = self._makeOne() @@ -344,6 +348,41 @@ def test_on_snapshot_unknown_listen_type(self): str(exc.exception) ) + def test_push_no_changes(self): + class DummyReadTime(object): + seconds = 1534858278 + inst = self._makeOne() + inst.push(DummyReadTime, 'token') + self.assertEqual( + self.snapshotted, + ([], [], datetime.datetime(2018, 8, 21, 9, 31, 18)), + ) + self.assertTrue(inst.has_pushed) + self.assertEqual(inst.resume_token, 'token') + + def test__current_size_empty(self): + inst = self._makeOne() + result = inst._current_size() + self.assertEqual(result, 0) + + def test__current_size_docmap_has_one(self): + inst = self._makeOne() + inst.doc_map['a'] = 1 + result = inst._current_size() + self.assertEqual(result, 1) + + def test__affects_target_target_id_None(self): + inst = self._makeOne() + self.assertTrue(inst._affects_target(None, [])) + + def test__affects_target_current_id_in_target_ids(self): + inst = self._makeOne() + self.assertTrue(inst._affects_target([1], 1)) + + def test__affects_target_current_id_not_in_target_ids(self): + inst = self._makeOne() + self.assertFalse(inst._affects_target([1], 2)) + class DummyFirestoreStub(object): def Listen(self): From 18532977ca121b53c048cab9c50e35dea0f31047 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Tue, 21 Aug 2018 12:54:55 -0700 Subject: [PATCH 089/148] Improve rpc_done and add early query support --- .../google/cloud/firestore_v1beta1/bidi.py | 7 +- .../google/cloud/firestore_v1beta1/query.py | 43 +++++----- .../google/cloud/firestore_v1beta1/watch.py | 83 ++++++++++++++----- 3 files changed, 89 insertions(+), 44 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/bidi.py b/firestore/google/cloud/firestore_v1beta1/bidi.py index e7629fc5df8e..53cfd7464c05 100644 --- a/firestore/google/cloud/firestore_v1beta1/bidi.py +++ b/firestore/google/cloud/firestore_v1beta1/bidi.py @@ -196,7 +196,6 @@ def open(self): request_generator = _RequestQueueGenerator( self._request_queue, initial_request=self._initial_request) - print('request generator created') call = self._start_rpc(iter(request_generator)) request_generator.call = call @@ -282,7 +281,7 @@ class ResumableBidiRpc(BidiRpc): def should_recover(exc): return ( isinstance(exc, grpc.RpcError) and - exc.code() == grpc.StatusCode.UNVAILABLE) + exc.code() == grpc.StatusCode.UNAVAILABLE) initial_request = example_pb2.StreamingRpcRequest( setting='example') @@ -429,7 +428,7 @@ class BackgroundConsumer(object): def should_recover(exc): return ( isinstance(exc, grpc.RpcError) and - exc.code() == grpc.StatusCode.UNVAILABLE) + exc.code() == grpc.StatusCode.UNAVAILABLE) initial_request = example_pb2.StreamingRpcRequest( setting='example') @@ -443,7 +442,7 @@ def on_response(response): print(response) consumer = BackgroundConsumer(rpc, on_response) - consumer.start() + consume.start() Note that error handling *must* be done by using the provided ``bidi_rpc``'s ``add_done_callback``. This helper will automatically exit diff --git a/firestore/google/cloud/firestore_v1beta1/query.py b/firestore/google/cloud/firestore_v1beta1/query.py index 2a88ad054678..229ae1afa8b1 100644 --- a/firestore/google/cloud/firestore_v1beta1/query.py +++ b/firestore/google/cloud/firestore_v1beta1/query.py @@ -29,7 +29,7 @@ from google.cloud.firestore_v1beta1 import document from google.cloud.firestore_v1beta1.gapic import enums from google.cloud.firestore_v1beta1.proto import query_pb2 - +from google.cloud.firestore_v1beta1.watch import Watch _EQ_OP = '==' _COMPARISON_OPERATORS = { @@ -601,33 +601,34 @@ def get(self, transaction=None): else: yield snapshot - # def on_snapshot(self, callback): - # """Monitor the documents in this collection that match this query. + def on_snapshot(self, callback): + """Monitor the documents in this collection that match this query. + + This starts a watch on this query using a background thread. The + provided callback is run on the snapshot of the documents. - # This starts a watch on this query using a background thread. The - # provided callback is run on the snapshot of the documents. + Args: + callback(~.firestore.query.QuerySnapshot): a callback to run when + a change occurs. - # Args: - # callback(~.firestore.query.QuerySnapshot): a callback to run when - # a change occurs. + Example: + from google.cloud import firestore - # Example: - # from google.cloud import firestore + db = firestore.Client() + query_ref = db.collection(u'users').where("user", "==", u'Ada') - # db = firestore.Client() - # query_ref = db.collection(u'users').where("user", "==", u'ada') + def on_snapshot(query_snapshot): + for doc in query_snapshot.documents: + print(u'{} => {}'.format(doc.id, doc.to_dict())) - # def on_snapshot(query_snapshot): - # for doc in query_snapshot.documents: - # print(u'{} => {}'.format(doc.id, doc.to_dict())) + # Watch this query + query_watch = query_ref.on_snapshot(on_snapshot) - # # Watch this query - # query_watch = query_ref.on_snapshot(on_snapshot) + # Terminate this watch + query_watch.unsubscribe() + """ + Watch.for_query(self, callback, document.DocumentSnapshot) - # # Terminate this watch - # query_watch.unsubscribe() - # """ - # raise NotImplemented def _enum_from_op_string(op_string): diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 577ce76212fd..859307ac5518 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -187,11 +187,13 @@ def __init__(self, self._comparator = comparator self.DocumentSnapshot = DocumentSnapshotCls self._snapshot_callback = snapshot_callback - + self._closing = threading.Lock() + self._closed = False + def should_recover(exc): return ( isinstance(exc, grpc.RpcError) and - exc.code() == grpc.StatusCode.UNVAILABLE) + exc.code() == grpc.StatusCode.UNAVAILABLE) initial_request = firestore_pb2.ListenRequest( database=self._firestore._database_string, @@ -238,6 +240,41 @@ def should_recover(exc): self._consumer = BackgroundConsumer(self.rpc, self.on_snapshot) self._consumer.start() + @property + def is_active(self): + """bool: True if this manager is actively streaming. + + Note that ``False`` does not indicate this is complete shut down, + just that it stopped getting new messages. + """ + return self._consumer is not None and self._consumer.is_active + + def close(self, reason=None): + """Stop consuming messages and shutdown all helper threads. + + This method is idempotent. Additional calls will have no effect. + + Args: + reason (Any): The reason to close this. If None, this is considered + an "intentional" shutdown. + """ + with self._closing: + if self._closed: + return + + # Stop consuming messages. + if self.is_active: + _LOGGER.debug('Stopping consumer.') + self._consumer.stop() + self._consumer = None + + # TODO: Verify we don't have other helper threads that need to be + # shut down here. + + self._rpc = None + self._closed = True + _LOGGER.debug('Finished stopping manager.') + def _on_rpc_done(self, future): """Triggered whenever the underlying RPC terminates without recovery. @@ -249,11 +286,10 @@ def _on_rpc_done(self, future): with shutting everything down. This is to prevent blocking in the background consumer and preventing it from being ``joined()``. """ - # TODO: look at pushing this down into the background consumer _LOGGER.info( - 'RPC termination has signaled shutdown.') + 'RPC termination has signaled manager shutdown.') future = _maybe_wrap_exception(future) - thread = self.threading.Thread( + thread = threading.Thread( name=_RPC_ERROR_THREAD_NAME, target=self.close, kwargs={'reason': future}) @@ -289,19 +325,25 @@ def for_document(cls, document_ref, snapshot_callback, snapshot_callback, snapshot_class_instance) - # @classmethod - # def for_query(cls, query, snapshot_callback): - # return cls(query._client, - # { - # 'query': query.to_proto(), - # 'target_id': WATCH_TARGET_ID - # }, - # query.comparator(), - # snapshot_callback, - # snapshot_class_instance) + @classmethod + def for_query(cls, query, snapshot_callback, snapshot_class_instance): + query_target = firestore_pb2.Target.QueryTarget( + parent=query._parent.id, + structured_query=query._to_protobuf(), + ) + return cls(query, + query._client, + { + 'query': query_target, + 'target_id': WATCH_TARGET_ID + }, + document_watch_comparator, + snapshot_callback, + snapshot_class_instance) # @classmethod - # def for_collection(cls, collection_ref, snapshot_callback): + # def for_collection(cls, collection_ref, snapshot_callback, + # snapshot_class_instance): # return cls(collection_ref._client, # { # 'collection': collection_ref.to_proto(), @@ -379,9 +421,9 @@ def on_snapshot(self, proto): target_change = proto.target_change - if str(target_change): # XXX why if str - _LOGGER.debug('on_snapshot: target change') + if str(target_change): # XXX why if str - if it doesn't exist it will be empty (falsy). Otherwise always true. target_change_type = target_change.target_change_type + _LOGGER.debug('on_snapshot: target change: ' + str(target_change_type)) meth = target_changetype_dispatch.get(target_change_type) if meth is None: _LOGGER.info('on_snapshot: Unknown target change ' + @@ -394,7 +436,10 @@ def on_snapshot(self, proto): raise Exception('Unknown target change type: %s ' % str(target_change_type)) # XXX Exception? else: - meth(proto) + try: + meth(proto) + except Exception as exc2: + _LOGGER.debug("meth(proto) exc: " + str(exc2)) # XXX this is currently a no-op # affects_target = self._affects_target( From 1e77943b5ff4151c0fa134c97683baad29f04547 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Tue, 21 Aug 2018 16:02:35 -0400 Subject: [PATCH 090/148] add more tests --- .../google/cloud/firestore_v1beta1/watch.py | 10 +-- firestore/tests/unit/test_watch.py | 71 +++++++++++++++++++ 2 files changed, 76 insertions(+), 5 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 859307ac5518..4851e775a09b 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -604,10 +604,10 @@ def delete_doc(name, updated_tree, updated_map): """ assert name in updated_map, 'Document to delete does not exist' old_document = updated_map.get(name) - existing = updated_tree.find(old_document) + existing = updated_tree.find(name) old_index = existing.index # TODO: was existing.remove returning tree (presumably immuatable?) - updated_tree = updated_tree.remove(old_document) + updated_tree = updated_tree.remove(name) del updated_map[name] return (DocumentChange(ChangeType.REMOVED, old_document, @@ -670,7 +670,7 @@ def modify_doc(new_document, updated_tree, updated_map): for name in delete_changes: change, updated_tree, updated_map = delete_doc( name, updated_tree, updated_map) - if change: + if change: # XXX will always be True appliedChanges.append(change) # TODO: SORT @@ -680,7 +680,7 @@ def modify_doc(new_document, updated_tree, updated_map): _LOGGER.debug('in add_changes') change, updated_tree, updated_map = add_doc( snapshot, updated_tree, updated_map) - if change: + if change: # XXX will always be True appliedChanges.append(change) # TODO: SORT @@ -688,7 +688,7 @@ def modify_doc(new_document, updated_tree, updated_map): for snapshot in update_changes: change, updated_tree, updated_map = modify_doc( snapshot, updated_tree, updated_map) - if change: + if change: # XXX will always be True appliedChanges.append(change) assert len(updated_tree) == len(updated_map), \ diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index d1044362a3ca..dfc2d7c378fb 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -383,6 +383,77 @@ def test__affects_target_current_id_not_in_target_ids(self): inst = self._makeOne() self.assertFalse(inst._affects_target([1], 2)) + def test__extract_changes_doc_removed(self): + from google.cloud.firestore_v1beta1.watch import ChangeType + inst = self._makeOne() + changes = {'name':ChangeType.REMOVED} + doc_map = {'name':True} + results = inst._extract_changes(doc_map, changes, None) + self.assertEqual(results, (['name'], [], [])) + + def test__extract_changes_doc_updated(self): + inst = self._makeOne() + class Dummy(object): + pass + doc = Dummy() + snapshot = Dummy() + changes = {'name':snapshot} + doc_map = {'name':doc} + results = inst._extract_changes(doc_map, changes, 1) + self.assertEqual(results, ([], [], [snapshot])) + self.assertEqual(snapshot.read_time, 1) + + def test__extract_changes_doc_added(self): + inst = self._makeOne() + class Dummy(object): + pass + snapshot = Dummy() + changes = {'name':snapshot} + doc_map = {} + results = inst._extract_changes(doc_map, changes, 1) + self.assertEqual(results, ([], [snapshot], [])) + self.assertEqual(snapshot.read_time, 1) + + def test__compute_snapshot_doctree_and_docmap_disagree_about_length(self): + inst = self._makeOne() + doc_tree = {} + doc_map = {None:None} + self.assertRaises( + AssertionError, + inst._compute_snapshot, doc_tree, doc_map, None, None, None, + ) + + def test__compute_snapshot_operation_relative_ordering(self): + from google.cloud.firestore_v1beta1.watch import WatchDocTree + doc_tree = WatchDocTree() + class DummyDoc(object): + pass + deleted_doc = DummyDoc() + added_doc = DummyDoc() + updated_doc = DummyDoc() + doc_tree = doc_tree.insert('deleted', deleted_doc) + doc_tree = doc_tree.insert('added', added_doc) + doc_tree = doc_tree.insert('updated', updated_doc) + doc_map = { + 'deleted':deleted_doc, + 'added':added_doc, + 'updated':updated_doc, + } + added_snapshot = DummyDocumentSnapshot() + updated_snapshot = DummyDocumentSnapshot() + updated_snapshot.reference = updated_doc + delete_changes = ['deleted'] + add_changes = [added_snapshot] + update_changes = [updated_snapshot] + inst = self._makeOne() + updated_tree, updated_map, applied_changes = inst._compute_snapshot( + doc_tree, + doc_map, + delete_changes, + add_changes, + update_changes + ) + self.assertEqual(updated_map, None) class DummyFirestoreStub(object): def Listen(self): From 27241321c4a24f3eb75dbd4645526b69f5c0c547 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Tue, 21 Aug 2018 16:23:40 -0400 Subject: [PATCH 091/148] add tests for close --- .../google/cloud/firestore_v1beta1/watch.py | 1 - firestore/tests/unit/test_watch.py | 37 +++++++++++++++++-- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 4851e775a09b..2ff8e500fafc 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -147,7 +147,6 @@ def document_watch_comparator(doc1, doc2): class Watch(object): - threading = threading # FBO unit tests BackgroundConsumer = BackgroundConsumer # FBO unit tests ResumableBidiRpc = ResumableBidiRpc # FBO unit tests MessageToDict = staticmethod(json_format.MessageToDict) # FBO unit tests diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index dfc2d7c378fb..cd7705c26483 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -149,14 +149,38 @@ def test_ctor(self): self.assertTrue(inst._consumer.started) self.assertTrue(inst.rpc.callbacks, [inst._on_rpc_done]) - def dont_test__on_rpc_done(self): # XXX fails + def test__on_rpc_done(self): inst = self._makeOne() threading = DummyThreading() - inst.threading = threading - inst._on_rpc_done(True) # no close method, fails + with mock.patch( + 'google.cloud.firestore_v1beta1.watch.threading', + threading + ): + inst._on_rpc_done(True) from google.cloud.firestore_v1beta1.watch import _RPC_ERROR_THREAD_NAME self.assertTrue(threading.threads[_RPC_ERROR_THREAD_NAME].started) + def test_close(self): + inst = self._makeOne() + inst.close() + self.assertEqual(inst._consumer, None) + self.assertEqual(inst._rpc, None) + self.assertTrue(inst._closed) + + def test_close_already_closed(self): + inst = self._makeOne() + inst._closed = True + old_consumer = inst._consumer + inst.close() + self.assertEqual(inst._consumer, old_consumer) + + def test_close_inactive(self): + inst = self._makeOne() + old_consumer = inst._consumer + old_consumer.is_active = False + inst.close() + self.assertEqual(old_consumer.stopped, False) + def test_unsubscribe(self): inst = self._makeOne() inst.unsubscribe() @@ -484,6 +508,7 @@ def __init__(self, **kw): class DummyBackgroundConsumer(object): started = False stopped = False + is_active = True def __init__(self, rpc, on_snapshot): self.rpc = rpc @@ -494,11 +519,17 @@ def start(self): def stop(self): self.stopped = True + self.is_active = False class DummyThread(object): started = False + def __init__(self, name, target, kwargs): + self.name = name + self.target = target + self.kwargs = kwargs + def start(self): self.started = True From d7258c886dc6a7711b7196d5113e088ca8db7c2a Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Tue, 21 Aug 2018 16:26:31 -0400 Subject: [PATCH 092/148] not reraising in except broke tests --- .../google/cloud/firestore_v1beta1/watch.py | 1 + firestore/noxfile.py | 21 ++++++++++--------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 2ff8e500fafc..7b9bf0a6e875 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -439,6 +439,7 @@ def on_snapshot(self, proto): meth(proto) except Exception as exc2: _LOGGER.debug("meth(proto) exc: " + str(exc2)) + raise # XXX this is currently a no-op # affects_target = self._affects_target( diff --git a/firestore/noxfile.py b/firestore/noxfile.py index 064f8044f182..624873d2c356 100644 --- a/firestore/noxfile.py +++ b/firestore/noxfile.py @@ -44,20 +44,21 @@ def default(session): session.run( 'py.test', '--quiet', - '--cov=google.cloud.firestore', - '--cov=google.cloud.firestore_v1beta1', - '--cov=tests.unit', - '--cov-append', - '--cov-config=.coveragerc', - '--cov-report=', - '--cov-fail-under=97', - os.path.join('tests', 'unit'), +# '--cov=google.cloud.firestore', +# '--cov=google.cloud.firestore_v1beta1', +# '--cov=tests.unit', +# '--cov-append', +# '--cov-config=.coveragerc', +# '--cov-report=', +# '--cov-fail-under=97', + os.path.join('tests', 'unit', 'test_watch.py'), *session.posargs ) -@nox.session(python=['2.7', '3.5', '3.6', '3.7']) -def unit(session): +@nox.session +@nox.parametrize('py', ['2.7', '3.7']) +def unit(session, py): """Run the unit test suite.""" default(session) From 1396b74a8ca6aa4f9e7d0a03e2aa50500066bcec Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Tue, 21 Aug 2018 17:24:43 -0400 Subject: [PATCH 093/148] compute_snapshot_ordering test still fails but fails later than it used to --- .../google/cloud/firestore_v1beta1/watch.py | 25 +++++++++---------- firestore/tests/unit/test_watch.py | 15 +++++------ 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 7b9bf0a6e875..bf462e10b838 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -604,6 +604,7 @@ def delete_doc(name, updated_tree, updated_map): """ assert name in updated_map, 'Document to delete does not exist' old_document = updated_map.get(name) + # XXX probably should not expose IndexError when doc doesnt exist existing = updated_tree.find(name) old_index = existing.index # TODO: was existing.remove returning tree (presumably immuatable?) @@ -638,18 +639,18 @@ def modify_doc(new_document, updated_tree, updated_map): document map. Returns the DocumentChange event for successful modifications. """ - name = new_document.reference.formattedName - assert updated_map.has(name), 'Document to modify does not exist' - oldDocument = updated_map.get(name) - if oldDocument.updateTime != new_document.updateTime: - removeChange, updated_tree, updated_map = delete_doc( + name = new_document.reference._document_path + assert name in updated_map, 'Document to modify does not exist' + old_document = updated_map.get(name) + if old_document.update_time != new_document.update_time: + remove_change, updated_tree, updated_map = delete_doc( name, updated_tree, updated_map) - addChange, updated_tree, updated_map = add_doc( + add_change, updated_tree, updated_map = add_doc( new_document, updated_tree, updated_map) return (DocumentChange(ChangeType.MODIFIED, new_document, - removeChange.old_index, - addChange.new_index), + remove_change.old_index, + add_change.new_index), updated_tree, updated_map) return None @@ -670,8 +671,7 @@ def modify_doc(new_document, updated_tree, updated_map): for name in delete_changes: change, updated_tree, updated_map = delete_doc( name, updated_tree, updated_map) - if change: # XXX will always be True - appliedChanges.append(change) + appliedChanges.append(change) # TODO: SORT # add_changes.sort(self._comparator) @@ -680,15 +680,14 @@ def modify_doc(new_document, updated_tree, updated_map): _LOGGER.debug('in add_changes') change, updated_tree, updated_map = add_doc( snapshot, updated_tree, updated_map) - if change: # XXX will always be True - appliedChanges.append(change) + appliedChanges.append(change) # TODO: SORT # update_changes.sort(self._comparator) for snapshot in update_changes: change, updated_tree, updated_map = modify_doc( snapshot, updated_tree, updated_map) - if change: # XXX will always be True + if change is not None: appliedChanges.append(change) assert len(updated_tree) == len(updated_map), \ diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index cd7705c26483..fd808b50a801 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -454,19 +454,20 @@ class DummyDoc(object): pass deleted_doc = DummyDoc() added_doc = DummyDoc() + added_doc._document_path = '/added' updated_doc = DummyDoc() - doc_tree = doc_tree.insert('deleted', deleted_doc) - doc_tree = doc_tree.insert('added', added_doc) - doc_tree = doc_tree.insert('updated', updated_doc) + updated_doc._document_path = '/updated' + doc_tree = doc_tree.insert('/deleted', deleted_doc) + doc_tree = doc_tree.insert('/updated', updated_doc) doc_map = { - 'deleted':deleted_doc, - 'added':added_doc, - 'updated':updated_doc, + '/deleted':deleted_doc, + '/updated':updated_doc, } added_snapshot = DummyDocumentSnapshot() + added_snapshot.reference = added_doc updated_snapshot = DummyDocumentSnapshot() updated_snapshot.reference = updated_doc - delete_changes = ['deleted'] + delete_changes = ['/deleted'] add_changes = [added_snapshot] update_changes = [updated_snapshot] inst = self._makeOne() From 1465bc2e7135ecac717abf003f80feac02d1c772 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Tue, 21 Aug 2018 17:25:29 -0400 Subject: [PATCH 094/148] remove incorrect comment --- firestore/google/cloud/firestore_v1beta1/watch.py | 1 - 1 file changed, 1 deletion(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index bf462e10b838..04cd1172379c 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -632,7 +632,6 @@ def add_doc(new_document, updated_tree, updated_map): new_index), updated_tree, updated_map) - # XXX modify_doc is broken via formattedName def modify_doc(new_document, updated_tree, updated_map): """ Applies a document modification to the document tree and the From 39b97db26fe27c992083e6cd640d2fd394aba69f Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Tue, 21 Aug 2018 15:19:59 -0700 Subject: [PATCH 095/148] parent is a fq path --- firestore/google/cloud/firestore_v1beta1/watch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 04cd1172379c..ec789f3eb044 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -188,7 +188,7 @@ def __init__(self, self._snapshot_callback = snapshot_callback self._closing = threading.Lock() self._closed = False - + def should_recover(exc): return ( isinstance(exc, grpc.RpcError) and @@ -327,7 +327,7 @@ def for_document(cls, document_ref, snapshot_callback, @classmethod def for_query(cls, query, snapshot_callback, snapshot_class_instance): query_target = firestore_pb2.Target.QueryTarget( - parent=query._parent.id, + parent=query._client._database_string, structured_query=query._to_protobuf(), ) return cls(query, From 39e429f3d7905d643c601d00c5d26e5e79608852 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Tue, 21 Aug 2018 18:34:32 -0400 Subject: [PATCH 096/148] fix and add test for _reset_docs --- .../google/cloud/firestore_v1beta1/watch.py | 10 ++++---- firestore/tests/unit/test_watch.py | 23 ++++++++++++++++++- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index ec789f3eb044..80a4967c4744 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -74,6 +74,9 @@ def __init__(self): def keys(self): return list(self._dict.keys()) + def items(self): + return list(self._dict.items()) + def _copy(self): wdt = WatchDocTree() wdt._dict = self._dict.copy() @@ -713,7 +716,7 @@ def _current_size(self): ) return len(self.doc_map) + len(adds) - len(deletes) - def _reset_docs(self): # XXX broken via formattedName + def _reset_docs(self): """ Helper to clear the docs on RESET or filter mismatch. """ @@ -723,8 +726,7 @@ def _reset_docs(self): # XXX broken via formattedName # TODO: mark each document as deleted. If documents are not delete # they will be sent again by the server. - for snapshot in self.doc_tree: - document_name = snapshot.reference.formattedName - self.change_map[document_name] = ChangeType.REMOVED + for name, snapshot in self.doc_tree.items(): + self.change_map[name] = ChangeType.REMOVED self.current = False diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index fd808b50a801..4cb86df238f0 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -478,7 +478,28 @@ class DummyDoc(object): add_changes, update_changes ) - self.assertEqual(updated_map, None) + # assertion is incorrect below, but we don't get here yet; the tested + # code raises an exception before we get a result + self.assertEqual(updated_map, None) + + def test__reset_docs(self): + from google.cloud.firestore_v1beta1.watch import ChangeType + inst = self._makeOne() + inst.change_map = {None:None} + from google.cloud.firestore_v1beta1.watch import WatchDocTree + doc = DummyDocumentReference() + doc._document_path = '/doc' + doc_tree = WatchDocTree() + doc_tree = doc_tree.insert('/doc', doc) + doc_tree = doc_tree.insert('/doc', doc) + snapshot = DummyDocumentSnapshot() + snapshot.reference = doc + inst.doc_tree = doc_tree + inst._reset_docs() + self.assertEqual(inst.change_map, {'/doc':ChangeType.REMOVED}) + self.assertEqual(inst.resume_token, None) + self.assertFalse(inst.current) + class DummyFirestoreStub(object): def Listen(self): From 397c6d682f33b03d864bbca1c7e255cf6b1007e1 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Wed, 22 Aug 2018 11:37:54 -0400 Subject: [PATCH 097/148] undo mistaken push --- firestore/noxfile.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/firestore/noxfile.py b/firestore/noxfile.py index 624873d2c356..5ead66979f87 100644 --- a/firestore/noxfile.py +++ b/firestore/noxfile.py @@ -44,14 +44,14 @@ def default(session): session.run( 'py.test', '--quiet', -# '--cov=google.cloud.firestore', -# '--cov=google.cloud.firestore_v1beta1', -# '--cov=tests.unit', -# '--cov-append', -# '--cov-config=.coveragerc', -# '--cov-report=', -# '--cov-fail-under=97', - os.path.join('tests', 'unit', 'test_watch.py'), + '--cov=google.cloud.firestore', + '--cov=google.cloud.firestore_v1beta1', + '--cov=tests.unit', + '--cov-append', + '--cov-config=.coveragerc', + '--cov-report=', + '--cov-fail-under=97', + os.path.join('tests', 'unit'), *session.posargs ) From 95a485a206eac0c3bde9af671b55a8ef57459039 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Wed, 22 Aug 2018 11:51:13 -0400 Subject: [PATCH 098/148] appease the linter --- .../google/cloud/firestore_v1beta1/query.py | 1 - .../google/cloud/firestore_v1beta1/watch.py | 9 +++--- firestore/tests/unit/test_watch.py | 31 ++++++++++--------- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/query.py b/firestore/google/cloud/firestore_v1beta1/query.py index 229ae1afa8b1..1ca4ffe8e2a5 100644 --- a/firestore/google/cloud/firestore_v1beta1/query.py +++ b/firestore/google/cloud/firestore_v1beta1/query.py @@ -630,7 +630,6 @@ def on_snapshot(query_snapshot): Watch.for_query(self, callback, document.DocumentSnapshot) - def _enum_from_op_string(op_string): """Convert a string representation of a binary operator to an enum. diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 80a4967c4744..78300bf693c5 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -270,7 +270,7 @@ def close(self, reason=None): self._consumer.stop() self._consumer = None - # TODO: Verify we don't have other helper threads that need to be + # TODO: Verify we don't have other helper threads that need to be # shut down here. self._rpc = None @@ -423,9 +423,10 @@ def on_snapshot(self, proto): target_change = proto.target_change - if str(target_change): # XXX why if str - if it doesn't exist it will be empty (falsy). Otherwise always true. + if str(target_change): target_change_type = target_change.target_change_type - _LOGGER.debug('on_snapshot: target change: ' + str(target_change_type)) + _LOGGER.debug( + 'on_snapshot: target change: ' + str(target_change_type)) meth = target_changetype_dispatch.get(target_change_type) if meth is None: _LOGGER.info('on_snapshot: Unknown target change ' + @@ -454,7 +455,7 @@ def on_snapshot(self, proto): # # bidi rpc to do its thing. # pass - elif str(proto.document_change): # XXX why if str + elif str(proto.document_change): _LOGGER.debug('on_snapshot: document change') # No other target_ids can show up here, but we still need to see diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index 4cb86df238f0..8332e1ab3046 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -410,29 +410,33 @@ def test__affects_target_current_id_not_in_target_ids(self): def test__extract_changes_doc_removed(self): from google.cloud.firestore_v1beta1.watch import ChangeType inst = self._makeOne() - changes = {'name':ChangeType.REMOVED} - doc_map = {'name':True} + changes = {'name': ChangeType.REMOVED} + doc_map = {'name': True} results = inst._extract_changes(doc_map, changes, None) self.assertEqual(results, (['name'], [], [])) def test__extract_changes_doc_updated(self): inst = self._makeOne() + class Dummy(object): pass + doc = Dummy() snapshot = Dummy() - changes = {'name':snapshot} - doc_map = {'name':doc} + changes = {'name': snapshot} + doc_map = {'name': doc} results = inst._extract_changes(doc_map, changes, 1) self.assertEqual(results, ([], [], [snapshot])) self.assertEqual(snapshot.read_time, 1) - + def test__extract_changes_doc_added(self): inst = self._makeOne() + class Dummy(object): pass + snapshot = Dummy() - changes = {'name':snapshot} + changes = {'name': snapshot} doc_map = {} results = inst._extract_changes(doc_map, changes, 1) self.assertEqual(results, ([], [snapshot], [])) @@ -441,7 +445,7 @@ class Dummy(object): def test__compute_snapshot_doctree_and_docmap_disagree_about_length(self): inst = self._makeOne() doc_tree = {} - doc_map = {None:None} + doc_map = {None: None} self.assertRaises( AssertionError, inst._compute_snapshot, doc_tree, doc_map, None, None, None, @@ -450,8 +454,10 @@ def test__compute_snapshot_doctree_and_docmap_disagree_about_length(self): def test__compute_snapshot_operation_relative_ordering(self): from google.cloud.firestore_v1beta1.watch import WatchDocTree doc_tree = WatchDocTree() + class DummyDoc(object): pass + deleted_doc = DummyDoc() added_doc = DummyDoc() added_doc._document_path = '/added' @@ -459,10 +465,7 @@ class DummyDoc(object): updated_doc._document_path = '/updated' doc_tree = doc_tree.insert('/deleted', deleted_doc) doc_tree = doc_tree.insert('/updated', updated_doc) - doc_map = { - '/deleted':deleted_doc, - '/updated':updated_doc, - } + doc_map = {'/deleted': deleted_doc, '/updated': updated_doc} added_snapshot = DummyDocumentSnapshot() added_snapshot.reference = added_doc updated_snapshot = DummyDocumentSnapshot() @@ -480,12 +483,12 @@ class DummyDoc(object): ) # assertion is incorrect below, but we don't get here yet; the tested # code raises an exception before we get a result - self.assertEqual(updated_map, None) + self.assertEqual(updated_map, None) def test__reset_docs(self): from google.cloud.firestore_v1beta1.watch import ChangeType inst = self._makeOne() - inst.change_map = {None:None} + inst.change_map = {None: None} from google.cloud.firestore_v1beta1.watch import WatchDocTree doc = DummyDocumentReference() doc._document_path = '/doc' @@ -496,7 +499,7 @@ def test__reset_docs(self): snapshot.reference = doc inst.doc_tree = doc_tree inst._reset_docs() - self.assertEqual(inst.change_map, {'/doc':ChangeType.REMOVED}) + self.assertEqual(inst.change_map, {'/doc': ChangeType.REMOVED}) self.assertEqual(inst.resume_token, None) self.assertFalse(inst.current) From 3424284fbcd2363c79110d4a7a0edcd005190520 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Wed, 22 Aug 2018 11:54:31 -0400 Subject: [PATCH 099/148] idiom --- firestore/google/cloud/firestore_v1beta1/watch.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 78300bf693c5..0aa2184a38d5 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -702,11 +702,8 @@ def _affects_target(self, target_ids, current_id): if target_ids is None: return True - if current_id in target_ids: - return True - - return False - + return current_id in target_ids + def _current_size(self): """ Returns the current count of all documents, including the changes from From 429f86e477cad27bb90f1b5d9139edcb4d39ce61 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Wed, 22 Aug 2018 10:24:09 -0700 Subject: [PATCH 100/148] enable collection watches --- .../cloud/firestore_v1beta1/collection.py | 57 ++++++++++--------- .../google/cloud/firestore_v1beta1/watch.py | 17 ++---- 2 files changed, 34 insertions(+), 40 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/collection.py b/firestore/google/cloud/firestore_v1beta1/collection.py index 090da0cb0151..2fd3f09680ac 100644 --- a/firestore/google/cloud/firestore_v1beta1/collection.py +++ b/firestore/google/cloud/firestore_v1beta1/collection.py @@ -22,7 +22,8 @@ from google.cloud.firestore_v1beta1 import _helpers from google.cloud.firestore_v1beta1 import query as query_mod from google.cloud.firestore_v1beta1.proto import document_pb2 - +from google.cloud.firestore_v1beta1.watch import Watch +from google.cloud.firestore_v1beta1.document import DocumentSnapshot _AUTO_ID_CHARS = ( 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789') @@ -371,33 +372,33 @@ def get(self, transaction=None): query = query_mod.Query(self) return query.get(transaction=transaction) - # def on_snapshot(self, callback): - # """Monitor the documents in this collection. - # - # This starts a watch on this collection using a background thread. The - # provided callback is run on the snapshot of the documents. - # - # Args: - # callback(~.firestore.collection.CollectionSnapshot): a callback - # to run when a change occurs. - # - # Example: - # from google.cloud import firestore - # - # db = firestore.Client() - # collection_ref = db.collection(u'users') - # - # def on_snapshot(collection_snapshot): - # for doc in collection_snapshot.documents: - # print(u'{} => {}'.format(doc.id, doc.to_dict())) - # - # # Watch this collection - # collection_watch = collection_ref.on_snapshot(on_snapshot) - # - # # Terminate this watch - # collection_watch.unsubscribe() - # """ - # raise NotImplemented + def on_snapshot(self, callback): + """Monitor the documents in this collection. + + This starts a watch on this collection using a background thread. The + provided callback is run on the snapshot of the documents. + + Args: + callback(~.firestore.collection.CollectionSnapshot): a callback + to run when a change occurs. + + Example: + from google.cloud import firestore + + db = firestore.Client() + collection_ref = db.collection(u'users') + + def on_snapshot(collection_snapshot): + for doc in collection_snapshot.documents: + print(u'{} => {}'.format(doc.id, doc.to_dict())) + + # Watch this collection + collection_watch = collection_ref.on_snapshot(on_snapshot) + + # Terminate this watch + collection_watch.unsubscribe() + """ + Watch.for_query(query_mod.Query(self), callback, DocumentSnapshot) def _auto_id(): diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 0aa2184a38d5..0815c80e4580 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -343,18 +343,6 @@ def for_query(cls, query, snapshot_callback, snapshot_class_instance): snapshot_callback, snapshot_class_instance) - # @classmethod - # def for_collection(cls, collection_ref, snapshot_callback, - # snapshot_class_instance): - # return cls(collection_ref._client, - # { - # 'collection': collection_ref.to_proto(), - # 'target_id': WATCH_TARGET_ID - # }, - # document_watch_comparator, - # snapshot_callback, - # snapshot_class_instance) - def _on_snapshot_target_change_no_change(self, proto): _LOGGER.debug('on_snapshot: target change: NO_CHANGE') change = proto.target_change @@ -424,6 +412,11 @@ def on_snapshot(self, proto): target_change = proto.target_change if str(target_change): +<<<<<<< HEAD +======= + # XXX why if str - if it doesn't exist it will be empty (falsy). + # Otherwise this was always true. +>>>>>>> enable collection watches target_change_type = target_change.target_change_type _LOGGER.debug( 'on_snapshot: target change: ' + str(target_change_type)) From 00de5d376b4df47211eef2518956e4eed4c0a309 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Wed, 22 Aug 2018 18:10:35 -0400 Subject: [PATCH 101/148] undo spurious changes --- firestore/noxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/firestore/noxfile.py b/firestore/noxfile.py index 5ead66979f87..999e57ca3291 100644 --- a/firestore/noxfile.py +++ b/firestore/noxfile.py @@ -57,7 +57,7 @@ def default(session): @nox.session -@nox.parametrize('py', ['2.7', '3.7']) +@nox.parametrize('py', ['2.7', '3.5', '3.6', '3.7']) def unit(session, py): """Run the unit test suite.""" From 4b96582b4cf3f0797a7c9b5fa56bc628d308d4dd Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Wed, 22 Aug 2018 18:11:23 -0400 Subject: [PATCH 102/148] add unfinished test --- firestore/tests/system.py | 692 ++------------------------------------ 1 file changed, 26 insertions(+), 666 deletions(-) diff --git a/firestore/tests/system.py b/firestore/tests/system.py index 48f8a24a269a..ce414196b24e 100644 --- a/firestore/tests/system.py +++ b/firestore/tests/system.py @@ -830,671 +830,31 @@ def on_response(*arg): if on_response.called_count != 1: raise AssertionError("Failed to get exactly one document change") +def test_watch_collection(client, cleanup): + db = client + doc_ref = db.collection(u'users').document( + u'alovelace' + unique_resource_id()) + collection_ref = db.collection(u'users') + def on_snapshot(snapshot): + for doc in snapshot.documents: + print(u'{} => {}'.format(doc.id, doc.to_dict())) + + collection_ref.on_snapshot(on_snapshot) + + sleep(1) + + # Initial setting + doc_ref.set({ + u'first': u'Jane', + u'last': u'Doe', + u'born': 1900 + }) + doc_ref.set({ + u'first': u'Ada', + u'last': u'Lovelace', + u'born': 1815 + }) -# def test_create_document(client, cleanup): -# now = datetime.datetime.utcnow().replace(tzinfo=UTC) -# document_id = 'shun' + unique_resource_id('-') -# document = client.document('collek', document_id) -# # Add to clean-up before API request (in case ``create()`` fails). -# cleanup(document) - -# data = { -# 'now': firestore.SERVER_TIMESTAMP, -# 'eenta-ger': 11, -# 'bites': b'\xe2\x98\x83 \xe2\x9b\xb5', -# 'also': { -# 'nestednow': firestore.SERVER_TIMESTAMP, -# 'quarter': 0.25, -# }, -# } -# write_result = document.create(data) -# updated = _pb_timestamp_to_datetime(write_result.update_time) -# delta = updated - now -# # Allow a bit of clock skew, but make sure timestamps are close. -# assert -300.0 < delta.total_seconds() < 300.0 - -# with pytest.raises(AlreadyExists): -# document.create(data) - -# # Verify the server times. -# snapshot = document.get() -# stored_data = snapshot.to_dict() -# server_now = stored_data['now'] - -# delta = updated - server_now -# # NOTE: We could check the ``transform_results`` from the write result -# # for the document transform, but this value gets dropped. Instead -# # we make sure the timestamps are close. -# assert 0.0 <= delta.total_seconds() < 5.0 -# expected_data = { -# 'now': server_now, -# 'eenta-ger': data['eenta-ger'], -# 'bites': data['bites'], -# 'also': { -# 'nestednow': server_now, -# 'quarter': data['also']['quarter'], -# }, -# } -# assert stored_data == expected_data - - -# def test_cannot_use_foreign_key(client, cleanup): -# document_id = 'cannot' + unique_resource_id('-') -# document = client.document('foreign-key', document_id) -# # Add to clean-up before API request (in case ``create()`` fails). -# cleanup(document) - -# other_client = firestore.Client( -# project='other-prahj', -# credentials=client._credentials, -# database='dee-bee') -# assert other_client._database_string != client._database_string -# fake_doc = other_client.document('foo', 'bar') -# with pytest.raises(InvalidArgument): -# document.create({'ref': fake_doc}) - - -# def assert_timestamp_less(timestamp_pb1, timestamp_pb2): -# dt_val1 = _pb_timestamp_to_datetime(timestamp_pb1) -# dt_val2 = _pb_timestamp_to_datetime(timestamp_pb2) -# assert dt_val1 < dt_val2 - - -# def test_no_document(client, cleanup): -# document_id = 'no_document' + unique_resource_id('-') -# document = client.document('abcde', document_id) -# snapshot = document.get() -# assert snapshot.to_dict() is None - - -# def test_document_set(client, cleanup): -# document_id = 'for-set' + unique_resource_id('-') -# document = client.document('i-did-it', document_id) -# # Add to clean-up before API request (in case ``set()`` fails). -# cleanup(document) - -# # 0. Make sure the document doesn't exist yet -# snapshot = document.get() -# assert snapshot.to_dict() is None - -# # 1. Use ``create()`` to create the document. -# data1 = {'foo': 88} -# write_result1 = document.create(data1) -# snapshot1 = document.get() -# assert snapshot1.to_dict() == data1 -# # Make sure the update is what created the document. -# assert snapshot1.create_time == snapshot1.update_time -# assert snapshot1.update_time == write_result1.update_time - -# # 2. Call ``set()`` again to overwrite. -# data2 = {'bar': None} -# write_result2 = document.set(data2) -# snapshot2 = document.get() -# assert snapshot2.to_dict() == data2 -# # Make sure the create time hasn't changed. -# assert snapshot2.create_time == snapshot1.create_time -# assert snapshot2.update_time == write_result2.update_time - - -# def test_document_integer_field(client, cleanup): -# document_id = 'for-set' + unique_resource_id('-') -# document = client.document('i-did-it', document_id) -# # Add to clean-up before API request (in case ``set()`` fails). -# cleanup(document) - -# data1 = { -# '1a': { -# '2b': '3c', -# 'ab': '5e'}, -# '6f': { -# '7g': '8h', -# 'cd': '0j'} -# } -# document.create(data1) - -# data2 = {'1a.ab': '4d', '6f.7g': '9h'} -# option2 = client.write_option(exists=True) -# document.update(data2, option=option2) -# snapshot = document.get() -# expected = { -# '1a': { -# '2b': '3c', -# 'ab': '4d'}, -# '6f': { -# '7g': '9h', -# 'cd': '0j'} -# } -# assert snapshot.to_dict() == expected - - -# def test_document_set_merge(client, cleanup): -# document_id = 'for-set' + unique_resource_id('-') -# document = client.document('i-did-it', document_id) -# # Add to clean-up before API request (in case ``set()`` fails). -# cleanup(document) - -# # 0. Make sure the document doesn't exist yet -# snapshot = document.get() -# assert not snapshot.exists - -# # 1. Use ``create()`` to create the document. -# data1 = {'name': 'Sam', -# 'address': {'city': 'SF', -# 'state': 'CA'}} -# write_result1 = document.create(data1) -# snapshot1 = document.get() -# assert snapshot1.to_dict() == data1 -# # Make sure the update is what created the document. -# assert snapshot1.create_time == snapshot1.update_time -# assert snapshot1.update_time == write_result1.update_time - -# # 2. Call ``set()`` to merge -# data2 = {'address': {'city': 'LA'}} -# write_result2 = document.set(data2, merge=True) -# snapshot2 = document.get() -# assert snapshot2.to_dict() == {'name': 'Sam', -# 'address': {'city': 'LA', -# 'state': 'CA'}} -# # Make sure the create time hasn't changed. -# assert snapshot2.create_time == snapshot1.create_time -# assert snapshot2.update_time == write_result2.update_time - - -# def test_update_document(client, cleanup): -# document_id = 'for-update' + unique_resource_id('-') -# document = client.document('made', document_id) -# # Add to clean-up before API request (in case ``create()`` fails). -# cleanup(document) - -# # 0. Try to update before the document exists. -# with pytest.raises(NotFound) as exc_info: -# document.update({'not': 'there'}) -# assert exc_info.value.message.startswith(MISSING_DOCUMENT) -# assert document_id in exc_info.value.message - -# # 1. Try to update before the document exists (now with an option). -# option1 = client.write_option(exists=True) -# with pytest.raises(NotFound) as exc_info: -# document.update({'still': 'not-there'}, option=option1) -# assert exc_info.value.message.startswith(MISSING_DOCUMENT) -# assert document_id in exc_info.value.message - -# # 2. Update and create the document (with an option). -# data = { -# 'foo': { -# 'bar': 'baz', -# }, -# 'scoop': { -# 'barn': 981, -# }, -# 'other': True, -# } -# option2 = client.write_option(exists=False) -# write_result2 = document.update(data, option=option2) - -# # 3. Send an update without a field path (no option). -# field_updates3 = {'foo': {'quux': 800}} -# write_result3 = document.update(field_updates3) -# assert_timestamp_less(write_result2.update_time, write_result3.update_time) -# snapshot3 = document.get() -# expected3 = { -# 'foo': field_updates3['foo'], -# 'scoop': data['scoop'], -# 'other': data['other'], -# } -# assert snapshot3.to_dict() == expected3 - -# # 4. Send an update **with** a field path and a delete and a valid -# # "last timestamp" option. -# field_updates4 = { -# 'scoop.silo': None, -# 'other': firestore.DELETE_FIELD, -# } -# option4 = client.write_option(last_update_time=snapshot3.update_time) -# write_result4 = document.update(field_updates4, option=option4) -# assert_timestamp_less(write_result3.update_time, write_result4.update_time) -# snapshot4 = document.get() -# expected4 = { -# 'foo': field_updates3['foo'], -# 'scoop': { -# 'barn': data['scoop']['barn'], -# 'silo': field_updates4['scoop.silo'], -# }, -# } -# assert snapshot4.to_dict() == expected4 - -# # 5. Call ``update()`` with invalid (in the past) "last timestamp" option. -# assert_timestamp_less(option4._last_update_time, snapshot4.update_time) -# with pytest.raises(FailedPrecondition) as exc_info: -# document.update({'bad': 'time-past'}, option=option4) - -# # 6. Call ``update()`` with invalid (in future) "last timestamp" option. -# timestamp_pb = timestamp_pb2.Timestamp( -# seconds=snapshot4.update_time.nanos + 3600, -# nanos=snapshot4.update_time.nanos, -# ) -# option6 = client.write_option(last_update_time=timestamp_pb) -# with pytest.raises(FailedPrecondition) as exc_info: -# document.update({'bad': 'time-future'}, option=option6) - - -# def check_snapshot(snapshot, document, data, write_result): -# assert snapshot.reference is document -# assert snapshot.to_dict() == data -# assert snapshot.exists -# assert snapshot.create_time == write_result.update_time -# assert snapshot.update_time == write_result.update_time - - -# def test_document_get(client, cleanup): -# now = datetime.datetime.utcnow().replace(tzinfo=UTC) -# document_id = 'for-get' + unique_resource_id('-') -# document = client.document('created', document_id) -# # Add to clean-up before API request (in case ``create()`` fails). -# cleanup(document) - -# # First make sure it doesn't exist. -# assert not document.get().exists - -# ref_doc = client.document('top', 'middle1', 'middle2', 'bottom') -# data = { -# 'turtle': 'power', -# 'cheese': 19.5, -# 'fire': 199099299, -# 'referee': ref_doc, -# 'gio': firestore.GeoPoint(45.5, 90.0), -# 'deep': [ -# u'some', -# b'\xde\xad\xbe\xef', -# ], -# 'map': { -# 'ice': True, -# 'water': None, -# 'vapor': { -# 'deeper': now, -# }, -# }, -# } -# write_result = document.create(data) -# snapshot = document.get() -# check_snapshot(snapshot, document, data, write_result) -# assert_timestamp_less(snapshot.create_time, snapshot.read_time) - - -# def test_document_delete(client, cleanup): -# document_id = 'deleted' + unique_resource_id('-') -# document = client.document('here-to-be', document_id) -# # Add to clean-up before API request (in case ``create()`` fails). -# cleanup(document) -# document.create({'not': 'much'}) - -# # 1. Call ``delete()`` with invalid (in the past) "last timestamp" option. -# snapshot1 = document.get() -# timestamp_pb = timestamp_pb2.Timestamp( -# seconds=snapshot1.update_time.nanos - 3600, -# nanos=snapshot1.update_time.nanos, -# ) -# option1 = client.write_option(last_update_time=timestamp_pb) -# with pytest.raises(FailedPrecondition): -# document.delete(option=option1) - -# # 2. Call ``delete()`` with invalid (in future) "last timestamp" option. -# timestamp_pb = timestamp_pb2.Timestamp( -# seconds=snapshot1.update_time.nanos + 3600, -# nanos=snapshot1.update_time.nanos, -# ) -# option2 = client.write_option(last_update_time=timestamp_pb) -# with pytest.raises(FailedPrecondition): -# document.delete(option=option2) - -# # 3. Actually ``delete()`` the document. -# delete_time3 = document.delete() - -# # 4. ``delete()`` again, even though we know the document is gone. -# delete_time4 = document.delete() -# assert_timestamp_less(delete_time3, delete_time4) - - -# def test_collection_add(client, cleanup): -# collection1 = client.collection('collek') -# collection2 = client.collection('collek', 'shun', 'child') -# explicit_doc_id = 'hula' + unique_resource_id('-') - -# # Auto-ID at top-level. -# data1 = {'foo': 'bar'} -# update_time1, document_ref1 = collection1.add(data1) -# cleanup(document_ref1) -# snapshot1 = document_ref1.get() -# assert snapshot1.to_dict() == data1 -# assert snapshot1.create_time == update_time1 -# assert snapshot1.update_time == update_time1 -# assert RANDOM_ID_REGEX.match(document_ref1.id) - -# # Explicit ID at top-level. -# data2 = {'baz': 999} -# update_time2, document_ref2 = collection1.add( -# data2, document_id=explicit_doc_id) -# cleanup(document_ref2) -# snapshot2 = document_ref2.get() -# assert snapshot2.to_dict() == data2 -# assert snapshot2.create_time == update_time2 -# assert snapshot2.update_time == update_time2 -# assert document_ref2.id == explicit_doc_id - -# # Auto-ID for nested collection. -# data3 = {'quux': b'\x00\x01\x02\x03'} -# update_time3, document_ref3 = collection2.add(data3) -# cleanup(document_ref3) -# snapshot3 = document_ref3.get() -# assert snapshot3.to_dict() == data3 -# assert snapshot3.create_time == update_time3 -# assert snapshot3.update_time == update_time3 -# assert RANDOM_ID_REGEX.match(document_ref3.id) - -# # Explicit for nested collection. -# data4 = {'kazaam': None, 'bad': False} -# update_time4, document_ref4 = collection2.add( -# data4, document_id=explicit_doc_id) -# cleanup(document_ref4) -# snapshot4 = document_ref4.get() -# assert snapshot4.to_dict() == data4 -# assert snapshot4.create_time == update_time4 -# assert snapshot4.update_time == update_time4 -# assert document_ref4.id == explicit_doc_id - - -# def test_query_get(client, cleanup): -# sub_collection = 'child' + unique_resource_id('-') -# collection = client.collection('collek', 'shun', sub_collection) - -# stored = {} -# num_vals = 5 -# allowed_vals = six.moves.xrange(num_vals) -# for a_val in allowed_vals: -# for b_val in allowed_vals: -# document_data = { -# 'a': a_val, -# 'b': b_val, -# 'stats': { -# 'sum': a_val + b_val, -# 'product': a_val * b_val, -# }, -# } -# _, doc_ref = collection.add(document_data) -# # Add to clean-up. -# cleanup(doc_ref) -# stored[doc_ref.id] = document_data - -# # 0. Limit to snapshots where ``a==1``. -# query0 = collection.where('a', '==', 1) -# values0 = { -# snapshot.id: snapshot.to_dict() -# for snapshot in query0.get() -# } -# assert len(values0) == num_vals -# for key, value in six.iteritems(values0): -# assert stored[key] == value -# assert value['a'] == 1 - -# # 1. Order by ``b``. -# query1 = collection.order_by('b', direction=query0.DESCENDING) -# values1 = [ -# (snapshot.id, snapshot.to_dict()) -# for snapshot in query1.get() -# ] -# assert len(values1) == len(stored) -# b_vals1 = [] -# for key, value in values1: -# assert stored[key] == value -# b_vals1.append(value['b']) -# # Make sure the ``b``-values are in DESCENDING order. -# assert sorted(b_vals1, reverse=True) == b_vals1 - -# # 2. Limit to snapshots where ``stats.sum > 1`` (a field path). -# query2 = collection.where('stats.sum', '>', 4) -# values2 = { -# snapshot.id: snapshot.to_dict() -# for snapshot in query2.get() -# } -# assert len(values2) == 10 -# ab_pairs2 = set() -# for key, value in six.iteritems(values2): -# assert stored[key] == value -# ab_pairs2.add((value['a'], value['b'])) - -# expected_ab_pairs = set([ -# (a_val, b_val) -# for a_val in allowed_vals -# for b_val in allowed_vals -# if a_val + b_val > 4 -# ]) -# assert expected_ab_pairs == ab_pairs2 - -# # 3. Use a start and end cursor. -# query3 = collection.start_at({'a': num_vals - 2}) -# query3 = query3.order_by('a') -# query3 = query3.end_before({'a': num_vals - 1}) -# values3 = [ -# (snapshot.id, snapshot.to_dict()) -# for snapshot in query3.get() -# ] -# assert len(values3) == num_vals -# for key, value in values3: -# assert stored[key] == value -# assert value['a'] == num_vals - 2 -# b_vals1.append(value['b']) - -# # 4. Send a query with no results. -# query4 = collection.where('b', '==', num_vals + 100) -# values4 = list(query4.get()) -# assert len(values4) == 0 - -# # 5. Select a subset of fields. -# query5 = collection.where('b', '<=', 1) -# query5 = query5.select(['a', 'stats.product']) -# values5 = { -# snapshot.id: snapshot.to_dict() -# for snapshot in query5.get() -# } -# assert len(values5) == num_vals * 2 # a ANY, b in (0, 1) -# for key, value in six.iteritems(values5): -# expected = { -# 'a': stored[key]['a'], -# 'stats': { -# 'product': stored[key]['stats']['product'], -# }, -# } -# assert expected == value - -# # 6. Add multiple filters via ``where()``. -# query6 = collection.where('stats.product', '>', 5) -# query6 = query6.where('stats.product', '<', 10) -# values6 = { -# snapshot.id: snapshot.to_dict() -# for snapshot in query6.get() -# } - -# matching_pairs = [ -# (a_val, b_val) -# for a_val in allowed_vals -# for b_val in allowed_vals -# if 5 < a_val * b_val < 10 -# ] -# assert len(values6) == len(matching_pairs) -# for key, value in six.iteritems(values6): -# assert stored[key] == value -# pair = (value['a'], value['b']) -# assert pair in matching_pairs - -# # 7. Skip the first three results, when ``b==2`` -# query7 = collection.where('b', '==', 2) -# offset = 3 -# query7 = query7.offset(offset) -# values7 = { -# snapshot.id: snapshot.to_dict() -# for snapshot in query7.get() -# } -# # NOTE: We don't check the ``a``-values, since that would require -# # an ``order_by('a')``, which combined with the ``b == 2`` -# # filter would necessitate an index. -# assert len(values7) == num_vals - offset -# for key, value in six.iteritems(values7): -# assert stored[key] == value -# assert value['b'] == 2 - - -# def test_query_unary(client, cleanup): -# collection_name = 'unary' + unique_resource_id('-') -# collection = client.collection(collection_name) -# field_name = 'foo' - -# _, document0 = collection.add({field_name: None}) -# # Add to clean-up. -# cleanup(document0) - -# nan_val = float('nan') -# _, document1 = collection.add({field_name: nan_val}) -# # Add to clean-up. -# cleanup(document1) - -# # 0. Query for null. -# query0 = collection.where(field_name, '==', None) -# values0 = list(query0.get()) -# assert len(values0) == 1 -# snapshot0 = values0[0] -# assert snapshot0.reference._path == document0._path -# assert snapshot0.to_dict() == {field_name: None} - -# # 1. Query for a NAN. -# query1 = collection.where(field_name, '==', nan_val) -# values1 = list(query1.get()) -# assert len(values1) == 1 -# snapshot1 = values1[0] -# assert snapshot1.reference._path == document1._path -# data1 = snapshot1.to_dict() -# assert len(data1) == 1 -# assert math.isnan(data1[field_name]) - - -# def test_get_all(client, cleanup): -# collection_name = 'get-all' + unique_resource_id('-') - -# document1 = client.document(collection_name, 'a') -# document2 = client.document(collection_name, 'b') -# document3 = client.document(collection_name, 'c') -# # Add to clean-up before API requests (in case ``create()`` fails). -# cleanup(document1) -# cleanup(document3) - -# data1 = { -# 'a': { -# 'b': 2, -# 'c': 3, -# }, -# 'd': 4, -# 'e': 0, -# } -# write_result1 = document1.create(data1) -# data3 = { -# 'a': { -# 'b': 5, -# 'c': 6, -# }, -# 'd': 7, -# 'e': 100, -# } -# write_result3 = document3.create(data3) - -# # 0. Get 3 unique documents, one of which is missing. -# snapshots = list(client.get_all( -# [document1, document2, document3])) - -# assert snapshots[0].exists -# assert snapshots[1].exists -# assert not snapshots[2].exists -# snapshots = [snapshot for snapshot in snapshots if snapshot.exists] -# id_attr = operator.attrgetter('id') -# snapshots.sort(key=id_attr) - -# snapshot1, snapshot3 = snapshots -# check_snapshot(snapshot1, document1, data1, write_result1) -# check_snapshot(snapshot3, document3, data3, write_result3) - -# # 1. Get 2 colliding documents. -# document1_also = client.document(collection_name, 'a') -# snapshots = list(client.get_all([document1, document1_also])) - -# assert len(snapshots) == 1 -# assert document1 is not document1_also -# check_snapshot(snapshots[0], document1_also, data1, write_result1) - -# # 2. Use ``field_paths`` / projection in ``get_all()``. -# snapshots = list(client.get_all( -# [document1, document3], field_paths=['a.b', 'd'])) - -# assert len(snapshots) == 2 -# snapshots.sort(key=id_attr) - -# snapshot1, snapshot3 = snapshots -# restricted1 = { -# 'a': {'b': data1['a']['b']}, -# 'd': data1['d'], -# } -# check_snapshot(snapshot1, document1, restricted1, write_result1) -# restricted3 = { -# 'a': {'b': data3['a']['b']}, -# 'd': data3['d'], -# } -# check_snapshot(snapshot3, document3, restricted3, write_result3) - - -# def test_batch(client, cleanup): -# collection_name = 'batch' + unique_resource_id('-') - -# document1 = client.document(collection_name, 'abc') -# document2 = client.document(collection_name, 'mno') -# document3 = client.document(collection_name, 'xyz') -# # Add to clean-up before API request (in case ``create()`` fails). -# cleanup(document1) -# cleanup(document2) -# cleanup(document3) - -# data2 = { -# 'some': { -# 'deep': 'stuff', -# 'and': 'here', -# }, -# 'water': 100.0, -# } -# document2.create(data2) -# document3.create({'other': 19}) - -# batch = client.batch() -# data1 = {'all': True} -# batch.create(document1, data1) -# new_value = 'there' -# batch.update(document2, {'some.and': new_value}) -# batch.delete(document3) -# write_results = batch.commit() - -# assert len(write_results) == 3 - -# write_result1 = write_results[0] -# write_result2 = write_results[1] -# write_result3 = write_results[2] -# assert not write_result3.HasField('update_time') - -# snapshot1 = document1.get() -# assert snapshot1.to_dict() == data1 -# assert snapshot1.create_time == write_result1.update_time -# assert snapshot1.update_time == write_result1.update_time - -# snapshot2 = document2.get() -# assert snapshot2.to_dict() != data2 -# data2['some']['and'] = new_value -# assert snapshot2.to_dict() == data2 -# assert_timestamp_less(snapshot2.create_time, write_result2.update_time) -# assert snapshot2.update_time == write_result2.update_time - -# assert not document3.get().exists + # CM: had to stop here, this test is totally unfinished, trying to formalize + # https://gist.github.com/crwilcox/ce05f3857adc7a0ed86ffbd039b1a035 From bcecb25d6ec6ce7268ed3a76bce87cabb4dba4c1 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Thu, 23 Aug 2018 14:52:08 -0700 Subject: [PATCH 103/148] modify the way we compute document references to support query and collection watches --- .../cloud/firestore_v1beta1/collection.py | 4 ++-- .../cloud/firestore_v1beta1/document.py | 2 +- .../google/cloud/firestore_v1beta1/query.py | 2 +- .../google/cloud/firestore_v1beta1/watch.py | 19 ++++++++++++++----- 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/collection.py b/firestore/google/cloud/firestore_v1beta1/collection.py index 2fd3f09680ac..81229120a1e5 100644 --- a/firestore/google/cloud/firestore_v1beta1/collection.py +++ b/firestore/google/cloud/firestore_v1beta1/collection.py @@ -23,7 +23,7 @@ from google.cloud.firestore_v1beta1 import query as query_mod from google.cloud.firestore_v1beta1.proto import document_pb2 from google.cloud.firestore_v1beta1.watch import Watch -from google.cloud.firestore_v1beta1.document import DocumentSnapshot +from google.cloud.firestore_v1beta1 import document _AUTO_ID_CHARS = ( 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789') @@ -398,7 +398,7 @@ def on_snapshot(collection_snapshot): # Terminate this watch collection_watch.unsubscribe() """ - Watch.for_query(query_mod.Query(self), callback, DocumentSnapshot) + Watch.for_query(query_mod.Query(self), callback, document) def _auto_id(): diff --git a/firestore/google/cloud/firestore_v1beta1/document.py b/firestore/google/cloud/firestore_v1beta1/document.py index b68aa93e93aa..c0ca6e81243b 100644 --- a/firestore/google/cloud/firestore_v1beta1/document.py +++ b/firestore/google/cloud/firestore_v1beta1/document.py @@ -475,7 +475,7 @@ def on_snapshot(document_snapshot): # Terminate this watch doc_watch.unsubscribe() """ - Watch.for_document(self, callback, DocumentSnapshot) + Watch.for_document(self, callback, __import__(__name__)) class DocumentSnapshot(object): diff --git a/firestore/google/cloud/firestore_v1beta1/query.py b/firestore/google/cloud/firestore_v1beta1/query.py index 1ca4ffe8e2a5..76712abc3c78 100644 --- a/firestore/google/cloud/firestore_v1beta1/query.py +++ b/firestore/google/cloud/firestore_v1beta1/query.py @@ -627,7 +627,7 @@ def on_snapshot(query_snapshot): # Terminate this watch query_watch.unsubscribe() """ - Watch.for_query(self, callback, document.DocumentSnapshot) + Watch.for_query(self, callback, document) def _enum_from_op_string(op_string): diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 0815c80e4580..548aad942189 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -160,7 +160,7 @@ def __init__(self, target, comparator, snapshot_callback, - DocumentSnapshotCls, + document_module, BackgroundConsumer=None, # FBO unit testing ResumableBidiRpc=None, # FBO unit testing ): @@ -180,14 +180,15 @@ def __init__(self, snapshot was obtained. # TODO: Go had an err here and node.js provided size. # TODO: do we want to include either? - DocumentSnapshotCls: instance of the DocumentSnapshot class + document_module: instance of the Document module """ self._document_reference = document_reference self._firestore = firestore self._api = firestore._firestore_api self._targets = target self._comparator = comparator - self.DocumentSnapshot = DocumentSnapshotCls + self.DocumentSnapshot = document_module.DocumentSnapshot + self.DocumentReference = document_module.DocumentReference self._snapshot_callback = snapshot_callback self._closing = threading.Lock() self._closed = False @@ -410,7 +411,6 @@ def on_snapshot(self, proto): } target_change = proto.target_change - if str(target_change): <<<<<<< HEAD ======= @@ -474,8 +474,17 @@ def on_snapshot(self, proto): data = self.MessageToDict(document) + # Create a snapshot. As Document and Query objects can be + # passed we need to get a Document Reference in a more manual + # fashion than self._document_reference + document_name = document.name + db_str = self._firestore._database_string + if document_name.startswith(db_str): + document_name = document_name[len(db_str):] + document_ref = self._firestore.document(document_name) + snapshot = self.DocumentSnapshot( - reference=self._document_reference, + reference=document_ref, data=data['fields'], exists=True, read_time=None, From 05db07c70276d61a48f3f58b152353b8bad228be Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Thu, 23 Aug 2018 21:41:37 -0700 Subject: [PATCH 104/148] add system tests for each variety of watch --- .../cloud/firestore_v1beta1/collection.py | 5 +- .../cloud/firestore_v1beta1/document.py | 2 +- .../google/cloud/firestore_v1beta1/query.py | 5 +- .../google/cloud/firestore_v1beta1/watch.py | 27 ++++-- firestore/tests/system.py | 95 ++++++++++++++++--- 5 files changed, 110 insertions(+), 24 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/collection.py b/firestore/google/cloud/firestore_v1beta1/collection.py index 81229120a1e5..1110858f4667 100644 --- a/firestore/google/cloud/firestore_v1beta1/collection.py +++ b/firestore/google/cloud/firestore_v1beta1/collection.py @@ -398,7 +398,10 @@ def on_snapshot(collection_snapshot): # Terminate this watch collection_watch.unsubscribe() """ - Watch.for_query(query_mod.Query(self), callback, document) + Watch.for_query(query_mod.Query(self), + callback, + document.DocumentSnapshot, + document.DocumentReference) def _auto_id(): diff --git a/firestore/google/cloud/firestore_v1beta1/document.py b/firestore/google/cloud/firestore_v1beta1/document.py index c0ca6e81243b..03b37a6363f7 100644 --- a/firestore/google/cloud/firestore_v1beta1/document.py +++ b/firestore/google/cloud/firestore_v1beta1/document.py @@ -475,7 +475,7 @@ def on_snapshot(document_snapshot): # Terminate this watch doc_watch.unsubscribe() """ - Watch.for_document(self, callback, __import__(__name__)) + Watch.for_document(self, callback, DocumentSnapshot, DocumentReference) class DocumentSnapshot(object): diff --git a/firestore/google/cloud/firestore_v1beta1/query.py b/firestore/google/cloud/firestore_v1beta1/query.py index 76712abc3c78..c39a5febea44 100644 --- a/firestore/google/cloud/firestore_v1beta1/query.py +++ b/firestore/google/cloud/firestore_v1beta1/query.py @@ -627,7 +627,10 @@ def on_snapshot(query_snapshot): # Terminate this watch query_watch.unsubscribe() """ - Watch.for_query(self, callback, document) + Watch.for_query(self, + callback, + document.DocumentSnapshot, + document.DocumentReference) def _enum_from_op_string(op_string): diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 548aad942189..61b2cee1c485 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -160,7 +160,8 @@ def __init__(self, target, comparator, snapshot_callback, - document_module, + document_snapshot_cls, + document_reference_cls, BackgroundConsumer=None, # FBO unit testing ResumableBidiRpc=None, # FBO unit testing ): @@ -180,15 +181,16 @@ def __init__(self, snapshot was obtained. # TODO: Go had an err here and node.js provided size. # TODO: do we want to include either? - document_module: instance of the Document module + document_snapshot_cls: instance of DocumentSnapshot + document_reference_cls: instance of DocumentReference """ self._document_reference = document_reference self._firestore = firestore self._api = firestore._firestore_api self._targets = target self._comparator = comparator - self.DocumentSnapshot = document_module.DocumentSnapshot - self.DocumentReference = document_module.DocumentReference + self.DocumentSnapshot = document_snapshot_cls + self.DocumentReference = document_reference_cls self._snapshot_callback = snapshot_callback self._closing = threading.Lock() self._closed = False @@ -304,7 +306,7 @@ def unsubscribe(self): # XXX should this be aliased to close? @classmethod def for_document(cls, document_ref, snapshot_callback, - snapshot_class_instance): + snapshot_class_instance, reference_class_instance): """ Creates a watch snapshot listener for a document. snapshot_callback receives a DocumentChange object, but may also start to get @@ -313,8 +315,10 @@ def for_document(cls, document_ref, snapshot_callback, Args: document_ref: Reference to Document snapshot_callback: callback to be called on snapshot - snapshot_class_instance: instance of snapshot cls to make + snapshot_class_instance: instance of DocumentSnapshot to make snapshots with to pass to snapshot_callback + reference_class_instance: instance of DocumentReference to make + references """ return cls(document_ref, @@ -326,10 +330,12 @@ def for_document(cls, document_ref, snapshot_callback, }, document_watch_comparator, snapshot_callback, - snapshot_class_instance) + snapshot_class_instance, + reference_class_instance) @classmethod - def for_query(cls, query, snapshot_callback, snapshot_class_instance): + def for_query(cls, query, snapshot_callback, snapshot_class_instance, + reference_class_instance): query_target = firestore_pb2.Target.QueryTarget( parent=query._client._database_string, structured_query=query._to_protobuf(), @@ -342,7 +348,8 @@ def for_query(cls, query, snapshot_callback, snapshot_class_instance): }, document_watch_comparator, snapshot_callback, - snapshot_class_instance) + snapshot_class_instance, + reference_class_instance) def _on_snapshot_target_change_no_change(self, proto): _LOGGER.debug('on_snapshot: target change: NO_CHANGE') @@ -705,7 +712,7 @@ def _affects_target(self, target_ids, current_id): return True return current_id in target_ids - + def _current_size(self): """ Returns the current count of all documents, including the changes from diff --git a/firestore/tests/system.py b/firestore/tests/system.py index ce414196b24e..d02ae9b14458 100644 --- a/firestore/tests/system.py +++ b/firestore/tests/system.py @@ -812,12 +812,12 @@ def test_watch_document(client, cleanup): sleep(1) # Setup listener - def on_response(*arg): - on_response.called_count += 1 + def on_snapshot(docs, changes, read_time): + on_snapshot.called_count += 1 - on_response.called_count = 0 + on_snapshot.called_count = 0 - doc_ref.on_snapshot(on_response) + doc_ref.on_snapshot(on_snapshot) # Alter document doc_ref.set({ @@ -827,22 +827,74 @@ def on_response(*arg): }) sleep(1) - if on_response.called_count != 1: - raise AssertionError("Failed to get exactly one document change") + + for _ in range(10): + if on_snapshot.called_count == 1: + return + sleep(1) + + if on_snapshot.called_count != 1: + raise AssertionError( + "Failed to get exactly one document change: count: " + + str(on_snapshot.called_count)) + def test_watch_collection(client, cleanup): db = client doc_ref = db.collection(u'users').document( u'alovelace' + unique_resource_id()) collection_ref = db.collection(u'users') - def on_snapshot(snapshot): - for doc in snapshot.documents: - print(u'{} => {}'.format(doc.id, doc.to_dict())) + + # Initial setting + doc_ref.set({ + u'first': u'Jane', + u'last': u'Doe', + u'born': 1900 + }) + + # Setup listener + def on_snapshot(docs, changes, read_time): + on_snapshot.called_count += 1 + + on_snapshot.called_count = 0 + + # def on_snapshot(docs, changes, read_time): + # for doc in docs: + # print(u'{} => {}'.format(doc.id, doc.to_dict())) collection_ref.on_snapshot(on_snapshot) sleep(1) + doc_ref.set({ + u'first': u'Ada', + u'last': u'Lovelace', + u'born': 1815 + }) + + sleep(1) + + for _ in range(10): + if on_snapshot.called_count == 1: + return + sleep(1) + + if on_snapshot.called_count != 1: + raise AssertionError( + "Failed to get exactly one document change: count: " + + str(on_snapshot.called_count)) + + # CM: had to stop here, this test is totally unfinished, trying to + # formalize + # https://gist.github.com/crwilcox/ce05f3857adc7a0ed86ffbd039b1a035 + + +def test_watch_query(client, cleanup): + db = client + doc_ref = db.collection(u'users').document( + u'alovelace' + unique_resource_id()) + query_ref = db.collection(u'users').where("first", "==", u'Ada') + # Initial setting doc_ref.set({ u'first': u'Jane', @@ -850,11 +902,32 @@ def on_snapshot(snapshot): u'born': 1900 }) + sleep(1) + + # Setup listener + def on_snapshot(docs, changes, read_time): + on_snapshot.called_count += 1 + print("docs: " + docs) + print("changes: " + changes) + print("read_time: " + read_time) + + on_snapshot.called_count = 0 + + query_ref.on_snapshot(on_snapshot) + + # Alter document doc_ref.set({ u'first': u'Ada', u'last': u'Lovelace', u'born': 1815 }) - # CM: had to stop here, this test is totally unfinished, trying to formalize - # https://gist.github.com/crwilcox/ce05f3857adc7a0ed86ffbd039b1a035 + for _ in range(10): + if on_snapshot.called_count == 1: + return + sleep(1) + + if on_snapshot.called_count != 1: + raise AssertionError( + "Failed to get exactly one document change: count: " + + str(on_snapshot.called_count)) From dfaacc770808c20c2874104aec126c21f37a3358 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Fri, 24 Aug 2018 10:42:52 -0400 Subject: [PATCH 105/148] fix most unit tests, 3 still fail --- firestore/tests/unit/test_watch.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index 8332e1ab3046..fb0708645530 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -106,8 +106,10 @@ def _makeOne( comparator=None, snapshot_callback=None, snapshot_class=None, + document_reference_class=None, ): from google.cloud.firestore_v1beta1.watch import Watch + from google.cloud.firestore_v1beta1.document import DocumentReference if document_reference is None: document_reference = DummyDocumentReference() if firestore is None: @@ -125,6 +127,8 @@ def _makeOne( snapshot_callback = self._snapshot_callback if snapshot_class is None: snapshot_class = DummyDocumentSnapshot + if document_reference_class is None: + document_reference_class = DocumentReference inst = Watch( document_reference, @@ -133,6 +137,7 @@ def _makeOne( comparator, snapshot_callback, snapshot_class, + document_reference_class, BackgroundConsumer=DummyBackgroundConsumer, ResumableBidiRpc=DummyRpc, ) From b59b12b9a155bdcc3c50f9b59543a3ebe6961e10 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Fri, 24 Aug 2018 09:24:34 -0700 Subject: [PATCH 106/148] merge and apply --- firestore/tests/unit/test_watch.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index fb0708645530..a1bd38fb080d 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -106,10 +106,9 @@ def _makeOne( comparator=None, snapshot_callback=None, snapshot_class=None, - document_reference_class=None, + reference_class=None ): from google.cloud.firestore_v1beta1.watch import Watch - from google.cloud.firestore_v1beta1.document import DocumentReference if document_reference is None: document_reference = DummyDocumentReference() if firestore is None: @@ -127,9 +126,8 @@ def _makeOne( snapshot_callback = self._snapshot_callback if snapshot_class is None: snapshot_class = DummyDocumentSnapshot - if document_reference_class is None: - document_reference_class = DocumentReference - + if reference_class is None: + reference_class = DummyDocumentReference inst = Watch( document_reference, firestore, @@ -137,7 +135,7 @@ def _makeOne( comparator, snapshot_callback, snapshot_class, - document_reference_class, + reference_class, BackgroundConsumer=DummyBackgroundConsumer, ResumableBidiRpc=DummyRpc, ) @@ -196,6 +194,7 @@ def test_for_document(self): docref = DummyDocumentReference() snapshot_callback = self._snapshot_callback snapshot_class_instance = DummyDocumentSnapshot + document_reference_class_instance = DummyDocumentReference modulename = 'google.cloud.firestore_v1beta1.watch' with mock.patch( '%s.Watch.ResumableBidiRpc' % modulename, @@ -208,7 +207,8 @@ def test_for_document(self): inst = Watch.for_document( docref, snapshot_callback, - snapshot_class_instance + snapshot_class_instance, + document_reference_class_instance ) self.assertTrue(inst._consumer.started) self.assertTrue(inst.rpc.callbacks, [inst._on_rpc_done]) @@ -528,6 +528,7 @@ def __init__(self): class DummyFirestore(object): _firestore_api = DummyFirestoreClient() _database_string = '' + document = DummyDocumentReference class DummyDocumentSnapshot(object): From 2a19765db8261cb70e05f2573d82bf135246e07b Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Fri, 24 Aug 2018 09:43:06 -0700 Subject: [PATCH 107/148] expected time of test was not the same as read time, so false failure --- firestore/tests/unit/test_watch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index a1bd38fb080d..3e8d75c5c400 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -384,7 +384,7 @@ class DummyReadTime(object): inst.push(DummyReadTime, 'token') self.assertEqual( self.snapshotted, - ([], [], datetime.datetime(2018, 8, 21, 9, 31, 18)), + ([], [], datetime.datetime(2018, 8, 21, 6, 31, 18)), ) self.assertTrue(inst.has_pushed) self.assertEqual(inst.resume_token, 'token') From 299fd94563aa43e5855991879d50c5f306042c7f Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Fri, 24 Aug 2018 10:11:09 -0700 Subject: [PATCH 108/148] tests passing --- firestore/tests/unit/test_watch.py | 52 +++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index 3e8d75c5c400..1644434916ab 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -461,7 +461,7 @@ def test__compute_snapshot_operation_relative_ordering(self): doc_tree = WatchDocTree() class DummyDoc(object): - pass + update_time = mock.sentinel deleted_doc = DummyDoc() added_doc = DummyDoc() @@ -471,9 +471,11 @@ class DummyDoc(object): doc_tree = doc_tree.insert('/deleted', deleted_doc) doc_tree = doc_tree.insert('/updated', updated_doc) doc_map = {'/deleted': deleted_doc, '/updated': updated_doc} - added_snapshot = DummyDocumentSnapshot() + added_snapshot = DummyDocumentSnapshot(added_doc, None, True, + None, None, None) added_snapshot.reference = added_doc - updated_snapshot = DummyDocumentSnapshot() + updated_snapshot = DummyDocumentSnapshot(updated_doc, None, True, + None, None, None) updated_snapshot.reference = updated_doc delete_changes = ['/deleted'] add_changes = [added_snapshot] @@ -486,9 +488,13 @@ class DummyDoc(object): add_changes, update_changes ) - # assertion is incorrect below, but we don't get here yet; the tested - # code raises an exception before we get a result - self.assertEqual(updated_map, None) + # TODO: + # Assertion is not verified correct below. Verify this test is good. + self.assertEqual(updated_map, + { + '/updated': updated_snapshot, + '/added': added_snapshot, + }) def test__reset_docs(self): from google.cloud.firestore_v1beta1.watch import ChangeType @@ -500,7 +506,7 @@ def test__reset_docs(self): doc_tree = WatchDocTree() doc_tree = doc_tree.insert('/doc', doc) doc_tree = doc_tree.insert('/doc', doc) - snapshot = DummyDocumentSnapshot() + snapshot = DummyDocumentSnapshot(doc, None, True, None, None, None) snapshot.reference = doc inst.doc_tree = doc_tree inst._reset_docs() @@ -520,20 +526,42 @@ def __init__(self): class DummyDocumentReference(object): - def __init__(self): - self._client = DummyFirestore() + def __init__(self, *document_path, **kw): + if 'client' not in kw: + self._client = DummyFirestore() + else: + self._client = kw['client'] + + self._path = document_path + self.__dict__.update(kw) + _document_path = '/' class DummyFirestore(object): _firestore_api = DummyFirestoreClient() _database_string = '' - document = DummyDocumentReference + + def document(self, *document_path): + if len(document_path) == 1: + path = document_path[0].split('/') + else: + path = document_path + + return DummyDocumentReference(*path, client=self) class DummyDocumentSnapshot(object): - def __init__(self, **kw): - self.__dict__.update(kw) + # def __init__(self, **kw): + # self.__dict__.update(kw) + def __init__(self, reference, data, exists, + read_time, create_time, update_time): + self.reference = reference + self.data = data + self.exists = exists + self.read_time = read_time + self.create_time = create_time + self.update_time = update_time class DummyBackgroundConsumer(object): From 9081664814c68bb0155795c4664b384534463a69 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Sat, 25 Aug 2018 13:10:54 -0400 Subject: [PATCH 109/148] make the datetime.datetime returned non-naive and assume it's in UTC --- firestore/google/cloud/firestore_v1beta1/watch.py | 4 +++- firestore/tests/unit/test_watch.py | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 61b2cee1c485..6d117dc53b32 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -18,6 +18,8 @@ import datetime from enum import Enum +import pytz + from google.cloud.firestore_v1beta1.bidi import ResumableBidiRpc from google.cloud.firestore_v1beta1.bidi import BackgroundConsumer from google.cloud.firestore_v1beta1.proto import firestore_pb2 @@ -566,7 +568,7 @@ def push(self, read_time, next_resume_token): self._snapshot_callback( updated_tree.keys(), appliedChanges, - datetime.datetime.fromtimestamp(read_time.seconds) + datetime.datetime.fromtimestamp(read_time.seconds, pytz.utc) ) self.has_pushed = True diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index 1644434916ab..a1fa9c987a9c 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -378,13 +378,18 @@ def test_on_snapshot_unknown_listen_type(self): ) def test_push_no_changes(self): + import pytz class DummyReadTime(object): seconds = 1534858278 inst = self._makeOne() inst.push(DummyReadTime, 'token') self.assertEqual( self.snapshotted, - ([], [], datetime.datetime(2018, 8, 21, 6, 31, 18)), + ( + [], + [], + datetime.datetime.fromtimestamp(DummyReadTime.seconds, pytz.utc) + ), ) self.assertTrue(inst.has_pushed) self.assertEqual(inst.resume_token, 'token') From 9489ec1683dabeb87178ab0344c6f666711a335d Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Sat, 25 Aug 2018 13:16:10 -0400 Subject: [PATCH 110/148] depends directly on pytz now --- firestore/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/firestore/setup.py b/firestore/setup.py index 73d2233a6fd9..763375cf42eb 100644 --- a/firestore/setup.py +++ b/firestore/setup.py @@ -31,6 +31,7 @@ dependencies = [ 'google-cloud-core<0.29dev,>=0.28.0', 'google-api-core[grpc]<2.0.0dev,>=0.1.1', + 'pytz', ] extras = { } From be51be82b9f8dc5f230f30a191ac5885c37575fa Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Tue, 28 Aug 2018 15:32:05 -0400 Subject: [PATCH 111/148] 100pct statement coverage for watch and test_watch --- .../google/cloud/firestore_v1beta1/watch.py | 4 +- firestore/tests/unit/test_watch.py | 106 +++++++++++++++++- 2 files changed, 104 insertions(+), 6 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 6d117dc53b32..2f911feda223 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -197,7 +197,7 @@ def __init__(self, self._closing = threading.Lock() self._closed = False - def should_recover(exc): + def should_recover(exc): # pragma: NO COVER return ( isinstance(exc, grpc.RpcError) and exc.code() == grpc.StatusCode.UNAVAILABLE) @@ -667,7 +667,7 @@ def modify_doc(new_document, updated_tree, updated_map): add_change.new_index), updated_tree, updated_map) - return None + return None, updated_tree, updated_map # Process the sorted changes in the order that is expected by our # clients (removals, additions, and then modifications). We also need diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index a1fa9c987a9c..9dc7861c04cd 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -38,6 +38,12 @@ def test___len__(self): inst = inst.insert('a', 2) self.assertEqual(len(inst), 2) + def test___iter__(self): + inst = self._makeOne() + inst = inst.insert('b', 1) + inst = inst.insert('a', 2) + self.assertEqual(sorted(list(inst)), ['a', 'b']) + class TestDocumentChange(unittest.TestCase): def _makeOne(self, type, document, old_index, new_index): @@ -107,7 +113,7 @@ def _makeOne( snapshot_callback=None, snapshot_class=None, reference_class=None - ): + ): # pragma: NO COVER from google.cloud.firestore_v1beta1.watch import Watch if document_reference is None: document_reference = DummyDocumentReference() @@ -141,7 +147,7 @@ def _makeOne( ) return inst - def _document_watch_comparator(self, doc1, doc2): + def _document_watch_comparator(self, doc1, doc2): # pragma: NO COVER return 0 def _snapshot_callback(self, docs, changes, read_time): @@ -213,6 +219,36 @@ def test_for_document(self): self.assertTrue(inst._consumer.started) self.assertTrue(inst.rpc.callbacks, [inst._on_rpc_done]) + def test_for_query(self): + from google.cloud.firestore_v1beta1.watch import Watch + snapshot_callback = self._snapshot_callback + snapshot_class_instance = DummyDocumentSnapshot + document_reference_class_instance = DummyDocumentReference + modulename = 'google.cloud.firestore_v1beta1.watch' + pb2 = DummyPb2() + with mock.patch( + '%s.firestore_pb2' % modulename, + pb2, + ): + with mock.patch( + '%s.Watch.ResumableBidiRpc' % modulename, + DummyRpc, + ): + with mock.patch( + '%s.Watch.BackgroundConsumer' % modulename, + DummyBackgroundConsumer, + ): + query = DummyQuery() + inst = Watch.for_query( + query, + snapshot_callback, + snapshot_class_instance, + document_reference_class_instance + ) + self.assertTrue(inst._consumer.started) + self.assertTrue(inst.rpc.callbacks, [inst._on_rpc_done]) + self.assertEqual(inst._targets['query'], 'dummy query target') + def test_on_snapshot_target_no_change_no_target_ids_not_current(self): inst = self._makeOne() proto = DummyProto() @@ -254,6 +290,16 @@ def test_on_snapshot_target_remove(self): inst.on_snapshot(proto) self.assertEqual(str(exc.exception), 'Error 1: hi') + def test_on_snapshot_target_remove_nocause(self): + inst = self._makeOne() + proto = DummyProto() + target_change = proto.target_change + target_change.cause = None + target_change.target_change_type = firestore_pb2.TargetChange.REMOVE + with self.assertRaises(Exception) as exc: + inst.on_snapshot(proto) + self.assertEqual(str(exc.exception), 'Error 13: internal error') + def test_on_snapshot_target_reset(self): inst = self._makeOne() @@ -501,6 +547,36 @@ class DummyDoc(object): '/added': added_snapshot, }) + def test__compute_snapshot_modify_docs_updated_doc_no_timechange(self): + from google.cloud.firestore_v1beta1.watch import WatchDocTree + doc_tree = WatchDocTree() + + class DummyDoc(object): + pass + + updated_doc_v1 = DummyDoc() + updated_doc_v1.update_time = 1 + updated_doc_v1._document_path = '/updated' + updated_doc_v2 = DummyDoc() + updated_doc_v2.update_time = 1 + updated_doc_v2._document_path = '/updated' + doc_tree = doc_tree.insert('/updated', updated_doc_v1) + doc_map = {'/updated': updated_doc_v1} + updated_snapshot = DummyDocumentSnapshot(updated_doc_v2, None, True, + None, None, 1) + delete_changes = [] + add_changes = [] + update_changes = [updated_snapshot] + inst = self._makeOne() + updated_tree, updated_map, applied_changes = inst._compute_snapshot( + doc_tree, + doc_map, + delete_changes, + add_changes, + update_changes + ) + self.assertEqual(updated_map, doc_map) # no change + def test__reset_docs(self): from google.cloud.firestore_v1beta1.watch import ChangeType inst = self._makeOne() @@ -521,7 +597,7 @@ def test__reset_docs(self): class DummyFirestoreStub(object): - def Listen(self): + def Listen(self): # pragma: NO COVER pass @@ -542,12 +618,22 @@ def __init__(self, *document_path, **kw): _document_path = '/' +class DummyQuery(object): # pragma: NO COVER + def __init__(self, **kw): + if 'client' not in kw: + self._client = DummyFirestore() + else: + self._client = kw['client'] + + def _to_protobuf(self): + return '' + class DummyFirestore(object): _firestore_api = DummyFirestoreClient() _database_string = '' - def document(self, *document_path): + def document(self, *document_path): # pragma: NO COVER if len(document_path) == 1: path = document_path[0].split('/') else: @@ -642,3 +728,15 @@ class DummyProto(object): def __init__(self): self.target_change = DummyChange() self.document_change = DummyChange() + + +class DummyTarget(object): + def QueryTarget(self, **kw): + self.kw = kw + return 'dummy query target' + + +class DummyPb2(object): + Target = DummyTarget() + def ListenRequest(self, **kw): + pass From 854d0512c6ecbc665ef67bb1fbab38a4f93791e2 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Tue, 28 Aug 2018 18:19:07 -0400 Subject: [PATCH 112/148] just cutnpaste this i guess --- firestore/tests/unit/test_bidi.py | 658 ++++++++++++++++++++++++++++++ 1 file changed, 658 insertions(+) create mode 100644 firestore/tests/unit/test_bidi.py diff --git a/firestore/tests/unit/test_bidi.py b/firestore/tests/unit/test_bidi.py new file mode 100644 index 000000000000..09a23b10405e --- /dev/null +++ b/firestore/tests/unit/test_bidi.py @@ -0,0 +1,658 @@ +# Copyright 2018, Google LLC All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import threading + +import grpc +import mock +import pytest +from six.moves import queue + +from google.api_core import exceptions +from google.cloud.firestore_v1beta1 import bidi + + +class Test_RequestQueueGenerator(object): + + def test_bounded_consume(self): + call = mock.create_autospec(grpc.Call, instance=True) + call.is_active.return_value = True + + def queue_generator(rpc): + yield mock.sentinel.A + yield queue.Empty() + yield mock.sentinel.B + rpc.is_active.return_value = False + yield mock.sentinel.C + + q = mock.create_autospec(queue.Queue, instance=True) + q.get.side_effect = queue_generator(call) + + generator = bidi._RequestQueueGenerator(q) + generator.call = call + + items = list(generator) + + assert items == [mock.sentinel.A, mock.sentinel.B] + + def test_yield_initial_and_exit(self): + q = mock.create_autospec(queue.Queue, instance=True) + q.get.side_effect = queue.Empty() + call = mock.create_autospec(grpc.Call, instance=True) + call.is_active.return_value = False + + generator = bidi._RequestQueueGenerator( + q, initial_request=mock.sentinel.A) + generator.call = call + + items = list(generator) + + assert items == [mock.sentinel.A] + + def test_yield_initial_callable_and_exit(self): + q = mock.create_autospec(queue.Queue, instance=True) + q.get.side_effect = queue.Empty() + call = mock.create_autospec(grpc.Call, instance=True) + call.is_active.return_value = False + + generator = bidi._RequestQueueGenerator( + q, initial_request=lambda: mock.sentinel.A) + generator.call = call + + items = list(generator) + + assert items == [mock.sentinel.A] + + def test_exit_when_inactive_with_item(self): + q = mock.create_autospec(queue.Queue, instance=True) + q.get.side_effect = [mock.sentinel.A, queue.Empty()] + call = mock.create_autospec(grpc.Call, instance=True) + call.is_active.return_value = False + + generator = bidi._RequestQueueGenerator(q) + generator.call = call + + items = list(generator) + + assert items == [] + # Make sure it put the item back. + q.put.assert_called_once_with(mock.sentinel.A) + + def test_exit_when_inactive_empty(self): + q = mock.create_autospec(queue.Queue, instance=True) + q.get.side_effect = queue.Empty() + call = mock.create_autospec(grpc.Call, instance=True) + call.is_active.return_value = False + + generator = bidi._RequestQueueGenerator(q) + generator.call = call + + items = list(generator) + + assert items == [] + + def test_exit_with_stop(self): + q = mock.create_autospec(queue.Queue, instance=True) + q.get.side_effect = [None, queue.Empty()] + call = mock.create_autospec(grpc.Call, instance=True) + call.is_active.return_value = True + + generator = bidi._RequestQueueGenerator(q) + generator.call = call + + items = list(generator) + + assert items == [] + + +class _CallAndFuture(grpc.Call, grpc.Future): + pass + + +def make_rpc(): + """Makes a mock RPC used to test Bidi classes.""" + call = mock.create_autospec(_CallAndFuture, instance=True) + rpc = mock.create_autospec(grpc.StreamStreamMultiCallable, instance=True) + + def rpc_side_effect(request): + call.is_active.return_value = True + call.request = request + return call + + rpc.side_effect = rpc_side_effect + + def cancel_side_effect(): + call.is_active.return_value = False + + call.cancel.side_effect = cancel_side_effect + + return rpc, call + + +class ClosedCall(object): + # NOTE: This is needed because defining `.next` on an **instance** + # rather than the **class** will not be iterable in Python 2. + # This is problematic since a `Mock` just sets members. + + def __init__(self, exception): + self.exception = exception + + def __next__(self): + raise self.exception + + next = __next__ # Python 2 + + def is_active(self): + return False + + +class TestBidiRpc(object): + def test_initial_state(self): + bidi_rpc = bidi.BidiRpc(None) + + assert bidi_rpc.is_active is False + + def test_done_callbacks(self): + bidi_rpc = bidi.BidiRpc(None) + callback = mock.Mock(spec=['__call__']) + + bidi_rpc.add_done_callback(callback) + bidi_rpc._on_call_done(mock.sentinel.future) + + callback.assert_called_once_with(mock.sentinel.future) + + def test_open(self): + rpc, call = make_rpc() + bidi_rpc = bidi.BidiRpc(rpc) + + bidi_rpc.open() + + assert bidi_rpc.call == call + assert bidi_rpc.is_active + call.add_done_callback.assert_called_once_with(bidi_rpc._on_call_done) + + def test_open_error_already_open(self): + rpc, _ = make_rpc() + bidi_rpc = bidi.BidiRpc(rpc) + + bidi_rpc.open() + + with pytest.raises(ValueError): + bidi_rpc.open() + + def test_close(self): + rpc, call = make_rpc() + bidi_rpc = bidi.BidiRpc(rpc) + bidi_rpc.open() + + bidi_rpc.close() + + call.cancel.assert_called_once() + assert bidi_rpc.call == call + assert bidi_rpc.is_active is False + # ensure the request queue was signaled to stop. + assert bidi_rpc.pending_requests == 1 + assert bidi_rpc._request_queue.get() is None + + def test_close_no_rpc(self): + bidi_rpc = bidi.BidiRpc(None) + bidi_rpc.close() + + def test_send(self): + rpc, call = make_rpc() + bidi_rpc = bidi.BidiRpc(rpc) + bidi_rpc.open() + + bidi_rpc.send(mock.sentinel.request) + + assert bidi_rpc.pending_requests == 1 + assert bidi_rpc._request_queue.get() is mock.sentinel.request + + def test_send_not_open(self): + rpc, call = make_rpc() + bidi_rpc = bidi.BidiRpc(rpc) + + with pytest.raises(ValueError): + bidi_rpc.send(mock.sentinel.request) + + def test_send_dead_rpc(self): + error = ValueError() + bidi_rpc = bidi.BidiRpc(None) + bidi_rpc.call = ClosedCall(error) + + with pytest.raises(ValueError) as exc_info: + bidi_rpc.send(mock.sentinel.request) + + assert exc_info.value == error + + def test_recv(self): + bidi_rpc = bidi.BidiRpc(None) + bidi_rpc.call = iter([mock.sentinel.response]) + + response = bidi_rpc.recv() + + assert response == mock.sentinel.response + + def test_recv_not_open(self): + rpc, call = make_rpc() + bidi_rpc = bidi.BidiRpc(rpc) + + with pytest.raises(ValueError): + bidi_rpc.recv() + + +class CallStub(object): + def __init__(self, values, active=True): + self.values = iter(values) + self._is_active = active + self.cancelled = False + + def __next__(self): + item = next(self.values) + if isinstance(item, Exception): + self._is_active = False + raise item + return item + + next = __next__ # Python 2 + + def is_active(self): + return self._is_active + + def add_done_callback(self, callback): + pass + + def cancel(self): + self.cancelled = True + + +class TestResumableBidiRpc(object): + def test_initial_state(self): + bidi_rpc = bidi.ResumableBidiRpc(None, lambda _: True) + + assert bidi_rpc.is_active is False + + def test_done_callbacks_recoverable(self): + start_rpc = mock.create_autospec( + grpc.StreamStreamMultiCallable, instance=True) + bidi_rpc = bidi.ResumableBidiRpc(start_rpc, lambda _: True) + callback = mock.Mock(spec=['__call__']) + + bidi_rpc.add_done_callback(callback) + bidi_rpc._on_call_done(mock.sentinel.future) + + callback.assert_not_called() + start_rpc.assert_called_once() + assert bidi_rpc.is_active + + def test_done_callbacks_non_recoverable(self): + bidi_rpc = bidi.ResumableBidiRpc(None, lambda _: False) + callback = mock.Mock(spec=['__call__']) + + bidi_rpc.add_done_callback(callback) + bidi_rpc._on_call_done(mock.sentinel.future) + + callback.assert_called_once_with(mock.sentinel.future) + + def test_send_recover(self): + error = ValueError() + call_1 = CallStub([error], active=False) + call_2 = CallStub([]) + start_rpc = mock.create_autospec( + grpc.StreamStreamMultiCallable, + instance=True, + side_effect=[call_1, call_2]) + should_recover = mock.Mock(spec=['__call__'], return_value=True) + bidi_rpc = bidi.ResumableBidiRpc(start_rpc, should_recover) + + bidi_rpc.open() + + bidi_rpc.send(mock.sentinel.request) + + assert bidi_rpc.pending_requests == 1 + assert bidi_rpc._request_queue.get() is mock.sentinel.request + + should_recover.assert_called_once_with(error) + assert bidi_rpc.call == call_2 + assert bidi_rpc.is_active is True + + def test_send_failure(self): + error = ValueError() + call = CallStub([error], active=False) + start_rpc = mock.create_autospec( + grpc.StreamStreamMultiCallable, + instance=True, + return_value=call) + should_recover = mock.Mock(spec=['__call__'], return_value=False) + bidi_rpc = bidi.ResumableBidiRpc(start_rpc, should_recover) + + bidi_rpc.open() + + with pytest.raises(ValueError) as exc_info: + bidi_rpc.send(mock.sentinel.request) + + assert exc_info.value == error + should_recover.assert_called_once_with(error) + assert bidi_rpc.call == call + assert bidi_rpc.is_active is False + assert call.cancelled is True + assert bidi_rpc.pending_requests == 1 + assert bidi_rpc._request_queue.get() is None + + def test_recv_recover(self): + error = ValueError() + call_1 = CallStub([1, error]) + call_2 = CallStub([2, 3]) + start_rpc = mock.create_autospec( + grpc.StreamStreamMultiCallable, + instance=True, + side_effect=[call_1, call_2]) + should_recover = mock.Mock(spec=['__call__'], return_value=True) + bidi_rpc = bidi.ResumableBidiRpc(start_rpc, should_recover) + + bidi_rpc.open() + + values = [] + for n in range(3): + values.append(bidi_rpc.recv()) + + assert values == [1, 2, 3] + should_recover.assert_called_once_with(error) + assert bidi_rpc.call == call_2 + assert bidi_rpc.is_active is True + + def test_recv_recover_race_condition(self): + # This test checks the race condition where two threads recv() and + # encounter an error and must re-open the stream. Only one thread + # should succeed in doing so. + error = ValueError() + call_1 = CallStub([error, error]) + call_2 = CallStub([1, 2]) + start_rpc = mock.create_autospec( + grpc.StreamStreamMultiCallable, + instance=True, + side_effect=[call_1, call_2]) + recovered_event = threading.Event() + + def second_thread_main(): + assert bidi_rpc.recv() == 2 + + second_thread = threading.Thread(target=second_thread_main) + + def should_recover(exception): + assert exception == error + if threading.current_thread() == second_thread: + recovered_event.wait() + return True + + bidi_rpc = bidi.ResumableBidiRpc(start_rpc, should_recover) + + bidi_rpc.open() + second_thread.start() + + assert bidi_rpc.recv() == 1 + recovered_event.set() + + assert bidi_rpc.call == call_2 + assert bidi_rpc.is_active is True + second_thread.join() + + def test_recv_failure(self): + error = ValueError() + call = CallStub([error]) + start_rpc = mock.create_autospec( + grpc.StreamStreamMultiCallable, + instance=True, + return_value=call) + should_recover = mock.Mock(spec=['__call__'], return_value=False) + bidi_rpc = bidi.ResumableBidiRpc(start_rpc, should_recover) + + bidi_rpc.open() + + with pytest.raises(ValueError) as exc_info: + bidi_rpc.recv() + + assert exc_info.value == error + should_recover.assert_called_once_with(error) + assert bidi_rpc.call == call + assert bidi_rpc.is_active is False + assert call.cancelled is True + + def test_reopen_failure_on_rpc_restart(self): + error1 = ValueError('1') + error2 = ValueError('2') + call = CallStub([error1]) + # Invoking start RPC a second time will trigger an error. + start_rpc = mock.create_autospec( + grpc.StreamStreamMultiCallable, + instance=True, + side_effect=[call, error2]) + should_recover = mock.Mock(spec=['__call__'], return_value=True) + callback = mock.Mock(spec=['__call__']) + + bidi_rpc = bidi.ResumableBidiRpc(start_rpc, should_recover) + bidi_rpc.add_done_callback(callback) + + bidi_rpc.open() + + with pytest.raises(ValueError) as exc_info: + bidi_rpc.recv() + + assert exc_info.value == error2 + should_recover.assert_called_once_with(error1) + assert bidi_rpc.call is None + assert bidi_rpc.is_active is False + callback.assert_called_once_with(error2) + + def test_finalize_idempotent(self): + error1 = ValueError('1') + error2 = ValueError('2') + callback = mock.Mock(spec=['__call__']) + should_recover = mock.Mock(spec=['__call__'], return_value=False) + + bidi_rpc = bidi.ResumableBidiRpc( + mock.sentinel.start_rpc, should_recover) + + bidi_rpc.add_done_callback(callback) + + bidi_rpc._on_call_done(error1) + bidi_rpc._on_call_done(error2) + + callback.assert_called_once_with(error1) + + +class TestBackgroundConsumer(object): + def test_consume_once_then_exit(self): + bidi_rpc = mock.create_autospec(bidi.BidiRpc, instance=True) + bidi_rpc.is_active = True + bidi_rpc.recv.side_effect = [mock.sentinel.response_1] + recved = threading.Event() + + def on_response(response): + assert response == mock.sentinel.response_1 + bidi_rpc.is_active = False + recved.set() + + consumer = bidi.BackgroundConsumer(bidi_rpc, on_response) + + consumer.start() + + recved.wait() + + bidi_rpc.recv.assert_called_once() + assert bidi_rpc.is_active is False + + consumer.stop() + + bidi_rpc.close.assert_called_once() + assert consumer.is_active is False + + def test_pause_resume_and_close(self): + # This test is relatively complex. It attempts to start the consumer, + # consume one item, pause the consumer, check the state of the world, + # then resume the consumer. Doing this in a deterministic fashion + # requires a bit more mocking and patching than usual. + + bidi_rpc = mock.create_autospec(bidi.BidiRpc, instance=True) + bidi_rpc.is_active = True + + def close_side_effect(): + bidi_rpc.is_active = False + + bidi_rpc.close.side_effect = close_side_effect + + # These are used to coordinate the two threads to ensure deterministic + # execution. + should_continue = threading.Event() + responses_and_events = { + mock.sentinel.response_1: threading.Event(), + mock.sentinel.response_2: threading.Event() + } + bidi_rpc.recv.side_effect = [ + mock.sentinel.response_1, mock.sentinel.response_2] + + recved_responses = [] + consumer = None + + def on_response(response): + if response == mock.sentinel.response_1: + consumer.pause() + + recved_responses.append(response) + responses_and_events[response].set() + should_continue.wait() + + consumer = bidi.BackgroundConsumer(bidi_rpc, on_response) + + consumer.start() + + # Wait for the first response to be recved. + responses_and_events[mock.sentinel.response_1].wait() + + # Ensure only one item has been recved and that the consumer is paused. + assert recved_responses == [mock.sentinel.response_1] + assert consumer.is_paused is True + assert consumer.is_active is True + + # Unpause the consumer, wait for the second item, then close the + # consumer. + should_continue.set() + consumer.resume() + + responses_and_events[mock.sentinel.response_2].wait() + + assert recved_responses == [ + mock.sentinel.response_1, mock.sentinel.response_2] + + consumer.stop() + + assert consumer.is_active is False + + def test_wake_on_error(self): + should_continue = threading.Event() + + bidi_rpc = mock.create_autospec(bidi.BidiRpc, instance=True) + bidi_rpc.is_active = True + bidi_rpc.add_done_callback.side_effect = ( + lambda _: should_continue.set()) + + consumer = bidi.BackgroundConsumer(bidi_rpc, mock.sentinel.on_response) + + # Start the consumer paused, which should immediately put it into wait + # state. + consumer.pause() + consumer.start() + + # Wait for add_done_callback to be called + should_continue.wait() + bidi_rpc.add_done_callback.assert_called_once_with( + consumer._on_call_done) + + # The consumer should now be blocked on waiting to be unpaused. + assert consumer.is_active + assert consumer.is_paused + + # Trigger the done callback, it should unpause the consumer and cause + # it to exit. + bidi_rpc.is_active = False + consumer._on_call_done(bidi_rpc) + + # It may take a few cycles for the thread to exit. + while consumer.is_active: + pass + + def test_consumer_expected_error(self, caplog): + caplog.set_level(logging.DEBUG) + + bidi_rpc = mock.create_autospec(bidi.BidiRpc, instance=True) + bidi_rpc.is_active = True + bidi_rpc.recv.side_effect = exceptions.ServiceUnavailable('Gone away') + + on_response = mock.Mock(spec=['__call__']) + + consumer = bidi.BackgroundConsumer(bidi_rpc, on_response) + + consumer.start() + + # Wait for the consumer's thread to exit. + while consumer.is_active: + pass + + on_response.assert_not_called() + bidi_rpc.recv.assert_called_once() + assert 'caught error' in caplog.text + + def test_consumer_unexpected_error(self, caplog): + caplog.set_level(logging.DEBUG) + + bidi_rpc = mock.create_autospec(bidi.BidiRpc, instance=True) + bidi_rpc.is_active = True + bidi_rpc.recv.side_effect = ValueError() + + on_response = mock.Mock(spec=['__call__']) + + consumer = bidi.BackgroundConsumer(bidi_rpc, on_response) + + consumer.start() + + # Wait for the consumer's thread to exit. + while consumer.is_active: + pass + + on_response.assert_not_called() + bidi_rpc.recv.assert_called_once() + assert 'caught unexpected exception' in caplog.text + + def test_double_stop(self, caplog): + caplog.set_level(logging.DEBUG) + bidi_rpc = mock.create_autospec(bidi.BidiRpc, instance=True) + bidi_rpc.is_active = True + on_response = mock.Mock(spec=['__call__']) + + def close_side_effect(): + bidi_rpc.is_active = False + + bidi_rpc.close.side_effect = close_side_effect + + consumer = bidi.BackgroundConsumer(bidi_rpc, on_response) + + consumer.start() + assert consumer.is_active is True + + consumer.stop() + assert consumer.is_active is False + + # calling stop twice should not result in an error. + consumer.stop() From 1059f2c43ca445284c4afa3566fc8cf908c0061a Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Tue, 28 Aug 2018 18:29:08 -0400 Subject: [PATCH 113/148] coverage for collection and bidi modules --- firestore/tests/unit/test_bidi.py | 2 +- firestore/tests/unit/test_collection.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/firestore/tests/unit/test_bidi.py b/firestore/tests/unit/test_bidi.py index 09a23b10405e..80d8ecf48389 100644 --- a/firestore/tests/unit/test_bidi.py +++ b/firestore/tests/unit/test_bidi.py @@ -279,7 +279,7 @@ def cancel(self): class TestResumableBidiRpc(object): - def test_initial_state(self): + def test_initial_state(self): # pragma: NO COVER bidi_rpc = bidi.ResumableBidiRpc(None, lambda _: True) assert bidi_rpc.is_active is False diff --git a/firestore/tests/unit/test_collection.py b/firestore/tests/unit/test_collection.py index b5d348412ed5..de12059bb61a 100644 --- a/firestore/tests/unit/test_collection.py +++ b/firestore/tests/unit/test_collection.py @@ -415,6 +415,12 @@ def test_get_with_transaction(self, query_class): self.assertIs(get_response, query_instance.get.return_value) query_instance.get.assert_called_once_with(transaction=transaction) + @mock.patch('google.cloud.firestore_v1beta1.collection.Watch',autospec=True) + def test_on_snapshot(self, watch): + collection = self._make_one('collection') + collection.on_snapshot(None) + watch.for_query.assert_called_once() + class Test__auto_id(unittest.TestCase): From 346e5d6c0d534fcc802e6b09f6e656018aac9248 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Tue, 28 Aug 2018 18:37:19 -0400 Subject: [PATCH 114/148] coverage for document and query methods added --- firestore/tests/unit/test_document.py | 9 +++++++++ firestore/tests/unit/test_query.py | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/firestore/tests/unit/test_document.py b/firestore/tests/unit/test_document.py index bf957b17aa47..f772e4421dad 100644 --- a/firestore/tests/unit/test_document.py +++ b/firestore/tests/unit/test_document.py @@ -563,6 +563,15 @@ def test_collections_wo_page_size(self): def test_collections_w_page_size(self): self._collections_helper(page_size=10) + @mock.patch('google.cloud.firestore_v1beta1.document.Watch', autospec=True) + def test_on_snapshot(self, watch): + client = mock.Mock( + _database_string='sprinklez', + spec=['_database_string']) + document = self._make_one('yellow', 'mellow', client=client) + document.on_snapshot(None) + watch.for_document.assert_called_once() + class TestDocumentSnapshot(unittest.TestCase): diff --git a/firestore/tests/unit/test_query.py b/firestore/tests/unit/test_query.py index 85f803c43fc3..e645408f537f 100644 --- a/firestore/tests/unit/test_query.py +++ b/firestore/tests/unit/test_query.py @@ -860,6 +860,12 @@ def test_get_empty_after_first_response(self): parent_path, query._to_protobuf(), transaction=None, metadata=client._rpc_metadata) + @mock.patch('google.cloud.firestore_v1beta1.query.Watch', autospec=True) + def test_on_snapshot(self, watch): + query = self._make_one(mock.sentinel.parent) + query.on_snapshot(None) + watch.for_query.assert_called_once() + class Test__enum_from_op_string(unittest.TestCase): From 6cb3e8588524ec522a89de5aea5cddc35b35d598 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Wed, 29 Aug 2018 14:17:01 -0400 Subject: [PATCH 115/148] 100 percent branch coverage --- .../google/cloud/firestore_v1beta1/watch.py | 3 + firestore/tests/unit/test_watch.py | 116 +++++++++++++++++- 2 files changed, 114 insertions(+), 5 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 2f911feda223..600bf38a8891 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -107,6 +107,9 @@ def __iter__(self): def __len__(self): return len(self._dict) + def __contains__(self, k): + return k in self._dict + class ChangeType(Enum): ADDED = 0 diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index 9dc7861c04cd..452398a22821 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -9,9 +9,6 @@ def _makeOne(self): from google.cloud.firestore_v1beta1.watch import WatchDocTree return WatchDocTree() - def setUp(self): - self.snapshotted = None - def test_insert_and_keys(self): inst = self._makeOne() inst = inst.insert('b', 1) @@ -44,6 +41,12 @@ def test___iter__(self): inst = inst.insert('a', 2) self.assertEqual(sorted(list(inst)), ['a', 'b']) + def test___contains__(self): + inst = self._makeOne() + inst = inst.insert('b', 1) + self.assertTrue('b' in inst) + self.assertFalse('a' in inst) + class TestDocumentChange(unittest.TestCase): def _makeOne(self, type, document, old_index, new_index): @@ -147,6 +150,9 @@ def _makeOne( ) return inst + def setUp(self): + self.snapshotted = None + def _document_watch_comparator(self, doc1, doc2): # pragma: NO COVER return 0 @@ -372,6 +378,40 @@ class DummyDocument: inst.on_snapshot(proto) self.assertEqual(inst.change_map['fred'].data, None) + def test_on_snapshot_document_change_changed_docname_db_prefix(self): + # XXX This test asserts the current behavior, but I have no level + # of confidence that the change map should contain the + # db-prefixed document name instead of the bare document name. + from google.cloud.firestore_v1beta1.watch import WATCH_TARGET_ID + inst = self._makeOne() + + def message_to_dict(document): + return {'fields': None} + + inst.MessageToDict = message_to_dict + proto = DummyProto() + proto.target_change = '' + proto.document_change.target_ids = [WATCH_TARGET_ID] + + class DummyDocument: + name = 'abc://foo/fred' + create_time = None + update_time = None + + proto.document_change.document = DummyDocument() + inst._firestore._database_string = 'abc://foo/' + inst.on_snapshot(proto) + self.assertEqual(inst.change_map['abc://foo/fred'].data, None) + + def test_on_snapshot_document_change_neither_changed_nor_removed(self): + inst = self._makeOne() + proto = DummyProto() + proto.target_change = '' + proto.document_change.target_ids = [] + + inst.on_snapshot(proto) + self.assertTrue(not inst.change_map) + def test_on_snapshot_document_removed(self): from google.cloud.firestore_v1beta1.watch import ChangeType inst = self._makeOne() @@ -408,6 +448,23 @@ def reset(): inst.on_snapshot(proto) self.assertTrue(inst._docs_reset) + def test_on_snapshot_filter_update_no_size_change(self): + inst = self._makeOne() + proto = DummyProto() + proto.target_change = '' + proto.document_change = '' + proto.document_remove = None + proto.document_delete = None + + class DummyFilter(object): + count = 0 + + proto.filter = DummyFilter() + inst._docs_reset = False + + inst.on_snapshot(proto) + self.assertFalse(inst._docs_reset) + def test_on_snapshot_unknown_listen_type(self): inst = self._makeOne() proto = DummyProto() @@ -423,7 +480,7 @@ def test_on_snapshot_unknown_listen_type(self): str(exc.exception) ) - def test_push_no_changes(self): + def test_push_callback_called_no_changes(self): import pytz class DummyReadTime(object): seconds = 1534858278 @@ -440,6 +497,18 @@ class DummyReadTime(object): self.assertTrue(inst.has_pushed) self.assertEqual(inst.resume_token, 'token') + def test_push_already_pushed(self): + class DummyReadTime(object): + seconds = 1534858278 + inst = self._makeOne() + inst.has_pushed = True + inst.push(DummyReadTime, 'token') + self.assertEqual( + self.snapshotted, + None) + self.assertTrue(inst.has_pushed) + self.assertEqual(inst.resume_token, 'token') + def test__current_size_empty(self): inst = self._makeOne() result = inst._current_size() @@ -471,6 +540,14 @@ def test__extract_changes_doc_removed(self): results = inst._extract_changes(doc_map, changes, None) self.assertEqual(results, (['name'], [], [])) + def test__extract_changes_doc_removed_docname_not_in_docmap(self): + from google.cloud.firestore_v1beta1.watch import ChangeType + inst = self._makeOne() + changes = {'name': ChangeType.REMOVED} + doc_map = {} + results = inst._extract_changes(doc_map, changes, None) + self.assertEqual(results, ([], [], [])) + def test__extract_changes_doc_updated(self): inst = self._makeOne() @@ -485,6 +562,21 @@ class Dummy(object): self.assertEqual(results, ([], [], [snapshot])) self.assertEqual(snapshot.read_time, 1) + def test__extract_changes_doc_updated_read_time_is_None(self): + inst = self._makeOne() + + class Dummy(object): + pass + + doc = Dummy() + snapshot = Dummy() + snapshot.read_time = None + changes = {'name': snapshot} + doc_map = {'name': doc} + results = inst._extract_changes(doc_map, changes, None) + self.assertEqual(results, ([], [], [snapshot])) + self.assertEqual(snapshot.read_time, None) + def test__extract_changes_doc_added(self): inst = self._makeOne() @@ -498,6 +590,20 @@ class Dummy(object): self.assertEqual(results, ([], [snapshot], [])) self.assertEqual(snapshot.read_time, 1) + def test__extract_changes_doc_added_read_time_is_None(self): + inst = self._makeOne() + + class Dummy(object): + pass + + snapshot = Dummy() + snapshot.read_time = None + changes = {'name': snapshot} + doc_map = {} + results = inst._extract_changes(doc_map, changes, None) + self.assertEqual(results, ([], [snapshot], [])) + self.assertEqual(snapshot.read_time, None) + def test__compute_snapshot_doctree_and_docmap_disagree_about_length(self): inst = self._makeOne() doc_tree = {} @@ -631,7 +737,7 @@ def _to_protobuf(self): class DummyFirestore(object): _firestore_api = DummyFirestoreClient() - _database_string = '' + _database_string = 'abc://bar/' def document(self, *document_path): # pragma: NO COVER if len(document_path) == 1: From 7c73721c0098802c85424491495d92fadd194189 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Wed, 29 Aug 2018 14:21:23 -0400 Subject: [PATCH 116/148] appease linter --- firestore/tests/unit/test_collection.py | 3 ++- firestore/tests/unit/test_query.py | 2 +- firestore/tests/unit/test_watch.py | 10 ++++++++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/firestore/tests/unit/test_collection.py b/firestore/tests/unit/test_collection.py index de12059bb61a..ab4da4ccee8f 100644 --- a/firestore/tests/unit/test_collection.py +++ b/firestore/tests/unit/test_collection.py @@ -415,7 +415,8 @@ def test_get_with_transaction(self, query_class): self.assertIs(get_response, query_instance.get.return_value) query_instance.get.assert_called_once_with(transaction=transaction) - @mock.patch('google.cloud.firestore_v1beta1.collection.Watch',autospec=True) + @mock.patch('google.cloud.firestore_v1beta1.collection.Watch', + autospec=True) def test_on_snapshot(self, watch): collection = self._make_one('collection') collection.on_snapshot(None) diff --git a/firestore/tests/unit/test_query.py b/firestore/tests/unit/test_query.py index e645408f537f..4e4619841438 100644 --- a/firestore/tests/unit/test_query.py +++ b/firestore/tests/unit/test_query.py @@ -865,7 +865,7 @@ def test_on_snapshot(self, watch): query = self._make_one(mock.sentinel.parent) query.on_snapshot(None) watch.for_query.assert_called_once() - + class Test__enum_from_op_string(unittest.TestCase): diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index 452398a22821..10f970861572 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -116,7 +116,7 @@ def _makeOne( snapshot_callback=None, snapshot_class=None, reference_class=None - ): # pragma: NO COVER + ): # pragma: NO COVER from google.cloud.firestore_v1beta1.watch import Watch if document_reference is None: document_reference = DummyDocumentReference() @@ -482,8 +482,10 @@ def test_on_snapshot_unknown_listen_type(self): def test_push_callback_called_no_changes(self): import pytz + class DummyReadTime(object): seconds = 1534858278 + inst = self._makeOne() inst.push(DummyReadTime, 'token') self.assertEqual( @@ -491,7 +493,8 @@ class DummyReadTime(object): ( [], [], - datetime.datetime.fromtimestamp(DummyReadTime.seconds, pytz.utc) + datetime.datetime.fromtimestamp( + DummyReadTime.seconds, pytz.utc) ), ) self.assertTrue(inst.has_pushed) @@ -724,6 +727,7 @@ def __init__(self, *document_path, **kw): _document_path = '/' + class DummyQuery(object): # pragma: NO COVER def __init__(self, **kw): if 'client' not in kw: @@ -843,6 +847,8 @@ def QueryTarget(self, **kw): class DummyPb2(object): + Target = DummyTarget() + def ListenRequest(self, **kw): pass From 7019fd549ead65005f4e9f532414261b9b82e896 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Wed, 19 Sep 2018 12:14:02 -0700 Subject: [PATCH 117/148] should return object on snapshot watching. This is needed to unsubscribe --- firestore/google/cloud/firestore_v1beta1/collection.py | 2 +- firestore/google/cloud/firestore_v1beta1/document.py | 2 +- firestore/google/cloud/firestore_v1beta1/query.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/collection.py b/firestore/google/cloud/firestore_v1beta1/collection.py index 1110858f4667..8234bee6a21a 100644 --- a/firestore/google/cloud/firestore_v1beta1/collection.py +++ b/firestore/google/cloud/firestore_v1beta1/collection.py @@ -398,7 +398,7 @@ def on_snapshot(collection_snapshot): # Terminate this watch collection_watch.unsubscribe() """ - Watch.for_query(query_mod.Query(self), + return Watch.for_query(query_mod.Query(self), callback, document.DocumentSnapshot, document.DocumentReference) diff --git a/firestore/google/cloud/firestore_v1beta1/document.py b/firestore/google/cloud/firestore_v1beta1/document.py index 03b37a6363f7..cb40b99dec91 100644 --- a/firestore/google/cloud/firestore_v1beta1/document.py +++ b/firestore/google/cloud/firestore_v1beta1/document.py @@ -475,7 +475,7 @@ def on_snapshot(document_snapshot): # Terminate this watch doc_watch.unsubscribe() """ - Watch.for_document(self, callback, DocumentSnapshot, DocumentReference) + return Watch.for_document(self, callback, DocumentSnapshot, DocumentReference) class DocumentSnapshot(object): diff --git a/firestore/google/cloud/firestore_v1beta1/query.py b/firestore/google/cloud/firestore_v1beta1/query.py index c39a5febea44..27e282d9ffb4 100644 --- a/firestore/google/cloud/firestore_v1beta1/query.py +++ b/firestore/google/cloud/firestore_v1beta1/query.py @@ -627,7 +627,7 @@ def on_snapshot(query_snapshot): # Terminate this watch query_watch.unsubscribe() """ - Watch.for_query(self, + return Watch.for_query(self, callback, document.DocumentSnapshot, document.DocumentReference) From 6f36b6f39955ca22ceb6875a126673eb997c1bb3 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Thu, 20 Sep 2018 11:13:41 -0700 Subject: [PATCH 118/148] Fix bug in deletion of document from map (using wrong key) --- .../google/cloud/firestore_v1beta1/watch.py | 11 ++++---- firestore/tests/system.py | 25 ++++++++++++++----- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 600bf38a8891..5b90dde22e0d 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -491,8 +491,10 @@ def on_snapshot(self, proto): # fashion than self._document_reference document_name = document.name db_str = self._firestore._database_string - if document_name.startswith(db_str): - document_name = document_name[len(db_str):] + db_str_documents = db_str + '/documents/' + if document_name.startswith(db_str_documents): + document_name = document_name[len(db_str_documents):] + document_ref = self._firestore.document(document_name) snapshot = self.DocumentSnapshot( @@ -552,7 +554,6 @@ def push(self, read_time, next_resume_token): """ # TODO: may need to lock here to avoid races on collecting snapshots # and sending them to the user. - deletes, adds, updates = Watch._extract_changes( self.doc_map, self.change_map, @@ -623,10 +624,10 @@ def delete_doc(name, updated_tree, updated_map): assert name in updated_map, 'Document to delete does not exist' old_document = updated_map.get(name) # XXX probably should not expose IndexError when doc doesnt exist - existing = updated_tree.find(name) + existing = updated_tree.find(old_document) old_index = existing.index # TODO: was existing.remove returning tree (presumably immuatable?) - updated_tree = updated_tree.remove(name) + updated_tree = updated_tree.remove(old_document) del updated_map[name] return (DocumentChange(ChangeType.REMOVED, old_document, diff --git a/firestore/tests/system.py b/firestore/tests/system.py index d02ae9b14458..019a894dee48 100644 --- a/firestore/tests/system.py +++ b/firestore/tests/system.py @@ -876,12 +876,24 @@ def on_snapshot(docs, changes, read_time): for _ in range(10): if on_snapshot.called_count == 1: - return + break sleep(1) - if on_snapshot.called_count != 1: + # Alter document + doc_ref.set({ + u'first': u'Ada', + u'last': u'Lovelace', + u'born': 0 + }) + + for _ in range(10): + if on_snapshot.called_count == 2: + break + sleep(1) + + if on_snapshot.called_count != 2: raise AssertionError( - "Failed to get exactly one document change: count: " + + "Failed to get exactly two document changes: count: " + str(on_snapshot.called_count)) # CM: had to stop here, this test is totally unfinished, trying to @@ -907,9 +919,10 @@ def test_watch_query(client, cleanup): # Setup listener def on_snapshot(docs, changes, read_time): on_snapshot.called_count += 1 - print("docs: " + docs) - print("changes: " + changes) - print("read_time: " + read_time) + + # A snapshot should return the same thing as if a query ran now. + query_ran = db.collection(u'users').where("first", "==", u'Ada').get() + assert len(docs) == len([i for i in query_ran]) on_snapshot.called_count = 0 From 23d467df1eee994e8d926a20a7913b5b4a128e0e Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Fri, 28 Sep 2018 16:38:55 -0700 Subject: [PATCH 119/148] startings of ordering --- .../google/cloud/firestore_v1beta1/order.py | 291 ++++++++++++++++++ firestore/tests/unit/test_order.py | 210 +++++++++++++ 2 files changed, 501 insertions(+) create mode 100644 firestore/google/cloud/firestore_v1beta1/order.py create mode 100644 firestore/tests/unit/test_order.py diff --git a/firestore/google/cloud/firestore_v1beta1/order.py b/firestore/google/cloud/firestore_v1beta1/order.py new file mode 100644 index 000000000000..4060673cf3f9 --- /dev/null +++ b/firestore/google/cloud/firestore_v1beta1/order.py @@ -0,0 +1,291 @@ +# Copyright 2017 Google LLC All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# package com.google.cloud.firestore; + +# import com.google.firestore.v1beta1.Value; +# import com.google.firestore.v1beta1.Value.ValueTypeCase; +# import com.google.protobuf.ByteString; +# import java.util.Comparator; +# import java.util.Iterator; +# import java.util.List; +# import java.util.Map.Entry; +# import java.util.SortedMap; +# import java.util.TreeMap; +# import javax.annotation.Nonnull; +from enum import Enum +from google.cloud.firestore_v1beta1._helpers import decode_value + +class TypeOrder(Enum): + # NOTE: This order is defined by the backend and cannot be changed. + NULL = 0 + BOOLEAN = 1 + NUMBER = 2 + TIMESTAMP = 3 + STRING = 4 + BLOB = 5 + REF = 6 + GEO_POINT = 7 + ARRAY = 8 + OBJECT = 9 + + def from_value(value): + v = value.WhichOneof('value_type') + + lut = { + 'null_value': NULL, + 'boolean_value': BOOLEAN, + 'integer_value': NUMBER, + 'double_value': NUMBER, + 'timestamp_value': TIMESTAMP, + 'string_value': STRING, + 'bytes_value': BLOB, + 'reference_value': REF, + 'geo_point_value': GEO_POINT, + 'array_value': ARRAY, + 'map_value': OBJECT, + } + + if v not in lut: + raise ArgumentException( + "Could not detect value type for " + value) + return lut[v] + + +class Order(object): + ''' + Order implements the ordering semantics of the backend. + ''' + def __init__(): + pass + + def compare(left, right): + ''' + Main comparison function for all Firestore types. + + @return -1 is left < right, 0 if left == right, otherwise 1 + ''' + + # First compare the types. + leftType = TypeOrder.from_value(left) + rightType = TypeOrder.from_value(right) + + if leftType != rightType: + if leftType < rightType: + return -1 + return 1 + + # TODO: may be able to use helpers.decode_value and do direct compares + # after converting to python types + value_type = value.WhichOneof('value_type') + + if value_type == 'null_value': + return 0 # nulls are all equal + elif value_type == 'boolean_value': + return _compareTo(decode_value(left), decode_value(right)) + elif value_type == 'integer_value': + return compare_numbers(left, right) + elif value_type == 'double_value': + return compare_numbers(left, right) + elif value_type == 'timestamp_value': + # NOTE: This conversion is "lossy", Python ``datetime.datetime`` + # has microsecond precision but ``timestamp_value`` has + # nanosecond precision. + #return _pb_timestamp_to_datetime(value.timestamp_value) + return compare_timestamps(left, right) + elif value_type == 'string_value': + #return value.string_value + return compare_strings(left, right) + elif value_type == 'bytes_value': + #return value.bytes_value + return compare_blobs(left, right) + elif value_type == 'reference_value': + #return reference_value_to_document(value.reference_value, client) + return compare_resource_paths(left, right) + elif value_type == 'geo_point_value': + #return GeoPoint( + # value.geo_point_value.latitude, + # value.geo_point_value.longitude) + return compare_geo_points(left, right) + elif value_type == 'array_value': + #return [decode_value(element, client) + # for element in value.array_value.values] + return compare_arrays(left, right) + elif value_type == 'map_value': + #return decode_dict(value.map_value.fields, client) + return compare_objects(left, right) + else: + raise ValueError('Unknown ``value_type``', value_type) + + +def compare_strings(left, right): + left_value = decode_value(left) + right_value = decode_value(right) + return _compareTo(left_value, right_value) + + +# private int compareBlobs(Value left, Value right) { +# ByteString leftBytes = left.getBytesValue(); +# ByteString rightBytes = right.getBytesValue(); + +# int size = Math.min(leftBytes.size(), rightBytes.size()); +# for (int i = 0; i < size; i++) { +# // Make sure the bytes are unsigned +# int thisByte = leftBytes.byteAt(i) & 0xff; +# int otherByte = rightBytes.byteAt(i) & 0xff; +# if (thisByte < otherByte) { +# return -1; +# } else if (thisByte > otherByte) { +# return 1; +# } +# // Byte values are equal, continue with comparison +# } +# return Integer.compare(leftBytes.size(), rightBytes.size()); +# } +def compare_blobs(left, right): + raise NotImplementedError() + + +def compare_timestamps(left, right): + left_value = left.timestamp_value + right_value = right.timestamp_value + + cmp = 0 + if left_value.seconds < right_value.seconds: + cmp = -1 + elif left_value.seconds == right_value.seconds: + cmp = 0 + else: + cmp = 0 + + if cmp != 0: + return cmp + else: + if left_value.nanos < right_value.nanos: + cmp = -1 + elif left_value.nanos == right_value.nanos: + cmp = 0 + else: + cmp = 1 + return cmp + + +def compare_geo_points(left, right): + left_value = decode_value(left) + right_value = decode_value(right) + cmp = 0 + if left_value.latitude < right_value.latitude: + cmp = -1 + elif left_value.latitude == right_value.latitude: + cmp = 0 + else: + cmp = 1 + + if cmp != 0: + return cmp + else: + if left.longitude < right.longitude: + cmp = -1 + elif left.longitude == right.longitude: + cmp = 0 + else: + cmp = 1 + return cmp + +# private int compareResourcePaths(Value left, Value right) { +# ResourcePath leftPath = ResourcePath.create(left.getReferenceValue()); +# ResourcePath rightPath = ResourcePath.create(right.getReferenceValue()); +# return leftPath.compareTo(rightPath); +# } +def compare_resource_paths(left, right): + raise NotImplementedError() + + +# private int compareArrays(Value left, Value right) { +# List leftValue = left.getArrayValue().getValuesList(); +# List rightValue = right.getArrayValue().getValuesList(); + +# int minLength = Math.min(leftValue.size(), rightValue.size()); +# for (int i = 0; i < minLength; i++) { +# int cmp = compare(leftValue.get(i), rightValue.get(i)); +# if (cmp != 0) { +# return cmp; +# } +# } +# return Integer.compare(leftValue.size(), rightValue.size()); +# } +def compare_arrays(left, right): + raise NotImplementedError() + + + +# private int compareObjects(Value left, Value right) { +# // This requires iterating over the keys in the object in order and doing a +# // deep comparison. +# SortedMap leftMap = new TreeMap<>(); +# leftMap.putAll(left.getMapValue().getFieldsMap()); +# SortedMap rightMap = new TreeMap<>(); +# rightMap.putAll(right.getMapValue().getFieldsMap()); + +# Iterator> leftIterator = leftMap.entrySet().iterator(); +# Iterator> rightIterator = rightMap.entrySet().iterator(); + +# while (leftIterator.hasNext() && rightIterator.hasNext()) { +# Entry leftEntry = leftIterator.next(); +# Entry rightEntry = rightIterator.next(); +# int keyCompare = leftEntry.getKey().compareTo(rightEntry.getKey()); +# if (keyCompare != 0) { +# return keyCompare; +# } +# int valueCompare = compare(leftEntry.getValue(), rightEntry.getValue()); +# if (valueCompare != 0) { +# return valueCompare; +# } +# } + +# // Only equal if both iterators are exhausted. +# return Boolean.compare(leftIterator.hasNext(), rightIterator.hasNext()); +# } +def compare_objects(left, right): + raise NotImplementedError() + +def compare_numbers(left, right): + left_value = decode_value(left) + right_value = decode_value(right) + return compare_doubles(left_value, right_value) + +def compare_doubles(left, right): + if math.isnan(left): + if math.isnan(right): + return 0 + return -1 + if math.isnan(right): + return 1 + + if left == -0.0: + left = 0 + if right == -0.0: + right = 0 + + return _compareTo(left, right) + + +def _compareTo(left, right): + if left < right: + return -1 + elif left == right: + return 0 + # left > right + return 1 diff --git a/firestore/tests/unit/test_order.py b/firestore/tests/unit/test_order.py new file mode 100644 index 000000000000..3f0cbd5348d2 --- /dev/null +++ b/firestore/tests/unit/test_order.py @@ -0,0 +1,210 @@ +# Copyright 2017 Google LLC All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import mock +from google.cloud.firestore_v1beta1._helpers import encode_value +from google.protobuf import timestamp_pb2 +from google.type import latlng_pb2 +import math + + +class TestOrder(unittest.TestCase): + @staticmethod + def _get_target_class(): + from google.cloud.firestore_v1beta1.order import Order + + return Order + + def _make_one(self, *args, **kwargs): + klass = self._get_target_class() + return klass(*args, **kwargs) + + def test_order(self): + + int_max_value = 10 ** 1000 + int_min_value = -10 ** 1000 + float_min_value = -10.0 ** 1000 + float_nan = float('nan') + + groups = [None] * 65 + + groups[0] = [nullValue()] + + groups[1] = [_booleanValue(False)] + groups[2] = [_booleanValue(True)] + + # numbers + groups[3] = [_doubleValue(float_nan), _doubleValue(float_nan)] + groups[4] = [_doubleValue(-math.inf)] + groups[5] = [_intValue(int_min_value - 1)] + groups[6] = [_intValue(int_min_value)] + groups[7] = [_doubleValue(-1.1)] + # Integers and Doubles order the same. + groups[8] = [_intValue(-1), _doubleValue(-1.0)] + groups[9] = [_doubleValue(-float_min_value)] + # zeros all compare the same. + groups[10] = [_intValue(0), _doubleValue(-0.0), + _doubleValue(0.0), _doubleValue(+0.0)] + groups[11] = [_doubleValue(float_min_value)] + groups[12] = [_intValue(1), _doubleValue(1.0)] + groups[13] = [_doubleValue(1.1)] + groups[14] = [_intValue(int_max_value)] + groups[15] = [_intValue(int_max_value + 1)] + groups[16] = [_doubleValue(math.inf)] + + groups[17] = [_timestampValue(123, 0)] + groups[18] = [_timestampValue(123, 123)] + groups[19] = [_timestampValue(345, 0)] + + # strings + groups[20] = [_stringValue("")] + groups[21] = [_stringValue("\u0000\ud7ff\ue000\uffff")] + groups[22] = [_stringValue("(╯°□°）╯︵ ┻━┻")] + groups[23] = [_stringValue("a")] + groups[24] = [_stringValue("abc def")] + # latin small letter e + combining acute accent + latin small letter b + groups[25] = [_stringValue("e\u0301b")] + groups[26] = [_stringValue("æ")] + # latin small letter e with acute accent + latin small letter a + groups[27] = [_stringValue("\u00e9a")] + + # blobs + groups[28] = [_blobValue(bytes())] + groups[29] = [_blobValue(bytes([0]))] + groups[30] = [_blobValue(bytes([0, 1, 2, 3, 4]))] + groups[31] = [_blobValue(bytes([0, 1, 2, 4, 3]))] + groups[32] = [_blobValue(bytes([127]))] + + # resource names + groups[33] = [ + _referenceValue("projects/p1/databases/d1/documents/c1/doc1")] + groups[34] = [ + _referenceValue("projects/p1/databases/d1/documents/c1/doc2")] + groups[35] = [ + _referenceValue( + "projects/p1/databases/d1/documents/c1/doc2/c2/doc1")] + groups[36] = [ + _referenceValue( + "projects/p1/databases/d1/documents/c1/doc2/c2/doc2")] + groups[37] = [ + _referenceValue("projects/p1/databases/d1/documents/c10/doc1")] + groups[38] = [ + _referenceValue("projects/p1/databases/d1/documents/c2/doc1")] + groups[39] = [ + _referenceValue("projects/p2/databases/d2/documents/c1/doc1")] + groups[40] = [ + _referenceValue("projects/p2/databases/d2/documents/c1-/doc1")] + groups[41] = [ + _referenceValue("projects/p2/databases/d3/documents/c1-/doc1")] + + # geo points + groups[42] = [_geoPointValue(-90, -180)] + groups[43] = [_geoPointValue(-90, 0)] + groups[44] = [_geoPointValue(-90, 180)] + groups[45] = [_geoPointValue(0, -180)] + groups[46] = [_geoPointValue(0, 0)] + groups[47] = [_geoPointValue(0, 180)] + groups[48] = [_geoPointValue(1, -180)] + groups[49] = [_geoPointValue(1, 0)] + groups[50] = [_geoPointValue(1, 180)] + groups[51] = [_geoPointValue(90, -180)] + groups[52] = [_geoPointValue(90, 0)] + groups[53] = [_geoPointValue(90, 180)] + + # arrays + groups[54] = [_arrayValue()] + groups[55] = [_arrayValue(_stringValue("bar"))] + groups[56] = [_arrayValue(_stringValue("foo"))] + groups[57] = [_arrayValue(_stringValue("foo"), _intValue(0))] + groups[58] = [_arrayValue(_stringValue("foo"), _intValue(1))] + groups[59] = [_arrayValue(_stringValue("foo"), _stringValue("0"))] + + # objects + groups[60] = [_objectValue({"bar": _intValue(0)})] + groups[61] = [_objectValue({ + "bar": _intValue(0), + "foo": _intValue(1) + })] + groups[62] = [_objectValue({"bar": _intValue(1)})] + groups[63] = [_objectValue({"bar": _intValue(2)})] + groups[64] = [_objectValue({"bar": _stringValue("0")})] + + target = self._make_one() + for left in groups: + for right in groups: + for i in groups[left]: + for j in groups[right]: + self.assertEqual( + _compare(left, right), + _compare( + target.compare( + groups[left][i], + groups[right][j]), 0), + "Order does not match for: groups[%d][%d] " + "and groups[%d][%d]".format(left, i, right, j) + ) + + +def _compare(left, right): + if left < right: + return -1 + elif left == right: + return 0 + return 1 + + +def _booleanValue(b): + return encode_value(b) + + +def _doubleValue(d): + return encode_value(d) + + +def _intValue(l): + return encode_value(l) + + +def _stringValue(s): + return encode_value(s) + + +def _referenceValue(r): + return encode_value(r) + + +def _blobValue(b): + return encode_value(b) + + +def nullValue(): + return encode_value(None) + + +def _timestampValue(seconds, nanos): + return timestamp_pb2.Timestamp(seconds=seconds, nanos=nanos) + + +def _geoPointValue(latitude, longitude): + return latlng_pb2.LatLng(latitude=latitude, + longitude=longitude) + + +def _arrayValue(values): + return encode_value(values) + + +def _objectValue(keysAndValues): + return encode_value(keysAndValues) From c8e293ad4944fff8c4f307bebe525ddcb60981fa Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Fri, 5 Oct 2018 16:14:18 -0700 Subject: [PATCH 120/148] update tests --- .../google/cloud/firestore_v1beta1/order.py | 419 ++++++++++-------- firestore/tests/unit/test_order.py | 93 ++-- 2 files changed, 289 insertions(+), 223 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/order.py b/firestore/google/cloud/firestore_v1beta1/order.py index 4060673cf3f9..274393a095a3 100644 --- a/firestore/google/cloud/firestore_v1beta1/order.py +++ b/firestore/google/cloud/firestore_v1beta1/order.py @@ -27,6 +27,7 @@ # import javax.annotation.Nonnull; from enum import Enum from google.cloud.firestore_v1beta1._helpers import decode_value +import math class TypeOrder(Enum): # NOTE: This order is defined by the backend and cannot be changed. @@ -45,17 +46,17 @@ def from_value(value): v = value.WhichOneof('value_type') lut = { - 'null_value': NULL, - 'boolean_value': BOOLEAN, - 'integer_value': NUMBER, - 'double_value': NUMBER, - 'timestamp_value': TIMESTAMP, - 'string_value': STRING, - 'bytes_value': BLOB, - 'reference_value': REF, - 'geo_point_value': GEO_POINT, - 'array_value': ARRAY, - 'map_value': OBJECT, + 'null_value': TypeOrder.NULL, + 'boolean_value': TypeOrder.BOOLEAN, + 'integer_value': TypeOrder.NUMBER, + 'double_value': TypeOrder.NUMBER, + 'timestamp_value': TypeOrder.TIMESTAMP, + 'string_value': TypeOrder.STRING, + 'bytes_value': TypeOrder.BLOB, + 'reference_value': TypeOrder.REF, + 'geo_point_value': TypeOrder.GEO_POINT, + 'array_value': TypeOrder.ARRAY, + 'map_value': TypeOrder.OBJECT, } if v not in lut: @@ -68,10 +69,10 @@ class Order(object): ''' Order implements the ordering semantics of the backend. ''' - def __init__(): + def __init__(self): pass - - def compare(left, right): + + def compare(self, left, right): ''' Main comparison function for all Firestore types. @@ -79,8 +80,8 @@ def compare(left, right): ''' # First compare the types. - leftType = TypeOrder.from_value(left) - rightType = TypeOrder.from_value(right) + leftType = TypeOrder.from_value(left).value + rightType = TypeOrder.from_value(right).value if leftType != rightType: if leftType < rightType: @@ -89,203 +90,245 @@ def compare(left, right): # TODO: may be able to use helpers.decode_value and do direct compares # after converting to python types - value_type = value.WhichOneof('value_type') + value_type = left.WhichOneof('value_type') if value_type == 'null_value': - return 0 # nulls are all equal + return 0 # nulls are all equal elif value_type == 'boolean_value': - return _compareTo(decode_value(left), decode_value(right)) + return self._compareTo(left.boolean_value, right.boolean_value) elif value_type == 'integer_value': - return compare_numbers(left, right) + return self.compare_numbers(left, right) elif value_type == 'double_value': - return compare_numbers(left, right) + return self.compare_numbers(left, right) elif value_type == 'timestamp_value': - # NOTE: This conversion is "lossy", Python ``datetime.datetime`` - # has microsecond precision but ``timestamp_value`` has - # nanosecond precision. - #return _pb_timestamp_to_datetime(value.timestamp_value) - return compare_timestamps(left, right) + return self.compare_timestamps(left, right) elif value_type == 'string_value': - #return value.string_value - return compare_strings(left, right) + return self._compareTo(left.string_value, right.string_value) elif value_type == 'bytes_value': - #return value.bytes_value - return compare_blobs(left, right) + return self.compare_blobs(left, right) elif value_type == 'reference_value': - #return reference_value_to_document(value.reference_value, client) - return compare_resource_paths(left, right) + return self.compare_resource_paths(left, right) elif value_type == 'geo_point_value': - #return GeoPoint( - # value.geo_point_value.latitude, - # value.geo_point_value.longitude) - return compare_geo_points(left, right) + return self.compare_geo_points(left, right) elif value_type == 'array_value': - #return [decode_value(element, client) - # for element in value.array_value.values] - return compare_arrays(left, right) + return self.compare_arrays(left, right) elif value_type == 'map_value': - #return decode_dict(value.map_value.fields, client) - return compare_objects(left, right) + return self.compare_objects(left, right) else: raise ValueError('Unknown ``value_type``', value_type) -def compare_strings(left, right): - left_value = decode_value(left) - right_value = decode_value(right) - return _compareTo(left_value, right_value) - - -# private int compareBlobs(Value left, Value right) { -# ByteString leftBytes = left.getBytesValue(); -# ByteString rightBytes = right.getBytesValue(); - -# int size = Math.min(leftBytes.size(), rightBytes.size()); -# for (int i = 0; i < size; i++) { -# // Make sure the bytes are unsigned -# int thisByte = leftBytes.byteAt(i) & 0xff; -# int otherByte = rightBytes.byteAt(i) & 0xff; -# if (thisByte < otherByte) { -# return -1; -# } else if (thisByte > otherByte) { -# return 1; -# } -# // Byte values are equal, continue with comparison -# } -# return Integer.compare(leftBytes.size(), rightBytes.size()); -# } -def compare_blobs(left, right): - raise NotImplementedError() - - -def compare_timestamps(left, right): - left_value = left.timestamp_value - right_value = right.timestamp_value - - cmp = 0 - if left_value.seconds < right_value.seconds: - cmp = -1 - elif left_value.seconds == right_value.seconds: - cmp = 0 - else: - cmp = 0 - - if cmp != 0: - return cmp - else: - if left_value.nanos < right_value.nanos: - cmp = -1 - elif left_value.nanos == right_value.nanos: - cmp = 0 - else: - cmp = 1 - return cmp + # private int compareBlobs(Value left, Value right) { + # ByteString leftBytes = left.getBytesValue(); + # ByteString rightBytes = right.getBytesValue(); + + # int size = Math.min(leftBytes.size(), rightBytes.size()); + # for (int i = 0; i < size; i++) { + # // Make sure the bytes are unsigned + # int thisByte = leftBytes.byteAt(i) & 0xff; + # int otherByte = rightBytes.byteAt(i) & 0xff; + # if (thisByte < otherByte) { + # return -1; + # } else if (thisByte > otherByte) { + # return 1; + # } + # // Byte values are equal, continue with comparison + # } + # return Integer.compare(leftBytes.size(), rightBytes.size()); + # } + @staticmethod + def compare_blobs(left, right): + left_bytes = left.bytes_value + right_bytes = right.bytes_value + + # TODO: verify this is okay. python can compare bytes so *shrugs* + return Order._compareTo(left_bytes, right_bytes) + @staticmethod + def compare_timestamps(left, right): + left = left.timestamp_value + right = right.timestamp_value -def compare_geo_points(left, right): - left_value = decode_value(left) - right_value = decode_value(right) - cmp = 0 - if left_value.latitude < right_value.latitude: - cmp = -1 - elif left_value.latitude == right_value.latitude: + seconds = Order._compareTo(left.seconds or 0, right.seconds or 0) + if seconds != 0: + return seconds + + return Order._compareTo(left.nanos or 0, right.nanos or 0) + + # cmp = 0 + # if left_value.seconds < right_value.seconds: + # cmp = -1 + # elif left_value.seconds == right_value.seconds: + # cmp = 0 + # else: + # cmp = 0 + + # if cmp != 0: + # return cmp + # else: + # if left_value.nanos < right_value.nanos: + # cmp = -1 + # elif left_value.nanos == right_value.nanos: + # cmp = 0 + # else: + # cmp = 1 + # return cmp + + @staticmethod + def compare_geo_points(left, right): + left_value = decode_value(left, None) + right_value = decode_value(right, None) cmp = 0 - else: - cmp = 1 - - if cmp != 0: - return cmp - else: - if left.longitude < right.longitude: + if left_value.latitude < right_value.latitude: cmp = -1 - elif left.longitude == right.longitude: + elif left_value.latitude == right_value.latitude: cmp = 0 else: cmp = 1 - return cmp - -# private int compareResourcePaths(Value left, Value right) { -# ResourcePath leftPath = ResourcePath.create(left.getReferenceValue()); -# ResourcePath rightPath = ResourcePath.create(right.getReferenceValue()); -# return leftPath.compareTo(rightPath); -# } -def compare_resource_paths(left, right): - raise NotImplementedError() - - -# private int compareArrays(Value left, Value right) { -# List leftValue = left.getArrayValue().getValuesList(); -# List rightValue = right.getArrayValue().getValuesList(); - -# int minLength = Math.min(leftValue.size(), rightValue.size()); -# for (int i = 0; i < minLength; i++) { -# int cmp = compare(leftValue.get(i), rightValue.get(i)); -# if (cmp != 0) { -# return cmp; -# } -# } -# return Integer.compare(leftValue.size(), rightValue.size()); -# } -def compare_arrays(left, right): - raise NotImplementedError() - - - -# private int compareObjects(Value left, Value right) { -# // This requires iterating over the keys in the object in order and doing a -# // deep comparison. -# SortedMap leftMap = new TreeMap<>(); -# leftMap.putAll(left.getMapValue().getFieldsMap()); -# SortedMap rightMap = new TreeMap<>(); -# rightMap.putAll(right.getMapValue().getFieldsMap()); - -# Iterator> leftIterator = leftMap.entrySet().iterator(); -# Iterator> rightIterator = rightMap.entrySet().iterator(); - -# while (leftIterator.hasNext() && rightIterator.hasNext()) { -# Entry leftEntry = leftIterator.next(); -# Entry rightEntry = rightIterator.next(); -# int keyCompare = leftEntry.getKey().compareTo(rightEntry.getKey()); -# if (keyCompare != 0) { -# return keyCompare; -# } -# int valueCompare = compare(leftEntry.getValue(), rightEntry.getValue()); -# if (valueCompare != 0) { -# return valueCompare; -# } -# } - -# // Only equal if both iterators are exhausted. -# return Boolean.compare(leftIterator.hasNext(), rightIterator.hasNext()); -# } -def compare_objects(left, right): - raise NotImplementedError() - -def compare_numbers(left, right): - left_value = decode_value(left) - right_value = decode_value(right) - return compare_doubles(left_value, right_value) - -def compare_doubles(left, right): - if math.isnan(left): - if math.isnan(right): - return 0 - return -1 - if math.isnan(right): - return 1 - if left == -0.0: - left = 0 - if right == -0.0: - right = 0 + if cmp != 0: + return cmp + else: + if left_value.longitude < right_value.longitude: + cmp = -1 + elif left_value.longitude == right_value.longitude: + cmp = 0 + else: + cmp = 1 + return cmp + + # private int compareResourcePaths(Value left, Value right) { + # ResourcePath leftPath = ResourcePath.create(left.getReferenceValue()); + # ResourcePath rightPath = ResourcePath.create(right.getReferenceValue()); + # return leftPath.compareTo(rightPath); + # } + @staticmethod + def compare_resource_paths(left, right): + """ + compareTo(other: Path): number { + const len = Math.min(left.segments.length, right.segments.length); + for (let i = 0; i < len; i++) { + if (left.segments[i] < right.segments[i]) { + return -1; + } + if (left.segments[i] > right.segments[i]) { + return 1; + } + } + if (left.segments.length < right.segments.length) { + return -1; + } + if (left.segments.length > right.segments.length) { + return 1; + } + return 0; + } + """ + left = left.reference_value + right = right.reference_value + + + left_segments = left.split('/') + right_segments = right.split('/') + shorter = min(len(left_segments), len(right_segments)) + # compare segments + for i in range(shorter): + if (left_segments[i] < right_segments[i]): + return -1 + + if (left_segments[i] > right_segments[i]): + return 1 + - return _compareTo(left, right) + left_length = len(left) + right_length = len(right) + if left_length < right_length: + return -1 + if left_length > right_length: + return 1 -def _compareTo(left, right): - if left < right: - return -1 - elif left == right: return 0 - # left > right - return 1 + #raise NotImplementedError() + + + # private int compareArrays(Value left, Value right) { + # List leftValue = left.getArrayValue().getValuesList(); + # List rightValue = right.getArrayValue().getValuesList(); + + # int minLength = Math.min(leftValue.size(), rightValue.size()); + # for (int i = 0; i < minLength; i++) { + # int cmp = compare(leftValue.get(i), rightValue.get(i)); + # if (cmp != 0) { + # return cmp; + # } + # } + # return Integer.compare(leftValue.size(), rightValue.size()); + # } + @staticmethod + def compare_arrays(left, right): + raise NotImplementedError() + + + + # private int compareObjects(Value left, Value right) { + # // This requires iterating over the keys in the object in order and doing a + # // deep comparison. + # SortedMap leftMap = new TreeMap<>(); + # leftMap.putAll(left.getMapValue().getFieldsMap()); + # SortedMap rightMap = new TreeMap<>(); + # rightMap.putAll(right.getMapValue().getFieldsMap()); + + # Iterator> leftIterator = leftMap.entrySet().iterator(); + # Iterator> rightIterator = rightMap.entrySet().iterator(); + + # while (leftIterator.hasNext() && rightIterator.hasNext()) { + # Entry leftEntry = leftIterator.next(); + # Entry rightEntry = rightIterator.next(); + # int keyCompare = leftEntry.getKey().compareTo(rightEntry.getKey()); + # if (keyCompare != 0) { + # return keyCompare; + # } + # int valueCompare = compare(leftEntry.getValue(), rightEntry.getValue()); + # if (valueCompare != 0) { + # return valueCompare; + # } + # } + + # // Only equal if both iterators are exhausted. + # return Boolean.compare(leftIterator.hasNext(), rightIterator.hasNext()); + # } + @staticmethod + def compare_objects(left, right): + raise NotImplementedError() + + @staticmethod + def compare_numbers(left, right): + left_value = decode_value(left, None) + right_value = decode_value(right, None) + return Order.compare_doubles(left_value, right_value) + + @staticmethod + def compare_doubles(left, right): + if math.isnan(left): + if math.isnan(right): + return 0 + return -1 + if math.isnan(right): + return 1 + + if left == -0.0: + left = 0 + if right == -0.0: + right = 0 + + return Order._compareTo(left, right) + + @staticmethod + def _compareTo(left, right): + if left < right: + return -1 + elif left == right: + return 0 + # left > right + return 1 diff --git a/firestore/tests/unit/test_order.py b/firestore/tests/unit/test_order.py index 3f0cbd5348d2..5f16842763ad 100644 --- a/firestore/tests/unit/test_order.py +++ b/firestore/tests/unit/test_order.py @@ -14,11 +14,13 @@ import unittest import mock -from google.cloud.firestore_v1beta1._helpers import encode_value +from google.cloud.firestore_v1beta1._helpers import encode_value, GeoPoint from google.protobuf import timestamp_pb2 from google.type import latlng_pb2 import math - +from google.cloud.firestore_v1beta1.document import DocumentReference +from google.cloud.firestore_v1beta1.order import Order +from google.cloud.firestore_v1beta1.proto import document_pb2 class TestOrder(unittest.TestCase): @staticmethod @@ -33,9 +35,10 @@ def _make_one(self, *args, **kwargs): def test_order(self): - int_max_value = 10 ** 1000 - int_min_value = -10 ** 1000 - float_min_value = -10.0 ** 1000 + # Constants used to represent min/max values of storage types. + int_max_value = 2 ** 31 - 1 + int_min_value = -(2 ** 31) + float_min_value = 1.175494351 ** -38 float_nan = float('nan') groups = [None] * 65 @@ -71,14 +74,14 @@ def test_order(self): # strings groups[20] = [_stringValue("")] groups[21] = [_stringValue("\u0000\ud7ff\ue000\uffff")] - groups[22] = [_stringValue("(╯°□°）╯︵ ┻━┻")] + groups[22] = [_stringValue(u"(╯°□°）╯︵ ┻━┻")] groups[23] = [_stringValue("a")] groups[24] = [_stringValue("abc def")] # latin small letter e + combining acute accent + latin small letter b groups[25] = [_stringValue("e\u0301b")] - groups[26] = [_stringValue("æ")] + groups[26] = [_stringValue(u"æ")] # latin small letter e with acute accent + latin small letter a - groups[27] = [_stringValue("\u00e9a")] + groups[27] = [_stringValue(u"\u00e9a")] # blobs groups[28] = [_blobValue(bytes())] @@ -124,38 +127,58 @@ def test_order(self): groups[53] = [_geoPointValue(90, 180)] # arrays + # groups[54] = [_arrayValue()] + # groups[55] = [_arrayValue([_stringValue("bar"))] + # groups[56] = [_arrayValue(_stringValue("foo"))] + # groups[57] = [_arrayValue(_stringValue("foo"), _intValue(0))] + # groups[58] = [_arrayValue(_stringValue("foo"), _intValue(1))] + # groups[59] = [_arrayValue(_stringValue("foo"), _stringValue("0"))] groups[54] = [_arrayValue()] - groups[55] = [_arrayValue(_stringValue("bar"))] - groups[56] = [_arrayValue(_stringValue("foo"))] - groups[57] = [_arrayValue(_stringValue("foo"), _intValue(0))] - groups[58] = [_arrayValue(_stringValue("foo"), _intValue(1))] - groups[59] = [_arrayValue(_stringValue("foo"), _stringValue("0"))] + groups[55] = [_arrayValue(["bar"])] + groups[56] = [_arrayValue(["foo"])] + groups[57] = [_arrayValue(["foo", 0])] + groups[58] = [_arrayValue(["foo", 1])] + groups[59] = [_arrayValue(["foo", "0"])] # objects - groups[60] = [_objectValue({"bar": _intValue(0)})] + # groups[60] = [_objectValue({"bar": _intValue(0)})] + # groups[61] = [_objectValue({ + # "bar": _intValue(0), + # "foo": _intValue(1) + # })] + # groups[62] = [_objectValue({"bar": _intValue(1)})] + # groups[63] = [_objectValue({"bar": _intValue(2)})] + # groups[64] = [_objectValue({"bar": _stringValue("0")})] + + groups[60] = [_objectValue({"bar": 0})] groups[61] = [_objectValue({ - "bar": _intValue(0), - "foo": _intValue(1) + "bar":0, + "foo": 1 })] - groups[62] = [_objectValue({"bar": _intValue(1)})] - groups[63] = [_objectValue({"bar": _intValue(2)})] - groups[64] = [_objectValue({"bar": _stringValue("0")})] + groups[62] = [_objectValue({"bar": 1})] + groups[63] = [_objectValue({"bar": 2})] + groups[64] = [_objectValue({"bar": "0"})] target = self._make_one() - for left in groups: - for right in groups: - for i in groups[left]: - for j in groups[right]: + + for i in range(len(groups)): + for left in groups[i]: + for j in range(len(groups)): + for right in groups[j]: + expected = Order._compareTo(i,j) + self.assertEqual( - _compare(left, right), - _compare( - target.compare( - groups[left][i], - groups[right][j]), 0), - "Order does not match for: groups[%d][%d] " - "and groups[%d][%d]".format(left, i, right, j) + target.compare(left, right), expected, + "comparing L->R {} ({}) to {} ({})".format(i, left, j, right) ) + + expected = Order._compareTo(j, i); + self.assertEqual( + target.compare(right, left), expected, + #"comparing R->L {} to {}".format(right, left) + "comparing R->L {} ({}) to {} ({})".format(j, right, i, left) + ) def _compare(left, right): if left < right: @@ -182,7 +205,7 @@ def _stringValue(s): def _referenceValue(r): - return encode_value(r) + return document_pb2.Value(reference_value=r) def _blobValue(b): @@ -194,15 +217,15 @@ def nullValue(): def _timestampValue(seconds, nanos): - return timestamp_pb2.Timestamp(seconds=seconds, nanos=nanos) + return document_pb2.Value( + timestamp_value=timestamp_pb2.Timestamp(seconds=seconds, nanos=nanos)) def _geoPointValue(latitude, longitude): - return latlng_pb2.LatLng(latitude=latitude, - longitude=longitude) + return encode_value(GeoPoint(latitude,longitude)) -def _arrayValue(values): +def _arrayValue(values=[]): return encode_value(values) From 6cd0ac6100bfd33e5521b02ab8c8fd56b9b8fbfa Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Mon, 8 Oct 2018 13:06:28 -0700 Subject: [PATCH 121/148] complete implementation of order --- .../google/cloud/firestore_v1beta1/order.py | 76 ++++++++----------- 1 file changed, 32 insertions(+), 44 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/order.py b/firestore/google/cloud/firestore_v1beta1/order.py index 274393a095a3..a16ba9b7fb39 100644 --- a/firestore/google/cloud/firestore_v1beta1/order.py +++ b/firestore/google/cloud/firestore_v1beta1/order.py @@ -249,58 +249,46 @@ def compare_resource_paths(left, right): return 1 return 0 - #raise NotImplementedError() - # private int compareArrays(Value left, Value right) { - # List leftValue = left.getArrayValue().getValuesList(); - # List rightValue = right.getArrayValue().getValuesList(); - - # int minLength = Math.min(leftValue.size(), rightValue.size()); - # for (int i = 0; i < minLength; i++) { - # int cmp = compare(leftValue.get(i), rightValue.get(i)); - # if (cmp != 0) { - # return cmp; - # } - # } - # return Integer.compare(leftValue.size(), rightValue.size()); - # } @staticmethod def compare_arrays(left, right): - raise NotImplementedError() - - - - # private int compareObjects(Value left, Value right) { - # // This requires iterating over the keys in the object in order and doing a - # // deep comparison. - # SortedMap leftMap = new TreeMap<>(); - # leftMap.putAll(left.getMapValue().getFieldsMap()); - # SortedMap rightMap = new TreeMap<>(); - # rightMap.putAll(right.getMapValue().getFieldsMap()); + l_values = left.array_value.values#.keys() + r_values = right.array_value.values#.keys() + + length = min(len(l_values), len(r_values)) + for i in range(length): + cmp = Order().compare(l_values[i], r_values[i]) + if cmp != 0: + return cmp + + return Order._compareTo(len(l_values), len(r_values)) - # Iterator> leftIterator = leftMap.entrySet().iterator(); - # Iterator> rightIterator = rightMap.entrySet().iterator(); - # while (leftIterator.hasNext() && rightIterator.hasNext()) { - # Entry leftEntry = leftIterator.next(); - # Entry rightEntry = rightIterator.next(); - # int keyCompare = leftEntry.getKey().compareTo(rightEntry.getKey()); - # if (keyCompare != 0) { - # return keyCompare; - # } - # int valueCompare = compare(leftEntry.getValue(), rightEntry.getValue()); - # if (valueCompare != 0) { - # return valueCompare; - # } - # } - - # // Only equal if both iterators are exhausted. - # return Boolean.compare(leftIterator.hasNext(), rightIterator.hasNext()); - # } @staticmethod def compare_objects(left, right): - raise NotImplementedError() + left_fields = left.map_value.fields + right_fields = right.map_value.fields + + l_iter = left_fields.__iter__() + r_iter = right_fields.__iter__() + try: + while True: + left_key = l_iter.__next__() + right_key = r_iter.__next__() + + keyCompare = Order._compareTo(left_key, right_key) + if keyCompare != 0: + return keyCompare + + value_compare = Order().compare( + left_fields[left_key], right_fields[right_key]) + if value_compare != 0: + return value_compare + except StopIteration: + pass + + return Order._compareTo(len(left_fields), len(right_fields)) @staticmethod def compare_numbers(left, right): From 41ef649562ccd5125c5be5b97f50a030dbd8aa13 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Mon, 8 Oct 2018 13:08:01 -0700 Subject: [PATCH 122/148] remove commented code areas in order --- .../google/cloud/firestore_v1beta1/order.py | 75 +------------------ 1 file changed, 1 insertion(+), 74 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/order.py b/firestore/google/cloud/firestore_v1beta1/order.py index a16ba9b7fb39..d5a68e2ebf49 100644 --- a/firestore/google/cloud/firestore_v1beta1/order.py +++ b/firestore/google/cloud/firestore_v1beta1/order.py @@ -12,19 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - -# package com.google.cloud.firestore; - -# import com.google.firestore.v1beta1.Value; -# import com.google.firestore.v1beta1.Value.ValueTypeCase; -# import com.google.protobuf.ByteString; -# import java.util.Comparator; -# import java.util.Iterator; -# import java.util.List; -# import java.util.Map.Entry; -# import java.util.SortedMap; -# import java.util.TreeMap; -# import javax.annotation.Nonnull; from enum import Enum from google.cloud.firestore_v1beta1._helpers import decode_value import math @@ -118,24 +105,6 @@ def compare(self, left, right): raise ValueError('Unknown ``value_type``', value_type) - # private int compareBlobs(Value left, Value right) { - # ByteString leftBytes = left.getBytesValue(); - # ByteString rightBytes = right.getBytesValue(); - - # int size = Math.min(leftBytes.size(), rightBytes.size()); - # for (int i = 0; i < size; i++) { - # // Make sure the bytes are unsigned - # int thisByte = leftBytes.byteAt(i) & 0xff; - # int otherByte = rightBytes.byteAt(i) & 0xff; - # if (thisByte < otherByte) { - # return -1; - # } else if (thisByte > otherByte) { - # return 1; - # } - # // Byte values are equal, continue with comparison - # } - # return Integer.compare(leftBytes.size(), rightBytes.size()); - # } @staticmethod def compare_blobs(left, right): left_bytes = left.bytes_value @@ -155,24 +124,6 @@ def compare_timestamps(left, right): return Order._compareTo(left.nanos or 0, right.nanos or 0) - # cmp = 0 - # if left_value.seconds < right_value.seconds: - # cmp = -1 - # elif left_value.seconds == right_value.seconds: - # cmp = 0 - # else: - # cmp = 0 - - # if cmp != 0: - # return cmp - # else: - # if left_value.nanos < right_value.nanos: - # cmp = -1 - # elif left_value.nanos == right_value.nanos: - # cmp = 0 - # else: - # cmp = 1 - # return cmp @staticmethod def compare_geo_points(left, right): @@ -197,33 +148,9 @@ def compare_geo_points(left, right): cmp = 1 return cmp - # private int compareResourcePaths(Value left, Value right) { - # ResourcePath leftPath = ResourcePath.create(left.getReferenceValue()); - # ResourcePath rightPath = ResourcePath.create(right.getReferenceValue()); - # return leftPath.compareTo(rightPath); - # } + @staticmethod def compare_resource_paths(left, right): - """ - compareTo(other: Path): number { - const len = Math.min(left.segments.length, right.segments.length); - for (let i = 0; i < len; i++) { - if (left.segments[i] < right.segments[i]) { - return -1; - } - if (left.segments[i] > right.segments[i]) { - return 1; - } - } - if (left.segments.length < right.segments.length) { - return -1; - } - if (left.segments.length > right.segments.length) { - return 1; - } - return 0; - } - """ left = left.reference_value right = right.reference_value From 6be3422d2633620f0edd4730c83dbe595c1244c2 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Mon, 8 Oct 2018 13:09:21 -0700 Subject: [PATCH 123/148] refactor order --- .../google/cloud/firestore_v1beta1/order.py | 20 +- firestore/tests/unit/test_order.py | 185 ++++++++---------- 2 files changed, 92 insertions(+), 113 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/order.py b/firestore/google/cloud/firestore_v1beta1/order.py index d5a68e2ebf49..b1e1722daaa1 100644 --- a/firestore/google/cloud/firestore_v1beta1/order.py +++ b/firestore/google/cloud/firestore_v1beta1/order.py @@ -82,7 +82,7 @@ def compare(self, left, right): if value_type == 'null_value': return 0 # nulls are all equal elif value_type == 'boolean_value': - return self._compareTo(left.boolean_value, right.boolean_value) + return self._compare_to(left.boolean_value, right.boolean_value) elif value_type == 'integer_value': return self.compare_numbers(left, right) elif value_type == 'double_value': @@ -90,7 +90,7 @@ def compare(self, left, right): elif value_type == 'timestamp_value': return self.compare_timestamps(left, right) elif value_type == 'string_value': - return self._compareTo(left.string_value, right.string_value) + return self._compare_to(left.string_value, right.string_value) elif value_type == 'bytes_value': return self.compare_blobs(left, right) elif value_type == 'reference_value': @@ -111,18 +111,18 @@ def compare_blobs(left, right): right_bytes = right.bytes_value # TODO: verify this is okay. python can compare bytes so *shrugs* - return Order._compareTo(left_bytes, right_bytes) + return Order._compare_to(left_bytes, right_bytes) @staticmethod def compare_timestamps(left, right): left = left.timestamp_value right = right.timestamp_value - seconds = Order._compareTo(left.seconds or 0, right.seconds or 0) + seconds = Order._compare_to(left.seconds or 0, right.seconds or 0) if seconds != 0: return seconds - return Order._compareTo(left.nanos or 0, right.nanos or 0) + return Order._compare_to(left.nanos or 0, right.nanos or 0) @staticmethod @@ -189,7 +189,7 @@ def compare_arrays(left, right): if cmp != 0: return cmp - return Order._compareTo(len(l_values), len(r_values)) + return Order._compare_to(len(l_values), len(r_values)) @staticmethod @@ -204,7 +204,7 @@ def compare_objects(left, right): left_key = l_iter.__next__() right_key = r_iter.__next__() - keyCompare = Order._compareTo(left_key, right_key) + keyCompare = Order._compare_to(left_key, right_key) if keyCompare != 0: return keyCompare @@ -215,7 +215,7 @@ def compare_objects(left, right): except StopIteration: pass - return Order._compareTo(len(left_fields), len(right_fields)) + return Order._compare_to(len(left_fields), len(right_fields)) @staticmethod def compare_numbers(left, right): @@ -237,10 +237,10 @@ def compare_doubles(left, right): if right == -0.0: right = 0 - return Order._compareTo(left, right) + return Order._compare_to(left, right) @staticmethod - def _compareTo(left, right): + def _compare_to(left, right): if left < right: return -1 elif left == right: diff --git a/firestore/tests/unit/test_order.py b/firestore/tests/unit/test_order.py index 5f16842763ad..2a9e55765fe2 100644 --- a/firestore/tests/unit/test_order.py +++ b/firestore/tests/unit/test_order.py @@ -45,119 +45,104 @@ def test_order(self): groups[0] = [nullValue()] - groups[1] = [_booleanValue(False)] - groups[2] = [_booleanValue(True)] + groups[1] = [_boolean_value(False)] + groups[2] = [_boolean_value(True)] # numbers - groups[3] = [_doubleValue(float_nan), _doubleValue(float_nan)] - groups[4] = [_doubleValue(-math.inf)] - groups[5] = [_intValue(int_min_value - 1)] - groups[6] = [_intValue(int_min_value)] - groups[7] = [_doubleValue(-1.1)] + groups[3] = [_double_value(float_nan), _double_value(float_nan)] + groups[4] = [_double_value(-math.inf)] + groups[5] = [_int_value(int_min_value - 1)] + groups[6] = [_int_value(int_min_value)] + groups[7] = [_double_value(-1.1)] # Integers and Doubles order the same. - groups[8] = [_intValue(-1), _doubleValue(-1.0)] - groups[9] = [_doubleValue(-float_min_value)] + groups[8] = [_int_value(-1), _double_value(-1.0)] + groups[9] = [_double_value(-float_min_value)] # zeros all compare the same. - groups[10] = [_intValue(0), _doubleValue(-0.0), - _doubleValue(0.0), _doubleValue(+0.0)] - groups[11] = [_doubleValue(float_min_value)] - groups[12] = [_intValue(1), _doubleValue(1.0)] - groups[13] = [_doubleValue(1.1)] - groups[14] = [_intValue(int_max_value)] - groups[15] = [_intValue(int_max_value + 1)] - groups[16] = [_doubleValue(math.inf)] - - groups[17] = [_timestampValue(123, 0)] - groups[18] = [_timestampValue(123, 123)] - groups[19] = [_timestampValue(345, 0)] + groups[10] = [_int_value(0), _double_value(-0.0), + _double_value(0.0), _double_value(+0.0)] + groups[11] = [_double_value(float_min_value)] + groups[12] = [_int_value(1), _double_value(1.0)] + groups[13] = [_double_value(1.1)] + groups[14] = [_int_value(int_max_value)] + groups[15] = [_int_value(int_max_value + 1)] + groups[16] = [_double_value(math.inf)] + + groups[17] = [_timestamp_value(123, 0)] + groups[18] = [_timestamp_value(123, 123)] + groups[19] = [_timestamp_value(345, 0)] # strings - groups[20] = [_stringValue("")] - groups[21] = [_stringValue("\u0000\ud7ff\ue000\uffff")] - groups[22] = [_stringValue(u"(╯°□°）╯︵ ┻━┻")] - groups[23] = [_stringValue("a")] - groups[24] = [_stringValue("abc def")] + groups[20] = [_string_value("")] + groups[21] = [_string_value("\u0000\ud7ff\ue000\uffff")] + groups[22] = [_string_value(u"(╯°□°）╯︵ ┻━┻")] + groups[23] = [_string_value("a")] + groups[24] = [_string_value("abc def")] # latin small letter e + combining acute accent + latin small letter b - groups[25] = [_stringValue("e\u0301b")] - groups[26] = [_stringValue(u"æ")] + groups[25] = [_string_value("e\u0301b")] + groups[26] = [_string_value(u"æ")] # latin small letter e with acute accent + latin small letter a - groups[27] = [_stringValue(u"\u00e9a")] + groups[27] = [_string_value(u"\u00e9a")] # blobs - groups[28] = [_blobValue(bytes())] - groups[29] = [_blobValue(bytes([0]))] - groups[30] = [_blobValue(bytes([0, 1, 2, 3, 4]))] - groups[31] = [_blobValue(bytes([0, 1, 2, 4, 3]))] - groups[32] = [_blobValue(bytes([127]))] + groups[28] = [_blob_value(bytes())] + groups[29] = [_blob_value(bytes([0]))] + groups[30] = [_blob_value(bytes([0, 1, 2, 3, 4]))] + groups[31] = [_blob_value(bytes([0, 1, 2, 4, 3]))] + groups[32] = [_blob_value(bytes([127]))] # resource names groups[33] = [ - _referenceValue("projects/p1/databases/d1/documents/c1/doc1")] + _reference_value("projects/p1/databases/d1/documents/c1/doc1")] groups[34] = [ - _referenceValue("projects/p1/databases/d1/documents/c1/doc2")] + _reference_value("projects/p1/databases/d1/documents/c1/doc2")] groups[35] = [ - _referenceValue( + _reference_value( "projects/p1/databases/d1/documents/c1/doc2/c2/doc1")] groups[36] = [ - _referenceValue( + _reference_value( "projects/p1/databases/d1/documents/c1/doc2/c2/doc2")] groups[37] = [ - _referenceValue("projects/p1/databases/d1/documents/c10/doc1")] + _reference_value("projects/p1/databases/d1/documents/c10/doc1")] groups[38] = [ - _referenceValue("projects/p1/databases/d1/documents/c2/doc1")] + _reference_value("projects/p1/databases/d1/documents/c2/doc1")] groups[39] = [ - _referenceValue("projects/p2/databases/d2/documents/c1/doc1")] + _reference_value("projects/p2/databases/d2/documents/c1/doc1")] groups[40] = [ - _referenceValue("projects/p2/databases/d2/documents/c1-/doc1")] + _reference_value("projects/p2/databases/d2/documents/c1-/doc1")] groups[41] = [ - _referenceValue("projects/p2/databases/d3/documents/c1-/doc1")] + _reference_value("projects/p2/databases/d3/documents/c1-/doc1")] # geo points - groups[42] = [_geoPointValue(-90, -180)] - groups[43] = [_geoPointValue(-90, 0)] - groups[44] = [_geoPointValue(-90, 180)] - groups[45] = [_geoPointValue(0, -180)] - groups[46] = [_geoPointValue(0, 0)] - groups[47] = [_geoPointValue(0, 180)] - groups[48] = [_geoPointValue(1, -180)] - groups[49] = [_geoPointValue(1, 0)] - groups[50] = [_geoPointValue(1, 180)] - groups[51] = [_geoPointValue(90, -180)] - groups[52] = [_geoPointValue(90, 0)] - groups[53] = [_geoPointValue(90, 180)] + groups[42] = [_geoPoint_value(-90, -180)] + groups[43] = [_geoPoint_value(-90, 0)] + groups[44] = [_geoPoint_value(-90, 180)] + groups[45] = [_geoPoint_value(0, -180)] + groups[46] = [_geoPoint_value(0, 0)] + groups[47] = [_geoPoint_value(0, 180)] + groups[48] = [_geoPoint_value(1, -180)] + groups[49] = [_geoPoint_value(1, 0)] + groups[50] = [_geoPoint_value(1, 180)] + groups[51] = [_geoPoint_value(90, -180)] + groups[52] = [_geoPoint_value(90, 0)] + groups[53] = [_geoPoint_value(90, 180)] # arrays - # groups[54] = [_arrayValue()] - # groups[55] = [_arrayValue([_stringValue("bar"))] - # groups[56] = [_arrayValue(_stringValue("foo"))] - # groups[57] = [_arrayValue(_stringValue("foo"), _intValue(0))] - # groups[58] = [_arrayValue(_stringValue("foo"), _intValue(1))] - # groups[59] = [_arrayValue(_stringValue("foo"), _stringValue("0"))] - groups[54] = [_arrayValue()] - groups[55] = [_arrayValue(["bar"])] - groups[56] = [_arrayValue(["foo"])] - groups[57] = [_arrayValue(["foo", 0])] - groups[58] = [_arrayValue(["foo", 1])] - groups[59] = [_arrayValue(["foo", "0"])] + groups[54] = [_array_value()] + groups[55] = [_array_value(["bar"])] + groups[56] = [_array_value(["foo"])] + groups[57] = [_array_value(["foo", 0])] + groups[58] = [_array_value(["foo", 1])] + groups[59] = [_array_value(["foo", "0"])] # objects - # groups[60] = [_objectValue({"bar": _intValue(0)})] - # groups[61] = [_objectValue({ - # "bar": _intValue(0), - # "foo": _intValue(1) - # })] - # groups[62] = [_objectValue({"bar": _intValue(1)})] - # groups[63] = [_objectValue({"bar": _intValue(2)})] - # groups[64] = [_objectValue({"bar": _stringValue("0")})] - - groups[60] = [_objectValue({"bar": 0})] - groups[61] = [_objectValue({ + groups[60] = [_object_value({"bar": 0})] + groups[61] = [_object_value({ "bar":0, "foo": 1 })] - groups[62] = [_objectValue({"bar": 1})] - groups[63] = [_objectValue({"bar": 2})] - groups[64] = [_objectValue({"bar": "0"})] + groups[62] = [_object_value({"bar": 1})] + groups[63] = [_object_value({"bar": 2})] + groups[64] = [_object_value({"bar": "0"})] target = self._make_one() @@ -165,50 +150,44 @@ def test_order(self): for left in groups[i]: for j in range(len(groups)): for right in groups[j]: - expected = Order._compareTo(i,j) + expected = Order._compare_to(i,j) self.assertEqual( target.compare(left, right), expected, - "comparing L->R {} ({}) to {} ({})".format(i, left, j, right) + "comparing L->R {} ({}) to {} ({})".format( + i, left, j, right) ) - expected = Order._compareTo(j, i); + expected = Order._compare_to(j, i); self.assertEqual( target.compare(right, left), expected, - #"comparing R->L {} to {}".format(right, left) - "comparing R->L {} ({}) to {} ({})".format(j, right, i, left) + "comparing R->L {} ({}) to {} ({})".format( + j, right, i, left) ) -def _compare(left, right): - if left < right: - return -1 - elif left == right: - return 0 - return 1 - -def _booleanValue(b): +def _boolean_value(b): return encode_value(b) -def _doubleValue(d): +def _double_value(d): return encode_value(d) -def _intValue(l): +def _int_value(l): return encode_value(l) -def _stringValue(s): +def _string_value(s): return encode_value(s) -def _referenceValue(r): +def _reference_value(r): return document_pb2.Value(reference_value=r) -def _blobValue(b): +def _blob_value(b): return encode_value(b) @@ -216,18 +195,18 @@ def nullValue(): return encode_value(None) -def _timestampValue(seconds, nanos): +def _timestamp_value(seconds, nanos): return document_pb2.Value( timestamp_value=timestamp_pb2.Timestamp(seconds=seconds, nanos=nanos)) -def _geoPointValue(latitude, longitude): +def _geoPoint_value(latitude, longitude): return encode_value(GeoPoint(latitude,longitude)) -def _arrayValue(values=[]): +def _array_value(values=[]): return encode_value(values) -def _objectValue(keysAndValues): +def _object_value(keysAndValues): return encode_value(keysAndValues) From b1c42e9ae44f09285df455b6fe93c0d9cb649bbf Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Mon, 8 Oct 2018 14:53:48 -0700 Subject: [PATCH 124/148] refactor order compare_objects --- .../google/cloud/firestore_v1beta1/order.py | 27 +++++++------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/order.py b/firestore/google/cloud/firestore_v1beta1/order.py index b1e1722daaa1..a327639e5654 100644 --- a/firestore/google/cloud/firestore_v1beta1/order.py +++ b/firestore/google/cloud/firestore_v1beta1/order.py @@ -165,8 +165,6 @@ def compare_resource_paths(left, right): if (left_segments[i] > right_segments[i]): return 1 - - left_length = len(left) right_length = len(right) @@ -197,23 +195,16 @@ def compare_objects(left, right): left_fields = left.map_value.fields right_fields = right.map_value.fields - l_iter = left_fields.__iter__() - r_iter = right_fields.__iter__() - try: - while True: - left_key = l_iter.__next__() - right_key = r_iter.__next__() - - keyCompare = Order._compare_to(left_key, right_key) - if keyCompare != 0: - return keyCompare - - value_compare = Order().compare( - left_fields[left_key], right_fields[right_key]) - if value_compare != 0: + for left_key, right_key in zip(left_fields, right_fields): + keyCompare = Order._compare_to(left_key, right_key) + if keyCompare != 0: + return keyCompare + + value_compare = Order().compare( + left_fields[left_key], right_fields[right_key]) + if value_compare != 0: return value_compare - except StopIteration: - pass + return Order._compare_to(len(left_fields), len(right_fields)) From 59f725b828086e1ae53f4bcd77429e98d9f175ce Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Mon, 8 Oct 2018 15:00:14 -0700 Subject: [PATCH 125/148] add system test for ordering (currently failing for non-ordering --- firestore/tests/system.py | 81 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/firestore/tests/system.py b/firestore/tests/system.py index 019a894dee48..ba8c3b5362bb 100644 --- a/firestore/tests/system.py +++ b/firestore/tests/system.py @@ -944,3 +944,84 @@ def on_snapshot(docs, changes, read_time): raise AssertionError( "Failed to get exactly one document change: count: " + str(on_snapshot.called_count)) + + +def test_watch_query_order(client, cleanup): + db = client + unique_id = unique_resource_id() + doc_ref = db.collection(u'users').document( + u'alovelace' + unique_id) + query_ref = db.collection(u'users').where( + "first", "==", u'Ada' + unique_id).order_by("last") + + + # Setup listener + def on_snapshot(docs, changes, read_time): + try: + on_snapshot.called_count += 1 + + # A snapshot should return the same thing as if a query ran now. + query_ran = query_ref.get() + query_ran_results = [i for i in query_ran] + assert len(docs) == len(query_ran_results) + print("doc length: {}".format(len(docs))) + print("changes length: {}".format(len(changes))) + print("readtime: {}".format(read_time)) + + # compare the order things are returned + for snapshot, query in zip(docs, query_ran_results): + assert snapshot.get('last')['stringValue'] == query.get( + 'last'), "expect the sort order to match" + + except Exception as e: + pytest.fail(e) + + on_snapshot.called_count = 0 + + # Initial setting + doc_ref.set({ + u'first': u'Jane', + u'last': u'Doe', + u'born': 1900 + }) + + sleep(1) + + query_ref.on_snapshot(on_snapshot) + + # Alter document + doc_ref.set({ + u'first': u'Ada' + unique_id, + u'last': u'Lovelace', + u'born': 1815 + }) + + for _ in range(10): + if on_snapshot.called_count == 1: + break + sleep(1) + + if on_snapshot.called_count != 1: + raise AssertionError( + "Initial set should have called on_snapshot 1 time: " + + str(on_snapshot.called_count)) + + # Create new document + doc_ref_2 = db.collection(u'users').document( + u'alovelace' + unique_resource_id()) + doc_ref_2.set({ + u'first': u'Ada' + unique_id, + u'last': u'ASecondLovelace', + u'born': 1815 + }) + + for _ in range(10): + if on_snapshot.called_count == 2: + break + sleep(1) + + + if on_snapshot.called_count != 2: + raise AssertionError( + "After new add on_snapshot should be called 2 times: " + + str(on_snapshot.called_count)) From 6888e8a2c87ee3617fd7e470942a55037655b597 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Tue, 9 Oct 2018 21:32:09 -0700 Subject: [PATCH 126/148] add system test for query and verify order --- .../google/cloud/firestore_v1beta1/order.py | 33 +++++++------ .../google/cloud/firestore_v1beta1/query.py | 47 +++++++++++++++++++ .../google/cloud/firestore_v1beta1/watch.py | 39 ++++++++------- firestore/tests/system.py | 21 +++++---- 4 files changed, 96 insertions(+), 44 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/order.py b/firestore/google/cloud/firestore_v1beta1/order.py index a327639e5654..2ad62c96b906 100644 --- a/firestore/google/cloud/firestore_v1beta1/order.py +++ b/firestore/google/cloud/firestore_v1beta1/order.py @@ -16,6 +16,7 @@ from google.cloud.firestore_v1beta1._helpers import decode_value import math + class TypeOrder(Enum): # NOTE: This order is defined by the backend and cannot be changed. NULL = 0 @@ -56,16 +57,14 @@ class Order(object): ''' Order implements the ordering semantics of the backend. ''' - def __init__(self): - pass - def compare(self, left, right): + @classmethod + def compare(cls, left, right): ''' Main comparison function for all Firestore types. @return -1 is left < right, 0 if left == right, otherwise 1 ''' - # First compare the types. leftType = TypeOrder.from_value(left).value rightType = TypeOrder.from_value(right).value @@ -82,25 +81,25 @@ def compare(self, left, right): if value_type == 'null_value': return 0 # nulls are all equal elif value_type == 'boolean_value': - return self._compare_to(left.boolean_value, right.boolean_value) + return cls._compare_to(left.boolean_value, right.boolean_value) elif value_type == 'integer_value': - return self.compare_numbers(left, right) + return cls.compare_numbers(left, right) elif value_type == 'double_value': - return self.compare_numbers(left, right) + return cls.compare_numbers(left, right) elif value_type == 'timestamp_value': - return self.compare_timestamps(left, right) + return cls.compare_timestamps(left, right) elif value_type == 'string_value': - return self._compare_to(left.string_value, right.string_value) + return cls._compare_to(left.string_value, right.string_value) elif value_type == 'bytes_value': - return self.compare_blobs(left, right) + return cls.compare_blobs(left, right) elif value_type == 'reference_value': - return self.compare_resource_paths(left, right) + return cls.compare_resource_paths(left, right) elif value_type == 'geo_point_value': - return self.compare_geo_points(left, right) + return cls.compare_geo_points(left, right) elif value_type == 'array_value': - return self.compare_arrays(left, right) + return cls.compare_arrays(left, right) elif value_type == 'map_value': - return self.compare_objects(left, right) + return cls.compare_objects(left, right) else: raise ValueError('Unknown ``value_type``', value_type) @@ -110,7 +109,7 @@ def compare_blobs(left, right): left_bytes = left.bytes_value right_bytes = right.bytes_value - # TODO: verify this is okay. python can compare bytes so *shrugs* + # TODO: Should verify bytes comparisons in python work as expected return Order._compare_to(left_bytes, right_bytes) @staticmethod @@ -183,7 +182,7 @@ def compare_arrays(left, right): length = min(len(l_values), len(r_values)) for i in range(length): - cmp = Order().compare(l_values[i], r_values[i]) + cmp = Order.compare(l_values[i], r_values[i]) if cmp != 0: return cmp @@ -200,7 +199,7 @@ def compare_objects(left, right): if keyCompare != 0: return keyCompare - value_compare = Order().compare( + value_compare = Order.compare( left_fields[left_key], right_fields[right_key]) if value_compare != 0: return value_compare diff --git a/firestore/google/cloud/firestore_v1beta1/query.py b/firestore/google/cloud/firestore_v1beta1/query.py index 27e282d9ffb4..3febb1705a13 100644 --- a/firestore/google/cloud/firestore_v1beta1/query.py +++ b/firestore/google/cloud/firestore_v1beta1/query.py @@ -30,6 +30,7 @@ from google.cloud.firestore_v1beta1.gapic import enums from google.cloud.firestore_v1beta1.proto import query_pb2 from google.cloud.firestore_v1beta1.watch import Watch +from google.cloud.firestore_v1beta1.watch import Order _EQ_OP = '==' _COMPARISON_OPERATORS = { @@ -632,6 +633,52 @@ def on_snapshot(query_snapshot): document.DocumentSnapshot, document.DocumentReference) + def comparator(self, doc1, doc2): + _orders = self._orders + + # Add implicit sorting by name, using the last specified direction. + if len(_orders) == 0: + lastDirection = Query.ASCENDING + else: + if _orders[-1].direction == 1: + lastDirection = Query.ASCENDING + else: + lastDirection = Query.DESCENDING + + orderBys = list(_orders) + + order_pb = query_pb2.StructuredQuery.Order( + field=query_pb2.StructuredQuery.FieldReference( + field_path='id', + ), + direction=_enum_from_direction(lastDirection), + ) + orderBys.append(order_pb) + + for orderBy in orderBys: + if orderBy.field.field_path == 'id': + # If ordering by docuent id, compare resource paths. + comp = Order()._compare_to( + doc1.reference._path, doc2.reference._path) + else: + if orderBy.field.field_path not in doc1._data or \ + orderBy.field.field_path not in doc2._data: + raise Exception( + "Can only compare fields that exist in the " + "DocumentSnapshot. Please include the fields you are " + " ordering on in your select() call.") + v1 = doc1._data[orderBy.field.field_path] + v2 = doc2._data[orderBy.field.field_path] + encoded_v1 = _helpers.encode_value(v1) + encoded_v2 = _helpers.encode_value(v2) + comp = Order().compare(encoded_v1, encoded_v2) + + if (comp != 0): + # 1 == Ascending, -1 == Descending + return orderBy.direction * comp + + return 0 + def _enum_from_op_string(op_string): """Convert a string representation of a binary operator to an enum. diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 5b90dde22e0d..91d17bb4f725 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -17,16 +17,20 @@ import threading import datetime from enum import Enum +import functools import pytz from google.cloud.firestore_v1beta1.bidi import ResumableBidiRpc from google.cloud.firestore_v1beta1.bidi import BackgroundConsumer +from google.cloud.firestore_v1beta1._helpers import encode_value from google.cloud.firestore_v1beta1.proto import firestore_pb2 +from google.cloud.firestore_v1beta1.order import Order from google.api_core import exceptions from google.protobuf import json_format + # from bidi import BidiRpc, ResumableBidiRpc import grpc @@ -344,14 +348,15 @@ def for_query(cls, query, snapshot_callback, snapshot_class_instance, query_target = firestore_pb2.Target.QueryTarget( parent=query._client._database_string, structured_query=query._to_protobuf(), - ) + ) + return cls(query, query._client, { 'query': query_target, 'target_id': WATCH_TARGET_ID }, - document_watch_comparator, + query.comparator, snapshot_callback, snapshot_class_instance, reference_class_instance) @@ -560,17 +565,21 @@ def push(self, read_time, next_resume_token): read_time, ) - updated_tree, updated_map, appliedChanges = Watch._compute_snapshot( + updated_tree, updated_map, appliedChanges = self._compute_snapshot( self.doc_tree, self.doc_map, deletes, adds, updates, - ) + ) if not self.has_pushed or len(appliedChanges): + # TODO: the tree should be ordered. Sort here for now. + key = functools.cmp_to_key(self._comparator) + keys = sorted(updated_tree.keys(), key=key) + self._snapshot_callback( - updated_tree.keys(), + keys, appliedChanges, datetime.datetime.fromtimestamp(read_time.seconds, pytz.utc) ) @@ -602,13 +611,11 @@ def _extract_changes(doc_map, changes, read_time): return (deletes, adds, updates) - @staticmethod - def _compute_snapshot(doc_tree, doc_map, delete_changes, add_changes, + def _compute_snapshot(self, doc_tree, doc_map, delete_changes, add_changes, update_changes): # TODO: ACTUALLY NEED TO CALCULATE # return {updated_tree, updated_map, appliedChanges}; # return doc_tree, doc_map, changes - updated_tree = doc_tree updated_map = doc_map @@ -679,20 +686,17 @@ def modify_doc(new_document, updated_tree, updated_map): # keep incrementing. appliedChanges = [] - # Deletes are sorted based on the order of the existing document. - - # TODO: SORT - # delete_changes.sort( - # lambda name1, name2: - # self._comparator(updated_map.get(name1), updated_map.get(name2))) + key = functools.cmp_to_key(self._comparator) + # Deletes are sorted based on the order of the existing document. + delete_changes = sorted(delete_changes, key=key) for name in delete_changes: change, updated_tree, updated_map = delete_doc( name, updated_tree, updated_map) appliedChanges.append(change) - # TODO: SORT - # add_changes.sort(self._comparator) + + add_changes = sorted(add_changes, key=key) _LOGGER.debug('walk over add_changes') for snapshot in add_changes: _LOGGER.debug('in add_changes') @@ -700,8 +704,7 @@ def modify_doc(new_document, updated_tree, updated_map): snapshot, updated_tree, updated_map) appliedChanges.append(change) - # TODO: SORT - # update_changes.sort(self._comparator) + update_changes = sorted(update_changes, key=key) for snapshot in update_changes: change, updated_tree, updated_map = modify_doc( snapshot, updated_tree, updated_map) diff --git a/firestore/tests/system.py b/firestore/tests/system.py index ba8c3b5362bb..e3c578037153 100644 --- a/firestore/tests/system.py +++ b/firestore/tests/system.py @@ -958,25 +958,23 @@ def test_watch_query_order(client, cleanup): # Setup listener def on_snapshot(docs, changes, read_time): try: - on_snapshot.called_count += 1 - # A snapshot should return the same thing as if a query ran now. query_ran = query_ref.get() query_ran_results = [i for i in query_ran] assert len(docs) == len(query_ran_results) - print("doc length: {}".format(len(docs))) - print("changes length: {}".format(len(changes))) - print("readtime: {}".format(read_time)) # compare the order things are returned for snapshot, query in zip(docs, query_ran_results): - assert snapshot.get('last')['stringValue'] == query.get( - 'last'), "expect the sort order to match" + print("snapshot: " + snapshot.get('last')['stringValue'] + ", " + "query: " + query.get('last')) + assert snapshot.get('last')['stringValue'] == query.get( + 'last'), "expect the sort order to match" + on_snapshot.called_count += 1 except Exception as e: - pytest.fail(e) + on_snapshot.failed = e on_snapshot.called_count = 0 + on_snapshot.failed = None # Initial setting doc_ref.set({ @@ -1008,7 +1006,7 @@ def on_snapshot(docs, changes, read_time): # Create new document doc_ref_2 = db.collection(u'users').document( - u'alovelace' + unique_resource_id()) + u'asecondlovelace' + unique_id) doc_ref_2.set({ u'first': u'Ada' + unique_id, u'last': u'ASecondLovelace', @@ -1025,3 +1023,8 @@ def on_snapshot(docs, changes, read_time): raise AssertionError( "After new add on_snapshot should be called 2 times: " + str(on_snapshot.called_count)) + if on_snapshot.failed: + raise AssertionError( + "on_snapshot failed while trying to compare sort order: " + str( + on_snapshot.failed) + ) From f77616c0d5ce8f020cba563aa62325afc507356b Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Wed, 10 Oct 2018 10:06:13 -0700 Subject: [PATCH 127/148] Improve test for order --- firestore/tests/system.py | 77 ++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 41 deletions(-) diff --git a/firestore/tests/system.py b/firestore/tests/system.py index e3c578037153..a32182ca08df 100644 --- a/firestore/tests/system.py +++ b/firestore/tests/system.py @@ -949,8 +949,17 @@ def on_snapshot(docs, changes, read_time): def test_watch_query_order(client, cleanup): db = client unique_id = unique_resource_id() - doc_ref = db.collection(u'users').document( + doc_ref1 = db.collection(u'users').document( u'alovelace' + unique_id) + doc_ref2 = db.collection(u'users').document( + u'asecondlovelace' + unique_id) + doc_ref3 = db.collection(u'users').document( + u'athirdlovelace' + unique_id) + doc_ref4 = db.collection(u'users').document( + u'afourthlovelace' + unique_id) + doc_ref5 = db.collection(u'users').document( + u'afifthlovelace' + unique_id) + query_ref = db.collection(u'users').where( "first", "==", u'Ada' + unique_id).order_by("last") @@ -958,6 +967,8 @@ def test_watch_query_order(client, cleanup): # Setup listener def on_snapshot(docs, changes, read_time): try: + if len(docs) != 5: + return # A snapshot should return the same thing as if a query ran now. query_ran = query_ref.get() query_ran_results = [i for i in query_ran] @@ -965,66 +976,50 @@ def on_snapshot(docs, changes, read_time): # compare the order things are returned for snapshot, query in zip(docs, query_ran_results): - print("snapshot: " + snapshot.get('last')['stringValue'] + ", " + "query: " + query.get('last')) - assert snapshot.get('last')['stringValue'] == query.get( 'last'), "expect the sort order to match" on_snapshot.called_count += 1 + on_snapshot.last_doc_count = len(docs) except Exception as e: on_snapshot.failed = e on_snapshot.called_count = 0 + on_snapshot.last_doc_count = 0 on_snapshot.failed = None - - # Initial setting - doc_ref.set({ - u'first': u'Jane', - u'last': u'Doe', - u'born': 1900 - }) - - sleep(1) - query_ref.on_snapshot(on_snapshot) - # Alter document - doc_ref.set({ + doc_ref1.set({ u'first': u'Ada' + unique_id, u'last': u'Lovelace', u'born': 1815 }) - - for _ in range(10): - if on_snapshot.called_count == 1: - break - sleep(1) - - if on_snapshot.called_count != 1: - raise AssertionError( - "Initial set should have called on_snapshot 1 time: " + - str(on_snapshot.called_count)) - - # Create new document - doc_ref_2 = db.collection(u'users').document( - u'asecondlovelace' + unique_id) - doc_ref_2.set({ + doc_ref2.set({ + u'first': u'Ada' + unique_id, + u'last': u'SecondLovelace', + u'born': 1815 + }) + doc_ref3.set({ u'first': u'Ada' + unique_id, - u'last': u'ASecondLovelace', + u'last': u'ThirdLovelace', + u'born': 1815 + }) + doc_ref4.set({ + u'first': u'Ada' + unique_id, + u'last': u'FourthLovelace', + u'born': 1815 + }) + doc_ref5.set({ + u'first': u'Ada' + unique_id, + u'last': u'lovelace', u'born': 1815 }) for _ in range(10): - if on_snapshot.called_count == 2: + if on_snapshot.last_doc_count == 5: break sleep(1) - - if on_snapshot.called_count != 2: - raise AssertionError( - "After new add on_snapshot should be called 2 times: " + - str(on_snapshot.called_count)) - if on_snapshot.failed: + if on_snapshot.last_doc_count != 5: raise AssertionError( - "on_snapshot failed while trying to compare sort order: " + str( - on_snapshot.failed) - ) + "5 docs expected in snapshot method " + + str(on_snapshot.last_doc_count)) From cc445f15d4010e5913cd4c933111906f128f136e Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Wed, 10 Oct 2018 10:06:51 -0700 Subject: [PATCH 128/148] Add comparator to kw creation of dummyquery, fix old test, move comparator to internal --- firestore/google/cloud/firestore_v1beta1/query.py | 9 +++++---- firestore/google/cloud/firestore_v1beta1/watch.py | 2 +- firestore/tests/unit/test_watch.py | 10 ++++++++-- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/query.py b/firestore/google/cloud/firestore_v1beta1/query.py index 3febb1705a13..d3b1d89e65b0 100644 --- a/firestore/google/cloud/firestore_v1beta1/query.py +++ b/firestore/google/cloud/firestore_v1beta1/query.py @@ -633,7 +633,7 @@ def on_snapshot(query_snapshot): document.DocumentSnapshot, document.DocumentReference) - def comparator(self, doc1, doc2): + def _comparator(self, doc1, doc2): _orders = self._orders # Add implicit sorting by name, using the last specified direction. @@ -664,9 +664,10 @@ def comparator(self, doc1, doc2): if orderBy.field.field_path not in doc1._data or \ orderBy.field.field_path not in doc2._data: raise Exception( - "Can only compare fields that exist in the " - "DocumentSnapshot. Please include the fields you are " - " ordering on in your select() call.") + "Can only compare fields that exist in the " + "DocumentSnapshot. Please include the fields you are " + "ordering on in your select() call." + ) v1 = doc1._data[orderBy.field.field_path] v2 = doc2._data[orderBy.field.field_path] encoded_v1 = _helpers.encode_value(v1) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 91d17bb4f725..b492c8a26b54 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -356,7 +356,7 @@ def for_query(cls, query, snapshot_callback, snapshot_class_instance, 'query': query_target, 'target_id': WATCH_TARGET_ID }, - query.comparator, + query._comparator, snapshot_callback, snapshot_class_instance, reference_class_instance) diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index 10f970861572..8bc33bd7a67f 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -628,8 +628,8 @@ class DummyDoc(object): added_doc._document_path = '/added' updated_doc = DummyDoc() updated_doc._document_path = '/updated' - doc_tree = doc_tree.insert('/deleted', deleted_doc) - doc_tree = doc_tree.insert('/updated', updated_doc) + doc_tree = doc_tree.insert(deleted_doc, None) + doc_tree = doc_tree.insert(updated_doc, None) doc_map = {'/deleted': deleted_doc, '/updated': updated_doc} added_snapshot = DummyDocumentSnapshot(added_doc, None, True, None, None, None) @@ -735,6 +735,12 @@ def __init__(self, **kw): else: self._client = kw['client'] + if 'comparator' not in kw: + # don't really do the comparison, just return 0 (equal) for all + self._comparator = lambda x,y: 1 + else: + self._comparator = kw['comparator'] + def _to_protobuf(self): return '' From 85c199b279f5ede668c0c7b08046e76855e9e01d Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Wed, 10 Oct 2018 12:50:14 -0700 Subject: [PATCH 129/148] flake8 fixes --- .../cloud/firestore_v1beta1/collection.py | 6 +- .../cloud/firestore_v1beta1/document.py | 3 +- .../google/cloud/firestore_v1beta1/order.py | 87 +++++++++---------- .../google/cloud/firestore_v1beta1/query.py | 10 +-- .../google/cloud/firestore_v1beta1/watch.py | 15 +--- firestore/tests/system.py | 32 ++----- firestore/tests/unit/test_order.py | 25 +++--- firestore/tests/unit/test_watch.py | 2 +- 8 files changed, 69 insertions(+), 111 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/collection.py b/firestore/google/cloud/firestore_v1beta1/collection.py index 8234bee6a21a..399766da7148 100644 --- a/firestore/google/cloud/firestore_v1beta1/collection.py +++ b/firestore/google/cloud/firestore_v1beta1/collection.py @@ -399,9 +399,9 @@ def on_snapshot(collection_snapshot): collection_watch.unsubscribe() """ return Watch.for_query(query_mod.Query(self), - callback, - document.DocumentSnapshot, - document.DocumentReference) + callback, + document.DocumentSnapshot, + document.DocumentReference) def _auto_id(): diff --git a/firestore/google/cloud/firestore_v1beta1/document.py b/firestore/google/cloud/firestore_v1beta1/document.py index cb40b99dec91..b702a7c4f103 100644 --- a/firestore/google/cloud/firestore_v1beta1/document.py +++ b/firestore/google/cloud/firestore_v1beta1/document.py @@ -475,7 +475,8 @@ def on_snapshot(document_snapshot): # Terminate this watch doc_watch.unsubscribe() """ - return Watch.for_document(self, callback, DocumentSnapshot, DocumentReference) + return Watch.for_document(self, callback, DocumentSnapshot, + DocumentReference) class DocumentSnapshot(object): diff --git a/firestore/google/cloud/firestore_v1beta1/order.py b/firestore/google/cloud/firestore_v1beta1/order.py index 2ad62c96b906..132416238474 100644 --- a/firestore/google/cloud/firestore_v1beta1/order.py +++ b/firestore/google/cloud/firestore_v1beta1/order.py @@ -18,51 +18,50 @@ class TypeOrder(Enum): - # NOTE: This order is defined by the backend and cannot be changed. - NULL = 0 - BOOLEAN = 1 - NUMBER = 2 - TIMESTAMP = 3 - STRING = 4 - BLOB = 5 - REF = 6 - GEO_POINT = 7 - ARRAY = 8 - OBJECT = 9 - - def from_value(value): - v = value.WhichOneof('value_type') - - lut = { - 'null_value': TypeOrder.NULL, - 'boolean_value': TypeOrder.BOOLEAN, - 'integer_value': TypeOrder.NUMBER, - 'double_value': TypeOrder.NUMBER, - 'timestamp_value': TypeOrder.TIMESTAMP, - 'string_value': TypeOrder.STRING, - 'bytes_value': TypeOrder.BLOB, - 'reference_value': TypeOrder.REF, - 'geo_point_value': TypeOrder.GEO_POINT, - 'array_value': TypeOrder.ARRAY, - 'map_value': TypeOrder.OBJECT, - } - - if v not in lut: - raise ArgumentException( - "Could not detect value type for " + value) - return lut[v] + # NOTE: This order is defined by the backend and cannot be changed. + NULL = 0 + BOOLEAN = 1 + NUMBER = 2 + TIMESTAMP = 3 + STRING = 4 + BLOB = 5 + REF = 6 + GEO_POINT = 7 + ARRAY = 8 + OBJECT = 9 + + def from_value(value): + v = value.WhichOneof('value_type') + + lut = { + 'null_value': TypeOrder.NULL, + 'boolean_value': TypeOrder.BOOLEAN, + 'integer_value': TypeOrder.NUMBER, + 'double_value': TypeOrder.NUMBER, + 'timestamp_value': TypeOrder.TIMESTAMP, + 'string_value': TypeOrder.STRING, + 'bytes_value': TypeOrder.BLOB, + 'reference_value': TypeOrder.REF, + 'geo_point_value': TypeOrder.GEO_POINT, + 'array_value': TypeOrder.ARRAY, + 'map_value': TypeOrder.OBJECT, + } + + if v not in lut: + raise ValueError( + "Could not detect value type for " + value) + return lut[v] class Order(object): ''' Order implements the ordering semantics of the backend. ''' - + @classmethod def compare(cls, left, right): ''' Main comparison function for all Firestore types. - @return -1 is left < right, 0 if left == right, otherwise 1 ''' # First compare the types. @@ -103,12 +102,11 @@ def compare(cls, left, right): else: raise ValueError('Unknown ``value_type``', value_type) - @staticmethod def compare_blobs(left, right): left_bytes = left.bytes_value right_bytes = right.bytes_value - + # TODO: Should verify bytes comparisons in python work as expected return Order._compare_to(left_bytes, right_bytes) @@ -120,9 +118,8 @@ def compare_timestamps(left, right): seconds = Order._compare_to(left.seconds or 0, right.seconds or 0) if seconds != 0: return seconds - - return Order._compare_to(left.nanos or 0, right.nanos or 0) + return Order._compare_to(left.nanos or 0, right.nanos or 0) @staticmethod def compare_geo_points(left, right): @@ -147,13 +144,11 @@ def compare_geo_points(left, right): cmp = 1 return cmp - @staticmethod def compare_resource_paths(left, right): left = left.reference_value right = right.reference_value - left_segments = left.split('/') right_segments = right.split('/') shorter = min(len(left_segments), len(right_segments)) @@ -161,7 +156,6 @@ def compare_resource_paths(left, right): for i in range(shorter): if (left_segments[i] < right_segments[i]): return -1 - if (left_segments[i] > right_segments[i]): return 1 @@ -174,20 +168,18 @@ def compare_resource_paths(left, right): return 0 - @staticmethod def compare_arrays(left, right): - l_values = left.array_value.values#.keys() - r_values = right.array_value.values#.keys() + l_values = left.array_value.values + r_values = right.array_value.values length = min(len(l_values), len(r_values)) for i in range(length): cmp = Order.compare(l_values[i], r_values[i]) if cmp != 0: return cmp - - return Order._compare_to(len(l_values), len(r_values)) + return Order._compare_to(len(l_values), len(r_values)) @staticmethod def compare_objects(left, right): @@ -204,7 +196,6 @@ def compare_objects(left, right): if value_compare != 0: return value_compare - return Order._compare_to(len(left_fields), len(right_fields)) @staticmethod diff --git a/firestore/google/cloud/firestore_v1beta1/query.py b/firestore/google/cloud/firestore_v1beta1/query.py index d3b1d89e65b0..85f9f7bbb264 100644 --- a/firestore/google/cloud/firestore_v1beta1/query.py +++ b/firestore/google/cloud/firestore_v1beta1/query.py @@ -29,8 +29,8 @@ from google.cloud.firestore_v1beta1 import document from google.cloud.firestore_v1beta1.gapic import enums from google.cloud.firestore_v1beta1.proto import query_pb2 +from google.cloud.firestore_v1beta1.order import Order from google.cloud.firestore_v1beta1.watch import Watch -from google.cloud.firestore_v1beta1.watch import Order _EQ_OP = '==' _COMPARISON_OPERATORS = { @@ -629,9 +629,9 @@ def on_snapshot(query_snapshot): query_watch.unsubscribe() """ return Watch.for_query(self, - callback, - document.DocumentSnapshot, - document.DocumentReference) + callback, + document.DocumentSnapshot, + document.DocumentReference) def _comparator(self, doc1, doc2): _orders = self._orders @@ -662,7 +662,7 @@ def _comparator(self, doc1, doc2): doc1.reference._path, doc2.reference._path) else: if orderBy.field.field_path not in doc1._data or \ - orderBy.field.field_path not in doc2._data: + orderBy.field.field_path not in doc2._data: raise Exception( "Can only compare fields that exist in the " "DocumentSnapshot. Please include the fields you are " diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index b492c8a26b54..01eebff57690 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -23,15 +23,10 @@ from google.cloud.firestore_v1beta1.bidi import ResumableBidiRpc from google.cloud.firestore_v1beta1.bidi import BackgroundConsumer -from google.cloud.firestore_v1beta1._helpers import encode_value from google.cloud.firestore_v1beta1.proto import firestore_pb2 -from google.cloud.firestore_v1beta1.order import Order from google.api_core import exceptions from google.protobuf import json_format - - -# from bidi import BidiRpc, ResumableBidiRpc import grpc """Python client for Google Cloud Firestore Watch.""" @@ -348,7 +343,7 @@ def for_query(cls, query, snapshot_callback, snapshot_class_instance, query_target = firestore_pb2.Target.QueryTarget( parent=query._client._database_string, structured_query=query._to_protobuf(), - ) + ) return cls(query, query._client, @@ -429,11 +424,6 @@ def on_snapshot(self, proto): target_change = proto.target_change if str(target_change): -<<<<<<< HEAD -======= - # XXX why if str - if it doesn't exist it will be empty (falsy). - # Otherwise this was always true. ->>>>>>> enable collection watches target_change_type = target_change.target_change_type _LOGGER.debug( 'on_snapshot: target change: ' + str(target_change_type)) @@ -575,7 +565,7 @@ def push(self, read_time, next_resume_token): if not self.has_pushed or len(appliedChanges): # TODO: the tree should be ordered. Sort here for now. - key = functools.cmp_to_key(self._comparator) + key = functools.cmp_to_key(self._comparator) keys = sorted(updated_tree.keys(), key=key) self._snapshot_callback( @@ -695,7 +685,6 @@ def modify_doc(new_document, updated_tree, updated_map): name, updated_tree, updated_map) appliedChanges.append(change) - add_changes = sorted(add_changes, key=key) _LOGGER.debug('walk over add_changes') for snapshot in add_changes: diff --git a/firestore/tests/system.py b/firestore/tests/system.py index a32182ca08df..9a3640149f84 100644 --- a/firestore/tests/system.py +++ b/firestore/tests/system.py @@ -858,12 +858,9 @@ def on_snapshot(docs, changes, read_time): on_snapshot.called_count = 0 - # def on_snapshot(docs, changes, read_time): - # for doc in docs: - # print(u'{} => {}'.format(doc.id, doc.to_dict())) - collection_ref.on_snapshot(on_snapshot) + # delay here so initial on_snapshot occurs and isn't combined with set sleep(1) doc_ref.set({ @@ -872,34 +869,16 @@ def on_snapshot(docs, changes, read_time): u'born': 1815 }) - sleep(1) - for _ in range(10): if on_snapshot.called_count == 1: break sleep(1) - # Alter document - doc_ref.set({ - u'first': u'Ada', - u'last': u'Lovelace', - u'born': 0 - }) - - for _ in range(10): - if on_snapshot.called_count == 2: - break - sleep(1) - if on_snapshot.called_count != 2: raise AssertionError( - "Failed to get exactly two document changes: count: " + + "Expected 2 snapshots, initial, and change: " + str(on_snapshot.called_count)) - # CM: had to stop here, this test is totally unfinished, trying to - # formalize - # https://gist.github.com/crwilcox/ce05f3857adc7a0ed86ffbd039b1a035 - def test_watch_query(client, cleanup): db = client @@ -954,16 +933,15 @@ def test_watch_query_order(client, cleanup): doc_ref2 = db.collection(u'users').document( u'asecondlovelace' + unique_id) doc_ref3 = db.collection(u'users').document( - u'athirdlovelace' + unique_id) + u'athirdlovelace' + unique_id) doc_ref4 = db.collection(u'users').document( u'afourthlovelace' + unique_id) doc_ref5 = db.collection(u'users').document( - u'afifthlovelace' + unique_id) + u'afifthlovelace' + unique_id) query_ref = db.collection(u'users').where( "first", "==", u'Ada' + unique_id).order_by("last") - # Setup listener def on_snapshot(docs, changes, read_time): try: @@ -984,7 +962,7 @@ def on_snapshot(docs, changes, read_time): on_snapshot.failed = e on_snapshot.called_count = 0 - on_snapshot.last_doc_count = 0 + on_snapshot.last_doc_count = 0 on_snapshot.failed = None query_ref.on_snapshot(on_snapshot) diff --git a/firestore/tests/unit/test_order.py b/firestore/tests/unit/test_order.py index 2a9e55765fe2..4c0aafd7923f 100644 --- a/firestore/tests/unit/test_order.py +++ b/firestore/tests/unit/test_order.py @@ -13,15 +13,15 @@ # limitations under the License. import unittest -import mock -from google.cloud.firestore_v1beta1._helpers import encode_value, GeoPoint -from google.protobuf import timestamp_pb2 -from google.type import latlng_pb2 import math -from google.cloud.firestore_v1beta1.document import DocumentReference + +from google.cloud.firestore_v1beta1._helpers import encode_value, GeoPoint from google.cloud.firestore_v1beta1.order import Order from google.cloud.firestore_v1beta1.proto import document_pb2 +from google.protobuf import timestamp_pb2 + + class TestOrder(unittest.TestCase): @staticmethod def _get_target_class(): @@ -34,7 +34,6 @@ def _make_one(self, *args, **kwargs): return klass(*args, **kwargs) def test_order(self): - # Constants used to represent min/max values of storage types. int_max_value = 2 ** 31 - 1 int_min_value = -(2 ** 31) @@ -135,9 +134,9 @@ def test_order(self): groups[59] = [_array_value(["foo", "0"])] # objects - groups[60] = [_object_value({"bar": 0})] + groups[60] = [_object_value({"bar": 0})] groups[61] = [_object_value({ - "bar":0, + "bar": 0, "foo": 1 })] groups[62] = [_object_value({"bar": 1})] @@ -150,15 +149,15 @@ def test_order(self): for left in groups[i]: for j in range(len(groups)): for right in groups[j]: - expected = Order._compare_to(i,j) - + expected = Order._compare_to(i, j) + self.assertEqual( target.compare(left, right), expected, "comparing L->R {} ({}) to {} ({})".format( i, left, j, right) ) - - expected = Order._compare_to(j, i); + + expected = Order._compare_to(j, i) self.assertEqual( target.compare(right, left), expected, "comparing R->L {} ({}) to {} ({})".format( @@ -201,7 +200,7 @@ def _timestamp_value(seconds, nanos): def _geoPoint_value(latitude, longitude): - return encode_value(GeoPoint(latitude,longitude)) + return encode_value(GeoPoint(latitude, longitude)) def _array_value(values=[]): diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index 8bc33bd7a67f..29ed28119efa 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -737,7 +737,7 @@ def __init__(self, **kw): if 'comparator' not in kw: # don't really do the comparison, just return 0 (equal) for all - self._comparator = lambda x,y: 1 + self._comparator = lambda x, y: 1 else: self._comparator = kw['comparator'] From f75a49ca2f78bce5805600ca6c97d7e1007fb31f Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Wed, 10 Oct 2018 15:33:27 -0700 Subject: [PATCH 130/148] coverage + tests --- .../cloud/firestore_v1beta1/_helpers.py | 2 +- .../google/cloud/firestore_v1beta1/order.py | 2 +- .../google/cloud/firestore_v1beta1/query.py | 2 +- firestore/tests/system.py | 11 ++-- firestore/tests/unit/test_order.py | 16 +++++ firestore/tests/unit/test_query.py | 62 +++++++++++++++++++ firestore/tests/unit/test_watch.py | 7 ++- 7 files changed, 92 insertions(+), 10 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/_helpers.py b/firestore/google/cloud/firestore_v1beta1/_helpers.py index 4e9f15b0ec25..1474e33b851e 100644 --- a/firestore/google/cloud/firestore_v1beta1/_helpers.py +++ b/firestore/google/cloud/firestore_v1beta1/_helpers.py @@ -745,7 +745,7 @@ def get_nested_value(field_path, data): nested_data = data for index, field_name in enumerate(field_names): - if isinstance(nested_data, collections.Mapping): + if isinstance(nested_data, collections.abc.Mapping): if field_name in nested_data: nested_data = nested_data[field_name] else: diff --git a/firestore/google/cloud/firestore_v1beta1/order.py b/firestore/google/cloud/firestore_v1beta1/order.py index 132416238474..b747e9bce7bc 100644 --- a/firestore/google/cloud/firestore_v1beta1/order.py +++ b/firestore/google/cloud/firestore_v1beta1/order.py @@ -100,7 +100,7 @@ def compare(cls, left, right): elif value_type == 'map_value': return cls.compare_objects(left, right) else: - raise ValueError('Unknown ``value_type``', value_type) + raise ValueError('Unknown ``value_type``', str(value_type)) @staticmethod def compare_blobs(left, right): diff --git a/firestore/google/cloud/firestore_v1beta1/query.py b/firestore/google/cloud/firestore_v1beta1/query.py index 85f9f7bbb264..2710e2e97026 100644 --- a/firestore/google/cloud/firestore_v1beta1/query.py +++ b/firestore/google/cloud/firestore_v1beta1/query.py @@ -663,7 +663,7 @@ def _comparator(self, doc1, doc2): else: if orderBy.field.field_path not in doc1._data or \ orderBy.field.field_path not in doc2._data: - raise Exception( + raise ValueError( "Can only compare fields that exist in the " "DocumentSnapshot. Please include the fields you are " "ordering on in your select() call." diff --git a/firestore/tests/system.py b/firestore/tests/system.py index 9a3640149f84..567604a9f1ad 100644 --- a/firestore/tests/system.py +++ b/firestore/tests/system.py @@ -855,8 +855,11 @@ def test_watch_collection(client, cleanup): # Setup listener def on_snapshot(docs, changes, read_time): on_snapshot.called_count += 1 + for doc in [doc for doc in docs if doc.id == doc_ref.id]: + on_snapshot.born = doc._data['born']['integerValue'] on_snapshot.called_count = 0 + on_snapshot.born = 0 collection_ref.on_snapshot(on_snapshot) @@ -870,14 +873,14 @@ def on_snapshot(docs, changes, read_time): }) for _ in range(10): - if on_snapshot.called_count == 1: + if on_snapshot.born == '1815': break sleep(1) - if on_snapshot.called_count != 2: + if on_snapshot.born != '1815': raise AssertionError( - "Expected 2 snapshots, initial, and change: " + - str(on_snapshot.called_count)) + "Expected the last document update to update born: " + + str(on_snapshot.born)) def test_watch_query(client, cleanup): diff --git a/firestore/tests/unit/test_order.py b/firestore/tests/unit/test_order.py index 4c0aafd7923f..b8490a8a4964 100644 --- a/firestore/tests/unit/test_order.py +++ b/firestore/tests/unit/test_order.py @@ -14,9 +14,13 @@ import unittest import math +import mock + from google.cloud.firestore_v1beta1._helpers import encode_value, GeoPoint from google.cloud.firestore_v1beta1.order import Order +from google.cloud.firestore_v1beta1.order import TypeOrder + from google.cloud.firestore_v1beta1.proto import document_pb2 from google.protobuf import timestamp_pb2 @@ -165,6 +169,18 @@ def test_order(self): ) + def test_failure_to_find_type(self): + target = self._make_one() + left = mock.Mock() + left.WhichOneOf.return_value = "imaginary-type" + right = mock.Mock() + # Patch from value to get to the deep compare. Since left is a bad type + # expect this to fail with value error. + with mock.patch.object(TypeOrder, 'from_value',) as to: + to.value = None + with self.assertRaisesRegex(ValueError, "'Unknown ``value_type``"): + target.compare(left, right) + def _boolean_value(b): return encode_value(b) diff --git a/firestore/tests/unit/test_query.py b/firestore/tests/unit/test_query.py index 4e4619841438..6cffafd8bc8f 100644 --- a/firestore/tests/unit/test_query.py +++ b/firestore/tests/unit/test_query.py @@ -866,6 +866,68 @@ def test_on_snapshot(self, watch): query.on_snapshot(None) watch.for_query.assert_called_once() + def test_comparator_no_ordering(self): + query = self._make_one(mock.sentinel.parent) + query._orders = [] + doc1 = mock.Mock() + doc1.reference._path = ('col', 'adocument1') + + doc2 = mock.Mock() + doc2.reference._path = ('col', 'adocument2') + + sort = query._comparator(doc1, doc2) + self.assertEqual(sort, -1) + + def test_comparator_no_ordering_same_id(self): + query = self._make_one(mock.sentinel.parent) + query._orders = [] + doc1 = mock.Mock() + doc1.reference._path = ('col', 'adocument1') + + doc2 = mock.Mock() + doc2.reference._path = ('col', 'adocument1') + + sort = query._comparator(doc1, doc2) + self.assertEqual(sort, 0) + + def test_comparator_ordering(self): + query = self._make_one(mock.sentinel.parent) + orderByMock = mock.Mock() + orderByMock.field.field_path = 'last' + orderByMock.direction = 1 # ascending + query._orders = [orderByMock] + + doc1 = mock.Mock() + doc1.reference._path = ('col', 'adocument1') + doc1._data = {'first': {'stringValue': 'Ada'}, + 'last': {'stringValue': 'secondlovelace'}} + doc2 = mock.Mock() + doc2.reference._path = ('col', 'adocument2') + doc2._data = {'first': {'stringValue': 'Ada'}, + 'last': {'stringValue': 'lovelace'}} + + sort = query._comparator(doc1, doc2) + self.assertEqual(sort, 1) + + def test_comparator_missing_order_by_field_in_data_raises(self): + query = self._make_one(mock.sentinel.parent) + orderByMock = mock.Mock() + orderByMock.field.field_path = 'last' + orderByMock.direction = 1 # ascending + query._orders = [orderByMock] + + doc1 = mock.Mock() + doc1.reference._path = ('col', 'adocument1') + doc1._data = {} + doc2 = mock.Mock() + doc2.reference._path = ('col', 'adocument2') + doc2._data = {'first': {'stringValue': 'Ada'}, + 'last': {'stringValue': 'lovelace'}} + + with self.assertRaisesRegex(ValueError, + "Can only compare fields "): + query._comparator(doc1, doc2) + class Test__enum_from_op_string(unittest.TestCase): diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index 29ed28119efa..ba5c306cbce8 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -394,14 +394,15 @@ def message_to_dict(document): proto.document_change.target_ids = [WATCH_TARGET_ID] class DummyDocument: - name = 'abc://foo/fred' + name = 'abc://foo/documents/fred' create_time = None update_time = None proto.document_change.document = DummyDocument() - inst._firestore._database_string = 'abc://foo/' + inst._firestore._database_string = 'abc://foo' inst.on_snapshot(proto) - self.assertEqual(inst.change_map['abc://foo/fred'].data, None) + self.assertEqual(inst.change_map['abc://foo/documents/fred'].data, + None) def test_on_snapshot_document_change_neither_changed_nor_removed(self): inst = self._makeOne() From 0125c8719fad161c3bb4a190b1851def74efcdc9 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Thu, 11 Oct 2018 10:39:02 -0700 Subject: [PATCH 131/148] Properly decode document snapshot data and fix tests --- firestore/google/cloud/firestore_v1beta1/watch.py | 8 ++++---- firestore/tests/system.py | 15 ++++++++++----- firestore/tests/unit/test_watch.py | 14 ++++---------- 3 files changed, 18 insertions(+), 19 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 01eebff57690..10c71cc24148 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -24,8 +24,9 @@ from google.cloud.firestore_v1beta1.bidi import ResumableBidiRpc from google.cloud.firestore_v1beta1.bidi import BackgroundConsumer from google.cloud.firestore_v1beta1.proto import firestore_pb2 +from google.cloud.firestore_v1beta1 import _helpers + from google.api_core import exceptions -from google.protobuf import json_format import grpc @@ -156,7 +157,6 @@ class Watch(object): BackgroundConsumer = BackgroundConsumer # FBO unit tests ResumableBidiRpc = ResumableBidiRpc # FBO unit tests - MessageToDict = staticmethod(json_format.MessageToDict) # FBO unit tests def __init__(self, document_reference, @@ -479,7 +479,7 @@ def on_snapshot(self, proto): # google.cloud.firestore_v1beta1.types.Document document = document_change.document - data = self.MessageToDict(document) + data = _helpers.decode_dict(document.fields, self._firestore) # Create a snapshot. As Document and Query objects can be # passed we need to get a Document Reference in a more manual @@ -494,7 +494,7 @@ def on_snapshot(self, proto): snapshot = self.DocumentSnapshot( reference=document_ref, - data=data['fields'], + data=data, exists=True, read_time=None, create_time=document.create_time, diff --git a/firestore/tests/system.py b/firestore/tests/system.py index 567604a9f1ad..12f69cc277ee 100644 --- a/firestore/tests/system.py +++ b/firestore/tests/system.py @@ -856,7 +856,7 @@ def test_watch_collection(client, cleanup): def on_snapshot(docs, changes, read_time): on_snapshot.called_count += 1 for doc in [doc for doc in docs if doc.id == doc_ref.id]: - on_snapshot.born = doc._data['born']['integerValue'] + on_snapshot.born = doc.get('born') on_snapshot.called_count = 0 on_snapshot.born = 0 @@ -873,11 +873,11 @@ def on_snapshot(docs, changes, read_time): }) for _ in range(10): - if on_snapshot.born == '1815': + if on_snapshot.born == 1815: break sleep(1) - if on_snapshot.born != '1815': + if on_snapshot.born != 1815: raise AssertionError( "Expected the last document update to update born: " + str(on_snapshot.born)) @@ -957,8 +957,10 @@ def on_snapshot(docs, changes, read_time): # compare the order things are returned for snapshot, query in zip(docs, query_ran_results): - assert snapshot.get('last')['stringValue'] == query.get( - 'last'), "expect the sort order to match" + assert snapshot.get('last') == query.get( + 'last'), "expect the sort order to match, last" + assert snapshot.get('born') == query.get( + 'born'), "expect the sort order to match, born" on_snapshot.called_count += 1 on_snapshot.last_doc_count = len(docs) except Exception as e: @@ -1000,6 +1002,9 @@ def on_snapshot(docs, changes, read_time): break sleep(1) + if on_snapshot.failed: + raise on_snapshot.failed + if on_snapshot.last_doc_count != 5: raise AssertionError( "5 docs expected in snapshot method " + diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index ba5c306cbce8..7c49b64c0076 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -361,22 +361,19 @@ def test_on_snapshot_document_change_changed(self): from google.cloud.firestore_v1beta1.watch import WATCH_TARGET_ID inst = self._makeOne() - def message_to_dict(document): - return {'fields': None} - - inst.MessageToDict = message_to_dict proto = DummyProto() proto.target_change = '' proto.document_change.target_ids = [WATCH_TARGET_ID] class DummyDocument: name = 'fred' + fields = {} create_time = None update_time = None proto.document_change.document = DummyDocument() inst.on_snapshot(proto) - self.assertEqual(inst.change_map['fred'].data, None) + self.assertEqual(inst.change_map['fred'].data, {}) def test_on_snapshot_document_change_changed_docname_db_prefix(self): # XXX This test asserts the current behavior, but I have no level @@ -385,16 +382,13 @@ def test_on_snapshot_document_change_changed_docname_db_prefix(self): from google.cloud.firestore_v1beta1.watch import WATCH_TARGET_ID inst = self._makeOne() - def message_to_dict(document): - return {'fields': None} - - inst.MessageToDict = message_to_dict proto = DummyProto() proto.target_change = '' proto.document_change.target_ids = [WATCH_TARGET_ID] class DummyDocument: name = 'abc://foo/documents/fred' + fields = {} create_time = None update_time = None @@ -402,7 +396,7 @@ class DummyDocument: inst._firestore._database_string = 'abc://foo' inst.on_snapshot(proto) self.assertEqual(inst.change_map['abc://foo/documents/fred'].data, - None) + {}) def test_on_snapshot_document_change_neither_changed_nor_removed(self): inst = self._makeOne() From cdc65942b9bf27a9d03a16d4986ecfa5dde14c06 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Thu, 11 Oct 2018 13:03:15 -0700 Subject: [PATCH 132/148] coverage --- .../google/cloud/firestore_v1beta1/Untitled-1 | 1 + .../google/cloud/firestore_v1beta1/order.py | 2 +- firestore/tests/unit/test_order.py | 17 ++++++++++++++++- firestore/tests/unit/test_query.py | 19 +++++++++++++++++++ 4 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 firestore/google/cloud/firestore_v1beta1/Untitled-1 diff --git a/firestore/google/cloud/firestore_v1beta1/Untitled-1 b/firestore/google/cloud/firestore_v1beta1/Untitled-1 new file mode 100644 index 000000000000..1af90026971e --- /dev/null +++ b/firestore/google/cloud/firestore_v1beta1/Untitled-1 @@ -0,0 +1 @@ +google/cloud/firestore_v1beta1/order.py 140 2 74 2 98% 51, 192, 50->51, 191->192 \ No newline at end of file diff --git a/firestore/google/cloud/firestore_v1beta1/order.py b/firestore/google/cloud/firestore_v1beta1/order.py index b747e9bce7bc..0c864114d05f 100644 --- a/firestore/google/cloud/firestore_v1beta1/order.py +++ b/firestore/google/cloud/firestore_v1beta1/order.py @@ -49,7 +49,7 @@ def from_value(value): if v not in lut: raise ValueError( - "Could not detect value type for " + value) + "Could not detect value type for " + v) return lut[v] diff --git a/firestore/tests/unit/test_order.py b/firestore/tests/unit/test_order.py index b8490a8a4964..c9ff1fd9186c 100644 --- a/firestore/tests/unit/test_order.py +++ b/firestore/tests/unit/test_order.py @@ -169,10 +169,18 @@ def test_order(self): ) + def test_typeorder_type_failure(self): + target = self._make_one() + left = mock.Mock() + left.WhichOneof.return_value = "imaginary-type" + + with self.assertRaisesRegex(ValueError, "Could not detect value"): + target.compare(left, mock.Mock()) + def test_failure_to_find_type(self): target = self._make_one() left = mock.Mock() - left.WhichOneOf.return_value = "imaginary-type" + left.WhichOneof.return_value = "imaginary-type" right = mock.Mock() # Patch from value to get to the deep compare. Since left is a bad type # expect this to fail with value error. @@ -181,6 +189,13 @@ def test_failure_to_find_type(self): with self.assertRaisesRegex(ValueError, "'Unknown ``value_type``"): target.compare(left, right) + def test_compare_objects_different_keys(self): + left = _object_value({"foo": 0}) + right = _object_value({"bar": 0}) + + target = self._make_one() + target.compare(left, right) + def _boolean_value(b): return encode_value(b) diff --git a/firestore/tests/unit/test_query.py b/firestore/tests/unit/test_query.py index 6cffafd8bc8f..16c3b4df2a5f 100644 --- a/firestore/tests/unit/test_query.py +++ b/firestore/tests/unit/test_query.py @@ -909,6 +909,25 @@ def test_comparator_ordering(self): sort = query._comparator(doc1, doc2) self.assertEqual(sort, 1) + def test_comparator_ordering_descending(self): + query = self._make_one(mock.sentinel.parent) + orderByMock = mock.Mock() + orderByMock.field.field_path = 'last' + orderByMock.direction = -1 # descending + query._orders = [orderByMock] + + doc1 = mock.Mock() + doc1.reference._path = ('col', 'adocument1') + doc1._data = {'first': {'stringValue': 'Ada'}, + 'last': {'stringValue': 'secondlovelace'}} + doc2 = mock.Mock() + doc2.reference._path = ('col', 'adocument2') + doc2._data = {'first': {'stringValue': 'Ada'}, + 'last': {'stringValue': 'lovelace'}} + + sort = query._comparator(doc1, doc2) + self.assertEqual(sort, -1) + def test_comparator_missing_order_by_field_in_data_raises(self): query = self._make_one(mock.sentinel.parent) orderByMock = mock.Mock() From 36e8772b38e8562591847c4d43086330eeda85e7 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Thu, 11 Oct 2018 13:43:50 -0700 Subject: [PATCH 133/148] noxfile from master --- firestore/noxfile.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/firestore/noxfile.py b/firestore/noxfile.py index 999e57ca3291..064f8044f182 100644 --- a/firestore/noxfile.py +++ b/firestore/noxfile.py @@ -56,9 +56,8 @@ def default(session): ) -@nox.session -@nox.parametrize('py', ['2.7', '3.5', '3.6', '3.7']) -def unit(session, py): +@nox.session(python=['2.7', '3.5', '3.6', '3.7']) +def unit(session): """Run the unit test suite.""" default(session) From 738d9382d78a51dd6c9f6a830838afa19d997b7e Mon Sep 17 00:00:00 2001 From: Christopher Wilcox Date: Thu, 11 Oct 2018 14:03:13 -0700 Subject: [PATCH 134/148] Delete Untitled-1 --- firestore/google/cloud/firestore_v1beta1/Untitled-1 | 1 - 1 file changed, 1 deletion(-) delete mode 100644 firestore/google/cloud/firestore_v1beta1/Untitled-1 diff --git a/firestore/google/cloud/firestore_v1beta1/Untitled-1 b/firestore/google/cloud/firestore_v1beta1/Untitled-1 deleted file mode 100644 index 1af90026971e..000000000000 --- a/firestore/google/cloud/firestore_v1beta1/Untitled-1 +++ /dev/null @@ -1 +0,0 @@ -google/cloud/firestore_v1beta1/order.py 140 2 74 2 98% 51, 192, 50->51, 191->192 \ No newline at end of file From 9c2ff84fdf186d988e9b2ba80a55c2224bb9a9a0 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Tue, 16 Oct 2018 12:24:13 -0700 Subject: [PATCH 135/148] modify to use bidi in api-core. one test has started to fail --- .../google/cloud/firestore_v1beta1/bidi.py | 559 --------------- .../google/cloud/firestore_v1beta1/watch.py | 4 +- firestore/tests/unit/test_bidi.py | 658 ------------------ 3 files changed, 2 insertions(+), 1219 deletions(-) delete mode 100644 firestore/google/cloud/firestore_v1beta1/bidi.py delete mode 100644 firestore/tests/unit/test_bidi.py diff --git a/firestore/google/cloud/firestore_v1beta1/bidi.py b/firestore/google/cloud/firestore_v1beta1/bidi.py deleted file mode 100644 index 53cfd7464c05..000000000000 --- a/firestore/google/cloud/firestore_v1beta1/bidi.py +++ /dev/null @@ -1,559 +0,0 @@ -# Copyright 2017, Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Bi-directional streaming RPC helpers.""" - -import logging -import threading - -from six.moves import queue - -from google.api_core import exceptions - -_LOGGER = logging.getLogger(__name__) -_BIDIRECTIONAL_CONSUMER_NAME = 'Thread-ConsumeBidirectionalStream' - - -class _RequestQueueGenerator(object): - """A helper for sending requests to a gRPC stream from a Queue. - - This generator takes requests off a given queue and yields them to gRPC. - - This helper is useful when you have an indeterminate, indefinite, or - otherwise open-ended set of requests to send through a request-streaming - (or bidirectional) RPC. - - The reason this is necessary is because gRPC takes an iterator as the - request for request-streaming RPCs. gRPC consumes this iterator in another - thread to allow it to block while generating requests for the stream. - However, if the generator blocks indefinitely gRPC will not be able to - clean up the thread as it'll be blocked on `next(iterator)` and not be able - to check the channel status to stop iterating. This helper mitigates that - by waiting on the queue with a timeout and checking the RPC state before - yielding. - - Finally, it allows for retrying without swapping queues because if it does - pull an item off the queue when the RPC is inactive, it'll immediately put - it back and then exit. This is necessary because yielding the item in this - case will cause gRPC to discard it. In practice, this means that the order - of messages is not guaranteed. If such a thing is necessary it would be - easy to use a priority queue. - - Example:: - - requests = request_queue_generator(q) - call = stub.StreamingRequest(iter(requests)) - requests.call = call - - for response in call: - print(response) - q.put(...) - - Note that it is possible to accomplish this behavior without "spinning" - (using a queue timeout). One possible way would be to use more threads to - multiplex the grpc end event with the queue, another possible way is to - use selectors and a custom event/queue object. Both of these approaches - are significant from an engineering perspective for small benefit - the - CPU consumed by spinning is pretty minuscule. - - Args: - queue (queue.Queue): The request queue. - period (float): The number of seconds to wait for items from the queue - before checking if the RPC is cancelled. In practice, this - determines the maximum amount of time the request consumption - thread will live after the RPC is cancelled. - initial_request (Union[protobuf.Message, - Callable[None, protobuf.Message]]): The initial request to - yield. This is done independently of the request queue to allow fo - easily restarting streams that require some initial configuration - request. - """ - def __init__(self, queue, period=1, initial_request=None): - self._queue = queue - self._period = period - self._initial_request = initial_request - self.call = None - - def _is_active(self): - # Note: there is a possibility that this starts *before* the call - # property is set. So we have to check if self.call is set before - # seeing if it's active. - if self.call is not None and not self.call.is_active(): - return False - else: - return True - - def __iter__(self): - if self._initial_request is not None: - if callable(self._initial_request): - yield self._initial_request() - else: - yield self._initial_request - - while True: - try: - item = self._queue.get(timeout=self._period) - except queue.Empty: - if not self._is_active(): - _LOGGER.debug( - 'Empty queue and inactive call, exiting request ' - 'generator.') - return - else: - # call is still active, keep waiting for queue items. - continue - - # The consumer explicitly sent "None", indicating that the request - # should end. - if item is None: - _LOGGER.debug('Cleanly exiting request generator.') - return - - if not self._is_active(): - # We have an item, but the call is closed. We should put the - # item back on the queue so that the next call can consume it. - self._queue.put(item) - _LOGGER.debug( - 'Inactive call, replacing item on queue and exiting ' - 'request generator.') - return - - yield item - - -class BidiRpc(object): - """A helper for consuming a bi-directional streaming RPC. - - This maps gRPC's built-in interface which uses a request iterator and a - response iterator into a socket-like :func:`send` and :func:`recv`. This - is a more useful pattern for long-running or asymmetric streams (streams - where there is not a direct correlation between the requests and - responses). - - Example:: - - initial_request = example_pb2.StreamingRpcRequest( - setting='example') - rpc = BidiRpc(stub.StreamingRpc, initial_request=initial_request) - - rpc.open() - - while rpc.is_active(): - print(rpc.recv()) - rpc.send(example_pb2.StreamingRpcRequest( - data='example')) - - This does *not* retry the stream on errors. See :class:`ResumableBidiRpc`. - - Args: - start_rpc (grpc.StreamStreamMultiCallable): The gRPC method used to - start the RPC. - initial_request (Union[protobuf.Message, - Callable[None, protobuf.Message]]): The initial request to - yield. This is useful if an initial request is needed to start the - stream. - """ - def __init__(self, start_rpc, initial_request=None): - self._start_rpc = start_rpc - self._initial_request = initial_request - self._request_queue = queue.Queue() - self._request_generator = None - self._is_active = False - self._callbacks = [] - self.call = None - - def add_done_callback(self, callback): - """Adds a callback that will be called when the RPC terminates. - - This occurs when the RPC errors or is successfully terminated. - - Args: - callback (Callable[[grpc.Future], None]): The callback to execute. - It will be provided with the same gRPC future as the underlying - stream which will also be a :class:`grpc.Call`. - """ - self._callbacks.append(callback) - - def _on_call_done(self, future): - for callback in self._callbacks: - callback(future) - - def open(self): - """Opens the stream.""" - if self.is_active: - raise ValueError('Can not open an already open stream.') - - request_generator = _RequestQueueGenerator( - self._request_queue, initial_request=self._initial_request) - call = self._start_rpc(iter(request_generator)) - - request_generator.call = call - - # TODO: api_core should expose the future interface for wrapped - # callables as well. - if hasattr(call, '_wrapped'): # pragma: NO COVER - call._wrapped.add_done_callback(self._on_call_done) - else: - call.add_done_callback(self._on_call_done) - - self._request_generator = request_generator - self.call = call - - def close(self): - """Closes the stream.""" - if self.call is None: - return - - self._request_queue.put(None) - self.call.cancel() - self._request_generator = None - # Don't set self.call to None. Keep it around so that send/recv can - # raise the error. - - def send(self, request): - """Queue a message to be sent on the stream. - - Send is non-blocking. - - If the underlying RPC has been closed, this will raise. - - Args: - request (protobuf.Message): The request to send. - """ - if self.call is None: - raise ValueError( - 'Can not send() on an RPC that has never been open()ed.') - - # Don't use self.is_active(), as ResumableBidiRpc will overload it - # to mean something semantically different. - if self.call.is_active(): - self._request_queue.put(request) - else: - # calling next should cause the call to raise. - next(self.call) - - def recv(self): - """Wait for a message to be returned from the stream. - - Recv is blocking. - - If the underlying RPC has been closed, this will raise. - - Returns: - protobuf.Message: The received message. - """ - if self.call is None: - raise ValueError( - 'Can not recv() on an RPC that has never been open()ed.') - - return next(self.call) - - @property - def is_active(self): - """bool: True if this stream is currently open and active.""" - return self.call is not None and self.call.is_active() - - @property - def pending_requests(self): - """int: Returns an estimate of the number of queued requests.""" - return self._request_queue.qsize() - - -class ResumableBidiRpc(BidiRpc): - """A :class:`BidiRpc` that can automatically resume the stream on errors. - - It uses the ``should_recover`` arg to determine if it should re-establish - the stream on error. - - Example:: - - def should_recover(exc): - return ( - isinstance(exc, grpc.RpcError) and - exc.code() == grpc.StatusCode.UNAVAILABLE) - - initial_request = example_pb2.StreamingRpcRequest( - setting='example') - - rpc = ResumeableBidiRpc( - stub.StreamingRpc, - initial_request=initial_request, - should_recover=should_recover) - - rpc.open() - - while rpc.is_active(): - print(rpc.recv()) - rpc.send(example_pb2.StreamingRpcRequest( - data='example')) - - Args: - start_rpc (grpc.StreamStreamMultiCallable): The gRPC method used to - start the RPC. - initial_request (Union[protobuf.Message, - Callable[None, protobuf.Message]]): The initial request to - yield. This is useful if an initial request is needed to start the - stream. - should_recover (Callable[[Exception], bool]): A function that returns - True if the stream should be recovered. This will be called - whenever an error is encountered on the stream. - """ - def __init__(self, start_rpc, should_recover, initial_request=None): - super(ResumableBidiRpc, self).__init__(start_rpc, initial_request) - self._should_recover = should_recover - self._operational_lock = threading.RLock() - self._finalized = False - self._finalize_lock = threading.Lock() - - def _finalize(self, result): - with self._finalize_lock: - if self._finalized: - return - - for callback in self._callbacks: - callback(result) - - self._finalized = True - - def _on_call_done(self, future): - # Unlike the base class, we only execute the callbacks on a terminal - # error, not for errors that we can recover from. Note that grpc's - # "future" here is also a grpc.RpcError. - if not self._should_recover(future): - self._finalize(future) - else: - _LOGGER.debug('Re-opening stream from gRPC callback.') - self._reopen() - - def _reopen(self): - with self._operational_lock: - # Another thread already managed to re-open this stream. - if self.call is not None and self.call.is_active(): - _LOGGER.debug('Stream was already re-established.') - return - - self.call = None - # Request generator should exit cleanly since the RPC its bound to - # has exited. - self.request_generator = None - - # Note: we do not currently do any sort of backoff here. The - # assumption is that re-establishing the stream under normal - # circumstances will happen in intervals greater than 60s. - # However, it is possible in a degenerative case that the server - # closes the stream rapidly which would lead to thrashing here, - # but hopefully in those cases the server would return a non- - # retryable error. - - try: - self.open() - # If re-opening or re-calling the method fails for any reason, - # consider it a terminal error and finalize the stream. - except Exception as exc: - self._finalize(exc) - raise - - _LOGGER.info('Re-established stream') - - def _recoverable(self, method, *args, **kwargs): - """Wraps a method to recover the stream and retry on error. - - If a retryable error occurs while making the call, then the stream will - be re-opened and the method will be retried. This happens indefinitely - so long as the error is a retryable one. If an error occurs while - re-opening the stream, then this method will raise immediately and - trigger finalization of this object. - - Args: - method (Callable[..., Any]): The method to call. - args: The args to pass to the method. - kwargs: The kwargs to pass to the method. - """ - while True: - try: - return method(*args, **kwargs) - - except Exception as exc: - _LOGGER.debug('Call to retryable %r caused %s.', method, exc) - if not self._should_recover(exc): - self.close() - _LOGGER.debug('Not retrying %r due to %s.', method, exc) - self._finalize(exc) - raise exc - - _LOGGER.debug('Re-opening stream from retryable %r.', method) - self._reopen() - - def send(self, request): - return self._recoverable( - super(ResumableBidiRpc, self).send, request) - - def recv(self): - return self._recoverable( - super(ResumableBidiRpc, self).recv) - - @property - def is_active(self): - """bool: True if this stream is currently open and active.""" - # Use the operational lock. It's entirely possible for something - # to check the active state *while* the RPC is being retried. - # Also, use finalized to track the actual terminal state here. - # This is because if the stream is re-established by the gRPC thread - # it's technically possible to check this between when gRPC marks the - # RPC as inactive and when gRPC executes our callback that re-opens - # the stream. - with self._operational_lock: - return self.call is not None and not self._finalized - - -class BackgroundConsumer(object): - """A bi-directional stream consumer that runs in a separate thread. - - This maps the consumption of a stream into a callback-based model. It also - provides :func:`pause` and :func:`resume` to allow for flow-control. - - Example:: - - def should_recover(exc): - return ( - isinstance(exc, grpc.RpcError) and - exc.code() == grpc.StatusCode.UNAVAILABLE) - - initial_request = example_pb2.StreamingRpcRequest( - setting='example') - - rpc = ResumeableBidiRpc( - stub.StreamingRpc, - initial_request=initial_request, - should_recover=should_recover) - - def on_response(response): - print(response) - - consumer = BackgroundConsumer(rpc, on_response) - consume.start() - - Note that error handling *must* be done by using the provided - ``bidi_rpc``'s ``add_done_callback``. This helper will automatically exit - whenever the RPC itself exits and will not provide any error details. - - Args: - bidi_rpc (BidiRpc): The RPC to consume. Should not have been - ``open()``ed yet. - on_response (Callable[[protobuf.Message], None]): The callback to - be called for every response on the stream. - """ - def __init__(self, bidi_rpc, on_response): - self._bidi_rpc = bidi_rpc - self._on_response = on_response - self._paused = False - self._wake = threading.Condition() - self._thread = None - self._operational_lock = threading.Lock() - - def _on_call_done(self, future): - # Resume the thread if it's paused, this prevents blocking forever - # when the RPC has terminated. - self.resume() - - def _thread_main(self): - try: - self._bidi_rpc.add_done_callback(self._on_call_done) - self._bidi_rpc.open() - - while self._bidi_rpc.is_active: - # Do not allow the paused status to change at all during this - # section. There is a condition where we could be resumed - # between checking if we are paused and calling wake.wait(), - # which means that we will miss the notification to wake up - # (oops!) and wait for a notification that will never come. - # Keeping the lock throughout avoids that. - # In the future, we could use `Condition.wait_for` if we drop - # Python 2.7. - with self._wake: - if self._paused: - _LOGGER.debug('paused, waiting for waking.') - self._wake.wait() - _LOGGER.debug('woken.') - - _LOGGER.debug('waiting for recv.') - response = self._bidi_rpc.recv() - _LOGGER.debug('recved response.') - self._on_response(response) - - except exceptions.GoogleAPICallError as exc: - _LOGGER.debug( - '%s caught error %s and will exit. Generally this is due to ' - 'the RPC itself being cancelled and the error will be ' - 'surfaced to the calling code.', - _BIDIRECTIONAL_CONSUMER_NAME, exc, exc_info=True) - - except Exception as exc: - _LOGGER.exception( - '%s caught unexpected exception %s and will exit.', - _BIDIRECTIONAL_CONSUMER_NAME, exc) - - else: - _LOGGER.error( - 'The bidirectional RPC unexpectedly exited. This is a truly ' - 'exceptional case. Please file a bug with your logs.') - - _LOGGER.info('%s exiting', _BIDIRECTIONAL_CONSUMER_NAME) - - def start(self): - """Start the background thread and begin consuming the thread.""" - with self._operational_lock: - thread = threading.Thread( - name=_BIDIRECTIONAL_CONSUMER_NAME, - target=self._thread_main) - thread.daemon = True - thread.start() - self._thread = thread - _LOGGER.debug('Started helper thread %s', thread.name) - - def stop(self): - """Stop consuming the stream and shutdown the background thread.""" - with self._operational_lock: - self._bidi_rpc.close() - - if self._thread is not None: - # Resume the thread to wake it up in case it is sleeping. - self.resume() - self._thread.join() - - self._thread = None - - @property - def is_active(self): - """bool: True if the background thread is active.""" - return self._thread is not None and self._thread.is_alive() - - def pause(self): - """Pauses the response stream. - - This does *not* pause the request stream. - """ - with self._wake: - self._paused = True - - def resume(self): - """Resumes the response stream.""" - with self._wake: - self._paused = False - self._wake.notifyAll() - - @property - def is_paused(self): - """bool: True if the response stream is paused.""" - return self._paused diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 10c71cc24148..b706dfc4c298 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -21,8 +21,8 @@ import pytz -from google.cloud.firestore_v1beta1.bidi import ResumableBidiRpc -from google.cloud.firestore_v1beta1.bidi import BackgroundConsumer +from google.api_core.bidi import ResumableBidiRpc +from google.api_core.bidi import BackgroundConsumer from google.cloud.firestore_v1beta1.proto import firestore_pb2 from google.cloud.firestore_v1beta1 import _helpers diff --git a/firestore/tests/unit/test_bidi.py b/firestore/tests/unit/test_bidi.py deleted file mode 100644 index 80d8ecf48389..000000000000 --- a/firestore/tests/unit/test_bidi.py +++ /dev/null @@ -1,658 +0,0 @@ -# Copyright 2018, Google LLC All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import threading - -import grpc -import mock -import pytest -from six.moves import queue - -from google.api_core import exceptions -from google.cloud.firestore_v1beta1 import bidi - - -class Test_RequestQueueGenerator(object): - - def test_bounded_consume(self): - call = mock.create_autospec(grpc.Call, instance=True) - call.is_active.return_value = True - - def queue_generator(rpc): - yield mock.sentinel.A - yield queue.Empty() - yield mock.sentinel.B - rpc.is_active.return_value = False - yield mock.sentinel.C - - q = mock.create_autospec(queue.Queue, instance=True) - q.get.side_effect = queue_generator(call) - - generator = bidi._RequestQueueGenerator(q) - generator.call = call - - items = list(generator) - - assert items == [mock.sentinel.A, mock.sentinel.B] - - def test_yield_initial_and_exit(self): - q = mock.create_autospec(queue.Queue, instance=True) - q.get.side_effect = queue.Empty() - call = mock.create_autospec(grpc.Call, instance=True) - call.is_active.return_value = False - - generator = bidi._RequestQueueGenerator( - q, initial_request=mock.sentinel.A) - generator.call = call - - items = list(generator) - - assert items == [mock.sentinel.A] - - def test_yield_initial_callable_and_exit(self): - q = mock.create_autospec(queue.Queue, instance=True) - q.get.side_effect = queue.Empty() - call = mock.create_autospec(grpc.Call, instance=True) - call.is_active.return_value = False - - generator = bidi._RequestQueueGenerator( - q, initial_request=lambda: mock.sentinel.A) - generator.call = call - - items = list(generator) - - assert items == [mock.sentinel.A] - - def test_exit_when_inactive_with_item(self): - q = mock.create_autospec(queue.Queue, instance=True) - q.get.side_effect = [mock.sentinel.A, queue.Empty()] - call = mock.create_autospec(grpc.Call, instance=True) - call.is_active.return_value = False - - generator = bidi._RequestQueueGenerator(q) - generator.call = call - - items = list(generator) - - assert items == [] - # Make sure it put the item back. - q.put.assert_called_once_with(mock.sentinel.A) - - def test_exit_when_inactive_empty(self): - q = mock.create_autospec(queue.Queue, instance=True) - q.get.side_effect = queue.Empty() - call = mock.create_autospec(grpc.Call, instance=True) - call.is_active.return_value = False - - generator = bidi._RequestQueueGenerator(q) - generator.call = call - - items = list(generator) - - assert items == [] - - def test_exit_with_stop(self): - q = mock.create_autospec(queue.Queue, instance=True) - q.get.side_effect = [None, queue.Empty()] - call = mock.create_autospec(grpc.Call, instance=True) - call.is_active.return_value = True - - generator = bidi._RequestQueueGenerator(q) - generator.call = call - - items = list(generator) - - assert items == [] - - -class _CallAndFuture(grpc.Call, grpc.Future): - pass - - -def make_rpc(): - """Makes a mock RPC used to test Bidi classes.""" - call = mock.create_autospec(_CallAndFuture, instance=True) - rpc = mock.create_autospec(grpc.StreamStreamMultiCallable, instance=True) - - def rpc_side_effect(request): - call.is_active.return_value = True - call.request = request - return call - - rpc.side_effect = rpc_side_effect - - def cancel_side_effect(): - call.is_active.return_value = False - - call.cancel.side_effect = cancel_side_effect - - return rpc, call - - -class ClosedCall(object): - # NOTE: This is needed because defining `.next` on an **instance** - # rather than the **class** will not be iterable in Python 2. - # This is problematic since a `Mock` just sets members. - - def __init__(self, exception): - self.exception = exception - - def __next__(self): - raise self.exception - - next = __next__ # Python 2 - - def is_active(self): - return False - - -class TestBidiRpc(object): - def test_initial_state(self): - bidi_rpc = bidi.BidiRpc(None) - - assert bidi_rpc.is_active is False - - def test_done_callbacks(self): - bidi_rpc = bidi.BidiRpc(None) - callback = mock.Mock(spec=['__call__']) - - bidi_rpc.add_done_callback(callback) - bidi_rpc._on_call_done(mock.sentinel.future) - - callback.assert_called_once_with(mock.sentinel.future) - - def test_open(self): - rpc, call = make_rpc() - bidi_rpc = bidi.BidiRpc(rpc) - - bidi_rpc.open() - - assert bidi_rpc.call == call - assert bidi_rpc.is_active - call.add_done_callback.assert_called_once_with(bidi_rpc._on_call_done) - - def test_open_error_already_open(self): - rpc, _ = make_rpc() - bidi_rpc = bidi.BidiRpc(rpc) - - bidi_rpc.open() - - with pytest.raises(ValueError): - bidi_rpc.open() - - def test_close(self): - rpc, call = make_rpc() - bidi_rpc = bidi.BidiRpc(rpc) - bidi_rpc.open() - - bidi_rpc.close() - - call.cancel.assert_called_once() - assert bidi_rpc.call == call - assert bidi_rpc.is_active is False - # ensure the request queue was signaled to stop. - assert bidi_rpc.pending_requests == 1 - assert bidi_rpc._request_queue.get() is None - - def test_close_no_rpc(self): - bidi_rpc = bidi.BidiRpc(None) - bidi_rpc.close() - - def test_send(self): - rpc, call = make_rpc() - bidi_rpc = bidi.BidiRpc(rpc) - bidi_rpc.open() - - bidi_rpc.send(mock.sentinel.request) - - assert bidi_rpc.pending_requests == 1 - assert bidi_rpc._request_queue.get() is mock.sentinel.request - - def test_send_not_open(self): - rpc, call = make_rpc() - bidi_rpc = bidi.BidiRpc(rpc) - - with pytest.raises(ValueError): - bidi_rpc.send(mock.sentinel.request) - - def test_send_dead_rpc(self): - error = ValueError() - bidi_rpc = bidi.BidiRpc(None) - bidi_rpc.call = ClosedCall(error) - - with pytest.raises(ValueError) as exc_info: - bidi_rpc.send(mock.sentinel.request) - - assert exc_info.value == error - - def test_recv(self): - bidi_rpc = bidi.BidiRpc(None) - bidi_rpc.call = iter([mock.sentinel.response]) - - response = bidi_rpc.recv() - - assert response == mock.sentinel.response - - def test_recv_not_open(self): - rpc, call = make_rpc() - bidi_rpc = bidi.BidiRpc(rpc) - - with pytest.raises(ValueError): - bidi_rpc.recv() - - -class CallStub(object): - def __init__(self, values, active=True): - self.values = iter(values) - self._is_active = active - self.cancelled = False - - def __next__(self): - item = next(self.values) - if isinstance(item, Exception): - self._is_active = False - raise item - return item - - next = __next__ # Python 2 - - def is_active(self): - return self._is_active - - def add_done_callback(self, callback): - pass - - def cancel(self): - self.cancelled = True - - -class TestResumableBidiRpc(object): - def test_initial_state(self): # pragma: NO COVER - bidi_rpc = bidi.ResumableBidiRpc(None, lambda _: True) - - assert bidi_rpc.is_active is False - - def test_done_callbacks_recoverable(self): - start_rpc = mock.create_autospec( - grpc.StreamStreamMultiCallable, instance=True) - bidi_rpc = bidi.ResumableBidiRpc(start_rpc, lambda _: True) - callback = mock.Mock(spec=['__call__']) - - bidi_rpc.add_done_callback(callback) - bidi_rpc._on_call_done(mock.sentinel.future) - - callback.assert_not_called() - start_rpc.assert_called_once() - assert bidi_rpc.is_active - - def test_done_callbacks_non_recoverable(self): - bidi_rpc = bidi.ResumableBidiRpc(None, lambda _: False) - callback = mock.Mock(spec=['__call__']) - - bidi_rpc.add_done_callback(callback) - bidi_rpc._on_call_done(mock.sentinel.future) - - callback.assert_called_once_with(mock.sentinel.future) - - def test_send_recover(self): - error = ValueError() - call_1 = CallStub([error], active=False) - call_2 = CallStub([]) - start_rpc = mock.create_autospec( - grpc.StreamStreamMultiCallable, - instance=True, - side_effect=[call_1, call_2]) - should_recover = mock.Mock(spec=['__call__'], return_value=True) - bidi_rpc = bidi.ResumableBidiRpc(start_rpc, should_recover) - - bidi_rpc.open() - - bidi_rpc.send(mock.sentinel.request) - - assert bidi_rpc.pending_requests == 1 - assert bidi_rpc._request_queue.get() is mock.sentinel.request - - should_recover.assert_called_once_with(error) - assert bidi_rpc.call == call_2 - assert bidi_rpc.is_active is True - - def test_send_failure(self): - error = ValueError() - call = CallStub([error], active=False) - start_rpc = mock.create_autospec( - grpc.StreamStreamMultiCallable, - instance=True, - return_value=call) - should_recover = mock.Mock(spec=['__call__'], return_value=False) - bidi_rpc = bidi.ResumableBidiRpc(start_rpc, should_recover) - - bidi_rpc.open() - - with pytest.raises(ValueError) as exc_info: - bidi_rpc.send(mock.sentinel.request) - - assert exc_info.value == error - should_recover.assert_called_once_with(error) - assert bidi_rpc.call == call - assert bidi_rpc.is_active is False - assert call.cancelled is True - assert bidi_rpc.pending_requests == 1 - assert bidi_rpc._request_queue.get() is None - - def test_recv_recover(self): - error = ValueError() - call_1 = CallStub([1, error]) - call_2 = CallStub([2, 3]) - start_rpc = mock.create_autospec( - grpc.StreamStreamMultiCallable, - instance=True, - side_effect=[call_1, call_2]) - should_recover = mock.Mock(spec=['__call__'], return_value=True) - bidi_rpc = bidi.ResumableBidiRpc(start_rpc, should_recover) - - bidi_rpc.open() - - values = [] - for n in range(3): - values.append(bidi_rpc.recv()) - - assert values == [1, 2, 3] - should_recover.assert_called_once_with(error) - assert bidi_rpc.call == call_2 - assert bidi_rpc.is_active is True - - def test_recv_recover_race_condition(self): - # This test checks the race condition where two threads recv() and - # encounter an error and must re-open the stream. Only one thread - # should succeed in doing so. - error = ValueError() - call_1 = CallStub([error, error]) - call_2 = CallStub([1, 2]) - start_rpc = mock.create_autospec( - grpc.StreamStreamMultiCallable, - instance=True, - side_effect=[call_1, call_2]) - recovered_event = threading.Event() - - def second_thread_main(): - assert bidi_rpc.recv() == 2 - - second_thread = threading.Thread(target=second_thread_main) - - def should_recover(exception): - assert exception == error - if threading.current_thread() == second_thread: - recovered_event.wait() - return True - - bidi_rpc = bidi.ResumableBidiRpc(start_rpc, should_recover) - - bidi_rpc.open() - second_thread.start() - - assert bidi_rpc.recv() == 1 - recovered_event.set() - - assert bidi_rpc.call == call_2 - assert bidi_rpc.is_active is True - second_thread.join() - - def test_recv_failure(self): - error = ValueError() - call = CallStub([error]) - start_rpc = mock.create_autospec( - grpc.StreamStreamMultiCallable, - instance=True, - return_value=call) - should_recover = mock.Mock(spec=['__call__'], return_value=False) - bidi_rpc = bidi.ResumableBidiRpc(start_rpc, should_recover) - - bidi_rpc.open() - - with pytest.raises(ValueError) as exc_info: - bidi_rpc.recv() - - assert exc_info.value == error - should_recover.assert_called_once_with(error) - assert bidi_rpc.call == call - assert bidi_rpc.is_active is False - assert call.cancelled is True - - def test_reopen_failure_on_rpc_restart(self): - error1 = ValueError('1') - error2 = ValueError('2') - call = CallStub([error1]) - # Invoking start RPC a second time will trigger an error. - start_rpc = mock.create_autospec( - grpc.StreamStreamMultiCallable, - instance=True, - side_effect=[call, error2]) - should_recover = mock.Mock(spec=['__call__'], return_value=True) - callback = mock.Mock(spec=['__call__']) - - bidi_rpc = bidi.ResumableBidiRpc(start_rpc, should_recover) - bidi_rpc.add_done_callback(callback) - - bidi_rpc.open() - - with pytest.raises(ValueError) as exc_info: - bidi_rpc.recv() - - assert exc_info.value == error2 - should_recover.assert_called_once_with(error1) - assert bidi_rpc.call is None - assert bidi_rpc.is_active is False - callback.assert_called_once_with(error2) - - def test_finalize_idempotent(self): - error1 = ValueError('1') - error2 = ValueError('2') - callback = mock.Mock(spec=['__call__']) - should_recover = mock.Mock(spec=['__call__'], return_value=False) - - bidi_rpc = bidi.ResumableBidiRpc( - mock.sentinel.start_rpc, should_recover) - - bidi_rpc.add_done_callback(callback) - - bidi_rpc._on_call_done(error1) - bidi_rpc._on_call_done(error2) - - callback.assert_called_once_with(error1) - - -class TestBackgroundConsumer(object): - def test_consume_once_then_exit(self): - bidi_rpc = mock.create_autospec(bidi.BidiRpc, instance=True) - bidi_rpc.is_active = True - bidi_rpc.recv.side_effect = [mock.sentinel.response_1] - recved = threading.Event() - - def on_response(response): - assert response == mock.sentinel.response_1 - bidi_rpc.is_active = False - recved.set() - - consumer = bidi.BackgroundConsumer(bidi_rpc, on_response) - - consumer.start() - - recved.wait() - - bidi_rpc.recv.assert_called_once() - assert bidi_rpc.is_active is False - - consumer.stop() - - bidi_rpc.close.assert_called_once() - assert consumer.is_active is False - - def test_pause_resume_and_close(self): - # This test is relatively complex. It attempts to start the consumer, - # consume one item, pause the consumer, check the state of the world, - # then resume the consumer. Doing this in a deterministic fashion - # requires a bit more mocking and patching than usual. - - bidi_rpc = mock.create_autospec(bidi.BidiRpc, instance=True) - bidi_rpc.is_active = True - - def close_side_effect(): - bidi_rpc.is_active = False - - bidi_rpc.close.side_effect = close_side_effect - - # These are used to coordinate the two threads to ensure deterministic - # execution. - should_continue = threading.Event() - responses_and_events = { - mock.sentinel.response_1: threading.Event(), - mock.sentinel.response_2: threading.Event() - } - bidi_rpc.recv.side_effect = [ - mock.sentinel.response_1, mock.sentinel.response_2] - - recved_responses = [] - consumer = None - - def on_response(response): - if response == mock.sentinel.response_1: - consumer.pause() - - recved_responses.append(response) - responses_and_events[response].set() - should_continue.wait() - - consumer = bidi.BackgroundConsumer(bidi_rpc, on_response) - - consumer.start() - - # Wait for the first response to be recved. - responses_and_events[mock.sentinel.response_1].wait() - - # Ensure only one item has been recved and that the consumer is paused. - assert recved_responses == [mock.sentinel.response_1] - assert consumer.is_paused is True - assert consumer.is_active is True - - # Unpause the consumer, wait for the second item, then close the - # consumer. - should_continue.set() - consumer.resume() - - responses_and_events[mock.sentinel.response_2].wait() - - assert recved_responses == [ - mock.sentinel.response_1, mock.sentinel.response_2] - - consumer.stop() - - assert consumer.is_active is False - - def test_wake_on_error(self): - should_continue = threading.Event() - - bidi_rpc = mock.create_autospec(bidi.BidiRpc, instance=True) - bidi_rpc.is_active = True - bidi_rpc.add_done_callback.side_effect = ( - lambda _: should_continue.set()) - - consumer = bidi.BackgroundConsumer(bidi_rpc, mock.sentinel.on_response) - - # Start the consumer paused, which should immediately put it into wait - # state. - consumer.pause() - consumer.start() - - # Wait for add_done_callback to be called - should_continue.wait() - bidi_rpc.add_done_callback.assert_called_once_with( - consumer._on_call_done) - - # The consumer should now be blocked on waiting to be unpaused. - assert consumer.is_active - assert consumer.is_paused - - # Trigger the done callback, it should unpause the consumer and cause - # it to exit. - bidi_rpc.is_active = False - consumer._on_call_done(bidi_rpc) - - # It may take a few cycles for the thread to exit. - while consumer.is_active: - pass - - def test_consumer_expected_error(self, caplog): - caplog.set_level(logging.DEBUG) - - bidi_rpc = mock.create_autospec(bidi.BidiRpc, instance=True) - bidi_rpc.is_active = True - bidi_rpc.recv.side_effect = exceptions.ServiceUnavailable('Gone away') - - on_response = mock.Mock(spec=['__call__']) - - consumer = bidi.BackgroundConsumer(bidi_rpc, on_response) - - consumer.start() - - # Wait for the consumer's thread to exit. - while consumer.is_active: - pass - - on_response.assert_not_called() - bidi_rpc.recv.assert_called_once() - assert 'caught error' in caplog.text - - def test_consumer_unexpected_error(self, caplog): - caplog.set_level(logging.DEBUG) - - bidi_rpc = mock.create_autospec(bidi.BidiRpc, instance=True) - bidi_rpc.is_active = True - bidi_rpc.recv.side_effect = ValueError() - - on_response = mock.Mock(spec=['__call__']) - - consumer = bidi.BackgroundConsumer(bidi_rpc, on_response) - - consumer.start() - - # Wait for the consumer's thread to exit. - while consumer.is_active: - pass - - on_response.assert_not_called() - bidi_rpc.recv.assert_called_once() - assert 'caught unexpected exception' in caplog.text - - def test_double_stop(self, caplog): - caplog.set_level(logging.DEBUG) - bidi_rpc = mock.create_autospec(bidi.BidiRpc, instance=True) - bidi_rpc.is_active = True - on_response = mock.Mock(spec=['__call__']) - - def close_side_effect(): - bidi_rpc.is_active = False - - bidi_rpc.close.side_effect = close_side_effect - - consumer = bidi.BackgroundConsumer(bidi_rpc, on_response) - - consumer.start() - assert consumer.is_active is True - - consumer.stop() - assert consumer.is_active is False - - # calling stop twice should not result in an error. - consumer.stop() From 3f8e0c8380a3978fd72b2dd7668151221c6267fa Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Thu, 18 Oct 2018 03:10:38 -0400 Subject: [PATCH 136/148] get tests passing (intermittently) on 2.7,3.5,3.6 --- .../cloud/firestore_v1beta1/_helpers.py | 8 +++- .../google/cloud/firestore_v1beta1/order.py | 1 + .../google/cloud/firestore_v1beta1/watch.py | 2 +- firestore/tests/unit/test_order.py | 37 +++++++++++-------- firestore/tests/unit/test_query.py | 4 +- 5 files changed, 32 insertions(+), 20 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/_helpers.py b/firestore/google/cloud/firestore_v1beta1/_helpers.py index 1474e33b851e..720e0111abd6 100644 --- a/firestore/google/cloud/firestore_v1beta1/_helpers.py +++ b/firestore/google/cloud/firestore_v1beta1/_helpers.py @@ -15,7 +15,11 @@ """Common helpers shared across Google Cloud Firestore modules.""" -import collections +try: + from collections import abc +except ImportError: # python 2.7 + import collections as abc + import datetime import re @@ -745,7 +749,7 @@ def get_nested_value(field_path, data): nested_data = data for index, field_name in enumerate(field_names): - if isinstance(nested_data, collections.abc.Mapping): + if isinstance(nested_data, abc.Mapping): if field_name in nested_data: nested_data = nested_data[field_name] else: diff --git a/firestore/google/cloud/firestore_v1beta1/order.py b/firestore/google/cloud/firestore_v1beta1/order.py index 0c864114d05f..81a3ecc9b0ad 100644 --- a/firestore/google/cloud/firestore_v1beta1/order.py +++ b/firestore/google/cloud/firestore_v1beta1/order.py @@ -30,6 +30,7 @@ class TypeOrder(Enum): ARRAY = 8 OBJECT = 9 + @staticmethod def from_value(value): v = value.WhichOneof('value_type') diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index b706dfc4c298..8e63808409f7 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -22,7 +22,7 @@ import pytz from google.api_core.bidi import ResumableBidiRpc -from google.api_core.bidi import BackgroundConsumer +from google.api_core.bidi import BackgroundConsumer from google.cloud.firestore_v1beta1.proto import firestore_pb2 from google.cloud.firestore_v1beta1 import _helpers diff --git a/firestore/tests/unit/test_order.py b/firestore/tests/unit/test_order.py index c9ff1fd9186c..af9488c7bcf0 100644 --- a/firestore/tests/unit/test_order.py +++ b/firestore/tests/unit/test_order.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # Copyright 2017 Google LLC All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,10 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import unittest -import math +import collections import mock - +import six +import sys +import unittest from google.cloud.firestore_v1beta1._helpers import encode_value, GeoPoint from google.cloud.firestore_v1beta1.order import Order @@ -43,6 +45,7 @@ def test_order(self): int_min_value = -(2 ** 31) float_min_value = 1.175494351 ** -38 float_nan = float('nan') + inf = float('inf') groups = [None] * 65 @@ -53,7 +56,7 @@ def test_order(self): # numbers groups[3] = [_double_value(float_nan), _double_value(float_nan)] - groups[4] = [_double_value(-math.inf)] + groups[4] = [_double_value(-inf)] groups[5] = [_int_value(int_min_value - 1)] groups[6] = [_int_value(int_min_value)] groups[7] = [_double_value(-1.1)] @@ -68,7 +71,7 @@ def test_order(self): groups[13] = [_double_value(1.1)] groups[14] = [_int_value(int_max_value)] groups[15] = [_int_value(int_max_value + 1)] - groups[16] = [_double_value(math.inf)] + groups[16] = [_double_value(inf)] groups[17] = [_timestamp_value(123, 0)] groups[18] = [_timestamp_value(123, 123)] @@ -77,21 +80,21 @@ def test_order(self): # strings groups[20] = [_string_value("")] groups[21] = [_string_value("\u0000\ud7ff\ue000\uffff")] - groups[22] = [_string_value(u"(╯°□°）╯︵ ┻━┻")] + groups[22] = [_string_value("(╯°□°）╯︵ ┻━┻")] groups[23] = [_string_value("a")] groups[24] = [_string_value("abc def")] # latin small letter e + combining acute accent + latin small letter b groups[25] = [_string_value("e\u0301b")] - groups[26] = [_string_value(u"æ")] + groups[26] = [_string_value("æ")] # latin small letter e with acute accent + latin small letter a - groups[27] = [_string_value(u"\u00e9a")] + groups[27] = [_string_value("\u00e9a")] # blobs - groups[28] = [_blob_value(bytes())] - groups[29] = [_blob_value(bytes([0]))] - groups[30] = [_blob_value(bytes([0, 1, 2, 3, 4]))] - groups[31] = [_blob_value(bytes([0, 1, 2, 4, 3]))] - groups[32] = [_blob_value(bytes([127]))] + groups[28] = [_blob_value(b'')] + groups[29] = [_blob_value(b'\x00')] + groups[30] = [_blob_value(b'\x00\x01\x02\x03\x04')] + groups[31] = [_blob_value(b'\x00\x01\x02\x04\x03')] + groups[32] = [_blob_value(b'\x7f')] # resource names groups[33] = [ @@ -174,7 +177,7 @@ def test_typeorder_type_failure(self): left = mock.Mock() left.WhichOneof.return_value = "imaginary-type" - with self.assertRaisesRegex(ValueError, "Could not detect value"): + with self.assertRaisesRegexp(ValueError, "Could not detect value"): target.compare(left, mock.Mock()) def test_failure_to_find_type(self): @@ -186,7 +189,9 @@ def test_failure_to_find_type(self): # expect this to fail with value error. with mock.patch.object(TypeOrder, 'from_value',) as to: to.value = None - with self.assertRaisesRegex(ValueError, "'Unknown ``value_type``"): + with self.assertRaisesRegexp( + ValueError, "'Unknown ``value_type``" + ): target.compare(left, right) def test_compare_objects_different_keys(self): @@ -210,6 +215,8 @@ def _int_value(l): def _string_value(s): + if not isinstance(s, six.text_type): + s = six.u(s) return encode_value(s) diff --git a/firestore/tests/unit/test_query.py b/firestore/tests/unit/test_query.py index 16c3b4df2a5f..98d3f7b4fdd8 100644 --- a/firestore/tests/unit/test_query.py +++ b/firestore/tests/unit/test_query.py @@ -943,8 +943,8 @@ def test_comparator_missing_order_by_field_in_data_raises(self): doc2._data = {'first': {'stringValue': 'Ada'}, 'last': {'stringValue': 'lovelace'}} - with self.assertRaisesRegex(ValueError, - "Can only compare fields "): + with self.assertRaisesRegexp(ValueError, + "Can only compare fields "): query._comparator(doc1, doc2) From 4a892a701e1ffa3b7e78513cb1c1f82e7a595f62 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Fri, 19 Oct 2018 00:23:51 -0400 Subject: [PATCH 137/148] fix failing test_order tests when left and right were dictionaries of differing sizes (groups\[61\] vs groups\[62\]) --- firestore/google/cloud/firestore_v1beta1/order.py | 4 +++- firestore/tests/unit/test_order.py | 2 -- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/order.py b/firestore/google/cloud/firestore_v1beta1/order.py index 81a3ecc9b0ad..de224f5ce039 100644 --- a/firestore/google/cloud/firestore_v1beta1/order.py +++ b/firestore/google/cloud/firestore_v1beta1/order.py @@ -187,7 +187,9 @@ def compare_objects(left, right): left_fields = left.map_value.fields right_fields = right.map_value.fields - for left_key, right_key in zip(left_fields, right_fields): + for left_key, right_key in zip( + sorted(left_fields), sorted(right_fields) + ): keyCompare = Order._compare_to(left_key, right_key) if keyCompare != 0: return keyCompare diff --git a/firestore/tests/unit/test_order.py b/firestore/tests/unit/test_order.py index af9488c7bcf0..9f1017b8807d 100644 --- a/firestore/tests/unit/test_order.py +++ b/firestore/tests/unit/test_order.py @@ -13,10 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import collections import mock import six -import sys import unittest from google.cloud.firestore_v1beta1._helpers import encode_value, GeoPoint From cb74bf7b4b5a5b0f72b0f94d622b666ce3ba973b Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Fri, 19 Oct 2018 11:44:49 -0700 Subject: [PATCH 138/148] fix transports layer use in watch --- firestore/google/cloud/firestore_v1beta1/watch.py | 2 +- firestore/tests/unit/test_watch.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 8e63808409f7..3b91e7f79d2b 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -213,7 +213,7 @@ def should_recover(exc): # pragma: NO COVER ResumableBidiRpc = self.ResumableBidiRpc # FBO unit tests self.rpc = ResumableBidiRpc( - self._api.firestore_stub.Listen, + self._api.transport._stubs['firestore_stub'].Listen, initial_request=initial_request, should_recover=should_recover) diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index 7c49b64c0076..533692c90975 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -704,11 +704,11 @@ class DummyFirestoreStub(object): def Listen(self): # pragma: NO COVER pass - class DummyFirestoreClient(object): def __init__(self): - self.firestore_stub = DummyFirestoreStub() - + self.transport = mock.Mock( + _stubs={'firestore_stub': DummyFirestoreStub()} + ) class DummyDocumentReference(object): def __init__(self, *document_path, **kw): @@ -739,7 +739,6 @@ def __init__(self, **kw): def _to_protobuf(self): return '' - class DummyFirestore(object): _firestore_api = DummyFirestoreClient() _database_string = 'abc://bar/' From d42f8c5019e8e6696beca12052edd9ccd9c34393 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Fri, 19 Oct 2018 14:37:38 -0700 Subject: [PATCH 139/148] try adding wait before doc setting. System tests failing on CI but not local --- firestore/tests/system.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/firestore/tests/system.py b/firestore/tests/system.py index 12f69cc277ee..4827ca5e08d4 100644 --- a/firestore/tests/system.py +++ b/firestore/tests/system.py @@ -971,6 +971,8 @@ def on_snapshot(docs, changes, read_time): on_snapshot.failed = None query_ref.on_snapshot(on_snapshot) + sleep(1) + doc_ref1.set({ u'first': u'Ada' + unique_id, u'last': u'Lovelace', From 3add82d00a3ecd7cb339133e50600d7105036fd0 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Sat, 20 Oct 2018 14:07:21 -0400 Subject: [PATCH 140/148] linting --- firestore/tests/unit/test_watch.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index 533692c90975..9e40943ea5f7 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -704,12 +704,14 @@ class DummyFirestoreStub(object): def Listen(self): # pragma: NO COVER pass + class DummyFirestoreClient(object): def __init__(self): self.transport = mock.Mock( _stubs={'firestore_stub': DummyFirestoreStub()} ) + class DummyDocumentReference(object): def __init__(self, *document_path, **kw): if 'client' not in kw: @@ -739,6 +741,7 @@ def __init__(self, **kw): def _to_protobuf(self): return '' + class DummyFirestore(object): _firestore_api = DummyFirestoreClient() _database_string = 'abc://bar/' From b48409823d602e620db6ef4674349706a623a741 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Sat, 20 Oct 2018 14:09:21 -0400 Subject: [PATCH 141/148] make a bit more idiomatic --- firestore/google/cloud/firestore_v1beta1/order.py | 10 ++++------ firestore/tests/system.py | 3 --- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/order.py b/firestore/google/cloud/firestore_v1beta1/order.py index de224f5ce039..17604557bb67 100644 --- a/firestore/google/cloud/firestore_v1beta1/order.py +++ b/firestore/google/cloud/firestore_v1beta1/order.py @@ -225,9 +225,7 @@ def compare_doubles(left, right): @staticmethod def _compare_to(left, right): - if left < right: - return -1 - elif left == right: - return 0 - # left > right - return 1 + # We can't just use cmp(left, right) because cmp doesn't exist + # in Python 3, so this is an equivalent suggested by + # https://docs.python.org/3.0/whatsnew/3.0.html#ordering-comparisons + return (left > right) - (left < right) diff --git a/firestore/tests/system.py b/firestore/tests/system.py index 8a512a8101ea..4827ca5e08d4 100644 --- a/firestore/tests/system.py +++ b/firestore/tests/system.py @@ -971,11 +971,8 @@ def on_snapshot(docs, changes, read_time): on_snapshot.failed = None query_ref.on_snapshot(on_snapshot) -<<<<<<< HEAD -======= sleep(1) ->>>>>>> crwilcox/firestore-watch doc_ref1.set({ u'first': u'Ada' + unique_id, u'last': u'Lovelace', From e14421bd325af87fad74407246aaeb186dcff72a Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Sat, 20 Oct 2018 18:39:50 -0400 Subject: [PATCH 142/148] more idiomatic, and remove special casing of -0.0 and 0.0 in compare_doubles --- .../google/cloud/firestore_v1beta1/order.py | 35 +++++-------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/order.py b/firestore/google/cloud/firestore_v1beta1/order.py index 17604557bb67..aea729a5cbe2 100644 --- a/firestore/google/cloud/firestore_v1beta1/order.py +++ b/firestore/google/cloud/firestore_v1beta1/order.py @@ -126,24 +126,17 @@ def compare_timestamps(left, right): def compare_geo_points(left, right): left_value = decode_value(left, None) right_value = decode_value(right, None) - cmp = 0 - if left_value.latitude < right_value.latitude: - cmp = -1 - elif left_value.latitude == right_value.latitude: - cmp = 0 - else: - cmp = 1 + cmp = ( + (left_value.latitude > right_value.latitude) - + (left_value.latitude < right_value.latitude) + ) if cmp != 0: return cmp - else: - if left_value.longitude < right_value.longitude: - cmp = -1 - elif left_value.longitude == right_value.longitude: - cmp = 0 - else: - cmp = 1 - return cmp + return ( + (left_value.longitude > right_value.longitude) - + (left_value.longitude < right_value.longitude) + ) @staticmethod def compare_resource_paths(left, right): @@ -162,12 +155,7 @@ def compare_resource_paths(left, right): left_length = len(left) right_length = len(right) - if left_length < right_length: - return -1 - if left_length > right_length: - return 1 - - return 0 + return (left_length > right_length) - (left_length < right_length) @staticmethod def compare_arrays(left, right): @@ -216,11 +204,6 @@ def compare_doubles(left, right): if math.isnan(right): return 1 - if left == -0.0: - left = 0 - if right == -0.0: - right = 0 - return Order._compare_to(left, right) @staticmethod From e089dacba78b010d21a72441e8866d2a67900bbd Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Mon, 22 Oct 2018 15:10:07 -0700 Subject: [PATCH 143/148] lint fixes --- firestore/tests/unit/test_watch.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index 533692c90975..9e40943ea5f7 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -704,12 +704,14 @@ class DummyFirestoreStub(object): def Listen(self): # pragma: NO COVER pass + class DummyFirestoreClient(object): def __init__(self): self.transport = mock.Mock( _stubs={'firestore_stub': DummyFirestoreStub()} ) + class DummyDocumentReference(object): def __init__(self, *document_path, **kw): if 'client' not in kw: @@ -739,6 +741,7 @@ def __init__(self, **kw): def _to_protobuf(self): return '' + class DummyFirestore(object): _firestore_api = DummyFirestoreClient() _database_string = 'abc://bar/' From 136a881c2ccea038980758fbe4966c02c79b4344 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Fri, 26 Oct 2018 14:50:16 -0700 Subject: [PATCH 144/148] Start using self.close where previously using self._consumer.stop --- .../google/cloud/firestore_v1beta1/order.py | 3 - .../google/cloud/firestore_v1beta1/watch.py | 64 ++++++------------- firestore/tests/unit/test_watch.py | 3 +- 3 files changed, 22 insertions(+), 48 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/order.py b/firestore/google/cloud/firestore_v1beta1/order.py index aea729a5cbe2..e5003df14091 100644 --- a/firestore/google/cloud/firestore_v1beta1/order.py +++ b/firestore/google/cloud/firestore_v1beta1/order.py @@ -74,8 +74,6 @@ def compare(cls, left, right): return -1 return 1 - # TODO: may be able to use helpers.decode_value and do direct compares - # after converting to python types value_type = left.WhichOneof('value_type') if value_type == 'null_value': @@ -108,7 +106,6 @@ def compare_blobs(left, right): left_bytes = left.bytes_value right_bytes = right.bytes_value - # TODO: Should verify bytes comparisons in python work as expected return Order._compare_to(left_bytes, right_bytes) @staticmethod diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 3b91e7f79d2b..2cb65a108299 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -69,6 +69,9 @@ class WatchDocTree(object): + # TODO: Currently this uses a dict. Other implementations us an rbtree. + # The performance of this implementation should be investigated and may + # require modifying the underlying datastructure to a rbtree. def __init__(self): self._dict = {} self._index = 0 @@ -183,8 +186,7 @@ def __init__(self, this watch. read_time (string): The ISO 8601 time at which this snapshot was obtained. - # TODO: Go had an err here and node.js provided size. - # TODO: do we want to include either? + document_snapshot_cls: instance of DocumentSnapshot document_reference_cls: instance of DocumentReference """ @@ -222,8 +224,7 @@ def should_recover(exc): # pragma: NO COVER # Initialize state for on_snapshot # The sorted tree of QueryDocumentSnapshots as sent in the last # snapshot. We only look at the keys. - # TODO: using ordered dict right now but not great maybe - self.doc_tree = WatchDocTree() # TODO: rbtree(this._comparator) + self.doc_tree = WatchDocTree() # A map of document names to QueryDocumentSnapshots for the last sent # snapshot. @@ -277,13 +278,18 @@ def close(self, reason=None): self._consumer.stop() self._consumer = None - # TODO: Verify we don't have other helper threads that need to be - # shut down here. - self._rpc = None self._closed = True _LOGGER.debug('Finished stopping manager.') + if reason: + # Raise an exception if a reason is provided + _LOGGER.debug("reason for closing: %s" % reason) + if isinstance(reason, Exception): + raise reason + raise RuntimeError(reason) + + def _on_rpc_done(self, future): """Triggered whenever the underlying RPC terminates without recovery. @@ -372,13 +378,6 @@ def _on_snapshot_target_change_add(self, proto): _LOGGER.debug("on_snapshot: target change: ADD") assert WATCH_TARGET_ID == proto.target_change.target_ids[0], \ 'Unexpected target ID sent by server' - # TODO : do anything here? Node didn't so I think this isn't - # the right thing to do - # wr = WatchResult( - # None, - # self._document_reference.id, - # ChangeType.ADDED) - # self._snapshot_callback(wr) def _on_snapshot_target_change_remove(self, proto): _LOGGER.debug("on_snapshot: target change: REMOVE") @@ -431,13 +430,8 @@ def on_snapshot(self, proto): if meth is None: _LOGGER.info('on_snapshot: Unknown target change ' + str(target_change_type)) - self._consumer.stop() - # closeStream( - # new Error('Unknown target change type: ' + - # JSON.stringify(change)) - # TODO : make this exit the inner function and stop processing? - raise Exception('Unknown target change type: %s ' % - str(target_change_type)) # XXX Exception? + self.close(reason='Unknown target change type: %s ' % + str(target_change_type)) else: try: meth(proto) @@ -501,11 +495,6 @@ def on_snapshot(self, proto): update_time=document.update_time) self.change_map[document.name] = snapshot - # TODO: ensure we call this later, on current returend. - # wr = WatchResult(snapshot, - # self._document_reference.id, - # ChangeType.MODIFIED) - # self._snapshot_callback(wr) elif removed: _LOGGER.debug('on_snapshot: document change: REMOVED') @@ -516,10 +505,6 @@ def on_snapshot(self, proto): _LOGGER.debug('on_snapshot: document change: DELETE/REMOVE') name = (proto.document_delete or proto.document_remove).document self.change_map[name] = ChangeType.REMOVED - # wr = WatchResult(None, - # self._document_reference.id, - # ChangeType.REMOVED) - # self._snapshot_callback(wr) elif (proto.filter): _LOGGER.debug('on_snapshot: filter update') @@ -532,23 +517,14 @@ def on_snapshot(self, proto): else: _LOGGER.debug("UNKNOWN TYPE. UHOH") - self._consumer.stop() - raise Exception( - 'Unknown listen response type: %s' % proto - ) # XXX Exception? - # TODO: can we stop but raise an error? - # closeStream( - # new Error('Unknown listen response type: ' + - # JSON.stringify(proto)) - # ) + self.close(reason=ValueError( + 'Unknown listen response type: %s' % proto)) def push(self, read_time, next_resume_token): """ Assembles a new snapshot from the current set of changes and invokes the user's callback. Clears the current changes on completion. """ - # TODO: may need to lock here to avoid races on collecting snapshots - # and sending them to the user. deletes, adds, updates = Watch._extract_changes( self.doc_map, self.change_map, @@ -564,7 +540,8 @@ def push(self, read_time, next_resume_token): ) if not self.has_pushed or len(appliedChanges): - # TODO: the tree should be ordered. Sort here for now. + # TODO: It is possible in the future we will have the tree order + # on insert. For now, we sort here. key = functools.cmp_to_key(self._comparator) keys = sorted(updated_tree.keys(), key=key) @@ -623,7 +600,6 @@ def delete_doc(name, updated_tree, updated_map): # XXX probably should not expose IndexError when doc doesnt exist existing = updated_tree.find(old_document) old_index = existing.index - # TODO: was existing.remove returning tree (presumably immuatable?) updated_tree = updated_tree.remove(old_document) del updated_map[name] return (DocumentChange(ChangeType.REMOVED, @@ -729,7 +705,7 @@ def _reset_docs(self): self.change_map.clear() self.resume_token = None - # TODO: mark each document as deleted. If documents are not delete + # Mark each document as deleted. If documents are not deleted # they will be sent again by the server. for name, snapshot in self.doc_tree.items(): self.change_map[name] = ChangeType.REMOVED diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index 9e40943ea5f7..fca507382c8a 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -334,7 +334,8 @@ def test_on_snapshot_target_unknown(self): proto.target_change.target_change_type = 'unknown' with self.assertRaises(Exception) as exc: inst.on_snapshot(proto) - self.assertTrue(inst._consumer.stopped) + self.assertTrue(inst._consumer is None) + self.assertTrue(inst._rpc is None) self.assertEqual( str(exc.exception), 'Unknown target change type: unknown ' From 195814b6c5560518c5e2018ef8d0b722b1bd8ff0 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Fri, 26 Oct 2018 15:13:52 -0700 Subject: [PATCH 145/148] lint --- firestore/google/cloud/firestore_v1beta1/watch.py | 1 - 1 file changed, 1 deletion(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 2cb65a108299..8826ccc2cfce 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -289,7 +289,6 @@ def close(self, reason=None): raise reason raise RuntimeError(reason) - def _on_rpc_done(self, future): """Triggered whenever the underlying RPC terminates without recovery. From 34b4da37e7ba75c04be3d597981fd910b84a96a1 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Wed, 7 Nov 2018 09:53:03 +0000 Subject: [PATCH 146/148] clean up comments --- .../google/cloud/firestore_v1beta1/watch.py | 24 +++++++------------ firestore/tests/unit/test_watch.py | 8 +++---- 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index 8826ccc2cfce..aac5a71ca796 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -278,7 +278,7 @@ def close(self, reason=None): self._consumer.stop() self._consumer = None - self._rpc = None + self._rpc = None # TODO: Everywhere else this is self.rpc. self._closed = True _LOGGER.debug('Finished stopping manager.') @@ -388,8 +388,9 @@ def _on_snapshot_target_change_remove(self, proto): code = change.cause.code message = change.cause.message - # TODO: Surface a .code property on the exception. - raise Exception('Error %s: %s' % (code, message)) # XXX Exception? + # TODO: Consider surfacing a code property on the exception. + # TODO: Consider a more exact exception + raise Exception('Error %s: %s' % (code, message)) def _on_snapshot_target_change_reset(self, proto): # Whatever changes have happened so far no longer matter. @@ -438,15 +439,9 @@ def on_snapshot(self, proto): _LOGGER.debug("meth(proto) exc: " + str(exc2)) raise - # XXX this is currently a no-op - # affects_target = self._affects_target( - # target_change.target_ids, WATCH_TARGET_ID - # ) - - # if target_change.resume_token and affects_target: - # # TODO: they node version resets backoff here. We allow - # # bidi rpc to do its thing. - # pass + # NOTE: + # in other implementations, such as node, the backoff is reset here + # in this version bidi rpc is just used and will control this. elif str(proto.document_change): _LOGGER.debug('on_snapshot: document change') @@ -579,9 +574,6 @@ def _extract_changes(doc_map, changes, read_time): def _compute_snapshot(self, doc_tree, doc_map, delete_changes, add_changes, update_changes): - # TODO: ACTUALLY NEED TO CALCULATE - # return {updated_tree, updated_map, appliedChanges}; - # return doc_tree, doc_map, changes updated_tree = doc_tree updated_map = doc_map @@ -596,7 +588,7 @@ def delete_doc(name, updated_tree, updated_map): """ assert name in updated_map, 'Document to delete does not exist' old_document = updated_map.get(name) - # XXX probably should not expose IndexError when doc doesnt exist + # TODO: If a document doesn't exist this raises IndexError. Handle? existing = updated_tree.find(old_document) old_index = existing.index updated_tree = updated_tree.remove(old_document) diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index fca507382c8a..beaad6271c0f 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -377,9 +377,8 @@ class DummyDocument: self.assertEqual(inst.change_map['fred'].data, {}) def test_on_snapshot_document_change_changed_docname_db_prefix(self): - # XXX This test asserts the current behavior, but I have no level - # of confidence that the change map should contain the - # db-prefixed document name instead of the bare document name. + # TODO: Verify the current behavior. The change map currently contains + # the db-prefixed document name and not the bare document name. from google.cloud.firestore_v1beta1.watch import WATCH_TARGET_ID inst = self._makeOne() @@ -644,8 +643,7 @@ class DummyDoc(object): add_changes, update_changes ) - # TODO: - # Assertion is not verified correct below. Verify this test is good. + # TODO: Verify that the assertion here is correct. self.assertEqual(updated_map, { '/updated': updated_snapshot, From 5622f3c7577be1ffa4cace72dbf2ae7916538112 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Wed, 7 Nov 2018 09:54:00 +0000 Subject: [PATCH 147/148] alias unsubscribe to close --- firestore/google/cloud/firestore_v1beta1/watch.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index aac5a71ca796..d5c83e4fd06a 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -278,7 +278,8 @@ def close(self, reason=None): self._consumer.stop() self._consumer = None - self._rpc = None # TODO: Everywhere else this is self.rpc. + self.rpc.close() + self.rpc = None self._closed = True _LOGGER.debug('Finished stopping manager.') @@ -310,8 +311,8 @@ def _on_rpc_done(self, future): thread.daemon = True thread.start() - def unsubscribe(self): # XXX should this be aliased to close? - self.rpc.close() + def unsubscribe(self): + self.close() @classmethod def for_document(cls, document_ref, snapshot_callback, From 67c9a477008d9e7517af4b153d6c3983f6b75c57 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Wed, 7 Nov 2018 10:43:30 +0000 Subject: [PATCH 148/148] move rpc to internal attr --- firestore/google/cloud/firestore_v1beta1/watch.py | 10 +++++----- firestore/tests/unit/test_watch.py | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/firestore/google/cloud/firestore_v1beta1/watch.py b/firestore/google/cloud/firestore_v1beta1/watch.py index d5c83e4fd06a..1cdfe56598f2 100644 --- a/firestore/google/cloud/firestore_v1beta1/watch.py +++ b/firestore/google/cloud/firestore_v1beta1/watch.py @@ -214,12 +214,12 @@ def should_recover(exc): # pragma: NO COVER if ResumableBidiRpc is None: ResumableBidiRpc = self.ResumableBidiRpc # FBO unit tests - self.rpc = ResumableBidiRpc( + self._rpc = ResumableBidiRpc( self._api.transport._stubs['firestore_stub'].Listen, initial_request=initial_request, should_recover=should_recover) - self.rpc.add_done_callback(self._on_rpc_done) + self._rpc.add_done_callback(self._on_rpc_done) # Initialize state for on_snapshot # The sorted tree of QueryDocumentSnapshots as sent in the last @@ -247,7 +247,7 @@ def should_recover(exc): # pragma: NO COVER if BackgroundConsumer is None: # FBO unit tests BackgroundConsumer = self.BackgroundConsumer - self._consumer = BackgroundConsumer(self.rpc, self.on_snapshot) + self._consumer = BackgroundConsumer(self._rpc, self.on_snapshot) self._consumer.start() @property @@ -278,8 +278,8 @@ def close(self, reason=None): self._consumer.stop() self._consumer = None - self.rpc.close() - self.rpc = None + self._rpc.close() + self._rpc = None self._closed = True _LOGGER.debug('Finished stopping manager.') diff --git a/firestore/tests/unit/test_watch.py b/firestore/tests/unit/test_watch.py index beaad6271c0f..b04a68ee9acf 100644 --- a/firestore/tests/unit/test_watch.py +++ b/firestore/tests/unit/test_watch.py @@ -162,7 +162,7 @@ def _snapshot_callback(self, docs, changes, read_time): def test_ctor(self): inst = self._makeOne() self.assertTrue(inst._consumer.started) - self.assertTrue(inst.rpc.callbacks, [inst._on_rpc_done]) + self.assertTrue(inst._rpc.callbacks, [inst._on_rpc_done]) def test__on_rpc_done(self): inst = self._makeOne() @@ -199,7 +199,7 @@ def test_close_inactive(self): def test_unsubscribe(self): inst = self._makeOne() inst.unsubscribe() - self.assertTrue(inst.rpc.closed) + self.assertTrue(inst._rpc is None) def test_for_document(self): from google.cloud.firestore_v1beta1.watch import Watch @@ -223,7 +223,7 @@ def test_for_document(self): document_reference_class_instance ) self.assertTrue(inst._consumer.started) - self.assertTrue(inst.rpc.callbacks, [inst._on_rpc_done]) + self.assertTrue(inst._rpc.callbacks, [inst._on_rpc_done]) def test_for_query(self): from google.cloud.firestore_v1beta1.watch import Watch @@ -252,7 +252,7 @@ def test_for_query(self): document_reference_class_instance ) self.assertTrue(inst._consumer.started) - self.assertTrue(inst.rpc.callbacks, [inst._on_rpc_done]) + self.assertTrue(inst._rpc.callbacks, [inst._on_rpc_done]) self.assertEqual(inst._targets['query'], 'dummy query target') def test_on_snapshot_target_no_change_no_target_ids_not_current(self): @@ -773,7 +773,7 @@ class DummyBackgroundConsumer(object): is_active = True def __init__(self, rpc, on_snapshot): - self.rpc = rpc + self._rpc = rpc self.on_snapshot = on_snapshot def start(self):