Skip to content

Commit

Permalink
fix: ⬆️ pymongo 4 fix and flake8 code style fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
newgene committed Jul 18, 2022
1 parent 6008828 commit 699f0d9
Showing 1 changed file with 10 additions and 5 deletions.
15 changes: 10 additions & 5 deletions biothings/utils/backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ def finalize(self):
Final optimization or compacting can be done here as well.
'''


class DocMemoryBackend(DocBackendBase):
name = 'memory'

Expand Down Expand Up @@ -81,6 +82,7 @@ def finalize(self):
from biothings.utils.common import dump
dump(self.target_dict, self.target_name + '.pyobj')


class DocMongoBackend(DocBackendBase):
name = 'mongo'

Expand All @@ -98,8 +100,8 @@ def __eq__(self, other):
if not isinstance(other, DocMongoBackend):
return False
return self.target_name == other.target_name and \
self.target_collection.database.name == other.target_collection.database.name and \
self.target_collection.database.client.address == other.target_collection.database.client.address
self.target_collection.database.name == other.target_collection.database.name and \
self.target_collection.database.client.address == other.target_collection.database.client.address

@property
def target_name(self):
Expand Down Expand Up @@ -149,7 +151,7 @@ def update(self, docs, upsert=False):
bulk = []
for doc in docs:
bulk.append(UpdateOne({'_id': doc["_id"]}, {"$set": doc}, upsert=upsert))

if bulk:
result = self.target_collection.bulk_write(bulk)
# if doc is the same, it'll be matched but not modified.
Expand Down Expand Up @@ -206,7 +208,7 @@ def count_from_ids(self, ids, step=100000):
total_cnt = 0
for i in range(0, len(ids), step):
_ids = ids[i:i + step]
_cnt = self.target_collection.find({'_id': {'$in': _ids}}).count()
_cnt = self.target_collection.count_documents({'_id': {'$in': _ids}})
total_cnt += _cnt
return total_cnt

Expand All @@ -223,9 +225,11 @@ def remove_from_ids(self, ids, step=10000):
deleted += res.deleted_count
return deleted


# backward-compatible
DocMongoDBBackend = DocMongoBackend


class DocESBackend(DocBackendBase):
name = 'es'

Expand All @@ -235,7 +239,7 @@ def __init__(self, esidxer=None):
self._target_esidxer_provider = esidxer
self._target_esidxer = None
else:
_target_esidxer_provider = None
self._target_esidxer_provider = None
self._target_esidxer = esidxer

@property
Expand Down Expand Up @@ -331,6 +335,7 @@ def create_from_options(cls, options):
raise Exception("Cannot create backend class from options, ensure that es_index, es_host, and es_doc_type are set")
return cls(ESIndexer(index=options.es_index, doc_type=options.es_doc_type, es_host=options.es_host))


class DocBackendOptions(object):
def __init__(self, cls, es_index=None, es_host=None, es_doc_type=None,
mongo_target_db=None, mongo_target_collection=None):
Expand Down

0 comments on commit 699f0d9

Please sign in to comment.