From 70ecaa6f6a6efd4b8fd6ee96e804f94725e73326 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20G=C3=A1lvez=20Mart=C3=ADnez?= Date: Mon, 1 Feb 2021 17:25:08 +0100 Subject: [PATCH 1/7] Add mget Improve get and delete methods --- .vscode/settings.json | 3 ++ elasticmock/fake_elasticsearch.py | 62 +++++++++++++++++++--------- tests/fake_elasticsearch/test_get.py | 8 ++++ 3 files changed, 53 insertions(+), 20 deletions(-) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..9efeb86 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python.pythonPath": "/home/Tiendeo/.envs/elasticmock/bin/python" +} \ No newline at end of file diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index 63cc9c0..14aa9f5 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -6,8 +6,9 @@ import dateutil.parser from elasticsearch import Elasticsearch -from elasticsearch.client.utils import query_params -from elasticsearch.exceptions import NotFoundError +from elasticsearch.client.utils import query_params, _normalize_hosts +from elasticsearch.transport import Transport +from elasticsearch.exceptions import NotFoundError, RequestError from elasticmock.behaviour.server_failure import server_failure from elasticmock.fake_cluster import FakeClusterClient @@ -246,6 +247,7 @@ class FakeElasticsearch(Elasticsearch): def __init__(self, hosts=None, transport_class=None, **kwargs): self.__documents_dict = {} self.__scrolls = {} + self.transport = Transport(_normalize_hosts(hosts), **kwargs) @property def indices(self): @@ -295,10 +297,10 @@ def index(self, index, body, doc_type='_doc', id=None, params=None, headers=None if id is None: id = get_random_id() - elif self.exists(index, doc_type, id, params=params): - doc = self.get(index, id, doc_type, params=params) + elif self.exists(index, id, doc_type=doc_type, params=params): + doc = self.get(index, id, doc_type=doc_type, params=params) version = doc['_version'] + 1 - self.delete(index, doc_type, id) + self.delete(index, id, doc_type=doc_type) self.__documents_dict[index].append({ '_type': doc_type, @@ -322,26 +324,23 @@ def bulk(self, body, index=None, doc_type=None, params=None, headers=None): version = 1 items = [] + ids = [] for line in body.splitlines(): if len(line.strip()) > 0: line = json.loads(line) if 'index' in line: - index = line['index']['_index'] - doc_type = line['index']['_type'] + index = line['index'].get('_index') or index + doc_type = line['index'].get('_type') \ + or doc_type or '_all' if index not in self.__documents_dict: self.__documents_dict[index] = list() + ids.append(line['index'].get('_id')) else: - document_id = get_random_id() + document_id = ids.pop() if ids else None - self.__documents_dict[index].append({ - '_type': doc_type, - '_id': document_id, - '_source': line, - '_index': index, - '_version': version - }) + self.index(index, line, doc_type=doc_type, id=document_id) items.append({'index': { '_type': doc_type, @@ -358,7 +357,7 @@ def bulk(self, body, index=None, doc_type=None, params=None, headers=None): } @query_params('parent', 'preference', 'realtime', 'refresh', 'routing') - def exists(self, index, doc_type, id, params=None, headers=None): + def exists(self, index, id, doc_type=None, params=None, headers=None): result = False if index in self.__documents_dict: for document in self.__documents_dict[index]: @@ -399,6 +398,26 @@ def get(self, index, id, doc_type='_all', params=None, headers=None): } raise NotFoundError(404, json.dumps(error_data)) + @query_params('_source', '_source_exclude', '_source_include', + 'preference', 'realtime', 'refresh', 'routing', + 'stored_fields') + def mget(self, body, index, doc_type='_all', params=None, headers=None): + ids = body.get('ids') + results = [] + for id in ids: + try: + results.append(self.get(index, id, doc_type=doc_type, + params=params, headers=headers)) + except: + pass + if not results: + raise RequestError( + 400, + 'action_request_validation_exception', + 'Validation Failed: 1: no documents to get;' + ) + return {'docs': results} + @query_params('_source', '_source_exclude', '_source_include', 'parent', 'preference', 'realtime', 'refresh', 'routing', 'version', 'version_type') @@ -574,17 +593,20 @@ def scroll(self, scroll_id, params=None, headers=None): @query_params('consistency', 'parent', 'refresh', 'replication', 'routing', 'timeout', 'version', 'version_type') - def delete(self, index, doc_type, id, params=None, headers=None): + def delete(self, index, id, doc_type=None, params=None, headers=None): found = False ignore = extract_ignore_as_iterable(params) if index in self.__documents_dict: for document in self.__documents_dict[index]: - if document.get('_type') == doc_type and document.get('_id') == id: + if document.get('_id') == id: found = True - self.__documents_dict[index].remove(document) - break + if doc_type and document.get('_type') != doc_type: + found = False + if found: + self.__documents_dict[index].remove(document) + break result_dict = { 'found': found, diff --git a/tests/fake_elasticsearch/test_get.py b/tests/fake_elasticsearch/test_get.py index 10220ec..d895370 100644 --- a/tests/fake_elasticsearch/test_get.py +++ b/tests/fake_elasticsearch/test_get.py @@ -60,3 +60,11 @@ def test_should_get_only_document_source_with_id(self): target_doc_source = self.es.get_source(index=INDEX_NAME, doc_type=DOC_TYPE, id=document_id) self.assertEqual(target_doc_source, BODY) + + def test_mget_get_several_documents_by_id(self): + ids = [] + for _ in range(0, 10): + data = self.es.index(index=INDEX_NAME, doc_type=DOC_TYPE, body=BODY) + ids.append(data.get('_id')) + results = self.es.mget(index=INDEX_NAME, body={'ids': ids}) + self.assertEqual(len(results['docs']), 10) From c66844b64e00caa03b615a83c7b2c1c0c05711d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20G=C3=A1lvez=20Mart=C3=ADnez?= Date: Mon, 1 Feb 2021 18:39:16 +0100 Subject: [PATCH 2/7] Add compatibility with old versions of elastic --- elasticmock/fake_elasticsearch.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index 14aa9f5..bb11b90 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -6,7 +6,8 @@ import dateutil.parser from elasticsearch import Elasticsearch -from elasticsearch.client.utils import query_params, _normalize_hosts +from elasticsearch.client.utils import query_params +from elasticsearch.client import _normalize_hosts from elasticsearch.transport import Transport from elasticsearch.exceptions import NotFoundError, RequestError From 462527ee4273057d5ca0efe5d5729bcd3d1aa796 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20G=C3=A1lvez=20Mart=C3=ADnez?= Date: Mon, 1 Feb 2021 19:41:43 +0100 Subject: [PATCH 3/7] Remove vscode settings --- .vscode/settings.json | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 9efeb86..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "python.pythonPath": "/home/Tiendeo/.envs/elasticmock/bin/python" -} \ No newline at end of file From 32b724ddc67f7f9ab5538a0f76f1ece4e1bfd2f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20G=C3=A1lvez=20Mart=C3=ADnez?= Date: Mon, 1 Feb 2021 19:47:10 +0100 Subject: [PATCH 4/7] add vstudio to gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index dd2017e..6e593e4 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,9 @@ bin/ share/ pyvenv.cfg +### Visual Studio ### +.vscode/ + ### Intellij ### # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 From 02fab89c75853f9d3f45f68cd2908c06172d7ec7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20G=C3=A1lvez=20Mart=C3=ADnez?= Date: Mon, 8 Feb 2021 13:51:23 +0100 Subject: [PATCH 5/7] Remove boosting for fields --- elasticmock/fake_elasticsearch.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index bb11b90..b7686a6 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -216,6 +216,8 @@ def _compare_value_for_field(self, doc_source, field, value, ignore_case): value = value.lower() doc_val = doc_source + # Remove boosting + field, *_ = field.split("*") for k in field.split("."): if hasattr(doc_val, k): doc_val = getattr(doc_val, k) From 49cfa8d0d624c51f1dd2943f7aa4f8842e26347a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20G=C3=A1lvez=20Mart=C3=ADnez?= Date: Wed, 24 Feb 2021 14:04:57 +0100 Subject: [PATCH 6/7] Fix multi-value fields --- elasticmock/fake_elasticsearch.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index b7686a6..a820d9a 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -221,8 +221,10 @@ def _compare_value_for_field(self, doc_source, field, value, ignore_case): for k in field.split("."): if hasattr(doc_val, k): doc_val = getattr(doc_val, k) + break elif k in doc_val: doc_val = doc_val[k] + break else: return False @@ -237,7 +239,7 @@ def _compare_value_for_field(self, doc_source, field, value, ignore_case): if value == val: return True - if isinstance(val, str) and value in val: + if isinstance(val, str) and str(value) in val: return True return False From c8aa31d35feedc29806e949b3c0792b6815d9ffb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20G=C3=A1lvez=20Mart=C3=ADnez?= Date: Tue, 2 Mar 2021 12:26:55 +0100 Subject: [PATCH 7/7] Allows in the bulk method to indicate the index only in the signature --- elasticmock/fake_elasticsearch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index bf9ea31..6b580ff 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -352,7 +352,7 @@ def bulk(self, body, index=None, doc_type=None, params=None, headers=None): action = next(iter(line.keys())) version = 1 - index = line[action]['_index'] + index = line[action].get('_index') or index doc_type = line[action].get('_type', "_doc") # _type is deprecated in 7.x if action in ['delete', 'update'] and not line[action].get("_id"):