From 369cf3209d024367d75b511fb599e1a4957d1147 Mon Sep 17 00:00:00 2001 From: Marcos Cardoso Date: Sat, 19 Sep 2020 17:16:28 -0300 Subject: [PATCH 01/44] Fix version --- Makefile | 2 +- setup.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 1627946..3ac04ef 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -ELASTICMOCK_VERSION='2.0' +ELASTICMOCK_VERSION='1.6.0' install: pip install -r requirements.txt diff --git a/setup.py b/setup.py index eafcc94..428b4ba 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ import setuptools -__version__ = '2.0' +__version__ = '1.6.0' # read the contents of your readme file from os import path @@ -29,8 +29,6 @@ 'Intended Audience :: Developers', 'Operating System :: OS Independent', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', From 84e6f76089dbe0397417ca22087632f0e32c252c Mon Sep 17 00:00:00 2001 From: Marcos Cardoso Date: Sat, 19 Sep 2020 17:18:10 -0300 Subject: [PATCH 02/44] Update Readme and Makefile --- Makefile | 6 +++++- README.md | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3ac04ef..c37bd79 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,11 @@ upload: create_dist twine upload dist/* git push -create_dist: create_dist_commit update_pip +create_dist: create_dist_no_commit update_pip + rm -rf dist + python3 setup.py sdist + +create_dist_no_commit: update_pip rm -rf dist python3 setup.py sdist diff --git a/README.md b/README.md index a1bb79d..573bc57 100644 --- a/README.md +++ b/README.md @@ -177,6 +177,9 @@ python setup.py test ## Changelog +#### 1.6.0: +- [Implements several basic search types](https://github.com/vrcmarcos/elasticmock/pull/42) (Thanks [@KyKoPho](https://github.com/KyKoPho)) + #### 1.5.1: - [Fix tests for es > 7](https://github.com/vrcmarcos/elasticmock/pull/38) (Thanks [@chesstrian](https://github.com/chesstrian)) From 675105b66b6f3daad5f89e1ce02a7d6248b30c09 Mon Sep 17 00:00:00 2001 From: Jasper Oosterman Date: Tue, 3 Nov 2020 21:44:23 +0100 Subject: [PATCH 03/44] Adhere to ignore NotFound 404 on get and delete --- elasticmock/fake_elasticsearch.py | 11 +++++++++-- elasticmock/utilities/__init__.py | 8 ++++++++ tests/fake_elasticsearch/test_delete.py | 8 ++++++++ tests/fake_elasticsearch/test_get.py | 8 ++++++++ 4 files changed, 33 insertions(+), 2 deletions(-) diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index 541a17a..cf41f60 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -8,7 +8,7 @@ from elasticsearch.exceptions import NotFoundError from elasticmock.behaviour.server_failure import server_failure -from elasticmock.utilities import get_random_id, get_random_scroll_id +from elasticmock.utilities import extract_ignore_as_iterable, get_random_id, get_random_scroll_id from elasticmock.utilities.decorator import for_all_methods from elasticmock.fake_indices import FakeIndicesClient from elasticmock.fake_cluster import FakeClusterClient @@ -280,7 +280,9 @@ def exists(self, index, doc_type, id, params=None, headers=None): 'parent', 'preference', 'realtime', 'refresh', 'routing', 'version', 'version_type') def get(self, index, id, doc_type='_all', params=None, headers=None): + ignore = extract_ignore_as_iterable(params) result = None + if index in self.__documents_dict: for document in self.__documents_dict[index]: if document.get('_id') == id: @@ -294,6 +296,9 @@ def get(self, index, id, doc_type='_all', params=None, headers=None): if result: result['found'] = True + return result + elif params and 404 in ignore: + return {'found': False} else: error_data = { '_index': index, @@ -303,7 +308,6 @@ def get(self, index, id, doc_type='_all', params=None, headers=None): } raise NotFoundError(404, json.dumps(error_data)) - return result @query_params('_source', '_source_exclude', '_source_include', 'parent', 'preference', 'realtime', 'refresh', 'routing', 'version', @@ -442,6 +446,7 @@ def scroll(self, scroll_id, params=None, headers=None): def delete(self, index, doc_type, id, params=None, headers=None): found = False + ignore = extract_ignore_as_iterable(params) if index in self.__documents_dict: for document in self.__documents_dict[index]: @@ -460,6 +465,8 @@ def delete(self, index, doc_type, id, params=None, headers=None): if found: return result_dict + elif params and 404 in ignore: + return {'found': False} else: raise NotFoundError(404, json.dumps(result_dict)) diff --git a/elasticmock/utilities/__init__.py b/elasticmock/utilities/__init__.py index d43390c..09aa690 100644 --- a/elasticmock/utilities/__init__.py +++ b/elasticmock/utilities/__init__.py @@ -16,3 +16,11 @@ def get_random_id(size=DEFAULT_ELASTICSEARCH_ID_SIZE): def get_random_scroll_id(size=DEFAULT_ELASTICSEARCH_SEARCHRESULTPHASE_COUNT): return base64.b64encode(''.join(get_random_id() for _ in range(size)).encode()) + + +def extract_ignore_as_iterable(params): + """Extracts the value of the ignore parameter as iterable""" + ignore = params.get('ignore', ()) + if isinstance(ignore, int): + ignore = (ignore,) + return ignore diff --git a/tests/fake_elasticsearch/test_delete.py b/tests/fake_elasticsearch/test_delete.py index d8b2b89..6cfc7d9 100644 --- a/tests/fake_elasticsearch/test_delete.py +++ b/tests/fake_elasticsearch/test_delete.py @@ -11,6 +11,14 @@ def test_should_raise_exception_when_delete_nonindexed_document(self): with self.assertRaises(NotFoundError): self.es.delete(index=INDEX_NAME, doc_type=DOC_TYPE, id=1) + def test_should_not_raise_exception_when_delete_nonindexed_document_if_ignored(self): + target_doc = self.es.delete(index=INDEX_NAME, doc_type=DOC_TYPE, id=1, ignore=404) + self.assertFalse(target_doc.get('found')) + + def test_should_not_raise_exception_when_delete_nonindexed_document_if_ignored_list(self): + target_doc = self.es.delete(index=INDEX_NAME, doc_type=DOC_TYPE, id=1, ignore=(401, 404)) + self.assertFalse(target_doc.get('found')) + def test_should_delete_indexed_document(self): doc_indexed = self.es.index(index=INDEX_NAME, doc_type=DOC_TYPE, body=BODY) search = self.es.search(index=INDEX_NAME) diff --git a/tests/fake_elasticsearch/test_get.py b/tests/fake_elasticsearch/test_get.py index 5434804..10220ec 100644 --- a/tests/fake_elasticsearch/test_get.py +++ b/tests/fake_elasticsearch/test_get.py @@ -11,6 +11,14 @@ def test_should_raise_notfounderror_when_nonindexed_id_is_used(self): with self.assertRaises(NotFoundError): self.es.get(index=INDEX_NAME, id='1') + def test_should_not_raise_notfounderror_when_nonindexed_id_is_used_and_ignored(self): + target_doc = self.es.get(index=INDEX_NAME, id='1', ignore=404) + self.assertFalse(target_doc.get('found')) + + def test_should_not_raise_notfounderror_when_nonindexed_id_is_used_and_ignored_list(self): + target_doc = self.es.get(index=INDEX_NAME, id='1', ignore=(401, 404)) + self.assertFalse(target_doc.get('found')) + def test_should_get_document_with_id(self): data = self.es.index(index=INDEX_NAME, doc_type=DOC_TYPE, body=BODY) From 062d615b7db0858e474bbdd652614bdd03ba72b4 Mon Sep 17 00:00:00 2001 From: Jasper Oosterman Date: Tue, 3 Nov 2020 21:44:44 +0100 Subject: [PATCH 04/44] Bump patch version --- Makefile | 2 +- README.md | 3 +++ setup.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index c37bd79..92c5fe0 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -ELASTICMOCK_VERSION='1.6.0' +ELASTICMOCK_VERSION='1.6.1' install: pip install -r requirements.txt diff --git a/README.md b/README.md index 573bc57..808d986 100644 --- a/README.md +++ b/README.md @@ -177,6 +177,9 @@ python setup.py test ## Changelog +### 1.6.1: + + #### 1.6.0: - [Implements several basic search types](https://github.com/vrcmarcos/elasticmock/pull/42) (Thanks [@KyKoPho](https://github.com/KyKoPho)) diff --git a/setup.py b/setup.py index 428b4ba..72337b1 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ import setuptools -__version__ = '1.6.0' +__version__ = '1.6.1' # read the contents of your readme file from os import path From 8b7888f07f84bfb4ef2627e26e4bce5a2edfae30 Mon Sep 17 00:00:00 2001 From: Marcos Cardoso Date: Sun, 22 Nov 2020 12:53:42 -0300 Subject: [PATCH 05/44] Update 1.6.0 Changelog --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 808d986..b2a0081 100644 --- a/README.md +++ b/README.md @@ -177,11 +177,9 @@ python setup.py test ## Changelog -### 1.6.1: - - #### 1.6.0: - [Implements several basic search types](https://github.com/vrcmarcos/elasticmock/pull/42) (Thanks [@KyKoPho](https://github.com/KyKoPho)) +- [Allow ignoring of missing documents (404) for get and delete](https://github.com/vrcmarcos/elasticmock/pull/44) (Thanks [@joosterman](https://github.com/joosterman)) #### 1.5.1: - [Fix tests for es > 7](https://github.com/vrcmarcos/elasticmock/pull/38) (Thanks [@chesstrian](https://github.com/chesstrian)) From 15cd0c622c77fb0448b80c567fab7d4da7f9dbb5 Mon Sep 17 00:00:00 2001 From: Marcos Cardoso Date: Sun, 22 Nov 2020 15:49:09 -0200 Subject: [PATCH 06/44] Fix CRLF in files and Twine upload --- .gitignore | 5 +++++ Makefile | 14 +++++++------- README.md | 3 +++ requirements.txt | 2 +- setup.py | 2 +- 5 files changed, 17 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 39a104e..dd2017e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,11 @@ # Created by https://www.gitignore.io/api/intellij,linux,osx,pycharm,python +# Custom +bin/ +share/ +pyvenv.cfg + ### Intellij ### # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 diff --git a/Makefile b/Makefile index 92c5fe0..71bda9e 100644 --- a/Makefile +++ b/Makefile @@ -1,30 +1,30 @@ ELASTICMOCK_VERSION='1.6.1' install: - pip install -r requirements.txt + pip3 install -r requirements.txt test_install: install - pip install -r requirements_test.txt + pip3 install -r requirements_test.txt test: test_install - python3 setup.py test + python3.6 setup.py test upload: create_dist - pip install twine + pip3 install twine twine upload dist/* git push create_dist: create_dist_no_commit update_pip rm -rf dist - python3 setup.py sdist + python3.6 setup.py sdist create_dist_no_commit: update_pip rm -rf dist - python3 setup.py sdist + python3.6 setup.py sdist create_dist_commit: git commit --all -m "Bump version ${ELASTICMOCK_VERSION}" git tag ${ELASTICMOCK_VERSION} update_pip: - pip install --upgrade pip + pip3 install --upgrade pip diff --git a/README.md b/README.md index b2a0081..4c3b258 100644 --- a/README.md +++ b/README.md @@ -177,6 +177,9 @@ python setup.py test ## Changelog +#### 1.6.1: +- Fix Twine README.md + #### 1.6.0: - [Implements several basic search types](https://github.com/vrcmarcos/elasticmock/pull/42) (Thanks [@KyKoPho](https://github.com/KyKoPho)) - [Allow ignoring of missing documents (404) for get and delete](https://github.com/vrcmarcos/elasticmock/pull/44) (Thanks [@joosterman](https://github.com/joosterman)) diff --git a/requirements.txt b/requirements.txt index 822a3ed..68083db 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ elasticsearch>=1.9.0,<8.0.0 -mock==3.0.5 +mock ipdb \ No newline at end of file diff --git a/setup.py b/setup.py index 72337b1..e1cc49a 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ # read the contents of your readme file from os import path this_directory = path.abspath(path.dirname(__file__)) -with open(path.join(this_directory, 'README.md'), 'r') as f: +with open(path.join(this_directory, 'README.md'), encoding='utf-8') as f: long_description = f.read() setuptools.setup( From affbc4c2fd7f64db2066c4458c20407b1fe0b546 Mon Sep 17 00:00:00 2001 From: Xavier Sumba Date: Wed, 2 Dec 2020 08:29:54 -0500 Subject: [PATCH 07/44] Add must to query type --- elasticmock/fake_elasticsearch.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index cf41f60..01cfd34 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -25,6 +25,7 @@ class QueryType: MATCH = 'MATCH' TERM = 'TERM' TERMS = 'TERMS' + MUST = 'MUST' @staticmethod def get_query_type(type_str): @@ -38,6 +39,8 @@ def get_query_type(type_str): return QueryType.TERM elif type_str == 'terms': return QueryType.TERMS + elif type_str == 'must': + return QueryType.MUST else: raise NotImplementedError(f'type {type_str} is not implemented for QueryType') From deb8c3bc584efb7a70300e4550a2b2b26a14dc89 Mon Sep 17 00:00:00 2001 From: Xavier Sumba Date: Wed, 2 Dec 2020 08:34:29 -0500 Subject: [PATCH 08/44] add match all query type --- elasticmock/fake_elasticsearch.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index cf41f60..a6f4ca0 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -23,6 +23,7 @@ class QueryType: BOOL = 'BOOL' FILTER = 'FILTER' MATCH = 'MATCH' + MATCH_ALL = 'MATCH_ALL' TERM = 'TERM' TERMS = 'TERMS' @@ -34,6 +35,8 @@ def get_query_type(type_str): return QueryType.FILTER elif type_str == 'match': return QueryType.MATCH + elif type_str == 'match_all': + return QueryType.MATCH_ALL elif type_str == 'term': return QueryType.TERM elif type_str == 'terms': From 99a313805344765737ab5caebe1545d1bf745b84 Mon Sep 17 00:00:00 2001 From: Marcos Cardoso Date: Sun, 6 Dec 2020 22:19:11 -0200 Subject: [PATCH 09/44] Update Readme and create v1.6.2 --- Makefile | 2 +- README.md | 4 ++++ setup.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 71bda9e..c032f89 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -ELASTICMOCK_VERSION='1.6.1' +ELASTICMOCK_VERSION='1.6.2' install: pip3 install -r requirements.txt diff --git a/README.md b/README.md index 4c3b258..22ae177 100644 --- a/README.md +++ b/README.md @@ -177,6 +177,10 @@ python setup.py test ## Changelog +#### 1.6.2: +- [Add must to query type](https://github.com/vrcmarcos/elasticmock/pull/47) (Thanks [@cuent](https://github.com/cuent)) +- [Add match all query type](https://github.com/vrcmarcos/elasticmock/pull/48) (Thanks [@cuent](https://github.com/cuent)) + #### 1.6.1: - Fix Twine README.md diff --git a/setup.py b/setup.py index e1cc49a..1089f46 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ import setuptools -__version__ = '1.6.1' +__version__ = '1.6.2' # read the contents of your readme file from os import path From a3301c0dce13f14ff486f2f6eaf9420afe95f4e0 Mon Sep 17 00:00:00 2001 From: Omer Gelbard Date: Wed, 16 Dec 2020 12:02:58 -0800 Subject: [PATCH 10/44] Fixed count not working with empty doc_type list This will help support Elasticsearch-dsl, which uses doc_type=[] (acceptable by elasticsearch client) --- elasticmock/fake_elasticsearch.py | 2 +- tests/fake_elasticsearch/test_count.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index 6152c06..98e5a6e 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -336,7 +336,7 @@ def count(self, index=None, doc_type=None, body=None, params=None, headers=None) i = 0 for searchable_index in searchable_indexes: for document in self.__documents_dict[searchable_index]: - if doc_type is not None and document.get('_type') != doc_type: + if doc_type and document.get('_type') != doc_type: continue i += 1 result = { diff --git a/tests/fake_elasticsearch/test_count.py b/tests/fake_elasticsearch/test_count.py index 373902e..0d4ae1a 100644 --- a/tests/fake_elasticsearch/test_count.py +++ b/tests/fake_elasticsearch/test_count.py @@ -20,3 +20,8 @@ def test_should_count_in_multiple_indexes(self): result = self.es.count(index=['users', 'pcs']) self.assertEqual(2, result.get('count')) + + def test_should_count_with_empty_doc_types(self): + self.es.index(index='index', doc_type=DOC_TYPE, body={'data': 'test'}) + count = self.es.count(doc_type=[]) + self.assertEqual(1, count.get('count')) From 49197481ae7d193d8d79ed13c64aa9304f336733 Mon Sep 17 00:00:00 2001 From: Omer Gelbard Date: Wed, 16 Dec 2020 12:05:05 -0800 Subject: [PATCH 11/44] Added a doc_type test to count --- tests/fake_elasticsearch/test_count.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/fake_elasticsearch/test_count.py b/tests/fake_elasticsearch/test_count.py index 0d4ae1a..6aed8ea 100644 --- a/tests/fake_elasticsearch/test_count.py +++ b/tests/fake_elasticsearch/test_count.py @@ -25,3 +25,9 @@ def test_should_count_with_empty_doc_types(self): self.es.index(index='index', doc_type=DOC_TYPE, body={'data': 'test'}) count = self.es.count(doc_type=[]) self.assertEqual(1, count.get('count')) + + def test_should_count_with_doc_types(self): + self.es.index(index='index', doc_type=DOC_TYPE, body={'data': 'test1'}) + self.es.index(index='index', doc_type='different-doc-type', body={'data': 'test2'}) + count = self.es.count(doc_type=DOC_TYPE) + self.assertEqual(1, count.get('count')) From 23530b551b66924f27874b2aa4068a6deb30d11a Mon Sep 17 00:00:00 2001 From: Jan Kislinger Date: Tue, 29 Dec 2020 14:39:48 +0100 Subject: [PATCH 12/44] allow 'match_all' queries in FakeSearch --- elasticmock/fake_elasticsearch.py | 2 ++ tests/fake_elasticsearch/test_search.py | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index 98e5a6e..f9980e8 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -62,6 +62,8 @@ def evaluate(self, document): def _evaluate_for_query_type(self, document): if self.type == QueryType.MATCH: return self._evaluate_for_match_query_type(document) + elif self.type == QueryType.MATCH_ALL: + return True elif self.type == QueryType.TERM: return self._evaluate_for_term_query_type(document) elif self.type == QueryType.TERMS: diff --git a/tests/fake_elasticsearch/test_search.py b/tests/fake_elasticsearch/test_search.py index 74079fe..df67f22 100644 --- a/tests/fake_elasticsearch/test_search.py +++ b/tests/fake_elasticsearch/test_search.py @@ -24,6 +24,14 @@ def test_should_return_all_documents(self): search = self.es.search() self.assertEqual(index_quantity, search.get('hits').get('total')) + def test_should_return_all_documents_match_all(self): + index_quantity = 10 + for i in range(0, index_quantity): + self.es.index(index='index_{0}'.format(i), doc_type=DOC_TYPE, body={'data': 'test_{0}'.format(i)}) + + search = self.es.search(body={'query': {'match_all': {}}}) + self.assertEqual(index_quantity, search.get('hits').get('total')) + def test_should_return_only_indexed_documents_on_index(self): index_quantity = 2 for i in range(0, index_quantity): From f897cca0b886e4dfa3ac93c81b885777f4fa71f2 Mon Sep 17 00:00:00 2001 From: Jan Kislinger Date: Tue, 29 Dec 2020 17:29:40 +0100 Subject: [PATCH 13/44] Query using nested attributes --- elasticmock/fake_elasticsearch.py | 46 ++++++++++++------------- tests/fake_elasticsearch/test_search.py | 12 +++++++ 2 files changed, 34 insertions(+), 24 deletions(-) diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index 98e5a6e..ea73923 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -123,32 +123,30 @@ def _evaluate_for_compound_query_type(self, document): return return_val def _compare_value_for_field(self, doc_source, field, value, ignore_case): - value = str(value).lower() if ignore_case and isinstance(value, str) \ - else value - doc_val = None - if hasattr(doc_source, field): - doc_val = getattr(doc_source, field) - elif field in doc_source: - doc_val = doc_source[field] - - if isinstance(doc_val, list): - for val in doc_val: - val = val if isinstance(val, (int, float, complex)) \ - else str(val) - if ignore_case and isinstance(val, str): + if ignore_case and isinstance(value, str): + value = value.lower() + + doc_val = doc_source + for k in field.split("."): + if hasattr(doc_val, k): + doc_val = getattr(doc_val, k) + elif k in doc_val: + doc_val = doc_val[k] + else: + return False + + if not isinstance(doc_val, list): + doc_val = [doc_val] + + for val in doc_val: + if not isinstance(val, (int, float, complex)) or val is None: + val = str(val) + if ignore_case: val = val.lower() - if isinstance(val, str) and value in val: - return True - if value == val: - return True - else: - doc_val = doc_val if isinstance(doc_val, (int, float, complex)) \ - else str(doc_val) - if ignore_case and isinstance(doc_val, str): - doc_val = doc_val.lower() - if isinstance(doc_val, str) and value in doc_val: + + if value == val: return True - if value == doc_val: + if isinstance(val, str) and value in val: return True return False diff --git a/tests/fake_elasticsearch/test_search.py b/tests/fake_elasticsearch/test_search.py index 74079fe..5de2a9d 100644 --- a/tests/fake_elasticsearch/test_search.py +++ b/tests/fake_elasticsearch/test_search.py @@ -132,3 +132,15 @@ def test_search_with_terms_query(self): self.assertEqual(response['hits']['total'], 3) hits = response['hits']['hits'] self.assertEqual(len(hits), 3) + + def test_query_on_nested_data(self): + for i, y in enumerate(['yes', 'no']): + self.es.index('index_for_search', doc_type=DOC_TYPE, + body={'id': i, 'data': {'x': i, 'y': y}}) + + for term, value, i in [('data.x', 1, 1), ('data.y', 'yes', 0)]: + response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, + body={'query': {'term': {term: value}}}) + self.assertEqual(1, response['hits']['total']) + doc = response['hits']['hits'][0]['_source'] + self.assertEqual(i, doc['id']) From 7c8efeda39c19487873795b8b8530a2dd2780f5c Mon Sep 17 00:00:00 2001 From: Phil Weir Date: Tue, 29 Dec 2020 22:10:45 +0000 Subject: [PATCH 14/44] add shards skipped to search and count --- elasticmock/fake_elasticsearch.py | 2 ++ tests/fake_elasticsearch/test_count.py | 5 +++++ tests/fake_elasticsearch/test_search.py | 4 ++++ 3 files changed, 11 insertions(+) diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index 98e5a6e..35ed315 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -343,6 +343,7 @@ def count(self, index=None, doc_type=None, body=None, params=None, headers=None) 'count': i, '_shards': { 'successful': 1, + 'skipped': 0, 'failed': 0, 'total': 1 } @@ -394,6 +395,7 @@ def search(self, index=None, doc_type=None, body=None, params=None, headers=None '_shards': { # Simulate indexes with 1 shard each 'successful': len(searchable_indexes), + 'skipped': 0, 'failed': 0, 'total': len(searchable_indexes) }, diff --git a/tests/fake_elasticsearch/test_count.py b/tests/fake_elasticsearch/test_count.py index 6aed8ea..a481f40 100644 --- a/tests/fake_elasticsearch/test_count.py +++ b/tests/fake_elasticsearch/test_count.py @@ -26,6 +26,11 @@ def test_should_count_with_empty_doc_types(self): count = self.es.count(doc_type=[]) self.assertEqual(1, count.get('count')) + def test_should_return_skipped_shards(self): + self.es.index(index='index', doc_type=DOC_TYPE, body={'data': 'test'}) + count = self.es.count(doc_type=[]) + self.assertEqual(0, count.get('_shards').get('skipped')) + def test_should_count_with_doc_types(self): self.es.index(index='index', doc_type=DOC_TYPE, body={'data': 'test1'}) self.es.index(index='index', doc_type='different-doc-type', body={'data': 'test2'}) diff --git a/tests/fake_elasticsearch/test_search.py b/tests/fake_elasticsearch/test_search.py index 74079fe..e482c1f 100644 --- a/tests/fake_elasticsearch/test_search.py +++ b/tests/fake_elasticsearch/test_search.py @@ -16,6 +16,10 @@ def test_should_return_hits_hits_even_when_no_result(self): self.assertEqual(0, search.get('hits').get('total')) self.assertListEqual([], search.get('hits').get('hits')) + def test_should_return_skipped_shards(self): + search = self.es.search() + self.assertEqual(0, search.get('_shards').get('skipped')) + def test_should_return_all_documents(self): index_quantity = 10 for i in range(0, index_quantity): From fe4281405c94bc0cf1ce09be4e5e89bc314c150c Mon Sep 17 00:00:00 2001 From: Jan Kislinger Date: Wed, 30 Dec 2020 11:22:49 +0100 Subject: [PATCH 15/44] filter using range --- elasticmock/fake_elasticsearch.py | 58 +++++++++++++++++- requirements_test.txt | 3 +- tests/fake_elasticsearch/test_search.py | 81 +++++++++++++++++++++++++ 3 files changed, 140 insertions(+), 2 deletions(-) diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index a2e5e11..3ff6468 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- - +import datetime import json import sys @@ -27,6 +27,7 @@ class QueryType: TERM = 'TERM' TERMS = 'TERMS' MUST = 'MUST' + RANGE = 'RANGE' @staticmethod def get_query_type(type_str): @@ -44,6 +45,8 @@ def get_query_type(type_str): return QueryType.TERMS elif type_str == 'must': return QueryType.MUST + elif type_str == 'range': + return QueryType.RANGE else: raise NotImplementedError(f'type {type_str} is not implemented for QueryType') @@ -68,6 +71,8 @@ def _evaluate_for_query_type(self, document): return self._evaluate_for_term_query_type(document) elif self.type == QueryType.TERMS: return self._evaluate_for_terms_query_type(document) + elif self.type == QueryType.RANGE: + return self._evaluate_for_range_query_type(document) elif self.type == QueryType.BOOL: return self._evaluate_for_compound_query_type(document) elif self.type == QueryType.FILTER: @@ -102,6 +107,46 @@ def _evaluate_for_field(self, document, ignore_case): break return return_val + def _evaluate_for_range_query_type(self, document): + for field, comparisons in self.condition.items(): + doc_val = document['_source'] + for k in field.split("."): + if hasattr(doc_val, k): + doc_val = getattr(doc_val, k) + elif k in doc_val: + doc_val = doc_val[k] + else: + return False + + if isinstance(doc_val, list): + return False + + for sign, value in comparisons.items(): + if isinstance(doc_val, datetime.datetime): + value = datetime.datetime.fromisoformat(value) + # we can also use this: + # try: + # if not getattr(doc_val, f"__{sign}__")(value): + # return False + # except AttributeError: + # raise ValueError(f"Invalid comparison type {sign}") from None + if sign == 'gte': + if doc_val < value: + return False + elif sign == 'gt': + if doc_val <= value: + return False + elif sign == 'lte': + if doc_val > value: + return False + elif sign == 'lt': + if doc_val >= value: + return False + else: + raise ValueError(f"Invalid comparison type {sign}") + return True + + def _evaluate_for_compound_query_type(self, document): return_val = False if isinstance(self.condition, dict): @@ -387,6 +432,9 @@ def search(self, index=None, doc_type=None, body=None, params=None, headers=None else: matches.append(document) + for match in matches: + self._find_and_convert_data_types(match['_source']) + result = { 'hits': { 'total': len(matches), @@ -522,3 +570,11 @@ def _normalize_index_to_list(self, index): raise NotFoundError(404, 'IndexMissingException[[{0}] missing]'.format(searchable_index)) return searchable_indexes + + @classmethod + def _find_and_convert_data_types(cls, document): + for key, value in document.items(): + if isinstance(value, dict): + cls._find_and_convert_data_types(value) + elif isinstance(value, datetime.datetime): + document[key] = value.isoformat() diff --git a/requirements_test.txt b/requirements_test.txt index e4379d2..84a5da3 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1 +1,2 @@ -tox==3.13.2 \ No newline at end of file +tox==3.13.2 +parameterized \ No newline at end of file diff --git a/tests/fake_elasticsearch/test_search.py b/tests/fake_elasticsearch/test_search.py index 1121aca..49adfb7 100644 --- a/tests/fake_elasticsearch/test_search.py +++ b/tests/fake_elasticsearch/test_search.py @@ -1,6 +1,9 @@ # -*- coding: utf-8 -*- +import datetime +import unittest from elasticsearch.exceptions import NotFoundError +from parameterized import parameterized from tests import TestElasticmock, INDEX_NAME, DOC_TYPE @@ -156,3 +159,81 @@ def test_query_on_nested_data(self): self.assertEqual(1, response['hits']['total']) doc = response['hits']['hits'][0]['_source'] self.assertEqual(i, doc['id']) + + @parameterized.expand( + [ + ( + 'timestamp gt', + {'timestamp': {'gt': datetime.datetime(2009, 1, 1, 10, 20, 0).isoformat()}}, + range(5, 12), + ), + ( + 'timestamp gte', + {'timestamp': {'gte': datetime.datetime(2009, 1, 1, 10, 20, 0).isoformat()}}, + range(4, 12), + ), + ( + 'timestamp lt', + {'timestamp': {'lt': datetime.datetime(2009, 1, 1, 10, 35, 0).isoformat()}}, + range(7), + ), + ( + 'timestamp lte', + {'timestamp': {'lte': datetime.datetime(2009, 1, 1, 10, 35, 0).isoformat()}}, + range(8), + ), + ( + 'timestamp combination', + { + 'timestamp': { + 'gt': datetime.datetime(2009, 1, 1, 10, 15, 0).isoformat(), + 'lte': datetime.datetime(2009, 1, 1, 10, 35, 0).isoformat(), + } + }, + range(4, 8), + ), + ( + 'data_int gt', + {'data_int': {'gt': 40}}, + range(5, 12), + ), + ( + 'data_int gte', + {'data_int': {'gte': 40}}, + range(4, 12), + ), + ( + 'data_int lt', + {'data_int': {'lt': 70}}, + range(7), + ), + ( + 'data_int lte', + {'data_int': {'lte': 70}}, + range(8), + ), + ( + 'data_int combination', + {'data_int': {'gt': 30, 'lte': 70}}, + range(4, 8), + ), + ] + ) + def test_search_with_range_query(self, _, query_range, expected_ids): + for i in range(0, 12): + body = { + 'id': i, + 'timestamp': datetime.datetime(2009, 1, 1, 10, 5 * i, 0), + 'data_int': 10 * i, + } + self.es.index(index='index_for_search', doc_type=DOC_TYPE, body=body) + + response = self.es.search( + index='index_for_search', + doc_type=DOC_TYPE, + body={'query': {'range': query_range}}, + ) + + self.assertEqual(len(expected_ids), response['hits']['total']) + hits = response['hits']['hits'] + self.assertEqual(set(expected_ids), set(hit['_source']['id'] for hit in hits)) From 0630dc6e3f3f1ad99810b07e827c1e457f456dd0 Mon Sep 17 00:00:00 2001 From: Jan Kislinger Date: Mon, 4 Jan 2021 12:38:39 +0100 Subject: [PATCH 16/44] use search size --- elasticmock/fake_elasticsearch.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index 3ff6468..aa3550c 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -418,7 +418,11 @@ def search(self, index=None, doc_type=None, body=None, params=None, headers=None for query_type_str, condition in query.items(): conditions.append(self._get_fake_query_condition(query_type_str, condition)) for searchable_index in searchable_indexes: + for document in self.__documents_dict[searchable_index]: + if len(matches) >= int(params.get('size', 10_000)): + break + if doc_type: if isinstance(doc_type, list) and document.get('_type') not in doc_type: continue From b7f919ca69330b8c6779eed16d204ef569989e12 Mon Sep 17 00:00:00 2001 From: Jan Kislinger Date: Tue, 5 Jan 2021 17:40:48 +0100 Subject: [PATCH 17/44] add simple mock for aggregated results (buckets) --- elasticmock/fake_elasticsearch.py | 73 ++++++++++--- tests/fake_elasticsearch/test_search.py | 134 ++++++++++++++++-------- 2 files changed, 150 insertions(+), 57 deletions(-) diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index aa3550c..a943150 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -2,16 +2,18 @@ import datetime import json import sys +from collections import defaultdict from elasticsearch import Elasticsearch from elasticsearch.client.utils import query_params from elasticsearch.exceptions import NotFoundError +from elasticsearch_dsl import AttrDict from elasticmock.behaviour.server_failure import server_failure +from elasticmock.fake_cluster import FakeClusterClient +from elasticmock.fake_indices import FakeIndicesClient from elasticmock.utilities import extract_ignore_as_iterable, get_random_id, get_random_scroll_id from elasticmock.utilities.decorator import for_all_methods -from elasticmock.fake_indices import FakeIndicesClient -from elasticmock.fake_cluster import FakeClusterClient PY3 = sys.version_info[0] == 3 if PY3: @@ -19,7 +21,6 @@ class QueryType: - BOOL = 'BOOL' FILTER = 'FILTER' MATCH = 'MATCH' @@ -38,7 +39,7 @@ def get_query_type(type_str): elif type_str == 'match': return QueryType.MATCH elif type_str == 'match_all': - return QueryType.MATCH_ALL + return QueryType.MATCH_ALL elif type_str == 'term': return QueryType.TERM elif type_str == 'terms': @@ -51,6 +52,17 @@ def get_query_type(type_str): raise NotImplementedError(f'type {type_str} is not implemented for QueryType') +class MetricType: + CARDINALITY = "CARDINALITY" + + @staticmethod + def get_metric_type(type_str): + if type_str == "cardinality": + return MetricType.CARDINALITY + else: + raise NotImplementedError(f'type {type_str} is not implemented for MetricType') + + class FakeQueryCondition: type = None condition = None @@ -146,7 +158,6 @@ def _evaluate_for_range_query_type(self, document): raise ValueError(f"Invalid comparison type {sign}") return True - def _evaluate_for_compound_query_type(self, document): return_val = False if isinstance(self.condition, dict): @@ -250,7 +261,7 @@ def info(self, params=None, headers=None): def index(self, index, body, doc_type='_doc', id=None, params=None, headers=None): if index not in self.__documents_dict: self.__documents_dict[index] = list() - + version = 1 if id is None: @@ -278,7 +289,7 @@ def index(self, index, body, doc_type='_doc', id=None, params=None, headers=None @query_params('consistency', 'op_type', 'parent', 'refresh', 'replication', 'routing', 'timeout', 'timestamp', 'ttl', 'version', 'version_type') - def bulk(self, body, index=None, doc_type=None, params=None, headers=None): + def bulk(self, body, index=None, doc_type=None, params=None, headers=None): version = 1 items = [] @@ -359,7 +370,6 @@ def get(self, index, id, doc_type='_all', params=None, headers=None): } raise NotFoundError(404, json.dumps(error_data)) - @query_params('_source', '_source_exclude', '_source_include', 'parent', 'preference', 'realtime', 'refresh', 'routing', 'version', 'version_type') @@ -465,10 +475,11 @@ def search(self, index=None, doc_type=None, body=None, params=None, headers=None aggregations = {} for aggregation, definition in body['aggs'].items(): + metrics = None aggregations[aggregation] = { "doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, - "buckets": [] + "buckets": self.make_aggregation_buckets(definition, matches) } if aggregations: @@ -485,9 +496,9 @@ def search(self, index=None, doc_type=None, body=None, params=None, headers=None 'params': params } hits = hits[params.get('from'):params.get('from') + params.get('size')] - + result['hits']['hits'] = hits - + return result @query_params('scroll') @@ -500,7 +511,7 @@ def scroll(self, scroll_id, params=None, headers=None): params=scroll.get('params') ) return result - + @query_params('consistency', 'parent', 'refresh', 'replication', 'routing', 'timeout', 'version', 'version_type') def delete(self, index, doc_type, id, params=None, headers=None): @@ -582,3 +593,41 @@ def _find_and_convert_data_types(cls, document): cls._find_and_convert_data_types(value) elif isinstance(value, datetime.datetime): document[key] = value.isoformat() + + def make_aggregation_buckets(self, aggregation, documents): + def make_key(doc_source, agg_source): + attr = list(agg_source.values())[0]["terms"]["field"] + return doc_source[attr] + + def make_bucket(bucket_key, bucket): + out = { + "key": AttrDict({k: v for k, v in zip(bucket_key_fields, bucket_key)}), + "doc_count": len(bucket), + } + + for metric_key, metric_definition in aggregation["aggs"].items(): + metric_type_str = list(metric_definition)[0] + metric_type = MetricType.get_metric_type(metric_type_str) + attr = metric_definition[metric_type_str]["field"] + data = [doc[attr] for doc in bucket] + + if metric_type == MetricType.CARDINALITY: + value = len(set(data)) + else: + raise NotImplementedError(f"Metric type '{metric_type}' not implemented") + + print(out, metric_key, value) + out[metric_key] = AttrDict({"value": value}) + return AttrDict(out) + + agg_sources = aggregation["composite"]["sources"] + buckets = defaultdict(list) + bucket_key_fields = [list(src)[0] for src in agg_sources] + for document in documents: + doc_src = document["_source"] + key = tuple(make_key(doc_src, agg_src) for agg_src in aggregation["composite"]["sources"]) + buckets[key].append(doc_src) + + buckets = sorted(((k, v) for k, v in buckets.items()), key=lambda x: x[0]) + buckets = [make_bucket(bucket_key, bucket) for bucket_key, bucket in buckets] + return buckets diff --git a/tests/fake_elasticsearch/test_search.py b/tests/fake_elasticsearch/test_search.py index 49adfb7..7403b21 100644 --- a/tests/fake_elasticsearch/test_search.py +++ b/tests/fake_elasticsearch/test_search.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- import datetime -import unittest from elasticsearch.exceptions import NotFoundError from parameterized import parameterized @@ -85,12 +84,13 @@ def test_search_with_match_query(self): for i in range(0, 10): self.es.index(index='index_for_search', doc_type=DOC_TYPE, body={'data': 'test_{0}'.format(i)}) - response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, body={'query': {'match': {'data': 'TEST' } } }) + response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, + body={'query': {'match': {'data': 'TEST'}}}) self.assertEqual(response['hits']['total'], 10) hits = response['hits']['hits'] self.assertEqual(len(hits), 10) - response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, body={'query': {'match': {'data': '3' } } }) + response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, body={'query': {'match': {'data': '3'}}}) self.assertEqual(response['hits']['total'], 1) hits = response['hits']['hits'] self.assertEqual(len(hits), 1) @@ -99,17 +99,17 @@ def test_search_with_match_query(self): def test_search_with_match_query_in_int_list(self): for i in range(0, 10): self.es.index(index='index_for_search', doc_type=DOC_TYPE, body={'data': [i, 11, 13]}) - response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, body={'query': {'match': {'data': 1 } } }) + response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, body={'query': {'match': {'data': 1}}}) self.assertEqual(response['hits']['total'], 1) hits = response['hits']['hits'] self.assertEqual(len(hits), 1) - self.assertEqual(hits[0]['_source'], {'data': [1, 11, 13] }) + self.assertEqual(hits[0]['_source'], {'data': [1, 11, 13]}) def test_search_with_match_query_in_string_list(self): for i in range(0, 10): self.es.index(index='index_for_search', doc_type=DOC_TYPE, body={'data': [str(i), 'two', 'three']}) - response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, body={'query': {'match': {'data': '1' } } }) + response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, body={'query': {'match': {'data': '1'}}}) self.assertEqual(response['hits']['total'], 1) hits = response['hits']['hits'] self.assertEqual(len(hits), 1) @@ -119,12 +119,13 @@ def test_search_with_term_query(self): for i in range(0, 10): self.es.index(index='index_for_search', doc_type=DOC_TYPE, body={'data': 'test_{0}'.format(i)}) - response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, body={'query': {'term': {'data': 'TEST' } } }) + response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, + body={'query': {'term': {'data': 'TEST'}}}) self.assertEqual(response['hits']['total'], 0) hits = response['hits']['hits'] self.assertEqual(len(hits), 0) - response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, body={'query': {'term': {'data': '3' } } }) + response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, body={'query': {'term': {'data': '3'}}}) self.assertEqual(response['hits']['total'], 1) hits = response['hits']['hits'] self.assertEqual(len(hits), 1) @@ -134,7 +135,8 @@ def test_search_with_bool_query(self): for i in range(0, 10): self.es.index(index='index_for_search', doc_type=DOC_TYPE, body={'id': i}) - response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, body={'query': {'bool': {'filter': [{'term': {'id': 1}}]}}}) + response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, + body={'query': {'bool': {'filter': [{'term': {'id': 1}}]}}}) self.assertEqual(response['hits']['total'], 1) hits = response['hits']['hits'] self.assertEqual(len(hits), 1) @@ -143,7 +145,8 @@ def test_search_with_terms_query(self): for i in range(0, 10): self.es.index(index='index_for_search', doc_type=DOC_TYPE, body={'id': i}) - response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, body={'query': {'terms': {'id': [1, 2, 3]}}}) + response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, + body={'query': {'terms': {'id': [1, 2, 3]}}}) self.assertEqual(response['hits']['total'], 3) hits = response['hits']['hits'] self.assertEqual(len(hits), 3) @@ -163,59 +166,59 @@ def test_query_on_nested_data(self): @parameterized.expand( [ ( - 'timestamp gt', - {'timestamp': {'gt': datetime.datetime(2009, 1, 1, 10, 20, 0).isoformat()}}, - range(5, 12), + 'timestamp gt', + {'timestamp': {'gt': datetime.datetime(2009, 1, 1, 10, 20, 0).isoformat()}}, + range(5, 12), ), ( - 'timestamp gte', - {'timestamp': {'gte': datetime.datetime(2009, 1, 1, 10, 20, 0).isoformat()}}, - range(4, 12), + 'timestamp gte', + {'timestamp': {'gte': datetime.datetime(2009, 1, 1, 10, 20, 0).isoformat()}}, + range(4, 12), ), ( - 'timestamp lt', - {'timestamp': {'lt': datetime.datetime(2009, 1, 1, 10, 35, 0).isoformat()}}, - range(7), + 'timestamp lt', + {'timestamp': {'lt': datetime.datetime(2009, 1, 1, 10, 35, 0).isoformat()}}, + range(7), ), ( - 'timestamp lte', - {'timestamp': {'lte': datetime.datetime(2009, 1, 1, 10, 35, 0).isoformat()}}, - range(8), + 'timestamp lte', + {'timestamp': {'lte': datetime.datetime(2009, 1, 1, 10, 35, 0).isoformat()}}, + range(8), ), ( - 'timestamp combination', - { - 'timestamp': { - 'gt': datetime.datetime(2009, 1, 1, 10, 15, 0).isoformat(), - 'lte': datetime.datetime(2009, 1, 1, 10, 35, 0).isoformat(), - } - }, - range(4, 8), + 'timestamp combination', + { + 'timestamp': { + 'gt': datetime.datetime(2009, 1, 1, 10, 15, 0).isoformat(), + 'lte': datetime.datetime(2009, 1, 1, 10, 35, 0).isoformat(), + } + }, + range(4, 8), ), ( - 'data_int gt', - {'data_int': {'gt': 40}}, - range(5, 12), + 'data_int gt', + {'data_int': {'gt': 40}}, + range(5, 12), ), ( - 'data_int gte', - {'data_int': {'gte': 40}}, - range(4, 12), + 'data_int gte', + {'data_int': {'gte': 40}}, + range(4, 12), ), ( - 'data_int lt', - {'data_int': {'lt': 70}}, - range(7), + 'data_int lt', + {'data_int': {'lt': 70}}, + range(7), ), ( - 'data_int lte', - {'data_int': {'lte': 70}}, - range(8), + 'data_int lte', + {'data_int': {'lte': 70}}, + range(8), ), ( - 'data_int combination', - {'data_int': {'gt': 30, 'lte': 70}}, - range(4, 8), + 'data_int combination', + {'data_int': {'gt': 30, 'lte': 70}}, + range(4, 8), ), ] ) @@ -237,3 +240,44 @@ def test_search_with_range_query(self, _, query_range, expected_ids): self.assertEqual(len(expected_ids), response['hits']['total']) hits = response['hits']['hits'] self.assertEqual(set(expected_ids), set(hit['_source']['id'] for hit in hits)) + + def test_bucket_aggregation(self): + data = [ + {"data_x": 1, "data_y": "a"}, + {"data_x": 1, "data_y": "a"}, + {"data_x": 2, "data_y": "a"}, + {"data_x": 2, "data_y": "b"}, + {"data_x": 3, "data_y": "b"}, + ] + for body in data: + self.es.index(index='index_for_search', doc_type=DOC_TYPE, body=body) + + response = self.es.search( + index="index_for_search", + doc_type=DOC_TYPE, + body={ + "query": {"match_all": {}}, + "aggs": { + "stats": { + "composite": { + "sources": [{"data_x": {"terms": {"field": "data_x"}}}], + "size": 10000, + }, + "aggs": { + "distinct_data_y": {"cardinality": {"field": "data_y"}} + }, + } + }, + }, + ) + + expected = [ + {"key": {"data_x": 1}, "doc_count": 2}, + {"key": {"data_x": 2}, "doc_count": 2}, + {"key": {"data_x": 3}, "doc_count": 1}, + ] + actual = response["aggregations"]["stats"]["buckets"] + + for x, y in zip(expected, actual): + self.assertDictEqual(x["key"], y["key"].to_dict()) + self.assertEqual(x["doc_count"], y["doc_count"]) From 7789a34cd723a213b6d0cda690bb3d748bc38fea Mon Sep 17 00:00:00 2001 From: Jan Kislinger Date: Wed, 6 Jan 2021 13:36:31 +0100 Subject: [PATCH 18/44] fix failing tests --- elasticmock/fake_elasticsearch.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index a943150..97bf2b4 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -430,8 +430,6 @@ def search(self, index=None, doc_type=None, body=None, params=None, headers=None for searchable_index in searchable_indexes: for document in self.__documents_dict[searchable_index]: - if len(matches) >= int(params.get('size', 10_000)): - break if doc_type: if isinstance(doc_type, list) and document.get('_type') not in doc_type: @@ -475,7 +473,6 @@ def search(self, index=None, doc_type=None, body=None, params=None, headers=None aggregations = {} for aggregation, definition in body['aggs'].items(): - metrics = None aggregations[aggregation] = { "doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, @@ -487,7 +484,7 @@ def search(self, index=None, doc_type=None, body=None, params=None, headers=None if 'scroll' in params: result['_scroll_id'] = str(get_random_scroll_id()) - params['size'] = int(params.get('size') if 'size' in params else 10) + params['size'] = int(params.get('size', 10)) params['from'] = int(params.get('from') + params.get('size') if 'from' in params else 0) self.__scrolls[result.get('_scroll_id')] = { 'index': index, @@ -496,6 +493,8 @@ def search(self, index=None, doc_type=None, body=None, params=None, headers=None 'params': params } hits = hits[params.get('from'):params.get('from') + params.get('size')] + elif 'size' in params: + hits = hits[:int(params['size'])] result['hits']['hits'] = hits @@ -595,6 +594,12 @@ def _find_and_convert_data_types(cls, document): document[key] = value.isoformat() def make_aggregation_buckets(self, aggregation, documents): + if 'composite' in aggregation: + return self.make_composite_aggregation_buckets(aggregation, documents) + return [] + + def make_composite_aggregation_buckets(self, aggregation, documents): + def make_key(doc_source, agg_source): attr = list(agg_source.values())[0]["terms"]["field"] return doc_source[attr] @@ -616,7 +621,6 @@ def make_bucket(bucket_key, bucket): else: raise NotImplementedError(f"Metric type '{metric_type}' not implemented") - print(out, metric_key, value) out[metric_key] = AttrDict({"value": value}) return AttrDict(out) From 34ea02bb5ae2e3f50add1bcdb8338e945e6c5e91 Mon Sep 17 00:00:00 2001 From: Jan Kislinger Date: Wed, 6 Jan 2021 13:43:23 +0100 Subject: [PATCH 19/44] remove elasticsearch_dsl dependency --- elasticmock/fake_elasticsearch.py | 7 +++---- tests/fake_elasticsearch/test_search.py | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index 97bf2b4..6ce5e85 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -7,7 +7,6 @@ from elasticsearch import Elasticsearch from elasticsearch.client.utils import query_params from elasticsearch.exceptions import NotFoundError -from elasticsearch_dsl import AttrDict from elasticmock.behaviour.server_failure import server_failure from elasticmock.fake_cluster import FakeClusterClient @@ -606,7 +605,7 @@ def make_key(doc_source, agg_source): def make_bucket(bucket_key, bucket): out = { - "key": AttrDict({k: v for k, v in zip(bucket_key_fields, bucket_key)}), + "key": {k: v for k, v in zip(bucket_key_fields, bucket_key)}, "doc_count": len(bucket), } @@ -621,8 +620,8 @@ def make_bucket(bucket_key, bucket): else: raise NotImplementedError(f"Metric type '{metric_type}' not implemented") - out[metric_key] = AttrDict({"value": value}) - return AttrDict(out) + out[metric_key] = {"value": value} + return out agg_sources = aggregation["composite"]["sources"] buckets = defaultdict(list) diff --git a/tests/fake_elasticsearch/test_search.py b/tests/fake_elasticsearch/test_search.py index 7403b21..465f7fc 100644 --- a/tests/fake_elasticsearch/test_search.py +++ b/tests/fake_elasticsearch/test_search.py @@ -279,5 +279,5 @@ def test_bucket_aggregation(self): actual = response["aggregations"]["stats"]["buckets"] for x, y in zip(expected, actual): - self.assertDictEqual(x["key"], y["key"].to_dict()) + self.assertDictEqual(x["key"], y["key"]) self.assertEqual(x["doc_count"], y["doc_count"]) From 45ef73a8d27b9fc05b828e306e31b6ab4733915a Mon Sep 17 00:00:00 2001 From: Jan Kislinger Date: Wed, 6 Jan 2021 13:48:19 +0100 Subject: [PATCH 20/44] install test requirements in travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 80f3649..6edce58 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,7 @@ python: install: - pip install tox-travis - pip install coveralls + - pip install -r requirements_test.txt script: tox after_success: From edef5d3c2dcd0fac1def5c3c7b5a02f5df6ec18c Mon Sep 17 00:00:00 2001 From: Jan Kislinger Date: Wed, 6 Jan 2021 13:51:06 +0100 Subject: [PATCH 21/44] drop version requirement for tox --- requirements_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_test.txt b/requirements_test.txt index 84a5da3..408b093 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,2 +1,2 @@ -tox==3.13.2 +tox parameterized \ No newline at end of file From f8dd9969f1bd8d98a38cec59a39ca4d0013ee765 Mon Sep 17 00:00:00 2001 From: Jan Kislinger Date: Wed, 6 Jan 2021 14:03:03 +0100 Subject: [PATCH 22/44] add parameterized to tox.ini --- .travis.yml | 1 - tox.ini | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6edce58..80f3649 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,6 @@ python: install: - pip install tox-travis - pip install coveralls - - pip install -r requirements_test.txt script: tox after_success: diff --git a/tox.ini b/tox.ini index cbc50e8..64a1368 100644 --- a/tox.ini +++ b/tox.ini @@ -7,6 +7,7 @@ envlist = [testenv] deps = + parameterized pytest==4.6.9 pytest-cov==2.8.1 elasticsearch1: elasticsearch ==1.9.0 From 6c2b408542c66c00ab99675a5c6975f3d045265d Mon Sep 17 00:00:00 2001 From: Jan Kislinger Date: Wed, 6 Jan 2021 14:09:58 +0100 Subject: [PATCH 23/44] use dateutil to support py3.6 --- elasticmock/fake_elasticsearch.py | 9 ++------- requirements.txt | 3 ++- setup.py | 3 ++- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index 6ce5e85..5cb7d4d 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -4,6 +4,7 @@ import sys from collections import defaultdict +import dateutil.parser from elasticsearch import Elasticsearch from elasticsearch.client.utils import query_params from elasticsearch.exceptions import NotFoundError @@ -134,13 +135,7 @@ def _evaluate_for_range_query_type(self, document): for sign, value in comparisons.items(): if isinstance(doc_val, datetime.datetime): - value = datetime.datetime.fromisoformat(value) - # we can also use this: - # try: - # if not getattr(doc_val, f"__{sign}__")(value): - # return False - # except AttributeError: - # raise ValueError(f"Invalid comparison type {sign}") from None + value = dateutil.parser.isoparse(value) if sign == 'gte': if doc_val < value: return False diff --git a/requirements.txt b/requirements.txt index 68083db..bb56dad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ elasticsearch>=1.9.0,<8.0.0 mock -ipdb \ No newline at end of file +ipdb +python-dateutil \ No newline at end of file diff --git a/setup.py b/setup.py index 1089f46..7aaef97 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,8 @@ packages=setuptools.find_packages(exclude=('tests')), install_requires=[ 'elasticsearch', - 'mock' + 'mock', + 'python-dateutil', ], classifiers=[ 'Environment :: Web Environment', From c77f82588baf797a659362fa67f2c08b0a8c2d4d Mon Sep 17 00:00:00 2001 From: Maluna Date: Sat, 16 Jan 2021 16:03:30 +0100 Subject: [PATCH 24/44] adding SHOULD and MINIMUM_SHOULD_MATCH to QueryType --- elasticmock/fake_elasticsearch.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index 5cb7d4d..63986db 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -29,6 +29,8 @@ class QueryType: TERMS = 'TERMS' MUST = 'MUST' RANGE = 'RANGE' + SHOULD = 'SHOULD' + MINIMUM_SHOULD_MATCH = 'MINIMUM_SHOULD_MATCH' @staticmethod def get_query_type(type_str): @@ -48,6 +50,10 @@ def get_query_type(type_str): return QueryType.MUST elif type_str == 'range': return QueryType.RANGE + elif type_str == 'should': + return QueryType.SHOULD + elif type_str == 'minimum_should_match': + return QueryType.MINIMUM_SHOULD_MATCH else: raise NotImplementedError(f'type {type_str} is not implemented for QueryType') From b24e50651f0bb28d72dddd300f0778860b1ff2bb Mon Sep 17 00:00:00 2001 From: Marcos Cardoso Date: Sun, 17 Jan 2021 00:37:09 -0300 Subject: [PATCH 25/44] Upgrading to Python 3.8 --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index c032f89..bcd77c8 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ test_install: install pip3 install -r requirements_test.txt test: test_install - python3.6 setup.py test + python3.8 setup.py test upload: create_dist pip3 install twine @@ -16,11 +16,11 @@ upload: create_dist create_dist: create_dist_no_commit update_pip rm -rf dist - python3.6 setup.py sdist + python3.8 setup.py sdist create_dist_no_commit: update_pip rm -rf dist - python3.6 setup.py sdist + python3.8 setup.py sdist create_dist_commit: git commit --all -m "Bump version ${ELASTICMOCK_VERSION}" From 68050eec513e8bd5a01f0fd3dfb7764244f85cda Mon Sep 17 00:00:00 2001 From: Marcos Cardoso Date: Sun, 17 Jan 2021 00:46:12 -0300 Subject: [PATCH 26/44] Bump to 1.7.0 --- Makefile | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index bcd77c8..89db61b 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -ELASTICMOCK_VERSION='1.6.2' +ELASTICMOCK_VERSION='1.7.0' install: pip3 install -r requirements.txt diff --git a/setup.py b/setup.py index 7aaef97..4887e03 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ import setuptools -__version__ = '1.6.2' +__version__ = '1.7.0' # read the contents of your readme file from os import path From 83e25ca660e9b6eab10f36ea496fd02248ad078f Mon Sep 17 00:00:00 2001 From: Marcos Cardoso Date: Sun, 17 Jan 2021 00:51:43 -0300 Subject: [PATCH 27/44] Update README.md --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 22ae177..dc18763 100644 --- a/README.md +++ b/README.md @@ -177,6 +177,13 @@ python setup.py test ## Changelog +#### 1.7.0: +- [Add shards skipped to search and count](https://github.com/vrcmarcos/elasticmock/pull/56) (Thanks [@philtweir](https://github.com/philtweir)) +- [Allow 'match_all' queries in FakeSearch](https://github.com/vrcmarcos/elasticmock/pull/54) (Thanks [@jankislinger](https://github.com/jankislinger)) +- [Query using nested attributes](https://github.com/vrcmarcos/elasticmock/pull/55) (Thanks [@jankislinger](https://github.com/jankislinger)) +- [New features: range, size, aggregations](https://github.com/vrcmarcos/elasticmock/pull/57) (Thanks [@jankislinger](https://github.com/jankislinger)) +- [Adding "should" and "minimum_should_match" to QueryType](https://github.com/vrcmarcos/elasticmock/pull/62) (Thanks [@lunarie16](https://github.com/lunarie16)) + #### 1.6.2: - [Add must to query type](https://github.com/vrcmarcos/elasticmock/pull/47) (Thanks [@cuent](https://github.com/cuent)) - [Add match all query type](https://github.com/vrcmarcos/elasticmock/pull/48) (Thanks [@cuent](https://github.com/cuent)) From 0c192b5dafebaeb9063b20cda04f558b62752014 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20G=C3=A1lvez=20Mart=C3=ADnez?= Date: Wed, 20 Jan 2021 17:14:14 +0100 Subject: [PATCH 28/44] Add multi_match Allow evaluation for query_type MUST --- elasticmock/fake_elasticsearch.py | 29 +++++++++++++++++++++++++ tests/fake_elasticsearch/test_search.py | 26 ++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index 63986db..5926677 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -31,6 +31,7 @@ class QueryType: RANGE = 'RANGE' SHOULD = 'SHOULD' MINIMUM_SHOULD_MATCH = 'MINIMUM_SHOULD_MATCH' + MULTI_MATCH = 'MULTI_MATCH' @staticmethod def get_query_type(type_str): @@ -54,6 +55,8 @@ def get_query_type(type_str): return QueryType.SHOULD elif type_str == 'minimum_should_match': return QueryType.MINIMUM_SHOULD_MATCH + elif type_str == 'multi_match': + return QueryType.MULTI_MATCH else: raise NotImplementedError(f'type {type_str} is not implemented for QueryType') @@ -95,6 +98,10 @@ def _evaluate_for_query_type(self, document): return self._evaluate_for_compound_query_type(document) elif self.type == QueryType.FILTER: return self._evaluate_for_compound_query_type(document) + elif self.type == QueryType.MUST: + return self._evaluate_for_compound_query_type(document) + elif self.type == QueryType.MULTI_MATCH: + return self._evaluate_for_multi_match_query_type(document) else: raise NotImplementedError('Fake query evaluation not implemented for query type: %s' % self.type) @@ -125,6 +132,25 @@ def _evaluate_for_field(self, document, ignore_case): break return return_val + def _evaluate_for_fields(self, document): + doc_source = document['_source'] + return_val = False + value = self.condition.get('query') + if not value: + return return_val + fields = self.condition.get('fields', []) + for field in fields: + return_val = self._compare_value_for_field( + doc_source, + field, + value, + True + ) + if return_val: + break + + return return_val + def _evaluate_for_range_query_type(self, document): for field, comparisons in self.condition.items(): doc_val = document['_source'] @@ -180,6 +206,9 @@ def _evaluate_for_compound_query_type(self, document): return return_val + def _evaluate_for_multi_match_query_type(self, document): + return self._evaluate_for_fields(document) + def _compare_value_for_field(self, doc_source, field, value, ignore_case): if ignore_case and isinstance(value, str): value = value.lower() diff --git a/tests/fake_elasticsearch/test_search.py b/tests/fake_elasticsearch/test_search.py index 465f7fc..98aac2b 100644 --- a/tests/fake_elasticsearch/test_search.py +++ b/tests/fake_elasticsearch/test_search.py @@ -163,6 +163,32 @@ def test_query_on_nested_data(self): doc = response['hits']['hits'][0]['_source'] self.assertEqual(i, doc['id']) + + def test_search_with_bool_query_and_multi_match(self): + for i in range(0, 10): + self.es.index(index='index_for_search', doc_type=DOC_TYPE, body={ + 'data': 'test_{0}'.format(i) if i % 2 == 0 else None, + 'data2': 'test_{0}'.format(i) if (i+1) % 2 == 0 else None + }) + + search_body = { + "query": { + "bool": { + "must": { + "multi_match": { + "query": "test", + "fields": ["data", "data2"] + } + } + } + } + } + response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, + body=search_body) + self.assertEqual(response['hits']['total'], 10) + hits = response['hits']['hits'] + self.assertEqual(len(hits), 10) + @parameterized.expand( [ ( From fb4f6ad51717fe1e9c6252cd9249ef01fa853650 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20G=C3=A1lvez=20Mart=C3=ADnez?= Date: Thu, 21 Jan 2021 10:54:34 +0100 Subject: [PATCH 29/44] Add msearch --- elasticmock/fake_elasticsearch.py | 32 ++++++++++++++++++++ tests/fake_elasticsearch/test_search.py | 39 +++++++++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index 5926677..63cc9c0 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -438,6 +438,38 @@ def count(self, index=None, doc_type=None, body=None, params=None, headers=None) def _get_fake_query_condition(self, query_type_str, condition): return FakeQueryCondition(QueryType.get_query_type(query_type_str), condition) + @query_params( + "ccs_minimize_roundtrips", + "max_concurrent_searches", + "max_concurrent_shard_requests", + "pre_filter_shard_size", + "rest_total_hits_as_int", + "search_type", + "typed_keys", + ) + def msearch(self, body, index=None, doc_type=None, params=None, headers=None): + def grouped(iterable): + if len(iterable) % 2 != 0: + raise Exception('Malformed body') + iterator = iter(iterable) + while True: + try: + yield (next(iterator)['index'], next(iterator)) + except StopIteration: + break + + responses = [] + took = 0 + for ind, query in grouped(body): + response = self.search(index=ind, body=query) + took += response['took'] + responses.append(response) + result = { + 'took': took, + 'responses': responses + } + return result + @query_params('_source', '_source_exclude', '_source_include', 'allow_no_indices', 'analyze_wildcard', 'analyzer', 'default_operator', 'df', 'expand_wildcards', 'explain', 'fielddata_fields', 'fields', diff --git a/tests/fake_elasticsearch/test_search.py b/tests/fake_elasticsearch/test_search.py index 98aac2b..348f82e 100644 --- a/tests/fake_elasticsearch/test_search.py +++ b/tests/fake_elasticsearch/test_search.py @@ -189,6 +189,45 @@ def test_search_with_bool_query_and_multi_match(self): hits = response['hits']['hits'] self.assertEqual(len(hits), 10) + def test_msearch(self): + for i in range(0, 10): + self.es.index(index='index_for_search1', doc_type=DOC_TYPE, body={ + 'data': 'test_{0}'.format(i) if i % 2 == 0 else None, + 'data2': 'test_{0}'.format(i) if (i+1) % 2 == 0 else None + }) + for i in range(0, 10): + self.es.index(index='index_for_search2', doc_type=DOC_TYPE, body={ + 'data': 'test_{0}'.format(i) if i % 2 == 0 else None, + 'data2': 'test_{0}'.format(i) if (i+1) % 2 == 0 else None + }) + + search_body = { + "query": { + "bool": { + "must": { + "multi_match": { + "query": "test", + "fields": ["data", "data2"] + } + } + } + } + } + body = [] + body.append({'index': 'index_for_search1'}) + body.append(search_body) + body.append({'index': 'index_for_search2'}) + body.append(search_body) + + result = self.es.msearch(index='index_for_search', body=body) + response1, response2 = result['responses'] + self.assertEqual(response1['hits']['total'], 10) + hits1 = response1['hits']['hits'] + self.assertEqual(len(hits1), 10) + self.assertEqual(response2['hits']['total'], 10) + hits2 = response2['hits']['hits'] + self.assertEqual(len(hits2), 10) + @parameterized.expand( [ ( From 70ecaa6f6a6efd4b8fd6ee96e804f94725e73326 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20G=C3=A1lvez=20Mart=C3=ADnez?= Date: Mon, 1 Feb 2021 17:25:08 +0100 Subject: [PATCH 30/44] Add mget Improve get and delete methods --- .vscode/settings.json | 3 ++ elasticmock/fake_elasticsearch.py | 62 +++++++++++++++++++--------- tests/fake_elasticsearch/test_get.py | 8 ++++ 3 files changed, 53 insertions(+), 20 deletions(-) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..9efeb86 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python.pythonPath": "/home/Tiendeo/.envs/elasticmock/bin/python" +} \ No newline at end of file diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index 63cc9c0..14aa9f5 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -6,8 +6,9 @@ import dateutil.parser from elasticsearch import Elasticsearch -from elasticsearch.client.utils import query_params -from elasticsearch.exceptions import NotFoundError +from elasticsearch.client.utils import query_params, _normalize_hosts +from elasticsearch.transport import Transport +from elasticsearch.exceptions import NotFoundError, RequestError from elasticmock.behaviour.server_failure import server_failure from elasticmock.fake_cluster import FakeClusterClient @@ -246,6 +247,7 @@ class FakeElasticsearch(Elasticsearch): def __init__(self, hosts=None, transport_class=None, **kwargs): self.__documents_dict = {} self.__scrolls = {} + self.transport = Transport(_normalize_hosts(hosts), **kwargs) @property def indices(self): @@ -295,10 +297,10 @@ def index(self, index, body, doc_type='_doc', id=None, params=None, headers=None if id is None: id = get_random_id() - elif self.exists(index, doc_type, id, params=params): - doc = self.get(index, id, doc_type, params=params) + elif self.exists(index, id, doc_type=doc_type, params=params): + doc = self.get(index, id, doc_type=doc_type, params=params) version = doc['_version'] + 1 - self.delete(index, doc_type, id) + self.delete(index, id, doc_type=doc_type) self.__documents_dict[index].append({ '_type': doc_type, @@ -322,26 +324,23 @@ def bulk(self, body, index=None, doc_type=None, params=None, headers=None): version = 1 items = [] + ids = [] for line in body.splitlines(): if len(line.strip()) > 0: line = json.loads(line) if 'index' in line: - index = line['index']['_index'] - doc_type = line['index']['_type'] + index = line['index'].get('_index') or index + doc_type = line['index'].get('_type') \ + or doc_type or '_all' if index not in self.__documents_dict: self.__documents_dict[index] = list() + ids.append(line['index'].get('_id')) else: - document_id = get_random_id() + document_id = ids.pop() if ids else None - self.__documents_dict[index].append({ - '_type': doc_type, - '_id': document_id, - '_source': line, - '_index': index, - '_version': version - }) + self.index(index, line, doc_type=doc_type, id=document_id) items.append({'index': { '_type': doc_type, @@ -358,7 +357,7 @@ def bulk(self, body, index=None, doc_type=None, params=None, headers=None): } @query_params('parent', 'preference', 'realtime', 'refresh', 'routing') - def exists(self, index, doc_type, id, params=None, headers=None): + def exists(self, index, id, doc_type=None, params=None, headers=None): result = False if index in self.__documents_dict: for document in self.__documents_dict[index]: @@ -399,6 +398,26 @@ def get(self, index, id, doc_type='_all', params=None, headers=None): } raise NotFoundError(404, json.dumps(error_data)) + @query_params('_source', '_source_exclude', '_source_include', + 'preference', 'realtime', 'refresh', 'routing', + 'stored_fields') + def mget(self, body, index, doc_type='_all', params=None, headers=None): + ids = body.get('ids') + results = [] + for id in ids: + try: + results.append(self.get(index, id, doc_type=doc_type, + params=params, headers=headers)) + except: + pass + if not results: + raise RequestError( + 400, + 'action_request_validation_exception', + 'Validation Failed: 1: no documents to get;' + ) + return {'docs': results} + @query_params('_source', '_source_exclude', '_source_include', 'parent', 'preference', 'realtime', 'refresh', 'routing', 'version', 'version_type') @@ -574,17 +593,20 @@ def scroll(self, scroll_id, params=None, headers=None): @query_params('consistency', 'parent', 'refresh', 'replication', 'routing', 'timeout', 'version', 'version_type') - def delete(self, index, doc_type, id, params=None, headers=None): + def delete(self, index, id, doc_type=None, params=None, headers=None): found = False ignore = extract_ignore_as_iterable(params) if index in self.__documents_dict: for document in self.__documents_dict[index]: - if document.get('_type') == doc_type and document.get('_id') == id: + if document.get('_id') == id: found = True - self.__documents_dict[index].remove(document) - break + if doc_type and document.get('_type') != doc_type: + found = False + if found: + self.__documents_dict[index].remove(document) + break result_dict = { 'found': found, diff --git a/tests/fake_elasticsearch/test_get.py b/tests/fake_elasticsearch/test_get.py index 10220ec..d895370 100644 --- a/tests/fake_elasticsearch/test_get.py +++ b/tests/fake_elasticsearch/test_get.py @@ -60,3 +60,11 @@ def test_should_get_only_document_source_with_id(self): target_doc_source = self.es.get_source(index=INDEX_NAME, doc_type=DOC_TYPE, id=document_id) self.assertEqual(target_doc_source, BODY) + + def test_mget_get_several_documents_by_id(self): + ids = [] + for _ in range(0, 10): + data = self.es.index(index=INDEX_NAME, doc_type=DOC_TYPE, body=BODY) + ids.append(data.get('_id')) + results = self.es.mget(index=INDEX_NAME, body={'ids': ids}) + self.assertEqual(len(results['docs']), 10) From c66844b64e00caa03b615a83c7b2c1c0c05711d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20G=C3=A1lvez=20Mart=C3=ADnez?= Date: Mon, 1 Feb 2021 18:39:16 +0100 Subject: [PATCH 31/44] Add compatibility with old versions of elastic --- elasticmock/fake_elasticsearch.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index 14aa9f5..bb11b90 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -6,7 +6,8 @@ import dateutil.parser from elasticsearch import Elasticsearch -from elasticsearch.client.utils import query_params, _normalize_hosts +from elasticsearch.client.utils import query_params +from elasticsearch.client import _normalize_hosts from elasticsearch.transport import Transport from elasticsearch.exceptions import NotFoundError, RequestError From 462527ee4273057d5ca0efe5d5729bcd3d1aa796 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20G=C3=A1lvez=20Mart=C3=ADnez?= Date: Mon, 1 Feb 2021 19:41:43 +0100 Subject: [PATCH 32/44] Remove vscode settings --- .vscode/settings.json | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 9efeb86..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "python.pythonPath": "/home/Tiendeo/.envs/elasticmock/bin/python" -} \ No newline at end of file From 32b724ddc67f7f9ab5538a0f76f1ece4e1bfd2f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20G=C3=A1lvez=20Mart=C3=ADnez?= Date: Mon, 1 Feb 2021 19:47:10 +0100 Subject: [PATCH 33/44] add vstudio to gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index dd2017e..6e593e4 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,9 @@ bin/ share/ pyvenv.cfg +### Visual Studio ### +.vscode/ + ### Intellij ### # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 From ff6fda352392d9b118fe4948902a1802dec470b5 Mon Sep 17 00:00:00 2001 From: Fenimore Love Date: Tue, 2 Feb 2021 13:06:21 -0500 Subject: [PATCH 34/44] Add create and update to bulk API --- elasticmock/fake_elasticsearch.py | 83 ++++++++++++++++-------- tests/__init__.py | 1 + tests/fake_elasticsearch/test_bulk.py | 90 ++++++++++++++++++++++++++- 3 files changed, 146 insertions(+), 28 deletions(-) diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index 63cc9c0..3e29a7e 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -7,7 +7,7 @@ import dateutil.parser from elasticsearch import Elasticsearch from elasticsearch.client.utils import query_params -from elasticsearch.exceptions import NotFoundError +from elasticsearch.exceptions import NotFoundError, RequestError from elasticmock.behaviour.server_failure import server_failure from elasticmock.fake_cluster import FakeClusterClient @@ -321,42 +321,73 @@ def index(self, index, body, doc_type='_doc', id=None, params=None, headers=None def bulk(self, body, index=None, doc_type=None, params=None, headers=None): version = 1 items = [] + errors = False - for line in body.splitlines(): - if len(line.strip()) > 0: - line = json.loads(line) + for raw_line in body.splitlines(): + if len(raw_line.strip()) > 0: + line = json.loads(raw_line) - if 'index' in line: - index = line['index']['_index'] - doc_type = line['index']['_type'] + if any(action in line for action in ['index', 'create', 'update']): + action = next(iter(line.keys())) + + index = line[action]['_index'] + doc_type = line[action].get('_type', "_doc") # _type is deprecated in 7.x + + if action == 'updated' and not line[action].get("_id"): + raise RequestError(400, 'action_request_validation_exception', 'missing id') + + document_id = line[action].get('_id', get_random_id()) if index not in self.__documents_dict: self.__documents_dict[index] = list() else: - document_id = get_random_id() - - self.__documents_dict[index].append({ - '_type': doc_type, - '_id': document_id, - '_source': line, - '_index': index, - '_version': version - }) - - items.append({'index': { - '_type': doc_type, - '_id': document_id, - '_index': index, - '_version': version, - 'result': 'created', - 'status': 201 - }}) + if 'doc' in line and action == 'update': + source = line['doc'] + else: + source = line + status, result, error = self._validate_action(action, index, document_id, doc_type) + item = { + action: { + '_type': doc_type, + '_id': document_id, + '_index': index, + '_version': version, + 'status': status, + } + } + + if not error: + item[action]["result"] = result + self.__documents_dict[index].append({ + '_type': doc_type, + '_id': document_id, + '_source': source, + '_index': index, + '_version': version + }) + else: + errors = True + item[action]["error"] = result + + items.append(item) return { - 'errors': False, + 'errors': errors, 'items': items } + def _validate_action(self, action, index, document_id, doc_type): + if action in ['index', 'update'] and self.exists(index, id=document_id, doc_type=doc_type): + return 200, 'updated', False + if action == 'created' and self.exists(index, id=document_id, doc_type=doc_type): + return 409, 'version_conflict_engine_exception', True + elif action in ['index', 'create'] and not self.exists(index, id=document_id, doc_type=doc_type): + return 201, 'created', False + elif action == 'update' and not self.exists(index, id=document_id, doc_type=doc_type): + return 404, 'document_missing_exception', True + else: + raise NotImplementedError(f"{action} behaviour hasn't been implemented") + @query_params('parent', 'preference', 'realtime', 'refresh', 'routing') def exists(self, index, doc_type, id, params=None, headers=None): result = False diff --git a/tests/__init__.py b/tests/__init__.py index 487ef12..8eccaed 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -9,6 +9,7 @@ INDEX_NAME = 'test_index' DOC_TYPE = 'doc-Type' +DOC_ID = 'doc-id' BODY = { 'author': 'kimchy', 'text': 'Elasticsearch: cool. bonsai cool.', diff --git a/tests/fake_elasticsearch/test_bulk.py b/tests/fake_elasticsearch/test_bulk.py index 5ae0603..bd86c81 100644 --- a/tests/fake_elasticsearch/test_bulk.py +++ b/tests/fake_elasticsearch/test_bulk.py @@ -2,12 +2,12 @@ import json -from tests import TestElasticmock, INDEX_NAME, DOC_TYPE, BODY +from tests import TestElasticmock, INDEX_NAME, DOC_TYPE, BODY, DOC_ID class TestBulk(TestElasticmock): - def test_should_bulk_index_documents(self): + def test_should_bulk_index_documents_index_creates(self): action = {'index': {'_index': INDEX_NAME, '_type': DOC_TYPE}} action_json = json.dumps(action) body_json = json.dumps(BODY, default=str) @@ -27,8 +27,94 @@ def test_should_bulk_index_documents(self): for item in items: index = item.get('index') + self.assertEqual(DOC_TYPE, index.get('_type')) + self.assertEqual(INDEX_NAME, index.get('_index')) + self.assertEqual('created', index.get('result')) + self.assertEqual(201, index.get('status')) + + def test_should_bulk_index_documents_create_creates(self): + action = {'create': {'_index': INDEX_NAME, '_type': DOC_TYPE}} + action_json = json.dumps(action) + body_json = json.dumps(BODY, default=str) + num_of_documents = 10 + + lines = [] + for count in range(0, num_of_documents): + lines.append(action_json) + lines.append(body_json) + body = '\n'.join(lines) + + data = self.es.bulk(body=body) + items = data.get('items') + + self.assertFalse(data.get('errors')) + self.assertEqual(num_of_documents, len(items)) + for item in items: + index = item.get('create') self.assertEqual(DOC_TYPE, index.get('_type')) self.assertEqual(INDEX_NAME, index.get('_index')) self.assertEqual('created', index.get('result')) self.assertEqual(201, index.get('status')) + + def test_should_bulk_index_documents_index_updates(self): + action = {'index': {'_index': INDEX_NAME, '_id': DOC_ID, '_type': DOC_TYPE}} + action_json = json.dumps(action) + body_json = json.dumps(BODY, default=str) + num_of_documents = 10 + + lines = [] + for count in range(0, num_of_documents): + lines.append(action_json) + lines.append(body_json) + body = '\n'.join(lines) + + data = self.es.bulk(body=body) + items = data.get('items') + + self.assertFalse(data.get('errors')) + self.assertEqual(num_of_documents, len(items)) + + first_item = items.pop(0) + self.assertEqual(first_item["index"]["status"], 201) + self.assertEqual(first_item["index"]["result"], "created") + + for item in items: + index = item.get('index') + self.assertEqual(DOC_TYPE, index.get('_type')) + self.assertEqual(INDEX_NAME, index.get('_index')) + self.assertEqual('updated', index.get('result')) + self.assertEqual(200, index.get('status')) + + def test_should_bulk_index_documents_update_updates(self): + action = {'update': {'_index': INDEX_NAME, '_id': DOC_ID, '_type': DOC_TYPE}} + action_json = json.dumps(action) + create_action_json = json.dumps( + {'create': {'_index': INDEX_NAME, '_id': DOC_ID, '_type': DOC_TYPE}} + ) + body_json = json.dumps({'doc': BODY}, default=str) + num_of_documents = 4 + + lines = [create_action_json, json.dumps(BODY, default=str)] + for count in range(0, num_of_documents): + lines.append(action_json) + lines.append(body_json) + body = '\n'.join(lines) + + data = self.es.bulk(body=body) + items = data.get('items') + + self.assertFalse(data.get('errors')) + print(items) + self.assertEqual(num_of_documents + 1, len(items)) + + first_item = items.pop(0) + self.assertEqual(first_item["create"]["status"], 201) + self.assertEqual(first_item["create"]["result"], "created") + + for item in items: + index = item.get('update') + self.assertEqual(DOC_TYPE, index.get('_type')) + self.assertEqual(INDEX_NAME, index.get('_index')) + self.assertEqual('updated', index.get('result')) + self.assertEqual(200, index.get('status')) From 37b16f775a43a491d304f864c06c3446c7f38db9 Mon Sep 17 00:00:00 2001 From: Fenimore Love Date: Wed, 3 Feb 2021 16:46:17 -0500 Subject: [PATCH 35/44] WIP add delete --- elasticmock/fake_elasticsearch.py | 29 ++++++++--- tests/fake_elasticsearch/test_bulk.py | 74 ++++++++++++++++++++++----- 2 files changed, 85 insertions(+), 18 deletions(-) diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index 3e29a7e..7e89769 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -327,17 +327,28 @@ def bulk(self, body, index=None, doc_type=None, params=None, headers=None): if len(raw_line.strip()) > 0: line = json.loads(raw_line) - if any(action in line for action in ['index', 'create', 'update']): + if any(action in line for action in ['index', 'create', 'update', 'delete']): action = next(iter(line.keys())) index = line[action]['_index'] doc_type = line[action].get('_type', "_doc") # _type is deprecated in 7.x - if action == 'updated' and not line[action].get("_id"): + if action in ['delete', 'updated'] and not line[action].get("_id"): raise RequestError(400, 'action_request_validation_exception', 'missing id') document_id = line[action].get('_id', get_random_id()) + if action == 'delete' and self.exists(index, id=document_id, doc_type=doc_type): + self.delete(index, doc_type=doc_type, id=document_id) + items.append({action: { + '_type': doc_type, + '_id': document_id, + '_index': index, + '_version': version, + 'result': 'deleted', + 'status': 200, + }}) + if index not in self.__documents_dict: self.__documents_dict[index] = list() else: @@ -356,7 +367,7 @@ def bulk(self, body, index=None, doc_type=None, params=None, headers=None): } } - if not error: + if not error and action != 'delete': item[action]["result"] = result self.__documents_dict[index].append({ '_type': doc_type, @@ -368,9 +379,7 @@ def bulk(self, body, index=None, doc_type=None, params=None, headers=None): else: errors = True item[action]["error"] = result - items.append(item) - return { 'errors': errors, 'items': items @@ -379,12 +388,16 @@ def bulk(self, body, index=None, doc_type=None, params=None, headers=None): def _validate_action(self, action, index, document_id, doc_type): if action in ['index', 'update'] and self.exists(index, id=document_id, doc_type=doc_type): return 200, 'updated', False - if action == 'created' and self.exists(index, id=document_id, doc_type=doc_type): + if action == 'create' and self.exists(index, id=document_id, doc_type=doc_type): return 409, 'version_conflict_engine_exception', True elif action in ['index', 'create'] and not self.exists(index, id=document_id, doc_type=doc_type): return 201, 'created', False + elif action == "delete" and self.exists(index, id=document_id, doc_type=doc_type): + return 200, 'deleted', False elif action == 'update' and not self.exists(index, id=document_id, doc_type=doc_type): return 404, 'document_missing_exception', True + elif action == 'delete' and not self.exists(index, id=document_id, doc_type=doc_type): + return 404, 'not_found', True else: raise NotImplementedError(f"{action} behaviour hasn't been implemented") @@ -610,6 +623,7 @@ def delete(self, index, doc_type, id, params=None, headers=None): found = False ignore = extract_ignore_as_iterable(params) + print(self.__documents_dict['test_index']) if index in self.__documents_dict: for document in self.__documents_dict[index]: if document.get('_type') == doc_type and document.get('_id') == id: @@ -617,6 +631,9 @@ def delete(self, index, doc_type, id, params=None, headers=None): self.__documents_dict[index].remove(document) break + print("WTF") + print(found) + print(self.__documents_dict[index]) result_dict = { 'found': found, '_index': index, diff --git a/tests/fake_elasticsearch/test_bulk.py b/tests/fake_elasticsearch/test_bulk.py index bd86c81..c04a0f1 100644 --- a/tests/fake_elasticsearch/test_bulk.py +++ b/tests/fake_elasticsearch/test_bulk.py @@ -33,23 +33,31 @@ def test_should_bulk_index_documents_index_creates(self): self.assertEqual(201, index.get('status')) def test_should_bulk_index_documents_create_creates(self): - action = {'create': {'_index': INDEX_NAME, '_type': DOC_TYPE}} - action_json = json.dumps(action) - body_json = json.dumps(BODY, default=str) - num_of_documents = 10 - - lines = [] - for count in range(0, num_of_documents): - lines.append(action_json) - lines.append(body_json) - body = '\n'.join(lines) + create_action = {'create': {'_index': INDEX_NAME, '_type': DOC_TYPE}} + create_with_id = {'create': {'_index': INDEX_NAME, '_type': DOC_TYPE, '_id': DOC_ID}} + actions = [ + json.dumps(create_action), + json.dumps(BODY, default=str), + json.dumps(create_action), + json.dumps(BODY, default=str), + json.dumps(create_with_id), + json.dumps(BODY, default=str), + # Will fail on created document with the same ID + json.dumps(create_with_id), + json.dumps(BODY, default=str), + ] + body = '\n'.join(actions) data = self.es.bulk(body=body) + items = data.get('items') - self.assertFalse(data.get('errors')) - self.assertEqual(num_of_documents, len(items)) + self.assertTrue(data.get('errors')) + self.assertEqual(4, len(items)) + last_item = items.pop() + self.assertEqual(last_item['create']['error'], 'version_conflict_engine_exception') + self.assertEqual(last_item['create']['status'], 409) for item in items: index = item.get('create') self.assertEqual(DOC_TYPE, index.get('_type')) @@ -118,3 +126,45 @@ def test_should_bulk_index_documents_update_updates(self): self.assertEqual(INDEX_NAME, index.get('_index')) self.assertEqual('updated', index.get('result')) self.assertEqual(200, index.get('status')) + + def test_should_bulk_index_documents_mixed_actions(self): + doc_body = json.dumps(BODY, default=str) + + doc_id_1 = 1 + doc_id_2 = 2 + actions = [ + json.dumps({'create': {'_index': INDEX_NAME, '_type': DOC_TYPE, '_id': doc_id_1}}), + doc_body, # 201 + json.dumps({'create': {'_index': INDEX_NAME, '_type': DOC_TYPE, '_id': doc_id_1}}), + doc_body, # 409 + json.dumps({'index': {'_index': INDEX_NAME, '_type': DOC_TYPE, '_id': doc_id_2}}), + doc_body, # 201 + json.dumps({'index': {'_index': INDEX_NAME, '_type': DOC_TYPE, '_id': doc_id_2}}), + doc_body, # 200 + json.dumps({'update': {'_index': INDEX_NAME, '_type': DOC_TYPE, '_id': doc_id_1}}), + doc_body, # 200 + json.dumps({'delete': {'_index': INDEX_NAME, '_type': DOC_TYPE, '_id': doc_id_1}}), + json.dumps({'update': {'_index': INDEX_NAME, '_type': DOC_TYPE, '_id': doc_id_1}}), + doc_body, + json.dumps({'delete': {'_index': INDEX_NAME, '_type': DOC_TYPE, '_id': doc_id_1}}), + ] + body = '\n'.join(actions) + + data = self.es.bulk(body=body) + + expected = [ + {'create': {'_type': 'doc-Type', '_id': 1, '_index': 'test_index', '_version': 1, 'status': 201, 'result': 'created'}}, + {'create': {'_type': 'doc-Type', '_id': 1, '_index': 'test_index', '_version': 1, 'status': 409, 'error': 'version_conflict_engine_exception'}}, + {'index': {'_type': 'doc-Type', '_id': 2, '_index': 'test_index', '_version': 1, 'status': 201, 'result': 'created'}}, + {'index': {'_type': 'doc-Type', '_id': 2, '_index': 'test_index', '_version': 1, 'status': 200, 'result': 'updated'}}, + {'update': {'_type': 'doc-Type', '_id': 1, '_index': 'test_index', '_version': 1, 'status': 200, 'result': 'updated'}}, + {'delete': {'_type': 'doc-Type', '_id': 1, '_index': 'test_index', '_version': 1, 'result': 'deleted', 'status': 200}}, + {'update': {'_type': 'doc-Type', '_id': 1, '_index': 'test_index', '_version': 1, 'status': 200, 'result': 'updated'}}, + {'delete': {'_type': 'doc-Type', '_id': 1, '_index': 'test_index', '_version': 1, 'result': 'deleted', 'status': 200}}, + ] + + items = data.get('items') + + for item in items: + print(item) + pass From 229a3ebced880fa25248773568c618d97368ca7b Mon Sep 17 00:00:00 2001 From: Fenimore Love Date: Thu, 4 Feb 2021 10:54:09 -0500 Subject: [PATCH 36/44] Add delete --- elasticmock/fake_elasticsearch.py | 55 ++++++++++++--------- tests/fake_elasticsearch/test_bulk.py | 69 +++++++++++++++++++++------ 2 files changed, 87 insertions(+), 37 deletions(-) diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index 7e89769..202339a 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -319,7 +319,6 @@ def index(self, index, body, doc_type='_doc', id=None, params=None, headers=None @query_params('consistency', 'op_type', 'parent', 'refresh', 'replication', 'routing', 'timeout', 'timestamp', 'ttl', 'version', 'version_type') def bulk(self, body, index=None, doc_type=None, params=None, headers=None): - version = 1 items = [] errors = False @@ -330,6 +329,7 @@ def bulk(self, body, index=None, doc_type=None, params=None, headers=None): if any(action in line for action in ['index', 'create', 'update', 'delete']): action = next(iter(line.keys())) + version = 1 index = line[action]['_index'] doc_type = line[action].get('_type', "_doc") # _type is deprecated in 7.x @@ -338,16 +338,24 @@ def bulk(self, body, index=None, doc_type=None, params=None, headers=None): document_id = line[action].get('_id', get_random_id()) - if action == 'delete' and self.exists(index, id=document_id, doc_type=doc_type): - self.delete(index, doc_type=doc_type, id=document_id) - items.append({action: { + if action == 'delete': + status, result, error = self._validate_action( + action, index, document_id, doc_type, params=params + ) + item = {action: { '_type': doc_type, '_id': document_id, '_index': index, '_version': version, - 'result': 'deleted', - 'status': 200, - }}) + 'status': status, + }} + if error: + errors = True + item[action]["error"] = result + else: + self.delete(index, doc_type, document_id, params=params) + item[action]["result"] = result + items.append(item) if index not in self.__documents_dict: self.__documents_dict[index] = list() @@ -356,7 +364,9 @@ def bulk(self, body, index=None, doc_type=None, params=None, headers=None): source = line['doc'] else: source = line - status, result, error = self._validate_action(action, index, document_id, doc_type) + status, result, error = self._validate_action( + action, index, document_id, doc_type, params=params + ) item = { action: { '_type': doc_type, @@ -366,9 +376,13 @@ def bulk(self, body, index=None, doc_type=None, params=None, headers=None): 'status': status, } } - - if not error and action != 'delete': + if not error: item[action]["result"] = result + if self.exists(index, doc_type, document_id, params=params): + doc = self.get(index, document_id, doc_type, params=params) + version = doc['_version'] + 1 + self.delete(index, doc_type, document_id, params=params) + self.__documents_dict[index].append({ '_type': doc_type, '_id': document_id, @@ -385,18 +399,18 @@ def bulk(self, body, index=None, doc_type=None, params=None, headers=None): 'items': items } - def _validate_action(self, action, index, document_id, doc_type): - if action in ['index', 'update'] and self.exists(index, id=document_id, doc_type=doc_type): + def _validate_action(self, action, index, document_id, doc_type, params=None): + if action in ['index', 'update'] and self.exists(index, id=document_id, doc_type=doc_type, params=params): return 200, 'updated', False - if action == 'create' and self.exists(index, id=document_id, doc_type=doc_type): + if action == 'create' and self.exists(index, id=document_id, doc_type=doc_type, params=params): return 409, 'version_conflict_engine_exception', True - elif action in ['index', 'create'] and not self.exists(index, id=document_id, doc_type=doc_type): + elif action in ['index', 'create'] and not self.exists(index, id=document_id, doc_type=doc_type, params=params): return 201, 'created', False - elif action == "delete" and self.exists(index, id=document_id, doc_type=doc_type): + elif action == "delete" and self.exists(index, id=document_id, doc_type=doc_type, params=params): return 200, 'deleted', False - elif action == 'update' and not self.exists(index, id=document_id, doc_type=doc_type): + elif action == 'update' and not self.exists(index, id=document_id, doc_type=doc_type, params=params): return 404, 'document_missing_exception', True - elif action == 'delete' and not self.exists(index, id=document_id, doc_type=doc_type): + elif action == 'delete' and not self.exists(index, id=document_id, doc_type=doc_type, params=params): return 404, 'not_found', True else: raise NotImplementedError(f"{action} behaviour hasn't been implemented") @@ -623,7 +637,6 @@ def delete(self, index, doc_type, id, params=None, headers=None): found = False ignore = extract_ignore_as_iterable(params) - print(self.__documents_dict['test_index']) if index in self.__documents_dict: for document in self.__documents_dict[index]: if document.get('_type') == doc_type and document.get('_id') == id: @@ -631,9 +644,6 @@ def delete(self, index, doc_type, id, params=None, headers=None): self.__documents_dict[index].remove(document) break - print("WTF") - print(found) - print(self.__documents_dict[index]) result_dict = { 'found': found, '_index': index, @@ -641,13 +651,12 @@ def delete(self, index, doc_type, id, params=None, headers=None): '_id': id, '_version': 1, } - if found: return result_dict elif params and 404 in ignore: return {'found': False} else: - raise NotFoundError(404, json.dumps(result_dict)) + raise NotFoundError(404, json.dumps(result_dict, default=str)) @query_params('allow_no_indices', 'expand_wildcards', 'ignore_unavailable', 'preference', 'routing') diff --git a/tests/fake_elasticsearch/test_bulk.py b/tests/fake_elasticsearch/test_bulk.py index c04a0f1..f342224 100644 --- a/tests/fake_elasticsearch/test_bulk.py +++ b/tests/fake_elasticsearch/test_bulk.py @@ -113,7 +113,6 @@ def test_should_bulk_index_documents_update_updates(self): items = data.get('items') self.assertFalse(data.get('errors')) - print(items) self.assertEqual(num_of_documents + 1, len(items)) first_item = items.pop(0) @@ -127,6 +126,38 @@ def test_should_bulk_index_documents_update_updates(self): self.assertEqual('updated', index.get('result')) self.assertEqual(200, index.get('status')) + def test_should_bulk_index_documents_delete_deletes(self): + delete_action = {'delete': {'_index': INDEX_NAME, '_id': DOC_ID, '_type': DOC_TYPE}} + delete_action_json = json.dumps(delete_action) + create_action_json = json.dumps( + {'create': {'_index': INDEX_NAME, '_id': DOC_ID, '_type': DOC_TYPE}} + ) + + lines = [ + create_action_json, + json.dumps(BODY, default=str), + delete_action_json, + ] + body = '\n'.join(lines) + + data = self.es.bulk(body=body) + items = data.get('items') + + self.assertFalse(data.get('errors')) + self.assertEqual(2, len(items)) + + first_item = items.pop(0) + self.assertEqual(first_item["create"]["status"], 201) + self.assertEqual(first_item["create"]["result"], "created") + self.assertEqual(first_item["create"]['_type'], DOC_TYPE) + self.assertEqual(first_item["create"]['_id'], DOC_ID) + + second_item = items.pop(0) + self.assertEqual(second_item["delete"]["status"], 200) + self.assertEqual(second_item["delete"]["result"], "deleted") + self.assertEqual(second_item["delete"]['_type'], DOC_TYPE) + self.assertEqual(second_item["delete"]['_id'], DOC_ID) + def test_should_bulk_index_documents_mixed_actions(self): doc_body = json.dumps(BODY, default=str) @@ -144,27 +175,37 @@ def test_should_bulk_index_documents_mixed_actions(self): json.dumps({'update': {'_index': INDEX_NAME, '_type': DOC_TYPE, '_id': doc_id_1}}), doc_body, # 200 json.dumps({'delete': {'_index': INDEX_NAME, '_type': DOC_TYPE, '_id': doc_id_1}}), + # 200 json.dumps({'update': {'_index': INDEX_NAME, '_type': DOC_TYPE, '_id': doc_id_1}}), - doc_body, + doc_body, # 404 json.dumps({'delete': {'_index': INDEX_NAME, '_type': DOC_TYPE, '_id': doc_id_1}}), + # 404 ] body = '\n'.join(actions) data = self.es.bulk(body=body) expected = [ - {'create': {'_type': 'doc-Type', '_id': 1, '_index': 'test_index', '_version': 1, 'status': 201, 'result': 'created'}}, - {'create': {'_type': 'doc-Type', '_id': 1, '_index': 'test_index', '_version': 1, 'status': 409, 'error': 'version_conflict_engine_exception'}}, - {'index': {'_type': 'doc-Type', '_id': 2, '_index': 'test_index', '_version': 1, 'status': 201, 'result': 'created'}}, - {'index': {'_type': 'doc-Type', '_id': 2, '_index': 'test_index', '_version': 1, 'status': 200, 'result': 'updated'}}, - {'update': {'_type': 'doc-Type', '_id': 1, '_index': 'test_index', '_version': 1, 'status': 200, 'result': 'updated'}}, - {'delete': {'_type': 'doc-Type', '_id': 1, '_index': 'test_index', '_version': 1, 'result': 'deleted', 'status': 200}}, - {'update': {'_type': 'doc-Type', '_id': 1, '_index': 'test_index', '_version': 1, 'status': 200, 'result': 'updated'}}, - {'delete': {'_type': 'doc-Type', '_id': 1, '_index': 'test_index', '_version': 1, 'result': 'deleted', 'status': 200}}, + {'create': {'_type': DOC_TYPE, '_id': 1, '_index': INDEX_NAME, + '_version': 1, 'status': 201, 'result': 'created'}}, + {'create': {'_type': DOC_TYPE, '_id': 1, '_index': INDEX_NAME, + '_version': 1, 'status': 409, + 'error': 'version_conflict_engine_exception'}}, + {'index': {'_type': DOC_TYPE, '_id': 2, '_index': INDEX_NAME, + '_version': 1, 'status': 201, 'result': 'created'}}, + {'index': {'_type': DOC_TYPE, '_id': 2, '_index': INDEX_NAME, + '_version': 1, 'status': 200, 'result': 'updated'}}, + {'update': {'_type': DOC_TYPE, '_id': 1, '_index': INDEX_NAME, + '_version': 1, 'status': 200, 'result': 'updated'}}, + {'delete': {'_type': DOC_TYPE, '_id': 1, '_index': INDEX_NAME, + '_version': 1, 'result': 'deleted', 'status': 200}}, + {'update': {'_type': DOC_TYPE, '_id': 1, '_index': INDEX_NAME, + '_version': 1, 'status': 404, 'error': 'document_missing_exception'}}, + {'delete': {'_type': DOC_TYPE, '_id': 1, '_index': INDEX_NAME, + '_version': 1, 'error': 'not_found', 'status': 404}}, ] - items = data.get('items') + actual = data.get('items') - for item in items: - print(item) - pass + self.assertTrue(data.get('errors')) + self.assertEqual(actual, expected) From ab4d2f5a849416ba486ebbe7d1b125c2cfd553da Mon Sep 17 00:00:00 2001 From: Fenimore Love Date: Thu, 4 Feb 2021 14:49:20 -0500 Subject: [PATCH 37/44] Add should for bool search query --- elasticmock/fake_elasticsearch.py | 14 ++++++++++++++ tests/fake_elasticsearch/test_search.py | 21 +++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index 63cc9c0..97996c4 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -100,6 +100,8 @@ def _evaluate_for_query_type(self, document): return self._evaluate_for_compound_query_type(document) elif self.type == QueryType.MUST: return self._evaluate_for_compound_query_type(document) + elif self.type == QueryType.SHOULD: + return self._evaluate_for_should_query_type(document) elif self.type == QueryType.MULTI_MATCH: return self._evaluate_for_multi_match_query_type(document) else: @@ -206,6 +208,18 @@ def _evaluate_for_compound_query_type(self, document): return return_val + def _evaluate_for_should_query_type(self, document): + return_val = False + for sub_condition in self.condition: + for sub_condition_key in sub_condition: + return_val = FakeQueryCondition( + QueryType.get_query_type(sub_condition_key), + sub_condition[sub_condition_key] + ).evaluate(document) + if return_val: + return True + return return_val + def _evaluate_for_multi_match_query_type(self, document): return self._evaluate_for_fields(document) diff --git a/tests/fake_elasticsearch/test_search.py b/tests/fake_elasticsearch/test_search.py index 348f82e..a9137c7 100644 --- a/tests/fake_elasticsearch/test_search.py +++ b/tests/fake_elasticsearch/test_search.py @@ -189,6 +189,27 @@ def test_search_with_bool_query_and_multi_match(self): hits = response['hits']['hits'] self.assertEqual(len(hits), 10) + def test_search_bool_should_match_query(self): + for i in range(0, 10): + self.es.index(index='index_for_search', doc_type=DOC_TYPE, body={'data': 'test_{0}'.format(i)}) + + response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, + body={ + 'query': { + 'bool': { + 'should': [ + {'match': {'data': 'test_0'}}, + {'match': {'data': 'test_1'}}, + {'match': {'data': 'test_2'}}, + ] + } + } + }) + self.assertEqual(response['hits']['total'], 3) + hits = response['hits']['hits'] + self.assertEqual(len(hits), 3) + self.assertEqual(hits[0]['_source'], {'data': 'test_0'}) + def test_msearch(self): for i in range(0, 10): self.es.index(index='index_for_search1', doc_type=DOC_TYPE, body={ From 842f8e0eb2855671f7021b4e359b169f6918b6af Mon Sep 17 00:00:00 2001 From: Fenimore Love Date: Thu, 4 Feb 2021 14:56:42 -0500 Subject: [PATCH 38/44] Update Search API return result --- elasticmock/fake_elasticsearch.py | 2 +- tests/fake_elasticsearch/test_delete.py | 4 +-- tests/fake_elasticsearch/test_index.py | 4 +-- tests/fake_elasticsearch/test_scroll.py | 2 +- tests/fake_elasticsearch/test_search.py | 40 ++++++++++++------------- 5 files changed, 26 insertions(+), 26 deletions(-) diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index 63cc9c0..c0bf33c 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -510,7 +510,7 @@ def search(self, index=None, doc_type=None, body=None, params=None, headers=None result = { 'hits': { - 'total': len(matches), + 'total': {'value': len(matches), 'relation': 'eq'}, 'max_score': 1.0 }, '_shards': { diff --git a/tests/fake_elasticsearch/test_delete.py b/tests/fake_elasticsearch/test_delete.py index 6cfc7d9..14ec435 100644 --- a/tests/fake_elasticsearch/test_delete.py +++ b/tests/fake_elasticsearch/test_delete.py @@ -22,12 +22,12 @@ def test_should_not_raise_exception_when_delete_nonindexed_document_if_ignored_l def test_should_delete_indexed_document(self): doc_indexed = self.es.index(index=INDEX_NAME, doc_type=DOC_TYPE, body=BODY) search = self.es.search(index=INDEX_NAME) - self.assertEqual(1, search.get('hits').get('total')) + self.assertEqual(1, search.get('hits').get('total').get('value')) doc_id = doc_indexed.get('_id') doc_deleted = self.es.delete(index=INDEX_NAME, doc_type=DOC_TYPE, id=doc_id) search = self.es.search(index=INDEX_NAME) - self.assertEqual(0, search.get('hits').get('total')) + self.assertEqual(0, search.get('hits').get('total').get('value')) expected_doc_deleted = { 'found': True, diff --git a/tests/fake_elasticsearch/test_index.py b/tests/fake_elasticsearch/test_index.py index 5c8b88c..911559c 100644 --- a/tests/fake_elasticsearch/test_index.py +++ b/tests/fake_elasticsearch/test_index.py @@ -35,10 +35,10 @@ def test_doc_type_can_be_list(self): self.es.index(index=INDEX_NAME, doc_type=doc_type, body={}) result = self.es.search(doc_type=[doc_types[0]]) - self.assertEqual(count_per_doc_type, result.get('hits').get('total')) + self.assertEqual(count_per_doc_type, result.get('hits').get('total').get('value')) result = self.es.search(doc_type=doc_types[:2]) - self.assertEqual(count_per_doc_type * 2, result.get('hits').get('total')) + self.assertEqual(count_per_doc_type * 2, result.get('hits').get('total').get('value')) def test_update_existing_doc(self): data = self.es.index(index=INDEX_NAME, doc_type=DOC_TYPE, body=BODY) diff --git a/tests/fake_elasticsearch/test_scroll.py b/tests/fake_elasticsearch/test_scroll.py index 22d400b..1b90191 100644 --- a/tests/fake_elasticsearch/test_scroll.py +++ b/tests/fake_elasticsearch/test_scroll.py @@ -24,4 +24,4 @@ def __assert_scroll(self, result, expected_scroll_hits): self.assertNotEqual(None, result.get('_scroll_id', None)) self.assertEqual(expected_scroll_hits, len(hits.get('hits'))) - self.assertEqual(100, hits.get('total')) + self.assertEqual(100, hits.get('total').get('value')) diff --git a/tests/fake_elasticsearch/test_search.py b/tests/fake_elasticsearch/test_search.py index 348f82e..227c7e8 100644 --- a/tests/fake_elasticsearch/test_search.py +++ b/tests/fake_elasticsearch/test_search.py @@ -15,7 +15,7 @@ def test_should_raise_notfounderror_when_search_for_unexistent_index(self): def test_should_return_hits_hits_even_when_no_result(self): search = self.es.search() - self.assertEqual(0, search.get('hits').get('total')) + self.assertEqual(0, search.get('hits').get('total').get('value')) self.assertListEqual([], search.get('hits').get('hits')) def test_should_return_skipped_shards(self): @@ -28,7 +28,7 @@ def test_should_return_all_documents(self): self.es.index(index='index_{0}'.format(i), doc_type=DOC_TYPE, body={'data': 'test_{0}'.format(i)}) search = self.es.search() - self.assertEqual(index_quantity, search.get('hits').get('total')) + self.assertEqual(index_quantity, search.get('hits').get('total').get('value')) def test_should_return_all_documents_match_all(self): index_quantity = 10 @@ -36,7 +36,7 @@ def test_should_return_all_documents_match_all(self): self.es.index(index='index_{0}'.format(i), doc_type=DOC_TYPE, body={'data': 'test_{0}'.format(i)}) search = self.es.search(body={'query': {'match_all': {}}}) - self.assertEqual(index_quantity, search.get('hits').get('total')) + self.assertEqual(index_quantity, search.get('hits').get('total').get('value')) def test_should_return_only_indexed_documents_on_index(self): index_quantity = 2 @@ -44,7 +44,7 @@ def test_should_return_only_indexed_documents_on_index(self): self.es.index(index=INDEX_NAME, doc_type=DOC_TYPE, body={'data': 'test_{0}'.format(i)}) search = self.es.search(index=INDEX_NAME) - self.assertEqual(index_quantity, search.get('hits').get('total')) + self.assertEqual(index_quantity, search.get('hits').get('total').get('value')) def test_should_return_only_indexed_documents_on_index_with_doc_type(self): index_quantity = 2 @@ -53,7 +53,7 @@ def test_should_return_only_indexed_documents_on_index_with_doc_type(self): self.es.index(index=INDEX_NAME, doc_type='another-Doctype', body={'data': 'test'}) search = self.es.search(index=INDEX_NAME, doc_type=DOC_TYPE) - self.assertEqual(index_quantity, search.get('hits').get('total')) + self.assertEqual(index_quantity, search.get('hits').get('total').get('value')) def test_should_search_in_multiple_indexes(self): self.es.index(index='groups', doc_type='groups', body={'budget': 1000}) @@ -61,7 +61,7 @@ def test_should_search_in_multiple_indexes(self): self.es.index(index='pcs', doc_type='pcs', body={'model': 'macbook'}) result = self.es.search(index=['users', 'pcs']) - self.assertEqual(2, result.get('hits').get('total')) + self.assertEqual(2, result.get('hits').get('total').get('value')) def test_usage_of_aggregations(self): self.es.index(index='index', doc_type='document', body={'genre': 'rock'}) @@ -78,7 +78,7 @@ def test_search_with_scroll_param(self): result = self.es.search(index='groups', params={'scroll': '1m', 'size': 30}) self.assertNotEqual(None, result.get('_scroll_id', None)) self.assertEqual(30, len(result.get('hits').get('hits'))) - self.assertEqual(100, result.get('hits').get('total')) + self.assertEqual(100, result.get('hits').get('total').get('value')) def test_search_with_match_query(self): for i in range(0, 10): @@ -86,12 +86,12 @@ def test_search_with_match_query(self): response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, body={'query': {'match': {'data': 'TEST'}}}) - self.assertEqual(response['hits']['total'], 10) + self.assertEqual(response['hits']['total']['value'], 10) hits = response['hits']['hits'] self.assertEqual(len(hits), 10) response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, body={'query': {'match': {'data': '3'}}}) - self.assertEqual(response['hits']['total'], 1) + self.assertEqual(response['hits']['total']['value'], 1) hits = response['hits']['hits'] self.assertEqual(len(hits), 1) self.assertEqual(hits[0]['_source'], {'data': 'test_3'}) @@ -100,7 +100,7 @@ def test_search_with_match_query_in_int_list(self): for i in range(0, 10): self.es.index(index='index_for_search', doc_type=DOC_TYPE, body={'data': [i, 11, 13]}) response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, body={'query': {'match': {'data': 1}}}) - self.assertEqual(response['hits']['total'], 1) + self.assertEqual(response['hits']['total']['value'], 1) hits = response['hits']['hits'] self.assertEqual(len(hits), 1) self.assertEqual(hits[0]['_source'], {'data': [1, 11, 13]}) @@ -110,7 +110,7 @@ def test_search_with_match_query_in_string_list(self): self.es.index(index='index_for_search', doc_type=DOC_TYPE, body={'data': [str(i), 'two', 'three']}) response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, body={'query': {'match': {'data': '1'}}}) - self.assertEqual(response['hits']['total'], 1) + self.assertEqual(response['hits']['total']['value'], 1) hits = response['hits']['hits'] self.assertEqual(len(hits), 1) self.assertEqual(hits[0]['_source'], {'data': ['1', 'two', 'three']}) @@ -121,12 +121,12 @@ def test_search_with_term_query(self): response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, body={'query': {'term': {'data': 'TEST'}}}) - self.assertEqual(response['hits']['total'], 0) + self.assertEqual(response['hits']['total']['value'], 0) hits = response['hits']['hits'] self.assertEqual(len(hits), 0) response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, body={'query': {'term': {'data': '3'}}}) - self.assertEqual(response['hits']['total'], 1) + self.assertEqual(response['hits']['total']['value'], 1) hits = response['hits']['hits'] self.assertEqual(len(hits), 1) self.assertEqual(hits[0]['_source'], {'data': 'test_3'}) @@ -137,7 +137,7 @@ def test_search_with_bool_query(self): response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, body={'query': {'bool': {'filter': [{'term': {'id': 1}}]}}}) - self.assertEqual(response['hits']['total'], 1) + self.assertEqual(response['hits']['total']['value'], 1) hits = response['hits']['hits'] self.assertEqual(len(hits), 1) @@ -147,7 +147,7 @@ def test_search_with_terms_query(self): response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, body={'query': {'terms': {'id': [1, 2, 3]}}}) - self.assertEqual(response['hits']['total'], 3) + self.assertEqual(response['hits']['total']['value'], 3) hits = response['hits']['hits'] self.assertEqual(len(hits), 3) @@ -159,7 +159,7 @@ def test_query_on_nested_data(self): for term, value, i in [('data.x', 1, 1), ('data.y', 'yes', 0)]: response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, body={'query': {'term': {term: value}}}) - self.assertEqual(1, response['hits']['total']) + self.assertEqual(1, response['hits']['total']['value']) doc = response['hits']['hits'][0]['_source'] self.assertEqual(i, doc['id']) @@ -185,7 +185,7 @@ def test_search_with_bool_query_and_multi_match(self): } response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, body=search_body) - self.assertEqual(response['hits']['total'], 10) + self.assertEqual(response['hits']['total']['value'], 10) hits = response['hits']['hits'] self.assertEqual(len(hits), 10) @@ -221,10 +221,10 @@ def test_msearch(self): result = self.es.msearch(index='index_for_search', body=body) response1, response2 = result['responses'] - self.assertEqual(response1['hits']['total'], 10) + self.assertEqual(response1['hits']['total']['value'], 10) hits1 = response1['hits']['hits'] self.assertEqual(len(hits1), 10) - self.assertEqual(response2['hits']['total'], 10) + self.assertEqual(response2['hits']['total']['value'], 10) hits2 = response2['hits']['hits'] self.assertEqual(len(hits2), 10) @@ -302,7 +302,7 @@ def test_search_with_range_query(self, _, query_range, expected_ids): body={'query': {'range': query_range}}, ) - self.assertEqual(len(expected_ids), response['hits']['total']) + self.assertEqual(len(expected_ids), response['hits']['total']['value']) hits = response['hits']['hits'] self.assertEqual(set(expected_ids), set(hit['_source']['id'] for hit in hits)) From 02fab89c75853f9d3f45f68cd2908c06172d7ec7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20G=C3=A1lvez=20Mart=C3=ADnez?= Date: Mon, 8 Feb 2021 13:51:23 +0100 Subject: [PATCH 39/44] Remove boosting for fields --- elasticmock/fake_elasticsearch.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index bb11b90..b7686a6 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -216,6 +216,8 @@ def _compare_value_for_field(self, doc_source, field, value, ignore_case): value = value.lower() doc_val = doc_source + # Remove boosting + field, *_ = field.split("*") for k in field.split("."): if hasattr(doc_val, k): doc_val = getattr(doc_val, k) From 49cfa8d0d624c51f1dd2943f7aa4f8842e26347a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20G=C3=A1lvez=20Mart=C3=ADnez?= Date: Wed, 24 Feb 2021 14:04:57 +0100 Subject: [PATCH 40/44] Fix multi-value fields --- elasticmock/fake_elasticsearch.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index b7686a6..a820d9a 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -221,8 +221,10 @@ def _compare_value_for_field(self, doc_source, field, value, ignore_case): for k in field.split("."): if hasattr(doc_val, k): doc_val = getattr(doc_val, k) + break elif k in doc_val: doc_val = doc_val[k] + break else: return False @@ -237,7 +239,7 @@ def _compare_value_for_field(self, doc_source, field, value, ignore_case): if value == val: return True - if isinstance(val, str) and value in val: + if isinstance(val, str) and str(value) in val: return True return False From c8aa31d35feedc29806e949b3c0792b6815d9ffb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20G=C3=A1lvez=20Mart=C3=ADnez?= Date: Tue, 2 Mar 2021 12:26:55 +0100 Subject: [PATCH 41/44] Allows in the bulk method to indicate the index only in the signature --- elasticmock/fake_elasticsearch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index bf9ea31..6b580ff 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -352,7 +352,7 @@ def bulk(self, body, index=None, doc_type=None, params=None, headers=None): action = next(iter(line.keys())) version = 1 - index = line[action]['_index'] + index = line[action].get('_index') or index doc_type = line[action].get('_type', "_doc") # _type is deprecated in 7.x if action in ['delete', 'update'] and not line[action].get("_id"): From efa6b1b05e7ecb9c588a514a833b5a53539786f6 Mon Sep 17 00:00:00 2001 From: Marcos Cardoso Date: Fri, 5 Mar 2021 23:22:11 -0300 Subject: [PATCH 42/44] Bump to 1.8.0 --- Makefile | 2 +- README.md | 7 +++++++ setup.py | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 89db61b..e4fd00f 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -ELASTICMOCK_VERSION='1.7.0' +ELASTICMOCK_VERSION='1.8.0' install: pip3 install -r requirements.txt diff --git a/README.md b/README.md index dc18763..cf064e9 100644 --- a/README.md +++ b/README.md @@ -177,6 +177,13 @@ python setup.py test ## Changelog +#### 1.8.0: +- [Add multi_match](https://github.com/vrcmarcos/elasticmock/pull/63) (Thanks [@carlosgalvez-tiendeo](https://github.com/carlosgalvez-tiendeo)) +- [Add mget](https://github.com/vrcmarcos/elasticmock/pull/64) (Thanks [@carlosgalvez-tiendeo](https://github.com/carlosgalvez-tiendeo)) +- [Add create, update, and delete to bulk API](https://github.com/vrcmarcos/elasticmock/pull/65) (Thanks [@fenimore](https://github.com/fenimore)) +- [Add Should to bool Query](https://github.com/vrcmarcos/elasticmock/pull/67) (Thanks [@fenimore](https://github.com/fenimore)) +- [Update Search API return result](https://github.com/vrcmarcos/elasticmock/pull/68) (Thanks [@fenimore](https://github.com/fenimore)) + #### 1.7.0: - [Add shards skipped to search and count](https://github.com/vrcmarcos/elasticmock/pull/56) (Thanks [@philtweir](https://github.com/philtweir)) - [Allow 'match_all' queries in FakeSearch](https://github.com/vrcmarcos/elasticmock/pull/54) (Thanks [@jankislinger](https://github.com/jankislinger)) diff --git a/setup.py b/setup.py index 4887e03..b792023 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ import setuptools -__version__ = '1.7.0' +__version__ = '1.8.0' # read the contents of your readme file from os import path From daf0142ee5b0b019d8f655f9bfb4bd37decf455d Mon Sep 17 00:00:00 2001 From: Balazs Lombosi Date: Mon, 22 Mar 2021 11:50:07 +0100 Subject: [PATCH 43/44] testing elasticsearch, new query types, sorting, unit tests, --- elasticmock/fake_elasticsearch.py | 88 +++++++++++++++++-------- tests/fake_elasticsearch/test_search.py | 19 ++++++ tests/fake_elasticsearch/test_sort.py | 33 ++++++++++ 3 files changed, 114 insertions(+), 26 deletions(-) create mode 100644 tests/fake_elasticsearch/test_sort.py diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index 6b580ff..30fc8e8 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -15,7 +15,7 @@ from elasticmock.fake_cluster import FakeClusterClient from elasticmock.fake_indices import FakeIndicesClient from elasticmock.utilities import (extract_ignore_as_iterable, get_random_id, - get_random_scroll_id) + get_random_scroll_id) from elasticmock.utilities.decorator import for_all_methods PY3 = sys.version_info[0] == 3 @@ -35,6 +35,9 @@ class QueryType: SHOULD = 'SHOULD' MINIMUM_SHOULD_MATCH = 'MINIMUM_SHOULD_MATCH' MULTI_MATCH = 'MULTI_MATCH' + MUST_NOT = 'MUST_NOT' + EXISTS = 'EXISTS' + FIELD = 'FIELD' @staticmethod def get_query_type(type_str): @@ -60,6 +63,12 @@ def get_query_type(type_str): return QueryType.MINIMUM_SHOULD_MATCH elif type_str == 'multi_match': return QueryType.MULTI_MATCH + elif type_str == 'must_not': + return QueryType.MUST_NOT + elif type_str == 'exists': + return QueryType.EXISTS + elif type_str == 'field': + return QueryType.FIELD else: raise NotImplementedError(f'type {type_str} is not implemented for QueryType') @@ -107,9 +116,27 @@ def _evaluate_for_query_type(self, document): return self._evaluate_for_should_query_type(document) elif self.type == QueryType.MULTI_MATCH: return self._evaluate_for_multi_match_query_type(document) + elif self.type == QueryType.MUST_NOT: + return self._evaluate_for_must_not_query_type(document) + elif self.type == QueryType.EXISTS: + return self._evaluate_for_exists_query_type(document) + elif self.type == QueryType.FIELD: + return self._evaluate_for_compound_query_type(document) + elif self.type == QueryType.MINIMUM_SHOULD_MATCH: + return True else: raise NotImplementedError('Fake query evaluation not implemented for query type: %s' % self.type) + def _evaluate_for_must_not_query_type(self, document): + return not self._evaluate_for_compound_query_type(document) + + def _evaluate_for_exists_query_type(self, document): + doc_source = document['_source'] + return_val = False + for field, value in self.condition.items(): + return_val = value in doc_source.keys() and doc_source[value] is not None + return return_val + def _evaluate_for_match_query_type(self, document): return self._evaluate_for_field(document, True) @@ -170,6 +197,9 @@ def _evaluate_for_range_query_type(self, document): if isinstance(doc_val, list): return False + if doc_val != 0 and not doc_val: + return True + for sign, value in comparisons.items(): if isinstance(doc_val, datetime.datetime): value = dateutil.parser.isoparse(value) @@ -488,7 +518,7 @@ def mget(self, body, index, doc_type='_all', params=None, headers=None): for id in ids: try: results.append(self.get(index, id, doc_type=doc_type, - params=params, headers=headers)) + params=params, headers=headers)) except: pass if not results: @@ -517,20 +547,11 @@ def get_source(self, index, doc_type, id, params=None, headers=None): def count(self, index=None, doc_type=None, body=None, params=None, headers=None): searchable_indexes = self._normalize_index_to_list(index) - i = 0 - for searchable_index in searchable_indexes: - for document in self.__documents_dict[searchable_index]: - if doc_type and document.get('_type') != doc_type: - continue - i += 1 + contents = self.search(index=index, doc_type=doc_type, body=body, params=params, headers=headers) + result = { - 'count': i, - '_shards': { - 'successful': 1, - 'skipped': 0, - 'failed': 0, - 'total': 1 - } + 'count': contents['hits']['hits'].__len__(), + '_shards': contents['_shards'] } return result @@ -643,6 +664,14 @@ def search(self, index=None, doc_type=None, body=None, params=None, headers=None if aggregations: result['aggregations'] = aggregations + if body is not None and 'sort' in body: + for key, value in body['sort'][0].items(): + if body['sort'][0][key]['order'] == 'desc': + hits = sorted(hits, key=lambda k: k['_source'][key], reverse=True) + else: + hits = sorted(hits, key=lambda k: k['_source'][key]) + + if 'scroll' in params: result['_scroll_id'] = str(get_random_scroll_id()) params['size'] = int(params.get('size', 10)) @@ -774,18 +803,19 @@ def make_bucket(bucket_key, bucket): "doc_count": len(bucket), } - for metric_key, metric_definition in aggregation["aggs"].items(): - metric_type_str = list(metric_definition)[0] - metric_type = MetricType.get_metric_type(metric_type_str) - attr = metric_definition[metric_type_str]["field"] - data = [doc[attr] for doc in bucket] + if "aggs" in aggregation.keys(): + for metric_key, metric_definition in aggregation["aggs"].items(): + metric_type_str = list(metric_definition)[0] + metric_type = MetricType.get_metric_type(metric_type_str) + attr = metric_definition[metric_type_str]["field"] + data = [doc[attr] for doc in bucket] - if metric_type == MetricType.CARDINALITY: - value = len(set(data)) - else: - raise NotImplementedError(f"Metric type '{metric_type}' not implemented") + if metric_type == MetricType.CARDINALITY: + value = len(set(data)) + else: + raise NotImplementedError(f"Metric type '{metric_type}' not implemented") - out[metric_key] = {"value": value} + out[metric_key] = {"value": value} return out agg_sources = aggregation["composite"]["sources"] @@ -793,7 +823,13 @@ def make_bucket(bucket_key, bucket): bucket_key_fields = [list(src)[0] for src in agg_sources] for document in documents: doc_src = document["_source"] - key = tuple(make_key(doc_src, agg_src) for agg_src in aggregation["composite"]["sources"]) + key = () + for agg_src in aggregation["composite"]["sources"]: + k=make_key(doc_src, agg_src) + if isinstance(k, list): + key += tuple(k) + else: + key += tuple([k]) buckets[key].append(doc_src) buckets = sorted(((k, v) for k, v in buckets.items()), key=lambda x: x[0]) diff --git a/tests/fake_elasticsearch/test_search.py b/tests/fake_elasticsearch/test_search.py index 7b71070..c7d123b 100644 --- a/tests/fake_elasticsearch/test_search.py +++ b/tests/fake_elasticsearch/test_search.py @@ -210,6 +210,25 @@ def test_search_bool_should_match_query(self): self.assertEqual(len(hits), 3) self.assertEqual(hits[0]['_source'], {'data': 'test_0'}) + def test_search_bool_must_not_match_query(self): + for i in range(0, 10): + self.es.index(index='index_for_search', doc_type=DOC_TYPE, body={'data': 'test_{0}'.format(i)}) + + response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, + body={ + 'query': { + 'bool': { + 'must_not': [ + {'match': {'data': 'test_0'}}, + ] + } + } + }) + self.assertEqual(response['hits']['total']['value'], 9) + hits = response['hits']['hits'] + self.assertEqual(len(hits), 9) + self.assertEqual(hits[0]['_source'], {'data': 'test_1'}) + def test_msearch(self): for i in range(0, 10): self.es.index(index='index_for_search1', doc_type=DOC_TYPE, body={ diff --git a/tests/fake_elasticsearch/test_sort.py b/tests/fake_elasticsearch/test_sort.py new file mode 100644 index 0000000..099217c --- /dev/null +++ b/tests/fake_elasticsearch/test_sort.py @@ -0,0 +1,33 @@ +from tests import TestElasticmock, INDEX_NAME, DOC_TYPE + + +class TestSearch(TestElasticmock): + + def test_sort_by_field_asc(self): + index_quantity = 10 + result = [] + for i in range(0, index_quantity): + body = {'data': 'test_{0}'.format(i), 'sort_param':'{0}'.format(i)} + result.append(body) + self.es.index(index=INDEX_NAME, doc_type=DOC_TYPE, body=body) + + search = self.es.search(body={'query': {'match_all': {}}, + 'sort': [{ "sort_param" : {"order" : "asc"}}] + }) + search_result = [hit.get('_source') for hit in search.get('hits').get('hits')] + self.assertListEqual(result, search_result) + + def test_sort_by_field_desc(self): + index_quantity = 10 + result = [] + for i in range(0, index_quantity): + body = {'data': 'test_{0}'.format(i), 'sort_param':'{0}'.format(i)} + result.append(body) + self.es.index(index=INDEX_NAME, doc_type=DOC_TYPE, body=body) + + search = self.es.search(body={'query': {'match_all': {}}, + 'sort': [{ "sort_param" : {"order" : "desc"}}] + }) + search_result = [hit.get('_source') for hit in search.get('hits').get('hits')] + result.reverse() + self.assertListEqual(result, search_result) From 4bde09aaf20e8062272ae545b63bb32748776ce0 Mon Sep 17 00:00:00 2001 From: Balazs Lombosi Date: Wed, 24 Mar 2021 11:16:13 +0100 Subject: [PATCH 44/44] HSS-191954 fix makefile, setup version --- Makefile | 16 ++++++++-------- setup.py | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index e4fd00f..005d06e 100644 --- a/Makefile +++ b/Makefile @@ -1,30 +1,30 @@ -ELASTICMOCK_VERSION='1.8.0' +ELASTICMOCK_VERSION='2.2.0' install: - pip3 install -r requirements.txt + pip install -r requirements.txt test_install: install - pip3 install -r requirements_test.txt + pip install -r requirements_test.txt test: test_install - python3.8 setup.py test + python3 setup.py test upload: create_dist - pip3 install twine + pip install twine twine upload dist/* git push create_dist: create_dist_no_commit update_pip rm -rf dist - python3.8 setup.py sdist + python3 setup.py sdist create_dist_no_commit: update_pip rm -rf dist - python3.8 setup.py sdist + python3 setup.py sdist create_dist_commit: git commit --all -m "Bump version ${ELASTICMOCK_VERSION}" git tag ${ELASTICMOCK_VERSION} update_pip: - pip3 install --upgrade pip + pip install --upgrade pip diff --git a/setup.py b/setup.py index b792023..b9211cc 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ import setuptools -__version__ = '1.8.0' +__version__ = '2.2.0' # read the contents of your readme file from os import path