From e574aa4a829c7792d8d0b0ef704565ba212d042e Mon Sep 17 00:00:00 2001 From: grgilad Date: Fri, 3 Sep 2021 17:08:14 +0300 Subject: [PATCH 01/16] Relevant changes for integration with monolith-django --- elasticmock/fake_elasticsearch.py | 50 +++++++++++++++---------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index 1372952..1637b40 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -5,17 +5,15 @@ from collections import defaultdict import dateutil.parser -from elasticsearch import Elasticsearch + from elasticsearch.client.utils import query_params -from elasticsearch.client import _normalize_hosts from elasticsearch.transport import Transport from elasticsearch.exceptions import NotFoundError, RequestError from elasticmock.behaviour.server_failure import server_failure from elasticmock.fake_cluster import FakeClusterClient from elasticmock.fake_indices import FakeIndicesClient -from elasticmock.utilities import (extract_ignore_as_iterable, get_random_id, - get_random_scroll_id) +from elasticmock.utilities import (extract_ignore_as_iterable, get_random_id, get_random_scroll_id) from elasticmock.utilities.decorator import for_all_methods PY3 = sys.version_info[0] == 3 @@ -286,13 +284,13 @@ def _compare_value_for_field(self, doc_source, field, value, ignore_case): @for_all_methods([server_failure]) -class FakeElasticsearch(Elasticsearch): +class FakeElasticsearch(): __documents_dict = None - def __init__(self, hosts=None, transport_class=None, **kwargs): + def __init__(self, hosts=None, **kwargs): self.__documents_dict = {} self.__scrolls = {} - self.transport = Transport(_normalize_hosts(hosts), **kwargs) + self.transport = Transport(hosts, **kwargs) @property def indices(self): @@ -604,7 +602,10 @@ def grouped(iterable): 'suggest_size', 'suggest_text', 'terminate_after', 'timeout', 'track_scores', 'version') def search(self, index=None, doc_type=None, body=None, params=None, headers=None): - searchable_indexes = self._normalize_index_to_list(index) + if params and params.get('ignore_unavailable'): + searchable_indexes = self._normalize_index_to_list(index, True) + else: + searchable_indexes = self._normalize_index_to_list(index) matches = [] conditions = [] @@ -614,21 +615,20 @@ def search(self, index=None, doc_type=None, body=None, params=None, headers=None for query_type_str, condition in query.items(): conditions.append(self._get_fake_query_condition(query_type_str, condition)) for searchable_index in searchable_indexes: - - for document in self.__documents_dict[searchable_index]: - - if doc_type: - if isinstance(doc_type, list) and document.get('_type') not in doc_type: - continue - if isinstance(doc_type, str) and document.get('_type') != doc_type: - continue - if conditions: - for condition in conditions: - if condition.evaluate(document): - matches.append(document) - break - else: - matches.append(document) + if self.__documents_dict and self.__documents_dict.get(searchable_index): + for document in self.__documents_dict[searchable_index]: + if doc_type: + if isinstance(doc_type, list) and document.get('_type') not in doc_type: + continue + if isinstance(doc_type, str) and document.get('_type') != doc_type: + continue + if conditions: + for condition in conditions: + if condition.evaluate(document): + matches.append(document) + break + else: + matches.append(document) for match in matches: self._find_and_convert_data_types(match['_source']) @@ -755,7 +755,7 @@ def suggest(self, body, index=None, params=None, headers=None): ] return result_dict - def _normalize_index_to_list(self, index): + def _normalize_index_to_list(self, index, ignore_unavailable: bool = False): # Ensure to have a list of index if index is None: searchable_indexes = self.__documents_dict.keys() @@ -769,7 +769,7 @@ def _normalize_index_to_list(self, index): # Check index(es) exists for searchable_index in searchable_indexes: - if searchable_index not in self.__documents_dict: + if searchable_index not in self.__documents_dict and not ignore_unavailable: raise NotFoundError(404, 'IndexMissingException[[{0}] missing]'.format(searchable_index)) return searchable_indexes From 7c29d20618c9e5301a116ca0582b99def1f59e1c Mon Sep 17 00:00:00 2001 From: grgilad Date: Fri, 3 Sep 2021 18:22:05 +0300 Subject: [PATCH 02/16] Fix versioning --- Makefile | 2 +- setup.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 3d3a8f6..11fcd10 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -ELASTICMOCK_VERSION='1.8.0' +ELASTICMOCK_VERSION='1.8.1' install: pip3 install -r requirements.txt diff --git a/setup.py b/setup.py index 4488395..f953db8 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ import setuptools -__version__ = '1.8.0' +__version__ = '1.8.1' # read the contents of your readme file from os import path @@ -18,7 +18,7 @@ description='Python Elasticsearch Mock for test purposes', long_description=long_description, long_description_content_type='text/markdown', - url='https://github.com/vrcmarcos/elasticmock', + url='https://github.com/monte-carlo-data/elasticmock', packages=setuptools.find_packages(exclude=('tests')), install_requires=[ 'elasticsearch', From 2b5fbd66d818ef06c6be2cfdc54cf6d0923fe2b5 Mon Sep 17 00:00:00 2001 From: grgilad Date: Thu, 9 Sep 2021 13:56:14 +0300 Subject: [PATCH 03/16] Added support for data_histogram in composite fields + test --- Makefile | 2 +- elasticmock/fake_elasticsearch.py | 6 +++- setup.py | 2 +- tests/fake_elasticsearch/test_search.py | 43 ++++++++++++++++++++++++- 4 files changed, 49 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 11fcd10..7d4852c 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -ELASTICMOCK_VERSION='1.8.1' +ELASTICMOCK_VERSION='1.8.2' install: pip3 install -r requirements.txt diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index 1637b40..9a3a7f1 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -790,7 +790,11 @@ def make_aggregation_buckets(self, aggregation, documents): def make_composite_aggregation_buckets(self, aggregation, documents): def make_key(doc_source, agg_source): - attr = list(agg_source.values())[0]["terms"]["field"] + agg_lst = list(agg_source.values())[0] + if agg_lst.get("terms"): + attr = agg_lst["terms"]["field"] + elif agg_lst.get("date_histogram"): + attr = agg_lst["date_histogram"]["field"] return doc_source[attr] def make_bucket(bucket_key, bucket): diff --git a/setup.py b/setup.py index f953db8..5612cfb 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ import setuptools -__version__ = '1.8.1' +__version__ = '1.8.2' # read the contents of your readme file from os import path diff --git a/tests/fake_elasticsearch/test_search.py b/tests/fake_elasticsearch/test_search.py index f31c64b..8659051 100644 --- a/tests/fake_elasticsearch/test_search.py +++ b/tests/fake_elasticsearch/test_search.py @@ -339,7 +339,7 @@ def test_search_with_range_query(self, _, query_range, expected_ids): hits = response['hits']['hits'] self.assertEqual(set(expected_ids), set(hit['_source']['id'] for hit in hits)) - def test_bucket_aggregation(self): + def test_bucket_aggregation_terms(self): data = [ {"data_x": 1, "data_y": "a"}, {"data_x": 1, "data_y": "a"}, @@ -379,3 +379,44 @@ def test_bucket_aggregation(self): for x, y in zip(expected, actual): self.assertDictEqual(x["key"], y["key"]) self.assertEqual(x["doc_count"], y["doc_count"]) + + def test_bucket_aggregation_date_histogram(self): + start_date = datetime.datetime(2021, 12, 1, 15) + data = [ + {"data_x": 1, "data_y": "a", "timestamp": start_date}, + {"data_x": 1, "data_y": "a", "timestamp": start_date}, + {"data_x": 1, "data_y": "a", "timestamp": start_date - datetime.timedelta(hours=1)}, + {"data_x": 1, "data_y": "b", "timestamp": start_date - datetime.timedelta(hours=1)}, + {"data_x": 1, "data_y": "b", "timestamp": start_date - datetime.timedelta(hours=1)}, + ] + for body in data: + self.es.index(index='index_for_search', doc_type=DOC_TYPE, body=body) + + response = self.es.search( + index="index_for_search", + doc_type=DOC_TYPE, + body={ + "query": {"match_all": {}}, + "aggs": { + "stats": { + "composite": { + "sources": [{"histo": {"date_histogram": {"field": "timestamp"}}}], + "size": 10000, + }, + "aggs": { + "distinct_data_y": {"cardinality": {"field": "data_y"}} + }, + } + }, + }, + ) + + expected = [ + {"key": {"histo": '2021-12-01T14:00:00'}, "doc_count": 3}, + {"key": {"histo": '2021-12-01T15:00:00'}, "doc_count": 2}, + ] + actual = response["aggregations"]["stats"]["buckets"] + print(actual) + for x, y in zip(expected, actual): + self.assertDictEqual(x["key"], y["key"]) + self.assertEqual(x["doc_count"], y["doc_count"]) From 1b48e1a337a36beaa73f0f79976dd536c13e7935 Mon Sep 17 00:00:00 2001 From: grgilad Date: Thu, 9 Sep 2021 14:28:22 +0300 Subject: [PATCH 04/16] Added check for no other aggs apart from date_histogram + test --- Makefile | 2 +- elasticmock/fake_elasticsearch.py | 23 ++++++++------- setup.py | 2 +- tests/fake_elasticsearch/test_search.py | 38 +++++++++++++++++++++++++ 4 files changed, 52 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 7d4852c..ce5ec1d 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -ELASTICMOCK_VERSION='1.8.2' +ELASTICMOCK_VERSION='1.8.3' install: pip3 install -r requirements.txt diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index 9a3a7f1..725b9e7 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -803,18 +803,19 @@ def make_bucket(bucket_key, bucket): "doc_count": len(bucket), } - for metric_key, metric_definition in aggregation["aggs"].items(): - metric_type_str = list(metric_definition)[0] - metric_type = MetricType.get_metric_type(metric_type_str) - attr = metric_definition[metric_type_str]["field"] - data = [doc[attr] for doc in bucket] - - if metric_type == MetricType.CARDINALITY: - value = len(set(data)) - else: - raise NotImplementedError(f"Metric type '{metric_type}' not implemented") + if aggregation.get("aggs"): + for metric_key, metric_definition in aggregation["aggs"].items(): + metric_type_str = list(metric_definition)[0] + metric_type = MetricType.get_metric_type(metric_type_str) + attr = metric_definition[metric_type_str]["field"] + data = [doc[attr] for doc in bucket] + + if metric_type == MetricType.CARDINALITY: + value = len(set(data)) + else: + raise NotImplementedError(f"Metric type '{metric_type}' not implemented") - out[metric_key] = {"value": value} + out[metric_key] = {"value": value} return out agg_sources = aggregation["composite"]["sources"] diff --git a/setup.py b/setup.py index 5612cfb..8d19d9e 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ import setuptools -__version__ = '1.8.2' +__version__ = '1.8.3' # read the contents of your readme file from os import path diff --git a/tests/fake_elasticsearch/test_search.py b/tests/fake_elasticsearch/test_search.py index 8659051..edf36a5 100644 --- a/tests/fake_elasticsearch/test_search.py +++ b/tests/fake_elasticsearch/test_search.py @@ -420,3 +420,41 @@ def test_bucket_aggregation_date_histogram(self): for x, y in zip(expected, actual): self.assertDictEqual(x["key"], y["key"]) self.assertEqual(x["doc_count"], y["doc_count"]) + + def test_bucket_aggregation_date_histogram_without_other_aggs(self): + start_date = datetime.datetime(2021, 12, 1, 15) + data = [ + {"data_x": 1, "data_y": "a", "timestamp": start_date}, + {"data_x": 1, "data_y": "a", "timestamp": start_date}, + {"data_x": 1, "data_y": "a", "timestamp": start_date - datetime.timedelta(hours=1)}, + {"data_x": 1, "data_y": "b", "timestamp": start_date - datetime.timedelta(hours=1)}, + {"data_x": 1, "data_y": "b", "timestamp": start_date - datetime.timedelta(hours=1)}, + ] + for body in data: + self.es.index(index='index_for_search', doc_type=DOC_TYPE, body=body) + + response = self.es.search( + index="index_for_search", + doc_type=DOC_TYPE, + body={ + "query": {"match_all": {}}, + "aggs": { + "stats": { + "composite": { + "sources": [{"histo": {"date_histogram": {"field": "timestamp"}}}], + "size": 10000, + }, + } + }, + }, + ) + + expected = [ + {"key": {"histo": '2021-12-01T14:00:00'}, "doc_count": 3}, + {"key": {"histo": '2021-12-01T15:00:00'}, "doc_count": 2}, + ] + actual = response["aggregations"]["stats"]["buckets"] + print(actual) + for x, y in zip(expected, actual): + self.assertDictEqual(x["key"], y["key"]) + self.assertEqual(x["doc_count"], y["doc_count"]) From 96d6b3f1ed92012ac934b381fae546cc374076db Mon Sep 17 00:00:00 2001 From: grgilad Date: Thu, 9 Sep 2021 17:51:37 +0300 Subject: [PATCH 05/16] Add support for 'keyword' in field in agg --- Makefile | 2 +- elasticmock/fake_elasticsearch.py | 3 +- setup.py | 2 +- tests/fake_elasticsearch/test_search.py | 38 +++++++++++++++++++++++++ 4 files changed, 42 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index ce5ec1d..908c5b6 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -ELASTICMOCK_VERSION='1.8.3' +ELASTICMOCK_VERSION='1.8.4' install: pip3 install -r requirements.txt diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index 725b9e7..404df55 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -511,7 +511,7 @@ def mget(self, body, index, doc_type='_all', params=None, headers=None): for id in ids: try: results.append(self.get(index, id, doc_type=doc_type, - params=params, headers=headers)) + params=params, headers=headers)) except: pass if not results: @@ -795,6 +795,7 @@ def make_key(doc_source, agg_source): attr = agg_lst["terms"]["field"] elif agg_lst.get("date_histogram"): attr = agg_lst["date_histogram"]["field"] + attr = attr.split('.')[0] # support for '.keyword' return doc_source[attr] def make_bucket(bucket_key, bucket): diff --git a/setup.py b/setup.py index 8d19d9e..f51f26d 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ import setuptools -__version__ = '1.8.3' +__version__ = '1.8.4' # read the contents of your readme file from os import path diff --git a/tests/fake_elasticsearch/test_search.py b/tests/fake_elasticsearch/test_search.py index edf36a5..9d5d747 100644 --- a/tests/fake_elasticsearch/test_search.py +++ b/tests/fake_elasticsearch/test_search.py @@ -458,3 +458,41 @@ def test_bucket_aggregation_date_histogram_without_other_aggs(self): for x, y in zip(expected, actual): self.assertDictEqual(x["key"], y["key"]) self.assertEqual(x["doc_count"], y["doc_count"]) + + def test_bucket_aggregation_date_histogram_with_keyword_arg(self): + start_date = datetime.datetime(2021, 12, 1, 15) + data = [ + {"data_x": 1, "data_y": "a", "timestamp": start_date}, + {"data_x": 1, "data_y": "a", "timestamp": start_date}, + {"data_x": 1, "data_y": "a", "timestamp": start_date - datetime.timedelta(hours=1)}, + {"data_x": 1, "data_y": "b", "timestamp": start_date - datetime.timedelta(hours=1)}, + {"data_x": 1, "data_y": "b", "timestamp": start_date - datetime.timedelta(hours=1)}, + ] + for body in data: + self.es.index(index='index_for_search', doc_type=DOC_TYPE, body=body) + + response = self.es.search( + index="index_for_search", + doc_type=DOC_TYPE, + body={ + "query": {"match_all": {}}, + "aggs": { + "stats": { + "composite": { + "sources": [{"histo": {"date_histogram": {"field": "timestamp.keyword"}}}], + "size": 10000, + }, + } + }, + }, + ) + + expected = [ + {"key": {"histo": '2021-12-01T14:00:00'}, "doc_count": 3}, + {"key": {"histo": '2021-12-01T15:00:00'}, "doc_count": 2}, + ] + actual = response["aggregations"]["stats"]["buckets"] + print(actual) + for x, y in zip(expected, actual): + self.assertDictEqual(x["key"], y["key"]) + self.assertEqual(x["doc_count"], y["doc_count"]) From b3cd8c0123f913021dce352abbbeb26301091dca Mon Sep 17 00:00:00 2001 From: grgilad Date: Sun, 17 Oct 2021 17:10:58 +0300 Subject: [PATCH 06/16] Add sort to supported elasticmock search params --- elasticmock/fake_elasticsearch.py | 80 +++++++++++++++++++++++-- tests/fake_elasticsearch/test_search.py | 59 +++++++++++++++--- 2 files changed, 127 insertions(+), 12 deletions(-) diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index 404df55..d950874 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -76,6 +76,61 @@ def get_metric_type(type_str): raise NotImplementedError(f'type {type_str} is not implemented for MetricType') +class SortOrder: + ASC = "asc" + DESC = "desc" + + @staticmethod + def get_sort_order(type_str): + if type_str == "asc": + return SortOrder.ASC + elif type_str == "desc": + return SortOrder.DESC + else: + raise NotImplementedError(f'type {type_str} is not implemented for SortOrder') + + +class SortMode: + MIN = "min" + MAX = "max" + SUM = "sum" + AVG = "avg" + MEDIAN = "median" + + @staticmethod + def get_sort_mode(type_str): + if type_str == "min": + return SortMode.MIN + elif type_str == "max": + return SortMode.MAX + elif type_str == "sum": + return SortMode.SUM + elif type_str == "avg": + return SortMode.AVG + elif type_str == "median": + return SortMode.MEDIAN + else: + raise NotImplementedError(f'type {type_str} is not implemented for SortMode') + + +class FakeSortRequest: + _order = None + _mode = None + _field = None + + def __init__(self, order, mode, field): + self._order = order + self._mode = mode + self._field = field + + def sort(self, hits): + reverse = True + if self._order == SortOrder.ASC: + reverse = False + return sorted(hits, key=lambda hit: hit['_source'][f'{self._field}'], reverse=reverse) + + + class FakeQueryCondition: type = None condition = None @@ -561,6 +616,10 @@ def count(self, index=None, doc_type=None, body=None, params=None, headers=None) def _get_fake_query_condition(self, query_type_str, condition): return FakeQueryCondition(QueryType.get_query_type(query_type_str), condition) + def _get_fake_sort(self, field, sort_condition): + # mode is not supported yet... + return FakeSortRequest(SortOrder.get_sort_order(sort_condition['order']), None, field) + @query_params( "ccs_minimize_roundtrips", "max_concurrent_searches", @@ -609,11 +668,19 @@ def search(self, index=None, doc_type=None, body=None, params=None, headers=None matches = [] conditions = [] + sorts = [] + + if body: + if 'query' in body: + query = body['query'] + for query_type_str, condition in query.items(): + conditions.append(self._get_fake_query_condition(query_type_str, condition)) + if 'sort' in body: + out_of_sorts = body['sort'] + for sortie in out_of_sorts: + for field, sort_condition in sortie.items(): + sorts.append((self._get_fake_sort(field, sort_condition))) - if body and 'query' in body: - query = body['query'] - for query_type_str, condition in query.items(): - conditions.append(self._get_fake_query_condition(query_type_str, condition)) for searchable_index in searchable_indexes: if self.__documents_dict and self.__documents_dict.get(searchable_index): for document in self.__documents_dict[searchable_index]: @@ -654,6 +721,11 @@ def search(self, index=None, doc_type=None, body=None, params=None, headers=None match['_score'] = 1.0 hits.append(match) + # apply sort + if hits: + for sortie in sorts: + hits = sortie.sort(hits) + # build aggregations if body is not None and 'aggs' in body: aggregations = {} diff --git a/tests/fake_elasticsearch/test_search.py b/tests/fake_elasticsearch/test_search.py index 9d5d747..b344ea2 100644 --- a/tests/fake_elasticsearch/test_search.py +++ b/tests/fake_elasticsearch/test_search.py @@ -148,7 +148,7 @@ def test_search_with_must_not_query(self): body={'query': {'bool': { 'filter': [{'terms': {'id': [1, 2]}}], 'must_not': [{'term': {'id': 1}}], - }}}) + }}}) self.assertEqual(response['hits']['total']['value'], 1) doc = response['hits']['hits'][0]['_source'] self.assertEqual(2, doc['id']) @@ -175,13 +175,12 @@ def test_query_on_nested_data(self): doc = response['hits']['hits'][0]['_source'] self.assertEqual(i, doc['id']) - def test_search_with_bool_query_and_multi_match(self): for i in range(0, 10): self.es.index(index='index_for_search', doc_type=DOC_TYPE, body={ 'data': 'test_{0}'.format(i) if i % 2 == 0 else None, - 'data2': 'test_{0}'.format(i) if (i+1) % 2 == 0 else None - }) + 'data2': 'test_{0}'.format(i) if (i + 1) % 2 == 0 else None + }) search_body = { "query": { @@ -226,13 +225,13 @@ def test_msearch(self): for i in range(0, 10): self.es.index(index='index_for_search1', doc_type=DOC_TYPE, body={ 'data': 'test_{0}'.format(i) if i % 2 == 0 else None, - 'data2': 'test_{0}'.format(i) if (i+1) % 2 == 0 else None - }) + 'data2': 'test_{0}'.format(i) if (i + 1) % 2 == 0 else None + }) for i in range(0, 10): self.es.index(index='index_for_search2', doc_type=DOC_TYPE, body={ 'data': 'test_{0}'.format(i) if i % 2 == 0 else None, - 'data2': 'test_{0}'.format(i) if (i+1) % 2 == 0 else None - }) + 'data2': 'test_{0}'.format(i) if (i + 1) % 2 == 0 else None + }) search_body = { "query": { @@ -339,6 +338,50 @@ def test_search_with_range_query(self, _, query_range, expected_ids): hits = response['hits']['hits'] self.assertEqual(set(expected_ids), set(hit['_source']['id'] for hit in hits)) + def test_search_with_sort_asc(self): + for i in range(0, 12): + body = { + 'id': i, + 'timestamp': datetime.datetime(2009, 1, 1, 10, 5, 0) - datetime.timedelta(minutes=i), + 'data': 'data' + } + self.es.index(index='index_for_search', doc_type=DOC_TYPE, body=body) + + response = self.es.search( + index='index_for_search', + doc_type=DOC_TYPE, + body={"query": { + "term": {"data": "data"} + }, + "sort": [ + {"timestamp": {"order": "asc"}} + ]}, + ) + + self.assertEqual(11, response['hits']['hits'][0]['_source']['id']) + + def test_search_with_sort_desc(self): + for i in range(0, 12): + body = { + 'id': i, + 'timestamp': datetime.datetime(2009, 1, 1, 10, 5, 0) + datetime.timedelta(minutes=i), + 'data': 'data' + } + self.es.index(index='index_for_search', doc_type=DOC_TYPE, body=body) + + response = self.es.search( + index='index_for_search', + doc_type=DOC_TYPE, + body={"query": { + "term": {"data": "data"} + }, + "sort": [ + {"timestamp": {"order": "desc"}} + ]}, + ) + + self.assertEqual(11, response['hits']['hits'][0]['_source']['id']) + def test_bucket_aggregation_terms(self): data = [ {"data_x": 1, "data_y": "a"}, From 9eda8ef1101c62ce5bbdc0995c65eb54b60614cd Mon Sep 17 00:00:00 2001 From: grgilad Date: Sun, 17 Oct 2021 17:11:34 +0300 Subject: [PATCH 07/16] Push version --- Makefile | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 908c5b6..f2e21be 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -ELASTICMOCK_VERSION='1.8.4' +ELASTICMOCK_VERSION='1.8.5' install: pip3 install -r requirements.txt diff --git a/setup.py b/setup.py index f51f26d..89cd760 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ import setuptools -__version__ = '1.8.4' +__version__ = '1.8.5' # read the contents of your readme file from os import path From 5041046a25b80dc5ccc6835572ffff7d6c407bb7 Mon Sep 17 00:00:00 2001 From: grgilad Date: Mon, 14 Mar 2022 14:53:02 +0200 Subject: [PATCH 08/16] Fix wrong range parsing --- Makefile | 2 +- elasticmock/fake_elasticsearch.py | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index f2e21be..5a43fa1 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -ELASTICMOCK_VERSION='1.8.5' +ELASTICMOCK_VERSION='1.8.6' install: pip3 install -r requirements.txt diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index d950874..b8366c5 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -229,7 +229,7 @@ def _evaluate_for_range_query_type(self, document): return False for sign, value in comparisons.items(): - if isinstance(doc_val, datetime.datetime): + if isinstance(doc_val, datetime.datetime) and not isinstance(value, datetime.datetime): value = dateutil.parser.isoparse(value) if sign == 'gte': if doc_val < value: diff --git a/setup.py b/setup.py index 89cd760..690bc3c 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ import setuptools -__version__ = '1.8.5' +__version__ = '1.8.6' # read the contents of your readme file from os import path From 02f076263abbda11700e393a6b22267377a24c8d Mon Sep 17 00:00:00 2001 From: grgilad Date: Mon, 14 Mar 2022 15:55:40 +0200 Subject: [PATCH 09/16] Cleanup and support None in lte/lt range --- Makefile | 2 +- elasticmock/fake_elasticsearch.py | 4 ++-- setup.py | 2 +- tests/fake_elasticsearch/test_search.py | 3 --- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 5a43fa1..bbf50ff 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -ELASTICMOCK_VERSION='1.8.6' +ELASTICMOCK_VERSION='1.8.7' install: pip3 install -r requirements.txt diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index b8366c5..fc5d567 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -238,10 +238,10 @@ def _evaluate_for_range_query_type(self, document): if doc_val <= value: return False elif sign == 'lte': - if doc_val > value: + if not value or doc_val > value: return False elif sign == 'lt': - if doc_val >= value: + if not value or doc_val >= value: return False else: raise ValueError(f"Invalid comparison type {sign}") diff --git a/setup.py b/setup.py index 690bc3c..0b69bba 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ import setuptools -__version__ = '1.8.6' +__version__ = '1.8.7' # read the contents of your readme file from os import path diff --git a/tests/fake_elasticsearch/test_search.py b/tests/fake_elasticsearch/test_search.py index b344ea2..8bfecb7 100644 --- a/tests/fake_elasticsearch/test_search.py +++ b/tests/fake_elasticsearch/test_search.py @@ -459,7 +459,6 @@ def test_bucket_aggregation_date_histogram(self): {"key": {"histo": '2021-12-01T15:00:00'}, "doc_count": 2}, ] actual = response["aggregations"]["stats"]["buckets"] - print(actual) for x, y in zip(expected, actual): self.assertDictEqual(x["key"], y["key"]) self.assertEqual(x["doc_count"], y["doc_count"]) @@ -497,7 +496,6 @@ def test_bucket_aggregation_date_histogram_without_other_aggs(self): {"key": {"histo": '2021-12-01T15:00:00'}, "doc_count": 2}, ] actual = response["aggregations"]["stats"]["buckets"] - print(actual) for x, y in zip(expected, actual): self.assertDictEqual(x["key"], y["key"]) self.assertEqual(x["doc_count"], y["doc_count"]) @@ -535,7 +533,6 @@ def test_bucket_aggregation_date_histogram_with_keyword_arg(self): {"key": {"histo": '2021-12-01T15:00:00'}, "doc_count": 2}, ] actual = response["aggregations"]["stats"]["buckets"] - print(actual) for x, y in zip(expected, actual): self.assertDictEqual(x["key"], y["key"]) self.assertEqual(x["doc_count"], y["doc_count"]) From ace2a17170871f5fe9ce4b078f798954146d248d Mon Sep 17 00:00:00 2001 From: grgilad Date: Mon, 14 Mar 2022 16:04:13 +0200 Subject: [PATCH 10/16] Return true if none in value field for range --- Makefile | 2 +- elasticmock/fake_elasticsearch.py | 8 ++++---- setup.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index bbf50ff..8d82a6e 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -ELASTICMOCK_VERSION='1.8.7' +ELASTICMOCK_VERSION='1.8.8' install: pip3 install -r requirements.txt diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index fc5d567..76db33d 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -237,11 +237,11 @@ def _evaluate_for_range_query_type(self, document): elif sign == 'gt': if doc_val <= value: return False - elif sign == 'lte': - if not value or doc_val > value: + elif sign == 'lte' and value: + if doc_val > value: return False - elif sign == 'lt': - if not value or doc_val >= value: + elif sign == 'lt' and value: + if doc_val >= value: return False else: raise ValueError(f"Invalid comparison type {sign}") diff --git a/setup.py b/setup.py index 0b69bba..b45bd40 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ import setuptools -__version__ = '1.8.7' +__version__ = '1.8.8' # read the contents of your readme file from os import path From 635b66057750cddf813997fdb38063b11de2e810 Mon Sep 17 00:00:00 2001 From: grgilad Date: Mon, 14 Mar 2022 16:15:04 +0200 Subject: [PATCH 11/16] Wrote a test... --- Makefile | 2 +- elasticmock/fake_elasticsearch.py | 2 +- setup.py | 2 +- tests/fake_elasticsearch/test_search.py | 18 ++++++++++++++++++ 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 8d82a6e..0edc4ea 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -ELASTICMOCK_VERSION='1.8.8' +ELASTICMOCK_VERSION='1.8.9' install: pip3 install -r requirements.txt diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index 76db33d..f146315 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -243,7 +243,7 @@ def _evaluate_for_range_query_type(self, document): elif sign == 'lt' and value: if doc_val >= value: return False - else: + elif sign not in ['gte', 'gt', 'lte', 'lt']: raise ValueError(f"Invalid comparison type {sign}") return True diff --git a/setup.py b/setup.py index b45bd40..fc2ac74 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ import setuptools -__version__ = '1.8.8' +__version__ = '1.8.9' # read the contents of your readme file from os import path diff --git a/tests/fake_elasticsearch/test_search.py b/tests/fake_elasticsearch/test_search.py index 8bfecb7..c45918e 100644 --- a/tests/fake_elasticsearch/test_search.py +++ b/tests/fake_elasticsearch/test_search.py @@ -338,6 +338,24 @@ def test_search_with_range_query(self, _, query_range, expected_ids): hits = response['hits']['hits'] self.assertEqual(set(expected_ids), set(hit['_source']['id'] for hit in hits)) + def test_search_with_range_query_with_null_lte(self): + body = { + 'id': 5, + 'timestamp': datetime.datetime(2009, 1, 1, 10, 0, 0), + 'data_int': 10, + } + self.es.index(index='index_for_search', doc_type=DOC_TYPE, body=body) + + response = self.es.search( + index='index_for_search', + doc_type=DOC_TYPE, + body={'query': {'range': {'data_int': {"gte": 10, "lte": None}}}}, + ) + + self.assertEqual(1, response['hits']['total']['value']) + hits = response['hits']['hits'] + self.assertEqual({5}, set(hit['_source']['id'] for hit in hits)) + def test_search_with_sort_asc(self): for i in range(0, 12): body = { From 2875e1512165ca8457ee12da5073d257438de905 Mon Sep 17 00:00:00 2001 From: grgilad Date: Sun, 26 Jun 2022 20:44:10 +0300 Subject: [PATCH 12/16] Add support for wildcard matching --- Makefile | 2 +- elasticmock/fake_elasticsearch.py | 30 ++++++++++++++++++++----- setup.py | 2 +- tests/fake_elasticsearch/test_search.py | 16 +++++++++++++ 4 files changed, 43 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 0edc4ea..bca2676 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -ELASTICMOCK_VERSION='1.8.9' +ELASTICMOCK_VERSION='1.9.0' install: pip3 install -r requirements.txt diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index f146315..cf21343 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import datetime import json +import re import sys from collections import defaultdict @@ -34,6 +35,7 @@ class QueryType: MINIMUM_SHOULD_MATCH = 'MINIMUM_SHOULD_MATCH' MULTI_MATCH = 'MULTI_MATCH' MUST_NOT = 'MUST_NOT' + WILDCARD = 'WILDCARD' @staticmethod def get_query_type(type_str): @@ -61,6 +63,8 @@ def get_query_type(type_str): return QueryType.MULTI_MATCH elif type_str == 'must_not': return QueryType.MUST_NOT + elif type_str == 'wildcard': + return QueryType.WILDCARD else: raise NotImplementedError(f'type {type_str} is not implemented for QueryType') @@ -151,6 +155,8 @@ def _evaluate_for_query_type(self, document): return self._evaluate_for_term_query_type(document) elif self.type == QueryType.TERMS: return self._evaluate_for_terms_query_type(document) + elif self.type == QueryType.WILDCARD: + return self._evaluate_for_wildcard_query_type(document) elif self.type == QueryType.RANGE: return self._evaluate_for_range_query_type(document) elif self.type == QueryType.BOOL: @@ -181,7 +187,16 @@ def _evaluate_for_terms_query_type(self, document): return True return False - def _evaluate_for_field(self, document, ignore_case): + def _evaluate_for_wildcard_query_type(self, document): + return_val = False + if isinstance(self.condition, dict): + for _, sub_query in self.condition.items(): + return_val = self._evaluate_for_field(document, True, True) + if not return_val: + return False + return return_val + + def _evaluate_for_field(self, document, ignore_case=True, is_wildcard=False): doc_source = document['_source'] return_val = False for field, value in self.condition.items(): @@ -189,7 +204,8 @@ def _evaluate_for_field(self, document, ignore_case): doc_source, field, value, - ignore_case + ignore_case, + is_wildcard ) if return_val: break @@ -304,13 +320,16 @@ def _evaluate_for_should_query_type(self, document): def _evaluate_for_multi_match_query_type(self, document): return self._evaluate_for_fields(document) - def _compare_value_for_field(self, doc_source, field, value, ignore_case): + def _compare_value_for_field(self, doc_source, field, value, ignore_case, is_wildcard=False): + if is_wildcard: + value = value['value'] if ignore_case and isinstance(value, str): value = value.lower() doc_val = doc_source # Remove boosting - field, *_ = field.split("*") + if not is_wildcard: + field, *_ = field.split("*") for k in field.split("."): if hasattr(doc_val, k): doc_val = getattr(doc_val, k) @@ -329,7 +348,8 @@ def _compare_value_for_field(self, doc_source, field, value, ignore_case): val = str(val) if ignore_case: val = val.lower() - + if is_wildcard: + return re.search(value.replace('*', '.*'), val) if value == val: return True if isinstance(val, str) and str(value) in val: diff --git a/setup.py b/setup.py index fc2ac74..007a24f 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ import setuptools -__version__ = '1.8.9' +__version__ = '1.9.0' # read the contents of your readme file from os import path diff --git a/tests/fake_elasticsearch/test_search.py b/tests/fake_elasticsearch/test_search.py index c45918e..c972027 100644 --- a/tests/fake_elasticsearch/test_search.py +++ b/tests/fake_elasticsearch/test_search.py @@ -96,6 +96,22 @@ def test_search_with_match_query(self): self.assertEqual(len(hits), 1) self.assertEqual(hits[0]['_source'], {'data': 'test_3'}) + def test_search_with_wildcard_query(self): + self.es.index(index='index_for_search', doc_type=DOC_TYPE, body={'data': 'test_20221010'}) + self.es.index(index='index_for_search', doc_type=DOC_TYPE, body={'data': 'test_20221011'}) + self.es.index(index='index_for_search', doc_type=DOC_TYPE, body={'data': 'test_20221012'}) + response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, + body={'query': {'wildcard': {'data': {'value': 'test_1*'}}}}) + self.assertEqual(response['hits']['total']['value'], 0) + hits = response['hits']['hits'] + self.assertEqual(len(hits), 0) + + response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, + body={'query': {'wildcard': {'data': {'value': 'test_*'}}}}) + self.assertEqual(response['hits']['total']['value'], 3) + hits = response['hits']['hits'] + self.assertEqual(len(hits), 3) + def test_search_with_match_query_in_int_list(self): for i in range(0, 10): self.es.index(index='index_for_search', doc_type=DOC_TYPE, body={'data': [i, 11, 13]}) From ec754b5a990be4fc0a307034304c7a3cbc8f0917 Mon Sep 17 00:00:00 2001 From: Santiago Aguiar Date: Thu, 18 Aug 2022 19:18:56 -0300 Subject: [PATCH 13/16] add prefix query --- Makefile | 2 +- elasticmock/fake_elasticsearch.py | 27 ++++++++++++++++++++----- setup.py | 2 +- tests/fake_elasticsearch/test_search.py | 16 +++++++++++++++ 4 files changed, 40 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index bca2676..eea0b97 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -ELASTICMOCK_VERSION='1.9.0' +ELASTICMOCK_VERSION='1.10.0' install: pip3 install -r requirements.txt diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index cf21343..dab9b74 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -36,6 +36,7 @@ class QueryType: MULTI_MATCH = 'MULTI_MATCH' MUST_NOT = 'MUST_NOT' WILDCARD = 'WILDCARD' + PREFIX = 'PREFIX' @staticmethod def get_query_type(type_str): @@ -65,6 +66,8 @@ def get_query_type(type_str): return QueryType.MUST_NOT elif type_str == 'wildcard': return QueryType.WILDCARD + elif type_str == 'prefix': + return QueryType.PREFIX else: raise NotImplementedError(f'type {type_str} is not implemented for QueryType') @@ -157,6 +160,8 @@ def _evaluate_for_query_type(self, document): return self._evaluate_for_terms_query_type(document) elif self.type == QueryType.WILDCARD: return self._evaluate_for_wildcard_query_type(document) + elif self.type == QueryType.PREFIX: + return self._evaluate_for_prefix_query_type(document) elif self.type == QueryType.RANGE: return self._evaluate_for_range_query_type(document) elif self.type == QueryType.BOOL: @@ -191,12 +196,21 @@ def _evaluate_for_wildcard_query_type(self, document): return_val = False if isinstance(self.condition, dict): for _, sub_query in self.condition.items(): - return_val = self._evaluate_for_field(document, True, True) + return_val = self._evaluate_for_field(document, True, is_wildcard=True) if not return_val: return False return return_val - def _evaluate_for_field(self, document, ignore_case=True, is_wildcard=False): + def _evaluate_for_prefix_query_type(self, document): + return_val = False + if isinstance(self.condition, dict): + for _, sub_query in self.condition.items(): + return_val = self._evaluate_for_field(document, ignore_case=False, is_prefix=True) + if not return_val: + return False + return return_val + + def _evaluate_for_field(self, document, ignore_case=True, is_wildcard=False, is_prefix=False): doc_source = document['_source'] return_val = False for field, value in self.condition.items(): @@ -205,7 +219,8 @@ def _evaluate_for_field(self, document, ignore_case=True, is_wildcard=False): field, value, ignore_case, - is_wildcard + is_wildcard=is_wildcard, + is_prefix=is_prefix ) if return_val: break @@ -320,8 +335,8 @@ def _evaluate_for_should_query_type(self, document): def _evaluate_for_multi_match_query_type(self, document): return self._evaluate_for_fields(document) - def _compare_value_for_field(self, doc_source, field, value, ignore_case, is_wildcard=False): - if is_wildcard: + def _compare_value_for_field(self, doc_source, field, value, ignore_case, is_wildcard=False, is_prefix=False): + if is_wildcard or is_prefix: value = value['value'] if ignore_case and isinstance(value, str): value = value.lower() @@ -350,6 +365,8 @@ def _compare_value_for_field(self, doc_source, field, value, ignore_case, is_wil val = val.lower() if is_wildcard: return re.search(value.replace('*', '.*'), val) + if is_prefix: + return val.startswith(value) if value == val: return True if isinstance(val, str) and str(value) in val: diff --git a/setup.py b/setup.py index 007a24f..2f122ab 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ import setuptools -__version__ = '1.9.0' +__version__ = '1.10.0' # read the contents of your readme file from os import path diff --git a/tests/fake_elasticsearch/test_search.py b/tests/fake_elasticsearch/test_search.py index c972027..c154add 100644 --- a/tests/fake_elasticsearch/test_search.py +++ b/tests/fake_elasticsearch/test_search.py @@ -112,6 +112,22 @@ def test_search_with_wildcard_query(self): hits = response['hits']['hits'] self.assertEqual(len(hits), 3) + def test_search_with_prefix_query(self): + self.es.index(index='index_for_search', doc_type=DOC_TYPE, body={'data': 'test_20221010'}) + self.es.index(index='index_for_search', doc_type=DOC_TYPE, body={'data': 'test_20221011'}) + self.es.index(index='index_for_search', doc_type=DOC_TYPE, body={'data': 'test_20221012'}) + response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, + body={'query': {'prefix': {'data': {'value': 'test_1'}}}}) + self.assertEqual(response['hits']['total']['value'], 0) + hits = response['hits']['hits'] + self.assertEqual(len(hits), 0) + + response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, + body={'query': {'prefix': {'data': {'value': 'test_2'}}}}) + self.assertEqual(response['hits']['total']['value'], 3) + hits = response['hits']['hits'] + self.assertEqual(len(hits), 3) + def test_search_with_match_query_in_int_list(self): for i in range(0, 10): self.es.index(index='index_for_search', doc_type=DOC_TYPE, body={'data': [i, 11, 13]}) From 57b9cd68330b3c0d20cc065ad65933459dc3d05e Mon Sep 17 00:00:00 2001 From: Santiago Aguiar Date: Mon, 22 Aug 2022 19:22:12 -0300 Subject: [PATCH 14/16] support shorthand wildcard/prefix queries --- elasticmock/fake_elasticsearch.py | 2 +- tests/fake_elasticsearch/test_search.py | 32 +++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index dab9b74..6a72948 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -336,7 +336,7 @@ def _evaluate_for_multi_match_query_type(self, document): return self._evaluate_for_fields(document) def _compare_value_for_field(self, doc_source, field, value, ignore_case, is_wildcard=False, is_prefix=False): - if is_wildcard or is_prefix: + if (is_wildcard or is_prefix) and type(value) == type({}): value = value['value'] if ignore_case and isinstance(value, str): value = value.lower() diff --git a/tests/fake_elasticsearch/test_search.py b/tests/fake_elasticsearch/test_search.py index c154add..7a98370 100644 --- a/tests/fake_elasticsearch/test_search.py +++ b/tests/fake_elasticsearch/test_search.py @@ -112,6 +112,22 @@ def test_search_with_wildcard_query(self): hits = response['hits']['hits'] self.assertEqual(len(hits), 3) + def test_search_with_wildcard_query_shorthand(self): + self.es.index(index='index_for_search', doc_type=DOC_TYPE, body={'data': 'test_20221010'}) + self.es.index(index='index_for_search', doc_type=DOC_TYPE, body={'data': 'test_20221011'}) + self.es.index(index='index_for_search', doc_type=DOC_TYPE, body={'data': 'test_20221012'}) + response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, + body={'query': {'wildcard': {'data': 'test_1*'}}}) + self.assertEqual(response['hits']['total']['value'], 0) + hits = response['hits']['hits'] + self.assertEqual(len(hits), 0) + + response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, + body={'query': {'wildcard': {'data':'test_*'}}}) + self.assertEqual(response['hits']['total']['value'], 3) + hits = response['hits']['hits'] + self.assertEqual(len(hits), 3) + def test_search_with_prefix_query(self): self.es.index(index='index_for_search', doc_type=DOC_TYPE, body={'data': 'test_20221010'}) self.es.index(index='index_for_search', doc_type=DOC_TYPE, body={'data': 'test_20221011'}) @@ -128,6 +144,22 @@ def test_search_with_prefix_query(self): hits = response['hits']['hits'] self.assertEqual(len(hits), 3) + def test_search_with_prefix_query_shorthand(self): + self.es.index(index='index_for_search', doc_type=DOC_TYPE, body={'data': 'test_20221010'}) + self.es.index(index='index_for_search', doc_type=DOC_TYPE, body={'data': 'test_20221011'}) + self.es.index(index='index_for_search', doc_type=DOC_TYPE, body={'data': 'test_20221012'}) + response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, + body={'query': {'prefix': {'data': 'test_1'}}}) + self.assertEqual(response['hits']['total']['value'], 0) + hits = response['hits']['hits'] + self.assertEqual(len(hits), 0) + + response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, + body={'query': {'prefix': {'data': 'test_2'}}}) + self.assertEqual(response['hits']['total']['value'], 3) + hits = response['hits']['hits'] + self.assertEqual(len(hits), 3) + def test_search_with_match_query_in_int_list(self): for i in range(0, 10): self.es.index(index='index_for_search', doc_type=DOC_TYPE, body={'data': [i, 11, 13]}) From cf3d2346d2f05fe316449c838f3154e9d1552da2 Mon Sep 17 00:00:00 2001 From: Jiri Tobisek Date: Thu, 9 Feb 2023 10:53:50 +0200 Subject: [PATCH 15/16] Implement minimum_should_match query --- elasticmock/fake_elasticsearch.py | 23 ++++++++++++++++++----- tests/fake_elasticsearch/test_search.py | 25 +++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index 6a72948..59d1050 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -142,9 +142,10 @@ class FakeQueryCondition: type = None condition = None - def __init__(self, type, condition): + def __init__(self, type, condition, parent_condition=None): self.type = type self.condition = condition + self.parent_condition = parent_condition def evaluate(self, document): return self._evaluate_for_query_type(document) @@ -176,6 +177,9 @@ def _evaluate_for_query_type(self, document): return self._evaluate_for_multi_match_query_type(document) elif self.type == QueryType.MUST_NOT: return self._evaluate_for_must_not_query_type(document) + elif self.type == QueryType.MINIMUM_SHOULD_MATCH: + # Deal with minimum_should_match as part of the should query + return True else: raise NotImplementedError('Fake query evaluation not implemented for query type: %s' % self.type) @@ -284,7 +288,8 @@ def _evaluate_for_compound_query_type(self, document): for query_type, sub_query in self.condition.items(): return_val = FakeQueryCondition( QueryType.get_query_type(query_type), - sub_query + sub_query, + self.condition ).evaluate(document) if not return_val: return False @@ -320,8 +325,14 @@ def _evaluate_for_must_not_query_type(self, document): return False return True + def _get_minimum_should_match(self): + return self.parent_condition.get("minimum_should_match", 1) \ + if self.parent_condition \ + else 1 + def _evaluate_for_should_query_type(self, document): - return_val = False + minimum_should_match = self._get_minimum_should_match() + match_count = 0 for sub_condition in self.condition: for sub_condition_key in sub_condition: return_val = FakeQueryCondition( @@ -329,8 +340,10 @@ def _evaluate_for_should_query_type(self, document): sub_condition[sub_condition_key] ).evaluate(document) if return_val: - return True - return return_val + match_count += 1 + if match_count >= minimum_should_match: + return True + return False def _evaluate_for_multi_match_query_type(self, document): return self._evaluate_for_fields(document) diff --git a/tests/fake_elasticsearch/test_search.py b/tests/fake_elasticsearch/test_search.py index 7a98370..2e85075 100644 --- a/tests/fake_elasticsearch/test_search.py +++ b/tests/fake_elasticsearch/test_search.py @@ -285,6 +285,31 @@ def test_search_bool_should_match_query(self): self.assertEqual(len(hits), 3) self.assertEqual(hits[0]['_source'], {'data': 'test_0'}) + def test_search_bool_minimum_should_match_query(self): + self.es.index(index='index_for_search', doc_type=DOC_TYPE, body={'field1': 'test_0', 'field2': 'test_0'}) + self.es.index(index='index_for_search', doc_type=DOC_TYPE, body={'field1': 'test_1', 'field2': 'test_0'}) + self.es.index(index='index_for_search', doc_type=DOC_TYPE, body={'field1': 'test_0', 'field2': 'test_1'}) + self.es.index(index='index_for_search', doc_type=DOC_TYPE, body={'field1': 'test_1', 'field2': 'test_1'}) + + response = self.es.search(index='index_for_search', doc_type=DOC_TYPE, + body={ + 'query': { + 'bool': { + 'should': [ + {'match': {'field1': 'test_1'}}, + {'match': {'field2': 'test_1'}}, + ], + 'minimum_should_match': 1 + } + } + }) + self.assertEqual(response['hits']['total']['value'], 3) + hits = response['hits']['hits'] + self.assertEqual(len(hits), 3) + self.assertEqual(hits[0]['_source'], {'field1': 'test_1', 'field2': 'test_0'}) + self.assertEqual(hits[1]['_source'], {'field1': 'test_0', 'field2': 'test_1'}) + self.assertEqual(hits[2]['_source'], {'field1': 'test_1', 'field2': 'test_1'}) + def test_msearch(self): for i in range(0, 10): self.es.index(index='index_for_search1', doc_type=DOC_TYPE, body={ From 61571af46ee0764cbad8871d424c3b51e30d88be Mon Sep 17 00:00:00 2001 From: Jiri Tobisek Date: Thu, 9 Feb 2023 11:03:21 +0200 Subject: [PATCH 16/16] Implement minimum_should_match query --- elasticmock/fake_elasticsearch.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/elasticmock/fake_elasticsearch.py b/elasticmock/fake_elasticsearch.py index 59d1050..0e25ae0 100644 --- a/elasticmock/fake_elasticsearch.py +++ b/elasticmock/fake_elasticsearch.py @@ -298,7 +298,8 @@ def _evaluate_for_compound_query_type(self, document): for sub_condition_key in sub_condition: return_val = FakeQueryCondition( QueryType.get_query_type(sub_condition_key), - sub_condition[sub_condition_key] + sub_condition[sub_condition_key], + self.condition ).evaluate(document) if not return_val: return False