Add place to advansed persons search (#182)

* Add place to advansed persons search

* 180/181 Add place params to general and  persons search

* Remove usless file

bhs_api/persons.py

@@ -27,6 +27,7 @@
                                      "pob": None, "pob_t": "exact",
                                      "pom": None, "pom_t": "exact",
                                      "pod": None, "pod_t": "exact",
+                                     "place": None, "place_t": "exact",
                                      "yob": None, "yob_t": "exact", "yob_v": None,
                                      "yom": None, "yom_t": "exact", "yom_v": None,
                                      "yod": None, "yod_t": "exact", "yod_v": None,
@@ -41,7 +42,8 @@
                                         ("last", "last_name_lc"),
                                         ("pob", "BIRT_PLAC_lc"),
                                         ("pom", "MARR_PLAC_lc"),
-                                        ("pod", "DEAT_PLAC_lc"),)
+                                        ("pod", "DEAT_PLAC_lc"),
+                                        ("place", ["BIRT_PLAC_lc", "MARR_PLAC_lc", "DEAT_PLAC_lc"]),)
 
 PERSONS_SEARCH_EXACT_PARAMS = (("sex", "gender"),
                                ("treenum", "tree_num"))

bhs_api/v1_endpoints.py

@@ -54,6 +54,22 @@ def custom_error(error):
     app.error_handler_spec[None][i] = custom_error
 '''
 
+def get_person_elastic_search_text_param_query(text_type, text_attr, text_value):
+    if isinstance(text_attr, list):
+        return {"bool": {"should": [
+            get_person_elastic_search_text_param_query(text_type, sub_text_attr, text_value)
+            for sub_text_attr in text_attr
+        ]}}
+    elif text_type == "exact":
+        return {"term": {text_attr: text_value}}
+    elif text_type == "like":
+        return {"match": {text_attr: {"query": text_value,
+                                                    "fuzziness": "AUTO"}}}
+    elif text_type == "starts":
+        return {"prefix": {text_attr: text_value}}
+    else:
+        raise Exception("invalid value for {} ({}): {}".format(text_type, text_attr, text_type))
+
 
 def es_search(q, size, collection=None, from_=0, sort=None, with_persons=False, **kwargs):
     if collection:
@@ -63,14 +79,19 @@ def es_search(q, size, collection=None, from_=0, sort=None, with_persons=False,
         # we consider the with_persons to decide whether to include persons collection or not
         collections = [collection for collection in SEARCHABLE_COLLECTIONS
                                   if with_persons or collection != "persons"]
+
+
+    fields = ["Header.En^2", "Header.He^2", "UnitText1.En", "UnitText1.He"]
     default_query = {
         "query_string": {
-            "fields": ["Header.En^2", "Header.He^2", "UnitText1.En", "UnitText1.He"],
+            "fields": fields,
             "query": q,
             "default_operator": "and"
         }
     }
-
+    for k, v in PERSONS_SEARCH_TEXT_PARAMS_LOWERCASE:
+        if not isinstance(v, list):
+            fields.append(v)
     if collection == "persons":
         must_queries = []
         if q:
@@ -99,15 +120,8 @@ def es_search(q, size, collection=None, from_=0, sort=None, with_persons=False,
                 text_value = kwargs[text_param].lower()
                 text_type_param = "{}_t".format(text_param)
                 text_type = kwargs[text_type_param]
-                if text_type == "exact":
-                    must_queries.append({"term": {text_attr: text_value}})
-                elif text_type == "like":
-                    must_queries.append({"match": {text_attr: {"query": text_value,
-                                                               "fuzziness": "AUTO"}}})
-                elif text_type == "starts":
-                    must_queries.append({"prefix": {text_attr: text_value}})
-                else:
-                    raise Exception("invalid value for {} ({}): {}".format(text_type, text_attr, text_type))
+                must_queries.append(get_person_elastic_search_text_param_query(text_type, text_attr, text_value))
+
         for exact_param, exact_attr in PERSONS_SEARCH_EXACT_PARAMS:
             if kwargs[exact_param]:
                 exact_value = kwargs[exact_param]
@@ -452,6 +466,7 @@ def general_search():
             try:
                 rv = es_search(**parameters)
             except Exception as e:
+                logging.exception(e)
                 return humanify({"error": e.message}, 500)
             for item in rv['hits']['hits']:
                 enrich_item(item['_source'], collection_name=item['_type'])

docs/_build/index.html

@@ -14,9 +14,9 @@ <h4>Example URI</h4><div class="definition"><span class="method get">GET</span>&
             ]</span>,
             "<span class="hljs-attribute">total</span>": <span class="hljs-value"><span class="hljs-number">1</span> // total number of results
         </span>}
-    </span>}</code></pre><div style="height: 1px;"></div></div></div></div></div><div id="database-search-persons-advanced-search" class="resource"><h3 class="resource-heading">Persons advanced search <a href="#database-search-persons-advanced-search" class="permalink">&nbsp;&para;</a></h3><div id="database-search-persons-advanced-search-get" class="action get"><h4 class="action-heading"><div class="name">Persons advanced search</div><a href="#database-search-persons-advanced-search-get" class="method get">GET</a><code class="uri">/v1/search{?collection,q,from_,size,first,first_t,last,last_t,pob,pob_t,pom,pom_t,pod,pod_t,yob,yob_t,yob_v,yom,yom_t,yom_v,yod,yod_t,yod_v,sex,treenum}</code></h4><p>Advanced search on persons, must have at last one of either q param (as in general search) or the person specific search params</p>
-<h4>Example URI</h4><div class="definition"><span class="method get">GET</span>&nbsp;<span class="uri"><span class="hostname"></span>/v1/search?<span class="hljs-attribute">collection=</span><span class="hljs-literal">persons</span>&<span class="hljs-attribute">q=</span><span class="hljs-literal"></span>&<span class="hljs-attribute">from_=</span><span class="hljs-literal"></span>&<span class="hljs-attribute">size=</span><span class="hljs-literal"></span>&<span class="hljs-attribute">first=</span><span class="hljs-literal"></span>&<span class="hljs-attribute">first_t=</span><span class="hljs-literal">like</span>&<span class="hljs-attribute">last=</span><span class="hljs-literal"></span>&<span class="hljs-attribute">last_t=</span><span class="hljs-literal">like</span>&<span class="hljs-attribute">pob=</span><span class="hljs-literal"></span>&<span class="hljs-attribute">pob_t=</span><span class="hljs-literal">like</span>&<span class="hljs-attribute">pom=</span><span class="hljs-literal"></span>&<span class="hljs-attribute">pom_t=</span><span class="hljs-literal">like</span>&<span class="hljs-attribute">pod=</span><span class="hljs-literal"></span>&<span class="hljs-attribute">pod_t=</span><span class="hljs-literal">like</span>&<span class="hljs-attribute">yob=</span><span class="hljs-literal"></span>&<span class="hljs-attribute">yob_t=</span><span class="hljs-literal">pmyears</span>&<span class="hljs-attribute">yob_v=</span><span class="hljs-literal">2</span>&<span class="hljs-attribute">yom=</span><span class="hljs-literal"></span>&<span class="hljs-attribute">yom_t=</span><span class="hljs-literal">pmyears</span>&<span class="hljs-attribute">yom_v=</span><span class="hljs-literal">2</span>&<span class="hljs-attribute">yod=</span><span class="hljs-literal"></span>&<span class="hljs-attribute">yod_t=</span><span class="hljs-literal">pmyears</span>&<span class="hljs-attribute">yod_v=</span><span class="hljs-literal">2</span>&<span class="hljs-attribute">sex=</span><span class="hljs-literal"></span>&<span class="hljs-attribute">treenum=</span><span class="hljs-literal"></span></span></div><div class="title"><strong>URI Parameters</strong><div class="collapse-button show"><span class="close">Hide</span><span class="open">Show</span></div></div><div class="collapse-content"><dl class="inner"><dt>collection</dt><dd><code>enum</code>&nbsp;<span class="required">(required)</span>&nbsp;<span class="text-muted example"><strong>Example:&nbsp;</strong><span>persons</span></span><p>The advanced persons search requires collection to be persons</p>
-<p class="choices"><strong>Choices:&nbsp;</strong><code>persons</code> </p></dd><dt>q</dt><dd><code>string</code>&nbsp;<span>(optional)</span>&nbsp;<p>same as in <a href="#database-search-general-search">general search</a>, but optional</p>
+    </span>}</code></pre><div style="height: 1px;"></div></div></div></div></div><div id="database-search-persons-advanced-search" class="resource"><h3 class="resource-heading">Persons advanced search <a href="#database-search-persons-advanced-search" class="permalink">&nbsp;&para;</a></h3><div id="database-search-persons-advanced-search-get" class="action get"><h4 class="action-heading"><div class="name">Persons advanced search</div><a href="#database-search-persons-advanced-search-get" class="method get">GET</a><code class="uri">/v1/search{?collection,q,from_,size,first,first_t,last,last_t,pob,pob_t,pom,pom_t,pod,pod_t,place,place_t,yob,yob_t,yob_v,yom,yom_t,yom_v,yod,yod_t,yod_v,sex,treenum}</code></h4><p>Advanced search on persons, must have at last one of either q param (as in general search) or the person specific search params</p>
+<h4>Example URI</h4><div class="definition"><span class="method get">GET</span>&nbsp;<span class="uri"><span class="hostname"></span>/v1/search?<span class="hljs-attribute">collection=</span><span class="hljs-literal">persons</span>&<span class="hljs-attribute">q=</span><span class="hljs-literal"></span>&<span class="hljs-attribute">from_=</span><span class="hljs-literal"></span>&<span class="hljs-attribute">size=</span><span class="hljs-literal"></span>&<span class="hljs-attribute">first=</span><span class="hljs-literal"></span>&<span class="hljs-attribute">first_t=</span><span class="hljs-literal">like</span>&<span class="hljs-attribute">last=</span><span class="hljs-literal"></span>&<span class="hljs-attribute">last_t=</span><span class="hljs-literal">like</span>&<span class="hljs-attribute">pob=</span><span class="hljs-literal"></span>&<span class="hljs-attribute">pob_t=</span><span class="hljs-literal">like</span>&<span class="hljs-attribute">pom=</span><span class="hljs-literal"></span>&<span class="hljs-attribute">pom_t=</span><span class="hljs-literal">like</span>&<span class="hljs-attribute">pod=</span><span class="hljs-literal"></span>&<span class="hljs-attribute">pod_t=</span><span class="hljs-literal">like</span>&<span class="hljs-attribute">place=</span><span class="hljs-literal"></span>&<span class="hljs-attribute">place_t=</span><span class="hljs-literal">like</span>&<span class="hljs-attribute">yob=</span><span class="hljs-literal"></span>&<span class="hljs-attribute">yob_t=</span><span class="hljs-literal">pmyears</span>&<span class="hljs-attribute">yob_v=</span><span class="hljs-literal">2</span>&<span class="hljs-attribute">yom=</span><span class="hljs-literal"></span>&<span class="hljs-attribute">yom_t=</span><span class="hljs-literal">pmyears</span>&<span class="hljs-attribute">yom_v=</span><span class="hljs-literal">2</span>&<span class="hljs-attribute">yod=</span><span class="hljs-literal"></span>&<span class="hljs-attribute">yod_t=</span><span class="hljs-literal">pmyears</span>&<span class="hljs-attribute">yod_v=</span><span class="hljs-literal">2</span>&<span class="hljs-attribute">sex=</span><span class="hljs-literal"></span>&<span class="hljs-attribute">treenum=</span><span class="hljs-literal"></span></span></div><div class="title"><strong>URI Parameters</strong><div class="collapse-button show"><span class="close">Hide</span><span class="open">Show</span></div></div><div class="collapse-content"><dl class="inner"><dt>collection</dt><dd><code>enum</code>&nbsp;<span class="required">(required)</span>&nbsp;<span class="text-muted example"><strong>Example:&nbsp;</strong><span>persons</span></span><p>The advanced persons search requires collection to be persons</p>
+<p class="choices"><strong>Choices:&nbsp;</strong><code>persons</code> </p></dd><dt>q</dt><dd><code>string</code>&nbsp;<span>(optional)</span>&nbsp;<p>same as in <a href="#database-search-general-search">general search</a> except: 1. it’s optional   2. it searches in the person places as well</p>
 </dd><dt>from_</dt><dd><code>number</code>&nbsp;<span>(optional)</span>&nbsp;<p>see <a href="#database-search-general-search">general search</a></p>
 </dd><dt>size</dt><dd><code>number</code>&nbsp;<span>(optional)</span>&nbsp;<p>see <a href="#database-search-general-search">general search</a></p>
 </dd><dt>first</dt><dd><code>string</code>&nbsp;<span>(optional)</span>&nbsp;<p>first name</p>
@@ -29,6 +29,8 @@ <h4>Example URI</h4><div class="definition"><span class="method get">GET</span>&
 </dd><dt>pom_t</dt><dd><code>enum</code>&nbsp;<span>(optional)</span>&nbsp;<span class="text-muted example"><strong>Example:&nbsp;</strong><span>like</span></span><p>place of marriage search type</p>
 <p class="choices"><strong>Choices:&nbsp;</strong><code>exact</code> <code>like</code> <code>starts</code> </p></dd><dt>pod</dt><dd><code>string</code>&nbsp;<span>(optional)</span>&nbsp;<p>place of death</p>
 </dd><dt>pod_t</dt><dd><code>enum</code>&nbsp;<span>(optional)</span>&nbsp;<span class="text-muted example"><strong>Example:&nbsp;</strong><span>like</span></span><p>place of death search type</p>
+<p class="choices"><strong>Choices:&nbsp;</strong><code>exact</code> <code>like</code> <code>starts</code> </p></dd><dt>place</dt><dd><code>string</code>&nbsp;<span>(optional)</span>&nbsp;<p>search over all place fields, if any matches it will return the result</p>
+</dd><dt>place_t</dt><dd><code>enum</code>&nbsp;<span>(optional)</span>&nbsp;<span class="text-muted example"><strong>Example:&nbsp;</strong><span>like</span></span><p>combined place field search type</p>
 <p class="choices"><strong>Choices:&nbsp;</strong><code>exact</code> <code>like</code> <code>starts</code> </p></dd><dt>yob</dt><dd><code>number</code>&nbsp;<span>(optional)</span>&nbsp;<p>year of birth</p>
 </dd><dt>yob_t</dt><dd><code>enum</code>&nbsp;<span>(optional)</span>&nbsp;<span class="text-muted example"><strong>Example:&nbsp;</strong><span>pmyears</span></span><p>year of birth search type</p>
 <p class="choices"><strong>Choices:&nbsp;</strong><code>exact</code> <code>pmyears</code> </p></dd><dt>yob_v</dt><dd><code>number</code>&nbsp;<span>(optional)</span>&nbsp;<span class="text-muted example"><strong>Example:&nbsp;</strong><span>2</span></span><p>if the corresponding search type is pmyears, this is the value to add / subtract</p>
@@ -243,7 +245,7 @@ <h4>Example URI</h4><div class="definition"><span class="method get">GET</span>&
     <span class="hljs-string">"Priekule"</span>,
     <span class="hljs-string">"Prienai"</span>
   ]
-</span>}</code></pre><div style="height: 1px;"></div></div></div></div></div></section></div></div></div><p style="text-align: center;" class="text-muted">Generated by&nbsp;<a href="https://github.com/danielgtaylor/aglio" class="aglio">aglio</a>&nbsp;on 16 May 2017</p><script>/* eslint-env browser */
+</span>}</code></pre><div style="height: 1px;"></div></div></div></div></div></section></div></div></div><p style="text-align: center;" class="text-muted">Generated by&nbsp;<a href="https://github.com/danielgtaylor/aglio" class="aglio">aglio</a>&nbsp;on 07 Jun 2017</p><script>/* eslint-env browser */
 /* eslint quotes: [2, "single"] */
 'use strict';
 

docs/_sources/search/persons_advanced_search_params.md

@@ -1 +1 @@
-{?collection,q,from_,size,first,first_t,last,last_t,pob,pob_t,pom,pom_t,pod,pod_t,yob,yob_t,yob_v,yom,yom_t,yom_v,yod,yod_t,yod_v,sex,treenum}
+{?collection,q,from_,size,first,first_t,last,last_t,pob,pob_t,pom,pom_t,pod,pod_t,place,place_t,yob,yob_t,yob_v,yom,yom_t,yom_v,yod,yod_t,yod_v,sex,treenum}

docs/_sources/search/persons_advanced_search_params_description.md

@@ -2,7 +2,7 @@
     + collection: `persons` (enum, required) - The advanced persons search requires collection to be persons
         + Members
             + persons
-    + q (string, optional) - same as in [general search](#database-search-general-search), but optional
+    + q (string, optional) - same as in [general search](#database-search-general-search) except: 1. it's optional   2. it searches in the person places as well
     + from_ (number, optional) - see [general search](#database-search-general-search)
     + size (number, optional) - see [general search](#database-search-general-search)
     + first (string, optional) - first name
@@ -15,6 +15,8 @@
     + pom_t: `like` (enum, optional) - place of marriage <!-- include(text_search_type_members.md) -->
     + pod (string, optional) - place of death
     + pod_t: `like` (enum, optional) - place of death <!-- include(text_search_type_members.md) -->
+    + place (string, optional) - search over all place fields, if any matches it will return the result
+    + place_t: `like` (enum, optional) - combined place field <!-- include(text_search_type_members.md) -->
     + yob (number, optional) - year of birth
     + yob_t: `pmyears` (enum, optional) - year of birth <!-- include(year_search_type_members.md) -->
     + yob_v: <!-- include(year_search_v_param.md) -->

tests/common.py

@@ -43,7 +43,7 @@ def given_local_elasticsearch_client_with_test_data(app, session_id=None):
             "familyNames": [FAMILY_NAMES_DERI, FAMILY_NAMES_EDREHY],
             "personalities": [PERSONALITIES_FERDINAND, PERSONALITIES_DAVIDOV],
             "movies": [MOVIES_MIDAGES, MOVIES_SPAIN],
-            "persons": [PERSON_EINSTEIN, PERSON_LIVING],
+            "persons": [PERSON_EINSTEIN, PERSON_LIVING, PERSON_MOSHE_A, PERSON_MOSHE_B],
         }, reuse_db)
 
 
@@ -69,7 +69,7 @@ def assert_no_results(res):
 
 def assert_search_results(res, num_expected):
     hits = assert_common_elasticsearch_search_results(res)
-    assert len(hits["hits"]) == num_expected and hits["total"] == num_expected
+    assert len(hits["hits"]) == num_expected and hits["total"] == num_expected, "unexpected number of hits: {} / {}".format(len(hits["hits"]), hits["total"])
     for hit in hits["hits"]:
         assert hit["_index"] == "bh_dbs_back_pytest"
         yield hit
@@ -80,9 +80,9 @@ def assert_search_hit_ids(client, search_params, expected_ids, ignore_order=Fals
                in assert_search_results(client.get(u"/v1/search?{}".format(search_params)),
                                         len(expected_ids))]
     if not ignore_order:
-        assert hit_ids == expected_ids
+        assert hit_ids == expected_ids, "hit_ids={}".format(hit_ids)
     else:
-        assert {id:id for id in hit_ids} == {id:id for id in expected_ids}
+        assert {id:id for id in hit_ids} == {id:id for id in expected_ids}, "actual IDs = {}".format(hit_ids)
 
 def assert_suggest_response(client, collection, string,
                             expected_http_status_code=200, expected_error_message=None, expected_json=None):

tests/mocks.py

@@ -1131,3 +1131,27 @@
                    },}
 
 PERSON_LIVING = {"person_id" : "I687", "Slug" : { "En" : "person_1196;0.I687" }, "deceased" : False, "tree_num": 1933, "tree_version": 0, "name": ["Mookie", "Shooki"]}
+
+PERSON_MOSHE_A = {
+  "person_id" : "I7787", "Slug" : { "En" : "person_1796;0.I787" }, "deceased" : True, 
+  "tree_num": 6654, "tree_version": 0,
+  "first_name_lc": "moshe", "last_name_lc": "a", 
+  "BIRT_PLAC_lc": "holon", "DEAT_PLAC_lc": "tel aviv",
+  "Header": {
+      "En": "Moshe A",
+      "He": "Moshe A"
+  }}
+PERSON_MOSHE_B = {
+  "person_id" : "I7788", "Slug" : { "En" : "person_1796;0.I787" }, "deceased" : True, 
+  "tree_num": 6654, "tree_version": 0,
+  "first_name_lc": "moshe", "last_name_lc": "b", 
+  "BIRT_PLAC_lc": "holon", "DEAT_PLAC_lc": "yaffo",
+  "Header": {
+      "En": "Moshe B",
+      "He": "Moshe B"
+  }}
+
+
+
+
+