Beit-Hatfutsot#118 some more fixes for elasticsearch search, sort and…

… suggest
OriHoch · Mar 13, 2017 · 724ef25 · 724ef25
1 parent 2e6d3c4
commit 724ef25
Show file tree

Hide file tree

Showing 4 changed files with 667 additions and 64 deletions.
diff --git a/bhs_api/v1_endpoints.py b/bhs_api/v1_endpoints.py
@@ -8,6 +8,7 @@
 import urllib
 import mimetypes
 from uuid import UUID
+import json
 
 from flask import Flask, Blueprint, request, abort, url_for, current_app
 from flask.ext.security import auth_token_required
@@ -74,6 +75,7 @@ def es_search(q, size, collection=None, from_=0, sort=None):
             collection = collection.split(',')
         except:
             pass
+        current_app.logger.debug("es.search index={}, doc_type={} body={}".format(current_app.es_data_db_index_name, collection, json.dumps(body)))
         results = current_app.es.search(index=current_app.es_data_db_index_name, body=body,
                             doc_type=collection, size=size, from_=from_)
     except elasticsearch.exceptions.ConnectionError as e:

diff --git a/scripts/dump_mongo_to_es.py b/scripts/dump_mongo_to_es.py
@@ -60,35 +60,37 @@ def header_mapping(self):
             ret["properties"]["{}_lc".format(lang)] = {"type": "keyword"}
         return ret
 
+    def _get_index_body(self):
+        body = {
+            "mappings": {
+                collection: {
+                    "properties": {
+                        "Header": self.header_mapping,
+                    }
+                }
+                for collection in SEARCHABLE_COLLECTIONS
+            }
+        }
+        body["mappings"]["familyNames"]["properties"]["dm_soundex"] = {
+            "type": "completion",
+            "max_input_length": 20,
+            "contexts": [{
+                "name": "collection",
+                "type": "category",
+                "path": "_type"
+            }]
+        }
+        return body
+
+
     def create_es_index(self, delete_existing=False):
         if self.es.indices.exists(self.es_index_name):
             if delete_existing:
                 self.es.indices.delete(self.es_index_name)
             else:
                 raise Exception()
         # set the mapping to support completion fields
-        self.es.indices.create(self.es_index_name, body={
-            "mappings": {
-                "places": {"properties": {
-                    "Header": self.header_mapping,
-                }},
-                "familyNames": {"properties": {
-                    "Header": self.header_mapping,
-                    "dm_soundex": {
-                        "type": "completion",
-                        "max_input_length": 20,
-                        "contexts": [{
-                            "name": "collection",
-                            "type": "category",
-                            "path": "_type"
-                        }]
-                    }
-                }},
-                "photoUnits": {"properties": {
-                    "Header": self.header_mapping
-                }}
-            }
-        })
+        self.es.indices.create(self.es_index_name, body=self._get_index_body())
 
     def _process_collection(self, collection):
         started = datetime.datetime.now()

diff --git a/scripts/kibana_ssh_tunnel.sh b/scripts/kibana_ssh_tunnel.sh
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+gcloud compute ssh mongo1-dev -- -L 25601:localhost:5601
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		#!/usr/bin/env bash

		gcloud compute ssh mongo1-dev -- -L 25601:localhost:5601