work in progress on generalizing opensearch connection params

mmguero-dev · Aug 18, 2022 · fe962ca · fe962ca
1 parent 175163d
commit fe962ca
Show file tree

Hide file tree

Showing 2 changed files with 46 additions and 11 deletions.
diff --git a/Dockerfiles/api.Dockerfile b/Dockerfiles/api.Dockerfile
@@ -12,7 +12,7 @@ RUN    apt-get update -q \
     && python3 -m pip install flake8
 
 COPY ./api /usr/src/app/
-COPY scripts/malcolm_common.py /usr/src/app/project/
+COPY scripts/malcolm_common.py /usr/src/app/
 WORKDIR /usr/src/app
 
 RUN python3 -m pip wheel --no-cache-dir --no-deps --wheel-dir /usr/src/app/wheels -r requirements.txt \
@@ -70,7 +70,7 @@ WORKDIR "${APP_HOME}"
 COPY --from=builder /usr/src/app/wheels /wheels
 COPY --from=builder /usr/src/app/requirements.txt .
 COPY ./api "${APP_HOME}"
-COPY scripts/malcolm_common.py "${APP_HOME}"/project/
+COPY scripts/malcolm_common.py "${APP_HOME}"/
 COPY shared/bin/opensearch_status.sh "${APP_HOME}"/
 
 ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/

diff --git a/api/project/__init__.py b/api/project/__init__.py
@@ -1,5 +1,6 @@
 import dateparser
 import json
+import malcolm_common
 import opensearch_dsl
 import opensearchpy
 import os
@@ -9,14 +10,16 @@
 import requests
 import string
 import traceback
+import urllib3
 import warnings
 
 from collections import defaultdict
 from collections.abc import Iterable
 from datetime import datetime
 from flask import Flask, jsonify, request
+from requests.auth import HTTPBasicAuth
+from urllib.parse import urlparse
 
-# TODO: handle OPENSEARCH_URL, OPENSEARCH_LOCAL, OPENSEARCH_SSL_CERTIFICATE_VERIFICATION, OPENSEARCH_CREDS_CONFIG_FILE
 
 # map categories of field names to OpenSearch dashboards
 fields_to_urls = []
@@ -142,7 +145,7 @@
 field_type_map['time'] = 'date'
 field_type_map['timestamp'] = 'date'
 
-
+urllib3.disable_warnings()
 warnings.filterwarnings(
     "ignore",
     message="The localize method is no longer necessary, as this time zone supports the fold attribute",
@@ -151,13 +154,29 @@
 app = Flask(__name__)
 app.url_map.strict_slashes = False
 app.config.from_object("project.config.Config")
+
+debugApi = app.config["MALCOLM_API_DEBUG"] == "true"
+
+opensearchUrl = app.config["OPENSEARCH_URL"]
+opensearchLocal = (app.config["OPENSEARCH_LOCAL"] == "true") or (opensearchUrl == 'http://opensearch:9200')
+opensearchSslVerify = app.config["OPENSEARCH_SSL_CERTIFICATE_VERIFICATION"] == "true"
+opensearchCreds = (
+    malcolm_common.ParseCurlFile(app.config["OPENSEARCH_URL"]) if (not opensearchLocal) else defaultdict(lambda: None)
+)
+if opensearchCreds['user'] is not None:
+    opensearchDslHttpAuth = f"{opensearchCreds['user']}:{opensearchCreds['password']}"
+    opensearchReqHttpAuth = HTTPBasicAuth(opensearchCreds['user'], opensearchCreds['password'])
+else:
+    opensearchDslHttpAuth = None
+    opensearchReqHttpAuth = None
+
 opensearch_dsl.connections.create_connection(
-    hosts=[app.config["OPENSEARCH_URL"]],
-    verify_certs=False,
+    hosts=[opensearchUrl],
+    http_auth=opensearchDslHttpAuth,
+    verify_certs=opensearchSslVerify,
     ssl_assert_hostname=False,
     ssl_show_warn=False,
 )
-debugApi = app.config["MALCOLM_API_DEBUG"] == "true"
 
 
 def deep_get(d, keys, default=None):
@@ -528,7 +547,13 @@ def indices():
     indices
         The output of _cat/indices?format=json from the OpenSearch API
     """
-    return jsonify(indices=requests.get(f'{app.config["OPENSEARCH_URL"]}/_cat/indices?format=json').json())
+    return jsonify(
+        indices=requests.get(
+            f'{opensearchUrl}/_cat/indices?format=json',
+            auth=opensearchReqHttpAuth,
+            verify=opensearchSslVerify,
+        ).json()
+    )
 
 
 @app.route("/fields", methods=['GET'])
@@ -576,7 +601,11 @@ def fields():
 
     # get fields from OpenSearch template (and descendant components)
     try:
-        getTemplateResponseJson = requests.get(f'{app.config["OPENSEARCH_URL"]}/_index_template/{templateName}').json()
+        getTemplateResponseJson = requests.get(
+            f'{opensearchUrl}/_index_template/{templateName}',
+            auth=opensearchReqHttpAuth,
+            verify=opensearchSslVerify,
+        ).json()
 
         for template in deep_get(getTemplateResponseJson, ["index_templates"]):
             # top-level fields
@@ -594,7 +623,9 @@ def fields():
             # descendant component fields
             for componentName in get_iterable(deep_get(template, ["index_template", "composed_of"])):
                 getComponentResponseJson = requests.get(
-                    f'{app.config["OPENSEARCH_URL"]}/_component_template/{componentName}'
+                    f'{opensearchUrl}/_component_template/{componentName}',
+                    auth=opensearchReqHttpAuth,
+                    verify=opensearchSslVerify,
                 ).json()
                 for component in get_iterable(deep_get(getComponentResponseJson, ["component_templates"])):
                     for fieldname, fieldinfo in deep_get(
@@ -665,7 +696,11 @@ def version():
         version=app.config["MALCOLM_VERSION"],
         built=app.config["BUILD_DATE"],
         sha=app.config["VCS_REVISION"],
-        opensearch=requests.get(app.config["OPENSEARCH_URL"]).json(),
+        opensearch=requests.get(
+            opensearchUrl,
+            auth=opensearchReqHttpAuth,
+            verify=opensearchSslVerify,
+        ).json(),
         opensearch_health=opensearch_dsl.connections.get_connection().cluster.health(),
     )