From 4dcc36bf604398f14513195ad4bbc17a256435f9 Mon Sep 17 00:00:00 2001
From: sellnat77 <sell_nat@yahoo.com>
Date: Sun, 23 Feb 2020 13:33:00 -0800
Subject: [PATCH] Removed redundant config loading

---
 server/src/app.py                |  6 +++---
 server/src/services/sqlIngest.py | 29 ++++++++++-------------------
 2 files changed, 13 insertions(+), 22 deletions(-)

diff --git a/server/src/app.py b/server/src/app.py
index 537b4dc8f..32a98a919 100644
--- a/server/src/app.py
+++ b/server/src/app.py
@@ -80,8 +80,7 @@ async def ingest(request):
     if not all(year in ALLOWED_YEARS for year in years):
         return json({"error":
                     f"'years' param values must be one of {ALLOWED_YEARS}"})
-    loader = DataHandler()
-    loader.loadConfig(configFilePath='./settings.cfg')
+    loader = DataHandler(app.config['Settings'])
     loader.populateFullDatabase(yearRange=years)
     return_data = {'response': 'ingest ok'}
     return json(return_data)
@@ -125,7 +124,8 @@ async def test_multiple_workers(request):
 
 if __name__ == '__main__':
     configure_app()
+    worker_count = max(cpu_count()//2, 1)
     app.run(host=app.config['Settings']['Server']['HOST'],
             port=int(app.config['Settings']['Server']['PORT']),
-            workers=cpu_count()//2,
+            workers=worker_count,
             debug=app.config['Settings']['Server']['DEBUG'])
diff --git a/server/src/services/sqlIngest.py b/server/src/services/sqlIngest.py
index 08cc9a155..6dff4011f 100644
--- a/server/src/services/sqlIngest.py
+++ b/server/src/services/sqlIngest.py
@@ -4,8 +4,12 @@
 import pandas as pd
 import sqlalchemy as db
 from sodapy import Socrata
-from .databaseOrm import tableFields, insertFields, readFields  # Contains db specs and field definitions
 from configparser import ConfigParser
+if __name__ == '__main__':
+    # Contains db specs and field definitions
+    from databaseOrm import tableFields, insertFields, readFields
+else:
+    from .databaseOrm import tableFields, insertFields, readFields
 
 
 class DataHandler:
@@ -14,29 +18,15 @@ def __init__(self, config=None, configFilePath=None, separator=','):
         self.config = config
         self.dbString = None if not self.config \
             else self.config['Database']['DB_CONNECTION_STRING']
+        self.token = None if config['Socrata']['TOKEN'] == 'None' \
+            else config['Socrata']['TOKEN']
         self.filePath = None
         self.configFilePath = configFilePath
         self.separator = separator
         self.fields = tableFields
         self.insertParams = insertFields
         self.readParams = readFields
-        self.dialect = None
-
-    def loadConfig(self, configFilePath):
-        '''Load and parse config data'''
-        if self.config:
-            print('Config already exists at %s. Nothing to load.' %
-                  self.configFilePath)
-            return
-        print('Loading config file %s' % configFilePath)
-        self.configFilePath = configFilePath
-        config = ConfigParser()
-        config.read(configFilePath)
-        self.config = config
-        self.dbString = config['Database']['DB_CONNECTION_STRING']
         self.dialect = self.dbString.split(':')[0]
-        self.token = None if config['Socrata']['TOKEN'] == 'None' \
-            else config['Socrata']['TOKEN']
 
     def loadData(self, fileName="2018_mini"):
         '''Load dataset into pandas object'''
@@ -242,8 +232,9 @@ def fix_nan_vals(resultDict):
 
 if __name__ == "__main__":
     '''Class DataHandler workflow from initial load to SQL population'''
-    loader = DataHandler()
-    loader.loadConfig(configFilePath='../settings.cfg')
+    config = ConfigParser()
+    config.read('../settings.cfg')
+    loader = DataHandler(config)
     loader.fetchSocrataFull()
     loader.cleanData()
     loader.ingestData('ingest_staging_table')