Adjusted ingest endpoint to be more dynamic

Forget the separator correction

server/requirements.txt

@@ -15,6 +15,7 @@ MarkupSafe==1.1.1
 multidict==4.5.2
 numpy==1.17.4
 pandas==0.25.3
+psycopg2==2.8.4
 python-dateutil==2.8.1
 pytz==2019.3
 requests==2.22.0

server/src/app.py

@@ -46,10 +46,19 @@ async def sample_route(request):
     return json(sample_dataset)
 
 
-@app.route('/ingest')
+@app.route('/ingest', methods=["POST"])
 async def ingest(request):
+    '''Accept POST requests with a list of datasets to import\
+       based on the YearMapping. Body parameter format is \
+       {"sets": ["YearMappingKey","YearMappingKey","YearMappingKey"]}'''
+
     ingress_worker = ingress_service(config=app.config['Settings'])
-    return_data = ingress_worker.ingest()
+    return_data = {'response':'ingest ok'}
+
+    for dataSet in request.json.get("sets", None):
+        target_data = app.config["Settings"]["YearMapping"][dataSet]
+        return_data = await ingress_worker.ingest(from_dataset=target_data)
+
     return json(return_data)
 
 

server/src/services/ingress_service.py

@@ -1,13 +1,13 @@
-from sqlIngest import DataHandler
+from .sqlIngest import DataHandler
 
 class ingress_service(object):
     def __init__(self, config=None):
         self.config = config
 
 
-    def ingest(self):
+    async def ingest(self, from_dataset=None):
         loader = DataHandler(config=self.config)
-        loader.loadData()
+        loader.loadData(fileName=from_dataset)
         loader.cleanData()
         loader.ingestData()
         return {'response':'ingest ok'}
@@ -24,6 +24,6 @@ def hello_world(self):
 if __name__ == "__main__":
     from configparser import ConfigParser
     config = ConfigParser()
-    config.read(configFilePath)
+    config.read('../settings.cfg')
     worker = ingress_service(config = config)
     worker.ingest()

server/src/services/sqlIngest.py

@@ -40,7 +40,7 @@ def loadData(self, fileName="311data"):
         self.filePath  = os.path.join(self.config['Database']['DATA_DIRECTORY'], dataFile )
         print('Loading dataset %s' % self.filePath)
         self.data = pd.read_table(self.filePath,
-                                    sep=',',
+                                    sep=self.separator,
                                     na_values=['nan'],
                                     dtype={
                                     'SRNumber':str,
@@ -156,10 +156,8 @@ def ingestData(self):
                        'closedcreatedd':DateTime})
 
 if __name__ == "__main__":
-    def do_stuff():
-        loader = DataHandler()
-        loader.loadConfig('../settings.cfg')
-        loader.loadData()
-        loader.cleanData()
-        loader.ingestData()
-    do_stuff()
+    loader = DataHandler()
+    loader.loadConfig('../settings.cfg')
+    loader.loadData()
+    loader.cleanData()
+    loader.ingestData()

server/src/settings.example.cfg

@@ -9,3 +9,7 @@ DATA_DIRECTORY = .
 
 [Api]
 REDACTED = REDACTED
+
+[YearMapping]
+2018_MINI = 2018_mini
+2018_FULL = 311data