Merge pull request #603 from hackforla/594-BACK-LastPulledDate

metadata table with last_pulled populated during ingest
hackforla · May 12, 2020 · 19e3109 · 19e3109
2 parents d89c848 + c3041b5
commit 19e3109
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 4 deletions.
diff --git a/server/src/services/dataService.py b/server/src/services/dataService.py
@@ -1,4 +1,3 @@
-import datetime
 import pandas as pd
 from .databaseOrm import Ingest
 from utils.database import db
@@ -8,8 +7,8 @@ class DataService(object):
     default_table = Ingest.__tablename__
 
     async def lastPulled(self):
-        # Will represent last time the ingest pipeline ran
-        return datetime.datetime.utcnow()
+        rows = db.exec_sql('SELECT last_pulled FROM metadata')
+        return rows.first()[0]
 
     def standardFilters(self,
                         startDate=None,

diff --git a/server/src/services/sqlIngest.py b/server/src/services/sqlIngest.py
@@ -28,7 +28,11 @@ def __del__(self):
     def resetDatabase(self):
         log('\nResetting database.')
         db.exec_sql(f"""
-            DROP TABLE IF EXISTS {Ingest.__tablename__} CASCADE
+            DROP TABLE IF EXISTS {Ingest.__tablename__} CASCADE;
+
+            DROP TABLE IF EXISTS metadata;
+            CREATE TABLE metadata AS
+            SELECT * FROM (VALUES (NOW())) as vals(last_pulled);
         """)
         Base.metadata.create_all(db.engine)