updating ejscreen data, try two (#1747)

usds · Aug 10, 2022 · e98282d · e98282d
1 parent 002cddf
commit e98282d
Show file tree

Hide file tree

Showing 4 changed files with 11 additions and 14 deletions.
diff --git a/data/data-pipeline/data_pipeline/etl/score/etl_score.py b/data/data-pipeline/data_pipeline/etl/score/etl_score.py
@@ -43,9 +43,7 @@ def extract(self) -> None:
         logger.info("Loading data sets from disk.")
 
         # EJSCreen csv Load
-        ejscreen_csv = (
-            constants.DATA_PATH / "dataset" / "ejscreen_2019" / "usa.csv"
-        )
+        ejscreen_csv = constants.DATA_PATH / "dataset" / "ejscreen" / "usa.csv"
         self.ejscreen_df = pd.read_csv(
             ejscreen_csv,
             dtype={self.GEOID_TRACT_FIELD_NAME: "string"},

diff --git a/data/data-pipeline/data_pipeline/etl/sources/ejscreen/etl.py b/data/data-pipeline/data_pipeline/etl/sources/ejscreen/etl.py
@@ -8,16 +8,14 @@
 
 
 class EJSCREENETL(ExtractTransformLoad):
-    """Load EJSCREEN data.
-
-    Data dictionary:
-        https://gaftp.epa.gov/EJSCREEN/2019/2019_EJSCREEN_columns_explained.csv
-    """
+    """Load updated EJSCREEN data."""
 
     def __init__(self):
-        self.EJSCREEN_FTP_URL = "https://edap-arcgiscloud-data-commons.s3.amazonaws.com/EJSCREEN2020/EJSCREEN_Tract_2020_USPR.csv.zip"
-        self.EJSCREEN_CSV = self.get_tmp_path() / "EJSCREEN_Tract_2020_USPR.csv"
-        self.CSV_PATH = self.DATA_PATH / "dataset" / "ejscreen_2019"
+        self.EJSCREEN_FTP_URL = "https://gaftp.epa.gov/EJSCREEN/2021/EJSCREEN_2021_USPR_Tracts.csv.zip"
+        self.EJSCREEN_CSV = (
+            self.get_tmp_path() / "EJSCREEN_2021_USPR_Tracts.csv"
+        )
+        self.CSV_PATH = self.DATA_PATH / "dataset" / "ejscreen"
         self.df: pd.DataFrame
 
         self.COLUMNS_TO_KEEP = [
@@ -39,6 +37,7 @@ def __init__(self):
             field_names.OVER_64_FIELD,
             field_names.UNDER_5_FIELD,
             field_names.LEAD_PAINT_FIELD,
+            field_names.UST_FIELD,
         ]
 
     def extract(self) -> None:
@@ -53,7 +52,7 @@ def transform(self) -> None:
         logger.info("Transforming EJScreen Data")
         self.df = pd.read_csv(
             self.EJSCREEN_CSV,
-            dtype={"ID": "string"},
+            dtype={"ID": str},
             # EJSCREEN writes the word "None" for NA data.
             na_values=["None"],
             low_memory=False,
@@ -63,8 +62,6 @@ def transform(self) -> None:
         self.df.rename(
             columns={
                 "ID": self.GEOID_TRACT_FIELD_NAME,
-                # Note: it is currently unorthodox to use `field_names` in an ETL class,
-                # but I think that's the direction we'd like to move all ETL classes. - LMB
                 "ACSTOTPOP": field_names.TOTAL_POP_FIELD,
                 "CANCER": field_names.AIR_TOXICS_CANCER_RISK_FIELD,
                 "RESP": field_names.RESPIRATORY_HAZARD_FIELD,
@@ -81,6 +78,7 @@ def transform(self) -> None:
                 "OVER64PCT": field_names.OVER_64_FIELD,
                 "UNDER5PCT": field_names.UNDER_5_FIELD,
                 "PRE1960PCT": field_names.LEAD_PAINT_FIELD,
+                "UST": field_names.UST_FIELD,  # added for 2021 update
             },
             inplace=True,
         )

diff --git a/data/data-pipeline/data_pipeline/score/field_names.py b/data/data-pipeline/data_pipeline/score/field_names.py
@@ -170,6 +170,7 @@
 NPL_FIELD = "Proximity to NPL sites"
 AIR_TOXICS_CANCER_RISK_FIELD = "Air toxics cancer risk"
 RESPIRATORY_HAZARD_FIELD = "Respiratory hazard index"
+UST_FIELD = "Underground storage tanks"
 
 LOW_INCOME_THRESHOLD = "Exceeds FPL200 threshold"
 

diff --git a/docs/decisions/0007-files/MapComparison.mp4 b/docs/decisions/0007-files/MapComparison.mp4