diff --git a/data/data-pipeline/data_pipeline/etl/score/etl_score.py b/data/data-pipeline/data_pipeline/etl/score/etl_score.py index e5674518f..c736e3fc8 100644 --- a/data/data-pipeline/data_pipeline/etl/score/etl_score.py +++ b/data/data-pipeline/data_pipeline/etl/score/etl_score.py @@ -43,9 +43,7 @@ def extract(self) -> None: logger.info("Loading data sets from disk.") # EJSCreen csv Load - ejscreen_csv = ( - constants.DATA_PATH / "dataset" / "ejscreen_2019" / "usa.csv" - ) + ejscreen_csv = constants.DATA_PATH / "dataset" / "ejscreen" / "usa.csv" self.ejscreen_df = pd.read_csv( ejscreen_csv, dtype={self.GEOID_TRACT_FIELD_NAME: "string"}, diff --git a/data/data-pipeline/data_pipeline/etl/sources/ejscreen/etl.py b/data/data-pipeline/data_pipeline/etl/sources/ejscreen/etl.py index 683dbcd26..0ba5db88c 100644 --- a/data/data-pipeline/data_pipeline/etl/sources/ejscreen/etl.py +++ b/data/data-pipeline/data_pipeline/etl/sources/ejscreen/etl.py @@ -8,16 +8,14 @@ class EJSCREENETL(ExtractTransformLoad): - """Load EJSCREEN data. - - Data dictionary: - https://gaftp.epa.gov/EJSCREEN/2019/2019_EJSCREEN_columns_explained.csv - """ + """Load updated EJSCREEN data.""" def __init__(self): - self.EJSCREEN_FTP_URL = "https://edap-arcgiscloud-data-commons.s3.amazonaws.com/EJSCREEN2020/EJSCREEN_Tract_2020_USPR.csv.zip" - self.EJSCREEN_CSV = self.get_tmp_path() / "EJSCREEN_Tract_2020_USPR.csv" - self.CSV_PATH = self.DATA_PATH / "dataset" / "ejscreen_2019" + self.EJSCREEN_FTP_URL = "https://gaftp.epa.gov/EJSCREEN/2021/EJSCREEN_2021_USPR_Tracts.csv.zip" + self.EJSCREEN_CSV = ( + self.get_tmp_path() / "EJSCREEN_2021_USPR_Tracts.csv" + ) + self.CSV_PATH = self.DATA_PATH / "dataset" / "ejscreen" self.df: pd.DataFrame self.COLUMNS_TO_KEEP = [ @@ -39,6 +37,7 @@ def __init__(self): field_names.OVER_64_FIELD, field_names.UNDER_5_FIELD, field_names.LEAD_PAINT_FIELD, + field_names.UST_FIELD, ] def extract(self) -> None: @@ -53,7 +52,7 @@ def transform(self) -> None: logger.info("Transforming EJScreen Data") self.df = pd.read_csv( self.EJSCREEN_CSV, - dtype={"ID": "string"}, + dtype={"ID": str}, # EJSCREEN writes the word "None" for NA data. na_values=["None"], low_memory=False, @@ -63,8 +62,6 @@ def transform(self) -> None: self.df.rename( columns={ "ID": self.GEOID_TRACT_FIELD_NAME, - # Note: it is currently unorthodox to use `field_names` in an ETL class, - # but I think that's the direction we'd like to move all ETL classes. - LMB "ACSTOTPOP": field_names.TOTAL_POP_FIELD, "CANCER": field_names.AIR_TOXICS_CANCER_RISK_FIELD, "RESP": field_names.RESPIRATORY_HAZARD_FIELD, @@ -81,6 +78,7 @@ def transform(self) -> None: "OVER64PCT": field_names.OVER_64_FIELD, "UNDER5PCT": field_names.UNDER_5_FIELD, "PRE1960PCT": field_names.LEAD_PAINT_FIELD, + "UST": field_names.UST_FIELD, # added for 2021 update }, inplace=True, ) diff --git a/data/data-pipeline/data_pipeline/score/field_names.py b/data/data-pipeline/data_pipeline/score/field_names.py index 3254f08ec..04bc62e4f 100644 --- a/data/data-pipeline/data_pipeline/score/field_names.py +++ b/data/data-pipeline/data_pipeline/score/field_names.py @@ -170,6 +170,7 @@ NPL_FIELD = "Proximity to NPL sites" AIR_TOXICS_CANCER_RISK_FIELD = "Air toxics cancer risk" RESPIRATORY_HAZARD_FIELD = "Respiratory hazard index" +UST_FIELD = "Underground storage tanks" LOW_INCOME_THRESHOLD = "Exceeds FPL200 threshold" diff --git a/data/data-pipeline/data_pipeline/score/score_narwhal.py b/data/data-pipeline/data_pipeline/score/score_narwhal.py index 461be8af9..bea5cbed0 100644 --- a/data/data-pipeline/data_pipeline/score/score_narwhal.py +++ b/data/data-pipeline/data_pipeline/score/score_narwhal.py @@ -315,7 +315,6 @@ def _housing_factor(self) -> bool: field_names.LEAD_PAINT_MEDIAN_HOUSE_VALUE_LOW_INCOME_FIELD, field_names.HOUSING_BURDEN_LOW_INCOME_FIELD, # Until we get confirmation -- NOT included - # Updated in a patch # field_names.HISTORIC_REDLINING_SCORE_EXCEEDED_LOW_INCOME_FIELD, ] diff --git a/docs/decisions/0007-files/MapComparison.mp4 b/docs/decisions/0007-files/MapComparison.mp4 deleted file mode 100644 index 273c93924..000000000 Binary files a/docs/decisions/0007-files/MapComparison.mp4 and /dev/null differ