Pypsa model

.env.template

@@ -1,24 +1,16 @@
-# This template comes from https://www.github.omc/kmax12/gridstatus
-
 # Register at https://www.eia.gov/opendata/register.php for an API key
 EIA_API_KEY=
 
-# Register at https://apiportal.pjm.com/ for an API key
-PJM_API_KEY=
-
-# Register at https://apiexplorer.ercot.com/ for username/password
-# and follow instructions at
-# https://developer.ercot.com/applications/pubapi/ERCOT%20Public%20API%20Registration%20and%20Authentication/
-# to get the subscription key
-ERCOT_API_USERNAME=
-ERCOT_API_PASSWORD=
-ERCOT_API_SUBSCRIPTION_KEY=
-
-# Request access at https://www.ncdc.noaa.gov/cdo-web/token
-NOAA_API_KEY=
+# Request an API key at https://developer.nrel.gov/signup/
+NREL_API_KEY=
+NAME=
+REASON=
+AFFIL=
+EMAIL=
+MAILING_LIST=
 
 # Request access at https://api.census.gov/data/key_signup.html
-CENCUS_API_KEY=
+CENSUS_API_KEY=
 
 # Request access at https://www.epa.gov/power-sector/cam-api-portal#/api-key-signup 
 CEMS_API_KEY=

.github/workflows/CI.yml

@@ -0,0 +1,47 @@
+name: Workflow Test
+
+on: 
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs: 
+  build:
+    name: Build environment
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: 
+        - "3.11"
+        os:
+        - windows-latest
+
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up conda
+      uses: conda-incubator/setup-miniconda@v2
+      with:
+        miniforge-variant: Mambaforge # mamba is faster than base conda
+        miniforge-version: latest
+        activate-environment: kansas-city
+        use-mamba: true
+        use-only-tar-bz2: false
+    - run: |
+        conda config --env --set pip_interop_enabled True
+
+    - name: Update environment
+      run: mamba env update -n kansas-city -f environment.yml
+
+    - name: Create .env file
+      run: |
+        echo ${{ secrets.ENV_FILE }} > .env
+
+    - name: Run Snakemake Workflow
+      run: |
+        snakemake --cores=1

.gitignore

@@ -1,5 +1,9 @@
 cjest-data
-
+.snakemake/
+data
+01-energy-utility.ipynb
+02-census.ipynb
+puma_maps.ipynb
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

.pep8speaks.yml

@@ -0,0 +1,30 @@
+# File : .pep8speaks.yml
+
+scanner:
+  diff_only: True # If False, the entire file touched by the Pull Request is scanned for errors. If True, only the diff is scanned.
+  linter: pycodestyle # Other option is flake8
+
+pycodestyle: # Same as scanner.linter value. Other option is flake8
+  max-line-length: 100 # Default is 79 in PEP 8
+  ignore: # Errors and warnings to ignore
+    - W504 # line break after binary operator
+    - E402 # module level import not at top of file
+    - E731 # do not assign a lambda expression, use a def
+    - C406 # Unnecessary list literal - rewrite as a dict literal.
+    - E741 # ambiguous variable name
+
+no_blank_comment: True # If True, no comment is made on PR without any errors.
+descending_issues_order: False # If True, PEP 8 issues in message will be displayed in descending order of line numbers in the file
+
+message: # Customize the comment made by the bot
+  opened: # Messages when a new PR is submitted
+    header:
+      "Hello @{name}! Thanks for opening this PR. "
+      # The keyword {name} is converted into the author's username
+    footer:
+      "Do see the [Hitchhiker's guide to code style](https://goo.gl/hqbW4r)"
+      # The messages can be written as they would over GitHub
+  updated: # Messages when new commits are added to the PR
+    header: "Hello @{name}! Thanks for updating this PR. "
+    footer: "" # Why to comment the link to the style guide everytime? :)
+  no_errors: "There are currently no PEP 8 issues detected in this Pull Request. Cheers! :beers: "

README.md

@@ -1,2 +1,94 @@
 # 2024 Kansas City Analysis
-This repository holds analysis for the energy system in Kansas City, Kansas. Located in Wyandotte County, Kansas.
+This repository holds analysis for the energy system in Kansas City, Kansas. Located in Wyandotte County, Kansas.
+
+# Installation
+
+## Requirements
+
+* `git` - version control software
+    * [Windows Installation instructions](https://git-scm.com/download/win)
+    * [MacOS Installation instructions](https://git-scm.com/download/mac)
+    * [Linux Installation instructions](https://git-scm.com/download/linux)
+* Python installed with either `conda` or `mamba`(recommended)
+    * Download `mamba` installer [here](https://github.com/conda-forge/miniforge).
+    * 'anaconda' ('conda') installation instructions [here](https://docs.anaconda.com/anaconda/install/windows/).
+
+> [!NOTE]
+> Make sure you add Python to PATH during installation.
+
+## Installation Steps
+0. Open command prompt or terminal window. Copy and paste the following commands.
+
+1. Clone the repository
+
+```bash
+git clone https://github.com/ucsusa/2024-kansas-city-analysis.git
+```
+
+2. Set up the environment
+
+```bash
+cd 2024-kansas-city-analysis
+mamba env create  # mamba and conda may be used interchangeably, here
+mamba activate kansas-city
+```
+
+3. Creating the `.env` file
+
+Users should copy the `.env.template` file into a new file simply called `.env`.
+This file contains "secret" information, such as API keys, emails, and other data
+that should remain local. In order to run the current model, users must have API keys
+from the following organizations:
+
+* [U.S. Census API](https://api.census.gov/data/key_signup.html)
+
+These keys may be added directly to the `.env` file.    
+
+## Running the model 
+
+This project uses the workflow management tool, `snakemake`, to create a reproducible data pipeline.
+Running the command
+
+```bash
+snakemake --cores=1
+```
+
+will run the workflow illustrated in the directed acyclic graph (DAG) shown below.
+
+# Workflow
+
+The flow of data through the modeling process is shown in the graph below.
+
+![DAG](dag.png)
+
+There are a few categories of steps:
+* **Retrieve**: In a `retrieve` step, data are primarily downloaded and lightly processed (e.g., ensuring good formatting and data types).
+* **Calculate**: In a `calculate` step, data are transformed through some calculation.
+* *place holder for future additions*
+
+
+## Steps
+
+### `retrieve_census_data`
+In this step, data from the U.S. Census Bureau are queried. The datasets gathered, here, are:
+* Total population and
+* the number and types of residential building units.
+
+### `retrieve_armourdale_shape`
+In this step, the "shape" of the community of interest is retrieved. This shape can be used as a cut-out
+to subset other geospatial data later.
+
+> [!NOTE]
+> This data is specific to the particular community of Armourdale in Kansas City, Kansas. If you 
+> wish to model a different community, should omit this step or replace it with a different shape.
+> For example, by specifying a few census tracts.
+
+### `retrieve_spatial_lut`
+This step downloads the spatial lookup table (LUT) for NREL's ResStock datasets. The spatial LUT 
+cross references census tracts, counties, and states with public use microdata areas (PUMAs). As
+well as how the data are stored within NREL's models.
+
+### `retreive_res_load`
+Simulated building load data is collected from NREL's ResStock database in this step. Currently, 
+the data collected are aggregated building data for the building types defined in the `config.yml` file.
+Future versions may include an option to specify individual buildings.

Snakefile

@@ -0,0 +1,161 @@
+configfile: "config.yml"
+
+from us import states
+from pathlib import Path
+from dotenv import load_dotenv
+
+state = config['state']
+state_abbr = states.lookup(state).abbr
+
+county_name = config['county']
+community_name = config['community_name']
+
+env_file = Path("./.env").resolve()
+load_dotenv(str(env_file))
+
+rule targets:
+    input: 
+        community = f"data/spatial_data/{community_name.lower()}_shape.gpkg",
+        census_data = "data/spatial_data/county_census_data.gpkg",
+        state_blockgroups = f"data/spatial_data/{state.lower()}_blockgroups.gpkg",
+        county_blockgroups = f"data/spatial_data/{config['county'].lower()}_blockgroups.gpkg",
+        elec_load = "data/timeseries/residential_elec_load.csv",
+        heat_load = "data/timeseries/residential_heat_load.csv",
+        weather = "data/timeseries/weather_year.csv",
+        res_structures = "data/residential_buildings.csv",
+        rates = "data/usrdb_rates.csv",
+        project_sunroof = f"data/spatial_data/project-sunroof-census_tract.csv",
+        utility="data/spatial_data/electric_utility.gpkg",
+        lead_data = f"data/spatial_data/{state_abbr}-2018-LEAD-data/{state_abbr} AMI Census Tracts 2018.csv",
+        res_energy_expenses = f"data/{community_name.lower()}_energy_expenses.csv",
+        zoning_data = f"data/spatial_data/{community_name.lower()}/zoning.gpkg",
+        rescaled_elec_load = "data/timeseries/residential_elec_load_rescaled.csv",
+        costs = "data/technology_costs.csv",
+        dag = "dag.png"
+
+rule retrieve_supply_regions:
+    input:
+        script = "scripts/retrieve_supply_regions.py",
+        community = f"data/spatial_data/{community_name.lower()}_shape.gpkg"
+    output: 
+        supply_regions = "data/spatial_data/supply_regions.shp"
+    script: "scripts/retrieve_supply_regions.py"
+
+rule retrieve_spatial_lut:
+    output: 
+        spatial_lut = "data/spatial_data/spatial_lut.csv"
+    script: "scripts/retrieve_lut.py"
+
+rule retrieve_outage_data:
+    input:
+        "scripts/retrieve_outage_data.py"
+    output: 
+        outages = "data/timeseries/outages.csv",
+        county_outages = f"data/timeseries/{county_name.lower()}_outages.csv"
+    script: f"{input}"
+
+rule retrieve_census_data:
+    output:
+        census_data = "data/spatial_data/county_census_data.gpkg",
+        state_blockgroups = f"data/spatial_data/{state.lower()}_blockgroups.gpkg",
+        county_blockgroups = f"data/spatial_data/{county_name.lower()}_blockgroups.gpkg"
+    script: "scripts/retrieve_census_data.py"
+
+rule retrieve_project_sunroof:
+    input: 
+        blockgroups = f"data/spatial_data/{state.lower()}_blockgroups.gpkg",
+        community = f"data/spatial_data/{community_name.lower()}_shape.gpkg"
+    output: 
+        project_sunroof = "data/spatial_data/project-sunroof-census_tract.csv",
+        local_potential = f"data/spatial_data/{community_name.lower()}_rooftop_potential.gpkg"
+    script: "scripts/retrieve_project_sunroof.py"
+
+# a bespoke step to make this analysis specific to community
+rule retrieve_community_shape:
+    output: 
+        community = f"data/spatial_data/{community_name.lower()}_shape.gpkg"
+    script: "scripts/retrieve_community_cutout.py"
+
+rule retrieve_electric_utility:
+    input: 
+      cutout=f"data/spatial_data/{community_name.lower()}_shape.gpkg"
+    output:
+      utility="data/spatial_data/electric_utility.gpkg"
+    script: "scripts/retrieve_electric_utility.py"
+
+rule retrieve_usrdb:
+    input: 
+        utility="data/spatial_data/electric_utility.gpkg"
+    output: 
+        rates="data/usrdb_rates.csv"
+    script: "scripts/retrieve_usrdb.py"
+
+rule calculate_res_structures:
+    input: 
+        census_data = "data/spatial_data/county_census_data.gpkg",
+        community = f"data/spatial_data/{community_name.lower()}_shape.gpkg"
+    output: 
+        res_structures = "data/residential_buildings.csv"
+    script: "scripts/calculate_res_structures.py"
+
+rule retrieve_res_load:
+    input:
+        spatial_lut = "data/spatial_data/spatial_lut.csv"
+    output: 
+        elec_load = "data/timeseries/residential_elec_load.csv",
+        heat_load = "data/timeseries/residential_heat_load.csv",
+        weather = "data/timeseries/weather_year.csv"
+    script: "scripts/retrieve_res_load.py"
+
+rule retrieve_lead_data:
+    input: 
+        community = f"data/spatial_data/{community_name.lower()}_shape.gpkg",
+        county_blockgroups = f"data/spatial_data/{config['county'].lower()}_blockgroups.gpkg"
+    output: 
+        lead_data = f"data/spatial_data/{state_abbr}-2018-LEAD-data/{state_abbr} AMI Census Tracts 2018.csv",
+        lead_community = f"data/spatial_data/{community_name.lower()}_lead.csv"
+    script: "scripts/retrieve_lead_data.py"
+
+rule retrieve_nrel_costs:
+    output: 
+        costs = "data/technology_costs.csv"
+    script: "scripts/retrieve_nrel_costs.py"
+
+rule retrieve_renewable_profiles:
+    input:
+        supply_regions = "data/spatial_data/supply_regions.shp"
+    output:
+        solar = "data/timeseries/solar.csv"
+    script: "scripts/retrieve_renewables.py"
+
+rule calculate_historical_expenses:
+    input:
+        lead_community = f"data/spatial_data/{community_name.lower()}_lead.csv" 
+    output: 
+        res_energy_expenses = f"data/{community_name.lower()}_energy_expenses.csv"
+    script: "scripts/calculate_historical_expenses.py"
+
+rule retrieve_community_spatial_data:
+    input: 
+        community = f"data/spatial_data/{community_name.lower()}_shape.gpkg"
+    output: 
+        zoning_data = f"data/spatial_data/{community_name.lower()}/zoning.gpkg"
+    script: "scripts/retrieve_shapefiles.py"
+
+rule calculate_rescaled_load:
+    input: 
+        res_energy_expenses = f"data/{community_name.lower()}_energy_expenses.csv",
+        elec_load = "data/timeseries/residential_elec_load.csv",
+        heat_load = "data/timeseries/residential_heat_load.csv",
+        res_structures = "data/residential_buildings.csv"
+    output: 
+        rescaled_elec_load = "data/timeseries/residential_elec_load_rescaled.csv",
+        rescaled_heat_load = "data/timeseries/residential_heat_load_rescaled.csv",
+    script: "scripts/calculate_residential_load.py"
+
+rule build_dag:
+    input: "Snakefile"
+    output:
+        "dag.png"
+    shell:
+        "snakemake --dag | dot -Tpng > {output}"