Update Workflow's with newer versions of Postgres, Ubuntu, and Checko…

…ut action (#440) * update ubuntu * update postgres versions * update checkout to v3 * add more shared mem * update shm size in pts workflow * update documentation and names * revert back to 20 * add new env var and update to 22.04 * rever to ubuntu 20.04
NYCPlanning · Apr 21, 2023 · 63e5934 · 63e5934
1 parent e93691a
commit 63e5934
Show file tree

Hide file tree

Showing 8 changed files with 56 additions and 51 deletions.
diff --git a/.github/workflows/input_cama.yml b/.github/workflows/input_cama.yml
@@ -1,4 +1,4 @@
-name: 📁 CAMA Processing
+name: 📁 CAMA Processing (Input data for PLUTO)
 
 on:
   workflow_dispatch:
@@ -7,10 +7,10 @@ on:
 
 jobs:
   process_cama:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     services:
       postgres:
-        image: postgis/postgis:12-3.0-alpine
+        image: postgis/postgis:15-3.3-alpine
         env:
           POSTGRES_PASSWORD: postgres
         options: >-
@@ -30,7 +30,7 @@ jobs:
       GINGER_USER: ${{ secrets.GINGER_USER }}
       GINGER_HOST: ${{ secrets.GINGER_HOST }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: install ssh key
         run: |
           mkdir -p ~/.ssh/

diff --git a/.github/workflows/input_numbldgs.yml b/.github/workflows/input_numbldgs.yml
@@ -1,4 +1,4 @@
-name: 📁 NumBldgs processing
+name: 📁 Number of Buildings Processing (input data for PLUTO)
 
 on:
   schedule:
@@ -7,15 +7,15 @@ on:
 
 jobs:
   process_numbldgs:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     env:
       AWS_S3_ENDPOINT: ${{ secrets.DO_S3_ENDPOINT }}
       AWS_ACCESS_KEY_ID: ${{ secrets.DO_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.DO_SECRET_ACCESS_KEY }}
       AWS_S3_BUCKET: edm-recipes
       API_TOKEN: ${{ secrets.API_TOKEN }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: install minio ..
         run: |

diff --git a/.github/workflows/input_pts.yml b/.github/workflows/input_pts.yml
@@ -1,10 +1,10 @@
-name: 📁 PTS processing
+name: 📁 Property Tax System (PTS) processing (input data for PLUTO)
 on:
   push:
     branches:
       - main
       - pts/**
-    paths: 
+    paths:
       - pluto_build/python/geocode.py
       - pluto_build/bin/pts.sh
       - pluto_build/sql/_load_pts.sql
@@ -17,14 +17,15 @@ jobs:
     runs-on: ubuntu-20.04
     services:
       postgres:
-        image: postgis/postgis:12-3.0-alpine
+        image: postgis/postgis:15-3.3-alpine
         env:
           POSTGRES_PASSWORD: postgres
         options: >-
           --health-cmd pg_isready
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
+          --shm-size=2gb
         ports:
           - 5432:5432
     env:
@@ -37,7 +38,7 @@ jobs:
       GINGER_USER: ${{ secrets.GINGER_USER }}
       GINGER_HOST: ${{ secrets.GINGER_HOST }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: install ssh key
         run: |
           mkdir -p ~/.ssh/
@@ -54,7 +55,7 @@ jobs:
       - name: process pts
         working-directory: pluto_build
         run: ./pluto import pts
-        
+
       - uses: NYCPlanning/[email protected]
         id: pluto_pts
         with:
@@ -68,7 +69,7 @@ jobs:
       - name: geocode pts
         working-directory: pluto_build
         run: ./pluto geocode pts
-      
+
       - uses: NYCPlanning/[email protected]
         id: pluto_input_geocodes
         with:
@@ -78,7 +79,7 @@ jobs:
           latest: true
           compress: true
           output_format: pgdump csv
-      
+
       - name: clean up pts
         shell: bash
         working-directory: pluto_build

diff --git a/.github/workflows/issue-comment-runner.yml b/.github/workflows/issue-comment-runner.yml
@@ -15,11 +15,11 @@ jobs:
         github.event.comment.user.login == 'AmandaDoyle' ||
         github.event.comment.user.login == 'td928'
       )
-      
-    runs-on: ubuntu-20.04
+
+    runs-on: ubuntu-22.04
     services:
       postgres:
-        image: postgis/postgis:12-3.0-alpine
+        image: postgis/postgis:15-3.3-alpine
         env:
           POSTGRES_PASSWORD: postgres
         options: >-
@@ -37,7 +37,7 @@ jobs:
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: install dependencies
         working-directory: pluto_build
@@ -51,22 +51,22 @@ jobs:
         working-directory: pluto_build
         shell: bash
         run: ./01_dataloading.sh
-      
+
       - name: building ...
         working-directory: pluto_build
         shell: bash
         run: ./02_build.sh
-      
+
       - name: applying corrections ...
         shell: bash
         working-directory: pluto_build
         run: ./03_corrections.sh
-      
+
       - name: archiving ...
         shell: bash
         working-directory: pluto_build
         run: ./04_archive.sh
-          
+
       - name: Creating QAQC tables ...
         shell: bash
         working-directory: pluto_build
@@ -77,7 +77,7 @@ jobs:
         shell: bash
         run: |
           ./06_export.sh
-      
+
       - name: create report
         id: report
         shell: bash
@@ -88,22 +88,22 @@ jobs:
           report="${report//$'\n'/'%0A'}"
           report="${report//$'\r'/'%0D'}"
           echo ::set-output name=report::$report
-              
+
       - name: report results in comment
         if: success()
         uses: peter-evans/create-or-update-comment@v1
         with:
           comment-id: ${{ github.event.comment.id }}
           body: |
-            
+
             ## PLUTO Build Complete! 
-            
+
             <summary> Source Info </summary>
             <details>
-            
+
             ${{ steps.report.outputs.report }}
-            
+
             </details>
-            
+
             for more details, check https://github.com/NYCPlanning/db-pluto/actions/runs/${{ github.run_id }}
           reactions: hooray
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -8,23 +8,23 @@ on:
 jobs:
   version:
     name: get version
-    runs-on: ubuntu-20.04
-    outputs: 
+    runs-on: ubuntu-22.04
+    outputs:
       version: ${{ steps.version.outputs.version }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Get Version
         id: version
         run: |
           source pluto_build/version.env
           echo "::set-output name=version::$VERSION"
           echo "Version is $VERSION"
-  
+
   publish:
     needs: [version]
     name: publishing
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     env:
       AWS_S3_ENDPOINT: ${{ secrets.DO_S3_ENDPOINT }}
       AWS_ACCESS_KEY_ID: ${{ secrets.DO_ACCESS_KEY_ID }}
@@ -39,7 +39,7 @@ jobs:
           - dcp_mappluto_clipped
           # - pluto_corrections
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: NYCPlanning/[email protected]
         # if: matrix.dataset != 'pluto_corrections'
         with:
@@ -49,7 +49,7 @@ jobs:
           compress: true
           output_format: pgdump csv shapefile
           version: ${{ needs.version.outputs.version }}
-      
+
       - uses: NYCPlanning/[email protected]
         # if: matrix.dataset == 'pluto_corrections'
         with:
@@ -67,8 +67,8 @@ jobs:
           project_id: ${{ secrets.GCP_PROJECT_ID_DATA_ENGINEERING }}
           service_account_key: ${{ secrets.GCP_GCS_BQ_SA_KEY }}
           export_default_credentials: true
-          
+
       - name: Upload to BigQuery
         if: matrix.dataset == 'dcp_mappluto'
         working-directory: pluto_build
-        run: ./pluto bq publish dcp_mappluto
+        run: ./pluto bq publish dcp_mappluto
diff --git a/.github/workflows/publish_historical.yml b/.github/workflows/publish_historical.yml
@@ -5,27 +5,27 @@ on:
       version:
         description: version name of the release
         required: false
-        default: '02b'
+        default: "02b"
       library:
         description: run data library load process?
         required: false
-        default: 'yes'
+        default: "yes"
       bq:
         description: run bq load process?
         required: false
-        default: 'yes'
+        default: "yes"
 jobs:
   library:
     name: load to data library
     if: github.event.inputs.library == 'yes'
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     env:
       AWS_S3_ENDPOINT: ${{ secrets.DO_S3_ENDPOINT }}
       AWS_ACCESS_KEY_ID: ${{ secrets.DO_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.DO_SECRET_ACCESS_KEY }}
       AWS_S3_BUCKET: edm-recipes
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: NYCPlanning/[email protected]
         if: github.event.inputs.library == 'yes'
         with:
@@ -38,17 +38,17 @@ jobs:
   bq:
     name: load to bigquery
     needs: library
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     if: always() && github.event.inputs.bq == 'yes'
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Set up Cloud SDK
         uses: google-github-actions/setup-gcloud@master
         with:
           project_id: ${{ secrets.GCP_PROJECT_ID_DATA_ENGINEERING }}
           service_account_key: ${{ secrets.GCP_GCS_BQ_SA_KEY }}
           export_default_credentials: true
-          
+
       - name: Upload to BigQuery
         working-directory: pluto_build
         run: ./pluto bq publish_historical dcp_mappluto ${{ github.event.inputs.version }}
diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml
@@ -15,17 +15,18 @@ on:
         default: false
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     services:
       postgres:
-        image: postgis/postgis:12-3.0-alpine
+        image: postgis/postgis:15-3.3-alpine
         env:
           POSTGRES_PASSWORD: postgres
         options: >-
           --health-cmd pg_isready
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
+          --shm-size=2gb
         ports:
           - 5432:5432
     env:
@@ -35,7 +36,7 @@ jobs:
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: install dependencies
         working-directory: pluto_build

diff --git a/pluto_build/templates/pluto_input_cama_dof.yml b/pluto_build/templates/pluto_input_cama_dof.yml
@@ -26,6 +26,9 @@ dataset:
 
   info:
     description: |
-      ### CAMA
+      ### CAMA - DOF uses the Computer-Assisted Mass Appraisal System (CAMA) for property
+      tax valuations. Assessors determine parcel market value using one of several valuation
+      methods based on the parcels tax class. Once market value is determined, the assessed
+      value is a percentage calculation of the market value.
     url: null
-    dependents: []
+    dependents: []