From 362cfa68e6103f5d701160b3866daad1d138f8d5 Mon Sep 17 00:00:00 2001 From: mbh329 Date: Thu, 20 Apr 2023 12:16:34 -0400 Subject: [PATCH 1/9] update ubuntu --- .github/workflows/input_cama.yml | 2 +- .github/workflows/input_numbldgs.yml | 2 +- .github/workflows/input_pts.yml | 10 ++++----- .github/workflows/issue-comment-runner.yml | 26 +++++++++++----------- .github/workflows/publish.yml | 14 ++++++------ .github/workflows/publish_historical.yml | 12 +++++----- .github/workflows/run.yml | 2 +- 7 files changed, 34 insertions(+), 34 deletions(-) diff --git a/.github/workflows/input_cama.yml b/.github/workflows/input_cama.yml index 2089b55a..628e421b 100644 --- a/.github/workflows/input_cama.yml +++ b/.github/workflows/input_cama.yml @@ -7,7 +7,7 @@ on: jobs: process_cama: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 services: postgres: image: postgis/postgis:12-3.0-alpine diff --git a/.github/workflows/input_numbldgs.yml b/.github/workflows/input_numbldgs.yml index 718cd45a..91820d74 100644 --- a/.github/workflows/input_numbldgs.yml +++ b/.github/workflows/input_numbldgs.yml @@ -7,7 +7,7 @@ on: jobs: process_numbldgs: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 env: AWS_S3_ENDPOINT: ${{ secrets.DO_S3_ENDPOINT }} AWS_ACCESS_KEY_ID: ${{ secrets.DO_ACCESS_KEY_ID }} diff --git a/.github/workflows/input_pts.yml b/.github/workflows/input_pts.yml index 942fd2ee..8d361c08 100644 --- a/.github/workflows/input_pts.yml +++ b/.github/workflows/input_pts.yml @@ -4,7 +4,7 @@ on: branches: - main - pts/** - paths: + paths: - pluto_build/python/geocode.py - pluto_build/bin/pts.sh - pluto_build/sql/_load_pts.sql @@ -14,7 +14,7 @@ on: jobs: process_pts: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 services: postgres: image: postgis/postgis:12-3.0-alpine @@ -54,7 +54,7 @@ jobs: - name: process pts working-directory: pluto_build run: ./pluto import pts - + - uses: NYCPlanning/action-library-archive@v1.1 id: pluto_pts with: @@ -68,7 +68,7 @@ jobs: - name: geocode pts working-directory: pluto_build run: ./pluto geocode pts - + - uses: NYCPlanning/action-library-archive@v1.1 id: pluto_input_geocodes with: @@ -78,7 +78,7 @@ jobs: latest: true compress: true output_format: pgdump csv - + - name: clean up pts shell: bash working-directory: pluto_build diff --git a/.github/workflows/issue-comment-runner.yml b/.github/workflows/issue-comment-runner.yml index 78178db5..2c682648 100644 --- a/.github/workflows/issue-comment-runner.yml +++ b/.github/workflows/issue-comment-runner.yml @@ -15,8 +15,8 @@ jobs: github.event.comment.user.login == 'AmandaDoyle' || github.event.comment.user.login == 'td928' ) - - runs-on: ubuntu-20.04 + + runs-on: ubuntu-22.04 services: postgres: image: postgis/postgis:12-3.0-alpine @@ -51,22 +51,22 @@ jobs: working-directory: pluto_build shell: bash run: ./01_dataloading.sh - + - name: building ... working-directory: pluto_build shell: bash run: ./02_build.sh - + - name: applying corrections ... shell: bash working-directory: pluto_build run: ./03_corrections.sh - + - name: archiving ... shell: bash working-directory: pluto_build run: ./04_archive.sh - + - name: Creating QAQC tables ... shell: bash working-directory: pluto_build @@ -77,7 +77,7 @@ jobs: shell: bash run: | ./06_export.sh - + - name: create report id: report shell: bash @@ -88,22 +88,22 @@ jobs: report="${report//$'\n'/'%0A'}" report="${report//$'\r'/'%0D'}" echo ::set-output name=report::$report - + - name: report results in comment if: success() uses: peter-evans/create-or-update-comment@v1 with: comment-id: ${{ github.event.comment.id }} body: | - + ## PLUTO Build Complete! - + Source Info
- + ${{ steps.report.outputs.report }} - +
- + for more details, check https://github.com/NYCPlanning/db-pluto/actions/runs/${{ github.run_id }} reactions: hooray diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index bf6ace9c..8d1d6e56 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -8,8 +8,8 @@ on: jobs: version: name: get version - runs-on: ubuntu-20.04 - outputs: + runs-on: ubuntu-22.04 + outputs: version: ${{ steps.version.outputs.version }} steps: - uses: actions/checkout@v2 @@ -20,11 +20,11 @@ jobs: source pluto_build/version.env echo "::set-output name=version::$VERSION" echo "Version is $VERSION" - + publish: needs: [version] name: publishing - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 env: AWS_S3_ENDPOINT: ${{ secrets.DO_S3_ENDPOINT }} AWS_ACCESS_KEY_ID: ${{ secrets.DO_ACCESS_KEY_ID }} @@ -49,7 +49,7 @@ jobs: compress: true output_format: pgdump csv shapefile version: ${{ needs.version.outputs.version }} - + - uses: NYCPlanning/action-library-archive@v1.1 # if: matrix.dataset == 'pluto_corrections' with: @@ -67,8 +67,8 @@ jobs: project_id: ${{ secrets.GCP_PROJECT_ID_DATA_ENGINEERING }} service_account_key: ${{ secrets.GCP_GCS_BQ_SA_KEY }} export_default_credentials: true - + - name: Upload to BigQuery if: matrix.dataset == 'dcp_mappluto' working-directory: pluto_build - run: ./pluto bq publish dcp_mappluto \ No newline at end of file + run: ./pluto bq publish dcp_mappluto diff --git a/.github/workflows/publish_historical.yml b/.github/workflows/publish_historical.yml index 0e0f07d9..6ec682cc 100644 --- a/.github/workflows/publish_historical.yml +++ b/.github/workflows/publish_historical.yml @@ -5,20 +5,20 @@ on: version: description: version name of the release required: false - default: '02b' + default: "02b" library: description: run data library load process? required: false - default: 'yes' + default: "yes" bq: description: run bq load process? required: false - default: 'yes' + default: "yes" jobs: library: name: load to data library if: github.event.inputs.library == 'yes' - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 env: AWS_S3_ENDPOINT: ${{ secrets.DO_S3_ENDPOINT }} AWS_ACCESS_KEY_ID: ${{ secrets.DO_ACCESS_KEY_ID }} @@ -38,7 +38,7 @@ jobs: bq: name: load to bigquery needs: library - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 if: always() && github.event.inputs.bq == 'yes' steps: - uses: actions/checkout@v2 @@ -48,7 +48,7 @@ jobs: project_id: ${{ secrets.GCP_PROJECT_ID_DATA_ENGINEERING }} service_account_key: ${{ secrets.GCP_GCS_BQ_SA_KEY }} export_default_credentials: true - + - name: Upload to BigQuery working-directory: pluto_build run: ./pluto bq publish_historical dcp_mappluto ${{ github.event.inputs.version }} diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index 9086b3de..e22a8b28 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -15,7 +15,7 @@ on: default: false jobs: build: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 services: postgres: image: postgis/postgis:12-3.0-alpine From a8d805d21d511df12139dafaa057a8268c0b46ef Mon Sep 17 00:00:00 2001 From: mbh329 Date: Thu, 20 Apr 2023 12:20:44 -0400 Subject: [PATCH 2/9] update postgres versions --- .github/workflows/input_cama.yml | 2 +- .github/workflows/input_pts.yml | 2 +- .github/workflows/issue-comment-runner.yml | 2 +- .github/workflows/run.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/input_cama.yml b/.github/workflows/input_cama.yml index 628e421b..379f37ba 100644 --- a/.github/workflows/input_cama.yml +++ b/.github/workflows/input_cama.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-22.04 services: postgres: - image: postgis/postgis:12-3.0-alpine + image: postgis/postgis:15-3.3-alpine env: POSTGRES_PASSWORD: postgres options: >- diff --git a/.github/workflows/input_pts.yml b/.github/workflows/input_pts.yml index 8d361c08..f3484358 100644 --- a/.github/workflows/input_pts.yml +++ b/.github/workflows/input_pts.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-22.04 services: postgres: - image: postgis/postgis:12-3.0-alpine + image: postgis/postgis:15-3.3-alpine env: POSTGRES_PASSWORD: postgres options: >- diff --git a/.github/workflows/issue-comment-runner.yml b/.github/workflows/issue-comment-runner.yml index 2c682648..51519e39 100644 --- a/.github/workflows/issue-comment-runner.yml +++ b/.github/workflows/issue-comment-runner.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-22.04 services: postgres: - image: postgis/postgis:12-3.0-alpine + image: postgis/postgis:15-3.3-alpine env: POSTGRES_PASSWORD: postgres options: >- diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index e22a8b28..ef26888c 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-22.04 services: postgres: - image: postgis/postgis:12-3.0-alpine + image: postgis/postgis:15-3.3-alpine env: POSTGRES_PASSWORD: postgres options: >- From 65caeb0f8a5ea21e853a1fb5146c4bea42508bd3 Mon Sep 17 00:00:00 2001 From: mbh329 Date: Thu, 20 Apr 2023 12:25:25 -0400 Subject: [PATCH 3/9] update checkout to v3 --- .github/workflows/input_cama.yml | 2 +- .github/workflows/input_numbldgs.yml | 2 +- .github/workflows/input_pts.yml | 2 +- .github/workflows/issue-comment-runner.yml | 2 +- .github/workflows/publish.yml | 4 ++-- .github/workflows/publish_historical.yml | 4 ++-- .github/workflows/run.yml | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/input_cama.yml b/.github/workflows/input_cama.yml index 379f37ba..c8c1c77d 100644 --- a/.github/workflows/input_cama.yml +++ b/.github/workflows/input_cama.yml @@ -30,7 +30,7 @@ jobs: GINGER_USER: ${{ secrets.GINGER_USER }} GINGER_HOST: ${{ secrets.GINGER_HOST }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: install ssh key run: | mkdir -p ~/.ssh/ diff --git a/.github/workflows/input_numbldgs.yml b/.github/workflows/input_numbldgs.yml index 91820d74..46414bc2 100644 --- a/.github/workflows/input_numbldgs.yml +++ b/.github/workflows/input_numbldgs.yml @@ -15,7 +15,7 @@ jobs: AWS_S3_BUCKET: edm-recipes API_TOKEN: ${{ secrets.API_TOKEN }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: install minio .. run: | diff --git a/.github/workflows/input_pts.yml b/.github/workflows/input_pts.yml index f3484358..3ec71558 100644 --- a/.github/workflows/input_pts.yml +++ b/.github/workflows/input_pts.yml @@ -37,7 +37,7 @@ jobs: GINGER_USER: ${{ secrets.GINGER_USER }} GINGER_HOST: ${{ secrets.GINGER_HOST }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: install ssh key run: | mkdir -p ~/.ssh/ diff --git a/.github/workflows/issue-comment-runner.yml b/.github/workflows/issue-comment-runner.yml index 51519e39..0e3afb5c 100644 --- a/.github/workflows/issue-comment-runner.yml +++ b/.github/workflows/issue-comment-runner.yml @@ -37,7 +37,7 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: install dependencies working-directory: pluto_build diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8d1d6e56..e4236084 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -12,7 +12,7 @@ jobs: outputs: version: ${{ steps.version.outputs.version }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Get Version id: version @@ -39,7 +39,7 @@ jobs: - dcp_mappluto_clipped # - pluto_corrections steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: NYCPlanning/action-library-archive@v1.1 # if: matrix.dataset != 'pluto_corrections' with: diff --git a/.github/workflows/publish_historical.yml b/.github/workflows/publish_historical.yml index 6ec682cc..7f29e5c6 100644 --- a/.github/workflows/publish_historical.yml +++ b/.github/workflows/publish_historical.yml @@ -25,7 +25,7 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.DO_SECRET_ACCESS_KEY }} AWS_S3_BUCKET: edm-recipes steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: NYCPlanning/action-library-archive@v1.1 if: github.event.inputs.library == 'yes' with: @@ -41,7 +41,7 @@ jobs: runs-on: ubuntu-22.04 if: always() && github.event.inputs.bq == 'yes' steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Cloud SDK uses: google-github-actions/setup-gcloud@master with: diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index ef26888c..e92e188d 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -35,7 +35,7 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: install dependencies working-directory: pluto_build From 2fb55fd49e48cf686860b1837ce55a8ecd9b7e51 Mon Sep 17 00:00:00 2001 From: mbh329 Date: Thu, 20 Apr 2023 14:24:42 -0400 Subject: [PATCH 4/9] add more shared mem --- .github/workflows/run.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index e92e188d..db70055d 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -26,6 +26,7 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 + --shm-size=2gb ports: - 5432:5432 env: From 85cad29a8db52bab9d236511e8b1689e51c68cf1 Mon Sep 17 00:00:00 2001 From: mbh329 Date: Thu, 20 Apr 2023 16:28:14 -0400 Subject: [PATCH 5/9] update shm size in pts workflow --- .github/workflows/input_pts.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/input_pts.yml b/.github/workflows/input_pts.yml index 3ec71558..c98b40a7 100644 --- a/.github/workflows/input_pts.yml +++ b/.github/workflows/input_pts.yml @@ -25,6 +25,7 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 + --shm-size=2gb ports: - 5432:5432 env: From 3ea1023b0cbb022d035c87541997ee993b723f21 Mon Sep 17 00:00:00 2001 From: mbh329 Date: Fri, 21 Apr 2023 10:22:25 -0400 Subject: [PATCH 6/9] update documentation and names --- .github/workflows/input_cama.yml | 2 +- .github/workflows/input_numbldgs.yml | 2 +- .github/workflows/input_pts.yml | 2 +- pluto_build/templates/pluto_input_cama_dof.yml | 7 +++++-- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/input_cama.yml b/.github/workflows/input_cama.yml index c8c1c77d..7669a7a3 100644 --- a/.github/workflows/input_cama.yml +++ b/.github/workflows/input_cama.yml @@ -1,4 +1,4 @@ -name: 📁 CAMA Processing +name: 📁 CAMA Processing (Input data for PLUTO) on: workflow_dispatch: diff --git a/.github/workflows/input_numbldgs.yml b/.github/workflows/input_numbldgs.yml index 46414bc2..66c88c18 100644 --- a/.github/workflows/input_numbldgs.yml +++ b/.github/workflows/input_numbldgs.yml @@ -1,4 +1,4 @@ -name: 📁 NumBldgs processing +name: 📁 Number of Buildings Processing (input data for PLUTO) on: schedule: diff --git a/.github/workflows/input_pts.yml b/.github/workflows/input_pts.yml index c98b40a7..99c8424d 100644 --- a/.github/workflows/input_pts.yml +++ b/.github/workflows/input_pts.yml @@ -1,4 +1,4 @@ -name: 📁 PTS processing +name: 📁 Property Tax System (PTS) processing (input data for PLUTO) on: push: branches: diff --git a/pluto_build/templates/pluto_input_cama_dof.yml b/pluto_build/templates/pluto_input_cama_dof.yml index 865c6c23..2ceae8a8 100644 --- a/pluto_build/templates/pluto_input_cama_dof.yml +++ b/pluto_build/templates/pluto_input_cama_dof.yml @@ -26,6 +26,9 @@ dataset: info: description: | - ### CAMA + ### CAMA - DOF uses the Computer-Assisted Mass Appraisal System (CAMA) for property + tax valuations. Assessors determine parcel market value using one of several valuation + methods based on the parcels tax class. Once market value is determined, the assessed + value is a percentage calculation of the market value. url: null - dependents: [] \ No newline at end of file + dependents: [] From dc2ee5d0d3b67d9ef9c5ef7f62bb21e8c83c5546 Mon Sep 17 00:00:00 2001 From: mbh329 Date: Fri, 21 Apr 2023 12:27:22 -0400 Subject: [PATCH 7/9] revert back to 20 --- .github/workflows/input_pts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/input_pts.yml b/.github/workflows/input_pts.yml index 99c8424d..227bc4f8 100644 --- a/.github/workflows/input_pts.yml +++ b/.github/workflows/input_pts.yml @@ -14,7 +14,7 @@ on: jobs: process_pts: - runs-on: ubuntu-22.04 + runs-on: ubuntu-20.04 services: postgres: image: postgis/postgis:15-3.3-alpine From 688f0abda815eb17ba8113ec349c9cff4fbd2029 Mon Sep 17 00:00:00 2001 From: mbh329 Date: Fri, 21 Apr 2023 12:51:51 -0400 Subject: [PATCH 8/9] add new env var and update to 22.04 --- .github/workflows/input_pts.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/input_pts.yml b/.github/workflows/input_pts.yml index 227bc4f8..b5ff9b88 100644 --- a/.github/workflows/input_pts.yml +++ b/.github/workflows/input_pts.yml @@ -14,12 +14,13 @@ on: jobs: process_pts: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 services: postgres: image: postgis/postgis:15-3.3-alpine env: POSTGRES_PASSWORD: postgres + AWS_EC2_METADATA_DISABLED: "true" options: >- --health-cmd pg_isready --health-interval 10s From 1d4181ff68c9cdfcc004ef23ecacfa1ef93ce91b Mon Sep 17 00:00:00 2001 From: mbh329 Date: Fri, 21 Apr 2023 13:11:38 -0400 Subject: [PATCH 9/9] rever to ubuntu 20.04 --- .github/workflows/input_pts.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/input_pts.yml b/.github/workflows/input_pts.yml index b5ff9b88..227bc4f8 100644 --- a/.github/workflows/input_pts.yml +++ b/.github/workflows/input_pts.yml @@ -14,13 +14,12 @@ on: jobs: process_pts: - runs-on: ubuntu-22.04 + runs-on: ubuntu-20.04 services: postgres: image: postgis/postgis:15-3.3-alpine env: POSTGRES_PASSWORD: postgres - AWS_EC2_METADATA_DISABLED: "true" options: >- --health-cmd pg_isready --health-interval 10s