diff --git a/python/Dockerfile b/python/Dockerfile index cc0a7a48..9ccf20c7 100644 --- a/python/Dockerfile +++ b/python/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/opendrr/python-env +FROM ghcr.io/anthonyfok/python-env:debian-sid-20201012 LABEL maintainer="Joost van Ulden " @@ -8,8 +8,6 @@ COPY . . #RUN chmod +x docker-entrypoint.sh && chmod +x add_data.sh RUN chmod +x add_data.sh && \ - apt-get update && \ - apt-get install -y dos2unix eatmydata git git-lfs jq time && \ dos2unix add_data.sh #ENTRYPOINT ["/usr/src/app/docker-entrypoint.sh"] CMD ./add_data.sh diff --git a/python/add_data.sh b/python/add_data.sh index b7b30431..0781fda8 100755 --- a/python/add_data.sh +++ b/python/add_data.sh @@ -568,7 +568,7 @@ post_process_mh_tables() { copy_ancillary_tables() { LOG '## Use python to run \copy from a system call' - RUN python3 copyAncillaryTables.py + RUN pypy3 copyAncillaryTables.py } post_process_all_tables_update() { @@ -630,7 +630,7 @@ import_raw_psra_tables() { # This was only needed when the cHazard data was divided by economic region # for PT in "${PT_LIST[@]}"; do # ( cd "cHazard/$PT" - # RUN python3 /usr/src/app/PSRA_hCurveTableCombine.py --hCurveDir="/usr/src/app/cHazard/$PT/" + # RUN pypy3 /usr/src/app/PSRA_hCurveTableCombine.py --hCurveDir="/usr/src/app/cHazard/$PT/" # ) # done @@ -657,7 +657,7 @@ import_raw_psra_tables() { RUN merge_csv ebR_*avg_losses-stats_r2.csv "ebR_${PT}_avg_losses-stats_r2.csv" # Combine source loss tables for runs that were split by economic region or sub-region - RUN python3 /usr/src/app/PSRA_combineSrcLossTable.py --srcLossDir="/usr/src/app/ebRisk/$PT" + RUN pypy3 /usr/src/app/PSRA_combineSrcLossTable.py --srcLossDir="/usr/src/app/ebRisk/$PT" ) done } @@ -668,12 +668,12 @@ post_process_psra_tables() { LOG "## PSRA_1-8" for PT in "${PT_LIST[@]}"; do - RUN python3 PSRA_runCreate_tables.py --province="$PT" --sqlScript="psra_1.Create_tables.sql" - RUN python3 PSRA_copyTables.py --province="$PT" - RUN python3 PSRA_sqlWrapper.py --province="$PT" --sqlScript="psra_2.Create_table_updates.sql" - RUN python3 PSRA_sqlWrapper.py --province="$PT" --sqlScript="psra_3.Create_psra_building_all_indicators.sql" - RUN python3 PSRA_sqlWrapper.py --province="$PT" --sqlScript="psra_4.Create_psra_sauid_all_indicators.sql" - RUN python3 PSRA_sqlWrapper.py --province="$PT" --sqlScript="psra_5.Create_psra_sauid_references_indicators.sql" + RUN pypy3 PSRA_runCreate_tables.py --province="$PT" --sqlScript="psra_1.Create_tables.sql" + RUN pypy3 PSRA_copyTables.py --province="$PT" + RUN pypy3 PSRA_sqlWrapper.py --province="$PT" --sqlScript="psra_2.Create_table_updates.sql" + RUN pypy3 PSRA_sqlWrapper.py --province="$PT" --sqlScript="psra_3.Create_psra_building_all_indicators.sql" + RUN pypy3 PSRA_sqlWrapper.py --province="$PT" --sqlScript="psra_4.Create_psra_sauid_all_indicators.sql" + RUN pypy3 PSRA_sqlWrapper.py --province="$PT" --sqlScript="psra_5.Create_psra_sauid_references_indicators.sql" done RUN run_psql psra_6.Create_psra_merge_into_national_indicators.sql @@ -698,7 +698,7 @@ import_earthquake_scenarios() { LOG "## Importing scenario outputs into PostGIS" for eqscenario in ${EQSCENARIO_LIST[*]}; do - RUN python3 DSRA_outputs2postgres_lfs.py --dsraModelDir=$DSRA_REPOSITORY --columnsINI=DSRA_outputs2postgres.ini --eqScenario="$eqscenario" + RUN pypy3 DSRA_outputs2postgres_lfs.py --dsraModelDir=$DSRA_REPOSITORY --columnsINI=DSRA_outputs2postgres.ini --eqScenario="$eqscenario" done } @@ -720,7 +720,7 @@ import_shakemap() { -L "$DOWNLOAD_URL" # Run Create_table_shakemap.sql - RUN python3 DSRA_runCreateTableShakemap.py --shakemapFile="$shakemap_filename" + RUN pypy3 DSRA_runCreateTableShakemap.py --shakemapFile="$shakemap_filename" done # Run Create_table_shakemap_update.sql or Create_table_shakemap_update_ste.sql @@ -736,11 +736,11 @@ import_shakemap() { case $SITE in s) echo "Site Model" - RUN python3 DSRA_runCreateTableShakemapUpdate.py --eqScenario="$eqscenario" --exposureAgg="$SITE" + RUN pypy3 DSRA_runCreateTableShakemapUpdate.py --eqScenario="$eqscenario" --exposureAgg="$SITE" ;; b) echo "Building Model" - RUN python3 DSRA_runCreateTableShakemapUpdate.py --eqScenario="$eqscenario" --exposureAgg="$SITE" + RUN pypy3 DSRA_runCreateTableShakemapUpdate.py --eqScenario="$eqscenario" --exposureAgg="$SITE" ;; esac echo " " # TODO: Find out the purpose of this echo statement @@ -749,7 +749,7 @@ import_shakemap() { import_rupture_model() { LOG "## Importing Rupture Model" -RUN python3 DSRA_ruptures2postgres.py --dsraRuptureDir="https://github.com/OpenDRR/earthquake-scenarios/tree/master/ruptures" +RUN pypy3 DSRA_ruptures2postgres.py --dsraRuptureDir="https://github.com/OpenDRR/earthquake-scenarios/tree/master/ruptures" LOG "## Generating indicator views" for item in ${EQSCENARIO_LIST_LONGFORM[*]}; do @@ -760,14 +760,14 @@ LOG "## Generating indicator views" case $SITE in s) #echo "Site Model" - RUN python3 DSRA_createRiskProfileIndicators.py --eqScenario="$eqscenario" --aggregation=site_level --exposureModel=site - RUN python3 DSRA_createRiskProfileIndicators.py --eqScenario="$eqscenario" --aggregation=building --exposureModel=site - RUN python3 DSRA_createRiskProfileIndicators.py --eqScenario="$eqscenario" --aggregation=sauid --exposureModel=site + RUN pypy3 DSRA_createRiskProfileIndicators.py --eqScenario="$eqscenario" --aggregation=site_level --exposureModel=site + RUN pypy3 DSRA_createRiskProfileIndicators.py --eqScenario="$eqscenario" --aggregation=building --exposureModel=site + RUN pypy3 DSRA_createRiskProfileIndicators.py --eqScenario="$eqscenario" --aggregation=sauid --exposureModel=site ;; b) #echo "Building Model" - RUN python3 DSRA_createRiskProfileIndicators.py --eqScenario="$eqscenario" --aggregation=building --exposureModel=building - RUN python3 DSRA_createRiskProfileIndicators.py --eqScenario="$eqscenario" --aggregation=sauid --exposureModel=building + RUN pypy3 DSRA_createRiskProfileIndicators.py --eqScenario="$eqscenario" --aggregation=building --exposureModel=building + RUN pypy3 DSRA_createRiskProfileIndicators.py --eqScenario="$eqscenario" --aggregation=sauid --exposureModel=building ;; esac done @@ -801,10 +801,10 @@ export_to_elasticsearch() { # shellcheck disable=SC2154 if [ "$loadPsraModels" = true ]; then LOG "Creating PSRA indices in Elasticsearch" - RUN python3 psra_postgres2es.py - RUN python3 hmaps_postgres2es.py - RUN python3 uhs_postgres2es.py - RUN python3 srcLoss_postgres2es.py + RUN pypy3 psra_postgres2es.py + RUN pypy3 hmaps_postgres2es.py + RUN pypy3 uhs_postgres2es.py + RUN pypy3 srcLoss_postgres2es.py LOG "Creating PSRA Kibana Index Patterns" RUN curl -X POST -H "Content-Type: application/json" "${KIBANA_ENDPOINT}/s/gsc-cgc/api/saved_objects/index-pattern/opendrr_psra_indicators_s" -H "kbn-xsrf: true" -d '{ "attributes": { "title":"opendrr_psra_indicators_s"}}' @@ -819,8 +819,8 @@ export_to_elasticsearch() { if [[ "$loadDsraScenario" = true ]]; then for eqscenario in ${EQSCENARIO_LIST[*]}; do LOG "Creating Elasticsearch indexes for DSRA" - RUN python3 dsra_postgres2es.py --eqScenario="$eqscenario" --dbview="all_indicators" --idField="building" - RUN python3 dsra_postgres2es.py --eqScenario="$eqscenario" --dbview="all_indicators" --idField="sauid" + RUN pypy3 dsra_postgres2es.py --eqScenario="$eqscenario" --dbview="all_indicators" --idField="building" + RUN pypy3 dsra_postgres2es.py --eqScenario="$eqscenario" --dbview="all_indicators" --idField="sauid" # LOG "Creating DSRA Kibana Index Patterns" # Need to develop saved object workflow for automated index patern generation @@ -835,7 +835,7 @@ done # if [[ "$loadHazardThreat" = true ]]; then # # All Indicators # LOG "Creating Elasticsearch indexes for Hazard Threat" - # RUN python3 hazardThreat_postgres2es.py --type="all_indicators" --aggregation="sauid" --geometry=geom_poly --idField="Sauid" + # RUN pypy3 hazardThreat_postgres2es.py --type="all_indicators" --aggregation="sauid" --geometry=geom_poly --idField="Sauid" # LOG "Creating Hazard Threat Kibana Index Patterns" # RUN curl -X POST -H "Content-Type: application/json" "${KIBANA_ENDPOINT}/s/gsc-cgc/api/saved_objects/index-pattern/opendrr_hazard_threat_all_indicators_s" -H "kbn-xsrf: true" -d '{ "attributes": { "title":"opendrr_hazard_threat_all_indicators_s"}}' @@ -845,8 +845,8 @@ done # shellcheck disable=SC2154 if [[ $loadPhysicalExposure = true ]]; then LOG "Creating Elasticsearch indexes for Physical Exposure" - RUN python3 exposure_postgres2es.py --aggregation="building" --geometry=geom_point - RUN python3 exposure_postgres2es.py --aggregation="sauid" --geometry=geom_poly + RUN pypy3 exposure_postgres2es.py --aggregation="building" --geometry=geom_point + RUN pypy3 exposure_postgres2es.py --aggregation="sauid" --geometry=geom_poly LOG "Creating Exposure Kibana Index Patterns" RUN curl -X POST -H "Content-Type: application/json" "${KIBANA_ENDPOINT}/s/gsc-cgc/api/saved_objects/index-pattern/opendrr_nhsl_physical_exposure_all_indicators_s" -H "kbn-xsrf: true" -d '{ "attributes": { "title":"opendrr_nhsl_physical_exposure_all_indicators_s"}}' @@ -858,7 +858,7 @@ done # 2021/09/21 DR - Keeping Hazard Threah and Risk Dynamics out of ES for the time being # if [[ $loadRiskDynamics = true ]]; then # LOG "Creating Elasticsearch indexes for Risk Dynamics" - # RUN python3 riskDynamics_postgres2es.py --type="all_indicators" --aggregation="sauid" --geometry=geom_point --idField="ghslID" + # RUN pypy3 riskDynamics_postgres2es.py --type="all_indicators" --aggregation="sauid" --geometry=geom_point --idField="ghslID" # LOG "Creating Risk Dynamics Kibana Index Patterns" # RUN curl -X POST -H "Content-Type: application/json" "${KIBANA_ENDPOINT}/s/gsc-cgc/api/saved_objects/index-pattern/opendrr_nhsl_risk_dynamics_all_indicators" -H "kbn-xsrf: true" -d '{ "attributes": { "title":"opendrr_nhsl_risk_dynamics_all_indicators"}}' @@ -868,12 +868,12 @@ done # shellcheck disable=SC2154 if [[ $loadSocialFabric = true ]]; then LOG "Creating Elasticsearch indexes for Social Fabric" - RUN python3 socialFabric_postgres2es.py --aggregation="sauid" --geometry=geom_poly --sortfield="Sauid" - RUN python3 socialFabric_postgres2es.py --aggregation="hexgrid_5km" --geometry=geom --sortfield="gridid_5" - RUN python3 socialFabric_postgres2es.py --aggregation="hexgrid_10km" --geometry=geom --sortfield="gridid_10" - RUN python3 socialFabric_postgres2es.py --aggregation="hexgrid_25km" --geometry=geom --sortfield="gridid_25" - RUN python3 socialFabric_postgres2es.py --aggregation="hexgrid_50km" --geometry=geom --sortfield="gridid_50" - RUN python3 socialFabric_postgres2es.py --aggregation="hexgrid_100km" --geometry=geom --sortfield="gridid_100" + RUN pypy3 socialFabric_postgres2es.py --aggregation="sauid" --geometry=geom_poly --sortfield="Sauid" + RUN pypy3 socialFabric_postgres2es.py --aggregation="hexgrid_5km" --geometry=geom --sortfield="gridid_5" + RUN pypy3 socialFabric_postgres2es.py --aggregation="hexgrid_10km" --geometry=geom --sortfield="gridid_10" + RUN pypy3 socialFabric_postgres2es.py --aggregation="hexgrid_25km" --geometry=geom --sortfield="gridid_25" + RUN pypy3 socialFabric_postgres2es.py --aggregation="hexgrid_50km" --geometry=geom --sortfield="gridid_50" + RUN pypy3 socialFabric_postgres2es.py --aggregation="hexgrid_100km" --geometry=geom --sortfield="gridid_100" LOG "Creating Social Fabric Kibana Index Patterns" RUN curl -X POST -H "Content-Type: application/json" "${KIBANA_ENDPOINT}/s/gsc-cgc/api/saved_objects/index-pattern/opendrr_nhsl_social_fabric_all_indicators_s" -H "kbn-xsrf: true" -d '{ "attributes": { "title":"opendrr_nhsl_social_fabric_all_indicators_s"}}' @@ -888,12 +888,12 @@ done # shellcheck disable=SC2154 if [[ $loadHexGrid = true ]]; then LOG "Creating Elasticsearch indexes for Hexgrids" - RUN python3 hexgrid_5km_postgres2es.py - RUN python3 hexgrid_10km_postgres2es.py - RUN python3 hexgrid_25km_postgres2es.py - RUN python3 hexgrid_50km_postgres2es.py - RUN python3 hexgrid_100km_postgres2es.py - RUN python3 hexgrid_sauid_postgres2es.py + RUN pypy3 hexgrid_5km_postgres2es.py + RUN pypy3 hexgrid_10km_postgres2es.py + RUN pypy3 hexgrid_25km_postgres2es.py + RUN pypy3 hexgrid_50km_postgres2es.py + RUN pypy3 hexgrid_100km_postgres2es.py + RUN pypy3 hexgrid_sauid_postgres2es.py LOG "Creating HexGrid Kibana Index Patterns" RUN curl -X POST -H "Content-Type: application/json" "${KIBANA_ENDPOINT}/s/gsc-cgc/api/saved_objects/index-pattern/opendrr_hexgrid_5km" -H "kbn-xsrf: true" -d '{ "attributes": { "title":"opendrr_hexgrid_5km"}}' diff --git a/python/utils.py b/python/utils.py index aae92048..e26e8384 100644 --- a/python/utils.py +++ b/python/utils.py @@ -1,4 +1,3 @@ -#!/usr/bin/python3 # ================================================================= # SPDX-License-Identifier: MIT # @@ -9,10 +8,15 @@ # ================================================================= import configparser -import psycopg2 import json import decimal +import platform +if platform.python_implementation() == "PyPy": + import psycopg2cffi as psycopg2 +else: + import psycopg2 + from elasticsearch import Elasticsearch from elasticsearch import helpers