From bf1ff2b4326ab213eeea8b886609172633f41e55 Mon Sep 17 00:00:00 2001 From: remi Date: Wed, 31 Mar 2021 10:30:56 -0400 Subject: [PATCH] update doc (draft) and more --- README_HPC.md | 35 ++++++++++ environment.yml | 174 ++++++++++++++++++++++++++++++++++++++++++++++++ post-process.py | 29 ++++++-- 3 files changed, 233 insertions(+), 5 deletions(-) create mode 100644 README_HPC.md create mode 100644 environment.yml diff --git a/README_HPC.md b/README_HPC.md new file mode 100644 index 0000000..c679935 --- /dev/null +++ b/README_HPC.md @@ -0,0 +1,35 @@ +Auteur: Rémi Tavon | Date: March 31 2021 + +# *Geo-Deep-Learning* - Pipeline d'inférence & post-traitement +> Note: These instruction apply only to HPC. + +Répertoire de travail sur HPC pour les inférences: +`/gpfs/fs3/nrcan/nrcan_geobase/work/transfer/work/deep_learning/inference/` + +## Configuration + +### 1. Inference with geo-deep-learning +Clone geo-deep-learning from github repo into desired local directory + + ```.sh + cd /gpfs/fs3/nrcan/nrcan_geobase/work/transfer/work/deep_learning/inference/ + git clone https://github.com/NRCan/geo-deep-learning/ + ``` + +### 2. Post-processing with GeoSim +2.1 GeoSim + +- Download GeoSim QGIS plugin from https://github.com/remtav/GeoSim) as .zip +- Open QGIS, then install QGIS plugin "geo-sim" from .zip +2.2 Postprocess-gdl + +- Clone postprocess-gdl from github repo into desired local directory + + ```.sh + cd /gpfs/fs3/nrcan/nrcan_geobase/work/transfer/work/deep_learning/inference/ + git https://github.com/remtav/postprocess-gdl + ``` + +## Execution +- Set config in `postprocess-gdl/confif_4class.yaml` +- Set path to config yaml in `postprocess-gdl/inference_pipeline_HPC.sbatch`, then run with `sbatch postprocess-gdl/inference_pipeline_HPC.sbatch` file! \ No newline at end of file diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..f175455 --- /dev/null +++ b/environment.yml @@ -0,0 +1,174 @@ +name: qgis_316 +channels: + - conda-forge + - pytorch + - defaults +dependencies: + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=1_gnu + - boost-cpp=1.74.0=hc6e9bd1_2 + - brotlipy=0.7.0=py38h497a2fe_1001 + - bzip2=1.0.8=h7f98852_4 + - c-ares=1.17.1=h36c2ea0_0 + - ca-certificates=2021.1.19=h06a4308_1 + - cairo=1.16.0=h7979940_1007 + - certifi=2020.12.5=py38h06a4308_0 + - cffi=1.14.5=py38ha65f79e_0 + - cfitsio=3.470=hb418390_7 + - chardet=4.0.0=py38h578d9bd_1 + - coverage=5.5=py38h497a2fe_0 + - cpuonly=1.0=0 + - cryptography=3.4.4=py38h3e25421_0 + - curl=7.71.1=he644dc0_8 + - dbus=1.13.6=hfdff14a_1 + - exiv2=0.27.1=had08079_0 + - expat=2.2.10=h9c3ff4c_0 + - fontconfig=2.13.1=hba837de_1004 + - freetype=2.10.4=h0708190_1 + - freexl=1.0.5=h516909a_1002 + - future=0.18.2=py38h578d9bd_3 + - gdal=3.2.1=py38hc0b2d6b_4 + - geos=3.9.0=h9c3ff4c_0 + - geotiff=1.6.0=h2b14fbe_4 + - gettext=0.19.8.1=h0b5b191_1005 + - giflib=5.2.1=h36c2ea0_2 + - glib=2.66.7=h9c3ff4c_0 + - glib-tools=2.66.7=h9c3ff4c_0 + - gsl=2.6=he838d99_2 + - gst-plugins-base=1.14.5=h0935bb2_2 + - gstreamer=1.18.3=h3560a44_0 + - hdf4=4.2.13=h10796ff_1004 + - hdf5=1.10.6=nompi_h6a2412b_1114 + - httplib2=0.19.0=pyhd8ed1ab_0 + - icu=68.1=h58526e2_0 + - idna=2.10=pyh9f0ad1d_0 + - intel-openmp=2020.2=254 + - jinja2=2.11.3=pyh44b312d_0 + - joblib=1.0.1=pyhd3eb1b0_0 + - jpeg=9d=h36c2ea0_0 + - json-c=0.13.1=hbfbb72e_1002 + - kealib=1.4.14=he4dc956_1 + - krb5=1.17.2=h926e7f8_0 + - ld_impl_linux-64=2.33.1=h53a641e_7 + - libblas=3.9.0=8_openblas + - libcblas=3.9.0=8_openblas + - libclang=11.0.1=default_ha53f305_1 + - libcurl=7.71.1=hcdd3856_8 + - libdap4=3.20.6=hd7c4107_1 + - libedit=3.1.20191231=h14c3975_1 + - libev=4.33=h516909a_1 + - libevent=2.1.10=hcdb4288_3 + - libffi=3.3=he6710b0_2 + - libgcc-ng=9.3.0=h2828fa1_18 + - libgdal=3.2.1=h0a4f56a_4 + - libgfortran-ng=9.3.0=hff62375_18 + - libgfortran5=9.3.0=hff62375_18 + - libglib=2.66.7=h1f3bc88_0 + - libgomp=9.3.0=h2828fa1_18 + - libiconv=1.16=h516909a_0 + - libkml=1.3.0=h02e6976_1012 + - liblapack=3.9.0=8_openblas + - libllvm11=11.0.1=hf817b99_0 + - libnetcdf=4.7.4=nompi_h56d31a8_107 + - libnghttp2=1.43.0=h812cca2_0 + - libopenblas=0.3.12=pthreads_h4812303_1 + - libpng=1.6.37=h21135ba_2 + - libpq=12.3=h255efa7_3 + - libprotobuf=3.14.0=h780b84a_0 + - librttopo=1.1.0=hccdd1c9_5 + - libspatialindex=1.9.3=h9c3ff4c_3 + - libspatialite=5.0.1=h04c9dda_2 + - libssh2=1.9.0=hab1572f_5 + - libstdcxx-ng=9.3.0=h6de172a_18 + - libtiff=4.2.0=hdc55705_0 + - libuuid=2.32.1=h7f98852_1000 + - libwebp=1.2.0=h3452ae3_0 + - libwebp-base=1.2.0=h7f98852_0 + - libxcb=1.13=h7f98852_1003 + - libxkbcommon=1.0.3=he3ba5ed_0 + - libxml2=2.9.10=h72842e0_3 + - libxslt=1.1.33=h15afd5d_2 + - libzip=1.7.3=h4de3113_0 + - lz4-c=1.9.3=h9c3ff4c_0 + - markupsafe=1.1.1=py38h497a2fe_3 + - mkl=2020.2=256 + - mock=4.0.3=py38h578d9bd_1 + - mysql-common=8.0.22=ha770c72_3 + - mysql-libs=8.0.22=h935591d_3 + - ncurses=6.2=he6710b0_1 + - ninja=1.10.2=py38hff7bd54_0 + - nose2=0.9.2=py_0 + - nspr=4.29=h9c3ff4c_1 + - nss=3.62=hb5efdd6_0 + - numpy=1.20.1=py38h18fd61f_0 + - openjpeg=2.4.0=hf7af979_0 + - openssl=1.1.1k=h27cfd23_0 + - owslib=0.23.0=pyhd8ed1ab_0 + - pcre=8.44=he1b5a44_0 + - pip=21.0.1=py38h06a4308_0 + - pixman=0.40.0=h36c2ea0_0 + - plotly=4.14.3=pyh44b312d_0 + - poppler=0.89.0=h2de54a5_5 + - poppler-data=0.4.10=0 + - postgresql=12.3=hc2f5b80_3 + - proj=7.2.0=h277dcde_2 + - psycopg2=2.8.6=py38h2b97feb_1 + - pthread-stubs=0.4=h36c2ea0_1001 + - pycparser=2.20=pyh9f0ad1d_2 + - pygments=2.8.0=pyhd8ed1ab_0 + - pyopenssl=20.0.1=pyhd8ed1ab_0 + - pyparsing=2.4.7=pyh9f0ad1d_0 + - pyproj=3.0.0.post1=py38h8df5ac0_0 + - pyqt=5.12.3=py38h578d9bd_7 + - pyqt-impl=5.12.3=py38h7400c14_7 + - pyqt5-sip=4.19.18=py38h709712a_7 + - pyqtchart=5.12=py38h7400c14_7 + - pyqtwebengine=5.12.1=py38h7400c14_7 + - pyqtwebkit=5.212=py38hd669dca_1 + - pysocks=1.7.1=py38h578d9bd_3 + - python=3.8.8=hdb3f193_4 + - python-dateutil=2.8.1=py_0 + - python_abi=3.8=1_cp38 + - pytorch=1.4.0=py3.8_cpu_0 + - pytz=2021.1=pyhd8ed1ab_0 + - pyyaml=5.4.1=py38h497a2fe_0 + - qca=2.2.1=h73816c6_3 + - qgis=3.16.4=py38hf2201f9_0 + - qjson=0.9.0=h73816c6_1006 + - qscintilla2=2.11.2=py38h63a9b5b_4 + - qt=5.12.9=h9d6b050_2 + - qtkeychain=0.12.0=h2264404_0 + - qtlocation=5.12.9=he1b5a44_0 + - qtserialport=5.9.8=h73816c6_1 + - qtwebkit=5.212=h8f65c2e_1 + - qwt=6.1.6=h7ec6b3e_0 + - qwtpolar=1.1.1=h73816c6_7 + - readline=8.1=h27cfd23_0 + - requests=2.25.1=pyhd3deb0d_0 + - retrying=1.3.3=py_2 + - ruamel_yaml=0.15.87=py38h7b6447c_1 + - setuptools=52.0.0=py38h06a4308_0 + - six=1.15.0=pyh9f0ad1d_0 + - sqlite=3.34.0=h74cdb3f_0 + - tiledb=2.2.4=hb9a9e87_1 + - tk=8.6.10=hbc83047_0 + - tzcode=2021a=h7f98852_0 + - urllib3=1.26.3=pyhd8ed1ab_0 + - wheel=0.36.2=pyhd3eb1b0_0 + - xerces-c=3.2.3=h9d8b166_2 + - xorg-kbproto=1.0.7=h7f98852_1002 + - xorg-libice=1.0.10=h516909a_0 + - xorg-libsm=1.2.3=h84519dc_1000 + - xorg-libx11=1.6.12=h516909a_0 + - xorg-libxau=1.0.9=h7f98852_0 + - xorg-libxdmcp=1.1.3=h7f98852_0 + - xorg-libxext=1.3.4=h516909a_0 + - xorg-libxrender=0.9.10=h516909a_1002 + - xorg-renderproto=0.11.1=h14c3975_1002 + - xorg-xextproto=7.3.0=h7f98852_1002 + - xorg-xproto=7.0.31=h7f98852_1007 + - xz=5.2.5=h7b6447c_0 + - yaml=0.2.5=h516909a_0 + - zlib=1.2.11=h7b6447c_3 + - zstd=1.4.8=ha95c52a_1 +prefix: /home/remi/miniconda3/envs/qgis_316 diff --git a/post-process.py b/post-process.py index ac9169f..be0c990 100644 --- a/post-process.py +++ b/post-process.py @@ -21,10 +21,18 @@ def main(img_path, params): print(f'Post-processing {img_path}') # post-processing parameters - # FIXME: as yaml input classes = get_key_def('classes', params['global'], expected_type=dict) + r2v_cellsize_resamp = get_key_def('r2vect_cellsize_resamp', params['post-processing'], default=0, expected_type=int) - orthog_ang_thres = get_key_def('orthogonalize_ang_thresh', params['post-processing'], default=20, expected_type=int) + removeholesunder = get_key_def('removeholesunder', params['post-processing'], default=0, expected_type=int) + simptol = get_key_def('simptol', params['post-processing'], default=0, expected_type=int) + redbenddiamtol = get_key_def('redbenddiamtol', params['post-processing'], default=0, expected_type=int) + recttol = get_key_def('recttol', params['post-processing']['buildings'], default=0, expected_type=int) + compacttol = get_key_def('compacttol', params['post-processing']['buildings'], default=0, expected_type=int) + patterntol = get_key_def('patterntol', params['post-processing']['buildings'], default=20, expected_type=int) + orthogonalize_ang_thresh = get_key_def('orthogonalize_ang_thresh', params['post-processing']['buildings'], + default=0, expected_type=int) + to_cog = get_key_def('to_cog', params['post-processing'], default=True, expected_type=bool) keep_non_cog = get_key_def('keep_non_cog', params['post-processing'], default=True, expected_type=bool) @@ -35,6 +43,7 @@ def main(img_path, params): classes = {cl_val + 1: name for cl_val, name in classes} # set name of output gpkg: myinference.tif will become myinference.gpkg + # FIXME: let user set output directory final_gpkg = Path(img_path).parent / f'{Path(img_path).stem}.gpkg' if final_gpkg.is_file(): warnings.warn(f'Output geopackage exists: {final_gpkg}. Skipping to next inference...') @@ -44,6 +53,11 @@ def main(img_path, params): f'inputraster="{img_path}" ' \ f'r2vcellsizeresamp={r2v_cellsize_resamp} ' \ f'native:package_1:dest-gpkg={final_gpkg}' + elif len(classes.keys()) == 1 and classes[1] == 'buildings': + command = f'qgis_process run model:gdl-buildings -- ' \ + f'srcinfraster="{img_path}" ' \ + f'r2vcellsizeresamp={r2v_cellsize_resamp} ' \ + f'native:package_1:dest-gpkg={final_gpkg}' elif len(classes) == 4: command = f'qgis_process run model:gdl-{len(classes)}classes -- ' \ f'srcinfraster="{img_path}" ' \ @@ -65,7 +79,12 @@ def main(img_path, params): f'-co COMPRESS=LZW' subprocess_command(cog_command) if keep_non_cog is False and img_path_cog.is_file(): - img_path.unlink(missing_ok=True) + try: + img_path.unlink(missing_ok=True) + except TypeError: + img_path.unlink() + except FileNotFoundError: + print(f'Could not delete non cog inference: {keep_non_cog}') if __name__ == '__main__': @@ -115,8 +134,8 @@ def main(img_path, params): state_dict_path = get_key_def('state_dict_path', params['inference']) working_folder = Path(state_dict_path).parent - ckpt_num_bands = get_key_def('num_bands', params['global'], expected_type=int) - glob_pattern = f"inference_{ckpt_num_bands}bands/*_inference.tif" + #ckpt_num_bands = get_key_def('num_bands', params['global'], expected_type=int) + #glob_pattern = f"inference_{ckpt_num_bands}bands/*_inference.tif" glob_pattern = f"**/*_inference.tif" globbed_imgs_paths = list(working_folder.glob(glob_pattern))