Move interpolation script (#12)

* Move interpolation script with models where it belong * Refactor name of data scripts and add some docu * Refactor name of data scripts and add some docu * add todo * Add try clause to comet imports * Remove trash file * Include default config to doc * Bump version to V1.7.0
IGNF · Apr 5, 2022 · c76615e · c76615e
1 parent 7c5dbec
commit c76615e
Show file tree

Hide file tree

Showing 29 changed files with 234 additions and 1,108 deletions.
diff --git a/=0.17 b/=0.17
diff --git a/bash/setup_environment/requirements.txt b/bash/setup_environment/requirements.txt
@@ -1,7 +1,6 @@
 # --------- hydra --------- #
 hydra-core==1.1.0
 hydra-colorlog==1.1.0
-optuna>=2.5
 
 # --------- RST Linter --------- #
 rstcheck==3.3.*
@@ -10,5 +9,8 @@ rstcheck==3.3.*
 sphinx==4.5.*
 sphinx_rtd_theme==1.0.*
 myst_parser==0.17.*
-sphinx_paramlinks
-recommonmark==0.7.*
+sphinx_paramlinks==0.5.*
+recommonmark==0.7.*
+sphinxnotes-mock==1.0.0b0  # still a beta
+sphinx-argparse==0.3.*  #  Using 
+docutils==0.17
diff --git a/configs/callbacks/default.yaml b/configs/callbacks/default.yaml
@@ -18,7 +18,7 @@ log_iou_by_class:
   _target_: lidar_multiclass.callbacks.logging_callbacks.LogIoUByClass
   classification_dict: ${datamodule.dataset_description.classification_dict}
   interpolator:  # only used at test time
-    _target_: lidar_multiclass.datamodules.interpolation.Interpolator
+    _target_: lidar_multiclass.models.interpolation.Interpolator
     interpolation_k: ${predict.interpolation_k}
     classification_dict: ${datamodule.dataset_description.classification_dict}
     probas_to_save: ${predict.probas_to_save}  # replace by a list of string of class names to select specific probas to save

diff --git a/configs/datamodule/datamodule.yaml b/configs/datamodule/datamodule.yaml
@@ -1,4 +1,4 @@
-_target_: lidar_multiclass.datamodules.datamodule.DataModule
+_target_: lidar_multiclass.data.datamodule.DataModule
 
 prepared_data_dir: ${oc.env:PREPARED_DATA_DIR}
 

diff --git a/configs/datamodule/dataset_description/20220204_BuildingValidation_and_Ground.yaml b/configs/datamodule/dataset_description/20220204_BuildingValidation_and_Ground.yaml
@@ -12,4 +12,4 @@ num_classes: 3
 load_las_func:
   _target_: functools.partial
   _args_:
-    - "${get_method:lidar_multiclass.datamodules.data.FrenchLidarDataLogic.load_las}"
+    - "${get_method:lidar_multiclass.data.loading.FrenchLidarDataLogic.load_las}"
diff --git a/configs/datamodule/dataset_description/SwissSURFACE3D_Building_class.yaml b/configs/datamodule/dataset_description/SwissSURFACE3D_Building_class.yaml
@@ -12,4 +12,4 @@ num_classes: 2
 load_las_func:
   _target_: functools.partial
   _args_:
-    - "${get_method:lidar_multiclass.datamodules.data.SwissTopoLidarDataLogic.load_las}"
+    - "${get_method:lidar_multiclass.data.loading.SwissTopoLidarDataLogic.load_las}"
diff --git a/configs/datamodule/dataset_description/SwissSURFACE3D_all_6_classes.yaml b/configs/datamodule/dataset_description/SwissSURFACE3D_all_6_classes.yaml
@@ -12,4 +12,4 @@ num_classes: 6
 load_las_func:
   _target_: functools.partial
   _args_:
-    - "${get_method:lidar_multiclass.datamodules.data.SwissTopoLidarDataLogic.load_las}"
+    - "${get_method:lidar_multiclass.data.loading.SwissTopoLidarDataLogic.load_las}"
diff --git a/configs/datamodule/subsampler/fps.yaml b/configs/datamodule/subsampler/fps.yaml
@@ -1,2 +1,2 @@
-_target_: lidar_multiclass.datamodules.transforms.FPSSampler
+_target_: lidar_multiclass.data.transforms.FPSSampler
 subsample_size: 12500
diff --git a/configs/datamodule/subsampler/grid.yaml b/configs/datamodule/subsampler/grid.yaml
@@ -1,3 +1,3 @@
-_target_: lidar_multiclass.datamodules.transforms.CustomGridSampler
+_target_: lidar_multiclass.data.transforms.CustomGridSampler
 subsample_size: 12500
 voxel_size: 0.25
diff --git a/configs/datamodule/subsampler/random.yaml b/configs/datamodule/subsampler/random.yaml
@@ -1,2 +1,2 @@
-_target_: lidar_multiclass.datamodules.transforms.RandomSampler
+_target_: lidar_multiclass.data.transforms.RandomSampler
 subsample_size: 12500
diff --git a/docs/requirements.txt b/docs/requirements.txt
@@ -3,17 +3,20 @@
 setuptools
 numpy
 numpydoc
+tqdm
+hydra-core==1.1.0
+hydra-colorlog==1.1.*
+rich
 
 sphinx==4.5.*
 sphinx_rtd_theme==1.0.*
 myst_parser==0.17.*
 sphinx_paramlinks==0.5.*
 recommonmark==0.7.*
 sphinxnotes-mock==1.0.0b0  # still a beta
-
+sphinx-argparse==0.3.*  #  Using 
 docutils==0.17
-hydra-core
-rich
+
 comet_ml
 
 torch==1.10.1

diff --git a/docs/source/apidoc/configs.rst b/docs/source/apidoc/configs.rst
@@ -0,0 +1,11 @@
+Default configuration
+===============================
+
+Configurations are managed with `hydra`. Here, we show the default configuration at a glance.
+
+Refer to source configurations files in folder `configs` for more information.
+
+.. _hydra: https://hydra.cc/
+
+.. literalinclude:: default_config.yml
+  :language: yaml
diff --git a/docs/source/apidoc/default_config.yml b/docs/source/apidoc/default_config.yml
@@ -0,0 +1,136 @@
+seed: 12345
+work_dir: ${hydra:runtime.cwd}
+debug: false
+print_config: true
+ignore_warnings: true
+trainer:
+  _target_: pytorch_lightning.Trainer
+  gpus: 0
+  min_epochs: 1
+  max_epochs: 30
+  log_every_n_steps: 1
+  check_val_every_n_epoch: 1
+  weights_summary: null
+  progress_bar_refresh_rate: 1
+  auto_lr_find: false
+  overfit_batches: 1
+  num_sanity_val_steps: 0
+datamodule:
+  dataset_description:
+    _convert_: all
+    classification_preprocessing_dict:
+      59: 6
+      50: 1
+    classification_dict:
+      1: unclassified
+      2: ground
+      6: building
+    d_in: 12
+    num_classes: 3
+    load_las_func:
+      _target_: functools.partial
+      _args_:
+      - ${get_method:lidar_multiclass.data.loading.FrenchLidarDataLogic.load_las}
+  subsampler:
+    _target_: lidar_multiclass.data.transforms.CustomGridSampler
+    subsample_size: 12500
+    voxel_size: 0.25
+  _target_: lidar_multiclass.data.datamodule.DataModule
+  prepared_data_dir: ${oc.env:PREPARED_DATA_DIR}
+  num_workers: 1
+  batch_size: 16
+  subtile_width_meters: 50
+  subtile_overlap: ${predict.subtile_overlap}
+  augment: false
+  subsample_size: 12500
+callbacks:
+  log_code:
+    _target_: lidar_multiclass.callbacks.comet_callbacks.LogCode
+    code_dir: ${work_dir}/lidar_multiclass
+  log_logs_dir:
+    _target_: lidar_multiclass.callbacks.comet_callbacks.LogLogsPath
+  lr_monitor:
+    _target_: pytorch_lightning.callbacks.LearningRateMonitor
+    logging_interval: step
+    log_momentum: true
+  log_iou_by_class:
+    _target_: lidar_multiclass.callbacks.logging_callbacks.LogIoUByClass
+    classification_dict: ${datamodule.dataset_description.classification_dict}
+    interpolator:
+      _target_: lidar_multiclass.models.interpolation.Interpolator
+      interpolation_k: ${predict.interpolation_k}
+      classification_dict: ${datamodule.dataset_description.classification_dict}
+      probas_to_save: ${predict.probas_to_save}
+      output_dir: null
+  model_checkpoint:
+    _target_: pytorch_lightning.callbacks.ModelCheckpoint
+    monitor: val/loss_epoch
+    mode: min
+    save_top_k: 1
+    save_last: true
+    verbose: true
+    dirpath: checkpoints/
+    filename: epoch_{epoch:03d}
+    auto_insert_metric_name: false
+  early_stopping:
+    _target_: pytorch_lightning.callbacks.EarlyStopping
+    monitor: val/loss_epoch
+    mode: min
+    patience: 6
+    min_delta: 0
+model:
+  optimizer:
+    _target_: functools.partial
+    _args_:
+    - ${get_method:torch.optim.Adam}
+    lr: ${model.lr}
+  lr_scheduler:
+    _target_: functools.partial
+    _args_:
+    - ${get_method:torch.optim.lr_scheduler.ReduceLROnPlateau}
+    mode: min
+    factor: 0.5
+    patience: 5
+    cooldown: 0
+  _target_: lidar_multiclass.models.model.Model
+  d_in: ${datamodule.dataset_description.d_in}
+  num_classes: ${datamodule.dataset_description.num_classes}
+  ckpt_path: null
+  neural_net_class_name: RandLANet
+  neural_net_hparams:
+    num_classes: ${model.num_classes}
+    d_in: ${model.d_in}
+    num_neighbors: 16
+    decimation: 4
+    dropout: 0.5
+  iou:
+    _target_: functools.partial
+    _args_:
+    - ${get_method:torchmetrics.JaccardIndex}
+    - ${model.num_classes}
+    absent_score: 1.0
+  criterion:
+    _target_: torch.nn.CrossEntropyLoss
+    label_smoothing: 0.0
+  lr: 0.004566395347136576
+  momentum: 0.9
+  monitor: val/loss_epoch
+logger:
+  comet:
+    _target_: pytorch_lightning.loggers.comet.CometLogger
+    api_key: ${oc.env:COMET_API_TOKEN}
+    workspace: ${oc.env:COMET_WORKSPACE}
+    project_name: ${oc.env:COMET_PROJECT_NAME}
+    experiment_name: RandLaNetDebug
+    auto_log_co2: false
+    disabled: false
+task:
+  task_name: fit
+predict:
+  src_las: /path/to/input.las
+  output_dir: /path/to/output_dir/
+  resume_from_checkpoint: /path/to/lightning_model.ckpt
+  gpus: 0
+  probas_to_save: all
+  subtile_overlap: 25
+  interpolation_k: 10
diff --git a/docs/source/apidoc/lidar_multiclass.data.rst b/docs/source/apidoc/lidar_multiclass.data.rst
@@ -0,0 +1,20 @@
+lidar\_multiclass.data
+=====================================
+
+lidar\_multiclass.datamodules.datamodule
+-----------------------------------------------
+
+.. automodule:: lidar_multiclass.data.datamodule
+   :members:
+
+lidar\_multiclass.datamodules.loading
+-----------------------------------------
+
+.. automodule:: lidar_multiclass.data.loading
+   :members:
+
+lidar\_multiclass.datamodules.transforms
+-----------------------------------------------
+
+.. automodule:: lidar_multiclass.data.transforms
+   :members:
diff --git a/docs/source/apidoc/lidar_multiclass.datamodules.rst b/docs/source/apidoc/lidar_multiclass.datamodules.rst
diff --git a/docs/source/apidoc/lidar_multiclass.model.rst b/docs/source/apidoc/lidar_multiclass.model.rst
@@ -4,5 +4,11 @@ lidar\_multiclass.models
 Model
 -------------------------------------
 
-.. autoclass:: lidar_multiclass.models.model.Model
+.. automodule:: lidar_multiclass.models.model
+   :members:
+
+Interpolation
+-------------------------------------
+
+.. automodule:: lidar_multiclass.models.interpolation
    :members:
diff --git a/docs/source/apidoc/lidar_multiclass.utils.rst b/docs/source/apidoc/lidar_multiclass.utils.rst
@@ -1,8 +1,6 @@
 lidar\_multiclass.utils
 ===============================
 
-Submodules
-----------
 
 lidar\_multiclass.utils.utils
 ------------------------------------
@@ -11,9 +9,3 @@ lidar\_multiclass.utils.utils
    :members:
    :show-inheritance:
 
-Module contents
----------------
-
-.. automodule:: lidar_multiclass.utils
-   :members:
-   :show-inheritance:
diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -12,23 +12,33 @@
 #
 import os
 import sys
+import yaml
 from unittest import mock
 
-root_path = os.path.abspath("./../../")
-sys.path.insert(0, root_path)
+from hydra.experimental import compose, initialize
+from omegaconf import OmegaConf
 
-import yaml
 
-with open(os.path.join(root_path, "package_metadata.yaml"), "r") as f:
-    pm = yaml.safe_load(f)
+rel_root_path = "./../../"
+abs_root_path = os.path.abspath(rel_root_path)
+sys.path.insert(0, abs_root_path)
+
 
 # -- Project information -----------------------------------------------------
+with open(os.path.join(abs_root_path, "package_metadata.yaml"), "r") as f:
+    pm = yaml.safe_load(f)
 
 release = pm["__version__"]
 project = pm["__name__"]
 author = pm["__author__"]
 copyright = "2021, Institut National de l'Information Géographique et Forestière"
 
+# -- YAML main to print the config into  ---------------------------------------------------
+# We need to concatenate configs into a single file using hydra
+with initialize(config_path=os.path.join(rel_root_path, "configs/"), job_name="config"):
+    cfg = compose(config_name="config")
+    print(OmegaConf.to_yaml(cfg))
+    OmegaConf.save(cfg, "./apidoc/default_config.yml", resolve=False)
 
 # -- General configuration ---------------------------------------------------
 

diff --git a/docs/source/index.rst b/docs/source/index.rst
@@ -29,13 +29,16 @@ Lidar-Deep-Segmentation > Documentation
    background/interpolation
    background/data_optimization
 
+.. TODO: assure that all dosctrings are in third-personn mode.
+.. TODO: find a way to document hydra config ; perhaps by switching to a full dataclasses mode.
 
 .. toctree::
    :maxdepth: 1
    :caption: Package Reference
 
    apidoc/scripts
-   apidoc/lidar_multiclass.datamodules
+   apidoc/configs
+   apidoc/lidar_multiclass.data
    apidoc/lidar_multiclass.model
    apidoc/lidar_multiclass.models.modules
    apidoc/lidar_multiclass.callbacks