From 74f1cca286cf59b1a5ff7bd11db31c9a46cdf712 Mon Sep 17 00:00:00 2001 From: daspk Date: Mon, 12 Aug 2024 21:28:53 +0700 Subject: [PATCH] [DOC] Update Docker build instructions and handle remote modules * Add documentation for examples on remote modules, * Add some note on OTBTF related to TF library --- README.md | 144 ++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 130 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 0e248a6..55dd233 100644 --- a/README.md +++ b/README.md @@ -70,11 +70,13 @@ compressor for `Zarr` dataset. [Related Issue](https://github.com/remicres/otbtf Nix pacakge solved those issues as `GDAL` on `Nixpkgs` already has those. - One can combine OTB with different geospatial python libraries such as `rasterio`, `geopandas` etc. which as already available with [Nixpkgs](https://search.nixos.org/packages) no need to create a separate environment. -- As of today July 2024, there isn't any conda package for OTB, although it is under plan; +- There isn't any conda package for OTB, although it is under plan; this Nixpkgs should help people who want to use `OTB` with a different python version or packages such as (gdal,rasterio, geopandas etc.), which should solve most of those use cases for OTB to be -imported in an custom python environment. - +imported in an custom python environment [As of today August 2024,]. +- One can also build multi-arch version of OTB package apart from `AMD64 (x86_64-linux)` i.e. +`ARM64 (aarch64-linux)` as currently OTB has no `aarch64-linux` version. +- One can easily build docker image as well. ## Advanced Configuration @@ -156,19 +158,26 @@ Here is an example of how to create an `flake.nix` with all the above python pac ## Docker - Docker Image for OTB: - Linux AMD64 - - Linux ARM64: To build OTB for `linux-aarach-64` there are 3 options: + - Linux ARM64: To build OTB for `linux-aarch-64` there are 3 options: - The easiest way would be to build natively with ARM64 (Still needs to be tested) - - Complie using an emulator (Tested with GitHub Action and it builds fine) + - Complie using an emulator (Tested with [GitHub Action](https://github.com/daspk04/otb-nix/actions/runs/10191427474) and it builds fine) - We can also via nix cross compiler (Still needs to be Tested) -One can build a docker image for OTB with python support (default). +1) One can build a docker image for OTB with python support and no remote modules (default). ``` 1) Clone this repo -2) make build_docker_x86_64 +2) make build_docker 2) docker run -it --rm otb ``` -Example: How build a docker image based on `OTB`, `pyotb`, `gdal` and `rasterio` -without cloning this repo. +2) One can build a docker image for OTB with python support and all the remote modules. +``` +1) Clone this repo +2) make build_dev_docker +2) docker run -it --rm otb-dev +``` + +### Example 1: +##### Build a docker image based on `OTB`, `pyotb`, `gdal` and `rasterio` without cloning this repo. 1) Create a local directory 2) Create a `flake.nix` as shown below which already has the required python packages and as @@ -184,10 +193,9 @@ docker.nix 4) Run command as mentioned below: ``` -1) nix run .\#otb-docker-x86_64.copyToDockerDaemon +1) nix run .\#otb-docker.copyToDockerDaemon 2) docker run -it --rm otb ``` - Example of `flake.nix` with `OTB`, `Gdal`, `Pyotb` and `Rasterio`: ```nix { @@ -245,7 +253,7 @@ Example of `flake.nix` with `OTB`, `Gdal`, `Pyotb` and `Rasterio`: }; in rec { packages = { - otb-docker-x86_64 = pkgs.callPackage ./docker.nix { + otb-docker = pkgs.callPackage ./docker.nix { inherit pkgs otb nix2containerPkgs; img-name = "otb"; img-tag = "latest"; @@ -272,11 +280,112 @@ Example of `flake.nix` with `OTB`, `Gdal`, `Pyotb` and `Rasterio`: } ); } +``` + +### Example2: +Build a docker image based on `OTB`, `pyotb`, `gdal` and `rasterio` without cloning this repo and some specific remote modules + +1) The steps and the command are same as `Example1` except we just change the `flake.nix` file +and the `otb` section for only remote modules. + + +Example of `flake.nix` with only activated remote modules such as `prefetch` and `otbtf` [note on OTBTF](#note) +```nix +{ + description = "A flake for Orfeo Toolbox"; + + inputs = { + nixpkgs.url = "github:nixos/nixpkgs/24.05"; + nixpkgs-unstable.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + otbpkgs.url = "github:daspk04/otb-nix"; + nix2container.url = "github:nlewo/nix2container"; + }; + + outputs = { + self, + nixpkgs, + nixpkgs-unstable, + nix2container, + flake-utils, + otbpkgs, + }: + flake-utils.lib.eachDefaultSystem ( + system: let + pkgs = import nixpkgs {inherit system;}; + nix2containerPkgs = nix2container.packages.${system}; + python = pkgs.python312; + pyPkgs = python.pkgs; + + ## we only enable Prefetch and Otbtf remote modules + otb = pkgs.callPackage ./pkgs/otb/. { + inherit system; + shark = packages.shark; + itk_4_13 = packages.itk_4_13; + gdal = gdal; + python3 = python; # build otb with fixed python version + enablePython = true; + enablePrefetch = true; + enableOtbtf = true; + }; + + otbPath = with pkgs; pkgs.lib.makeLibraryPath [otb]; + + pyotb = pyPkgs.buildPythonPackage rec { + pname = "pyotb"; + version = "2.0.3.dev2"; + format = "pyproject"; + docheck = false; + + src = builtins.fetchGit { + name = pname; + url = "https://github.com/orfeotoolbox/pyotb.git"; + ref = "refs/tags/${version}"; + rev = "de801eae7e2bd80706801df4a48b23998136a5cd"; + }; + + nativeBuildInputs = with pyPkgs; [ + setuptools + ]; + + propagatedBuildInputs = with pyPkgs; [ + numpy + ]; + }; + in rec { + packages = { + otb-docker = pkgs.callPackage ./docker.nix { + inherit pkgs otb nix2containerPkgs; + img-name = "otb"; + img-tag = "latest"; + python3 = python; + extra-python-packages = with pyPkgs; [pyotb gdal rasterio]; + }; + }; + devShells.default = pkgs.mkShell rec { + packages = with pkgs; [ + bashInteractive + pyPkgs.gdal + pypkgs.rasterio + pyPkgs.python + pyPkgs.venvShellHook + pyotb + otb + ]; + venvDir = "./.venv"; + postShellHook = '' + export PYTHONPATH="$PYTHONPATH:${otbPath}/otb/python" + ''; + }; + } + ); +} ``` + ## Develop - In case one needs to develop or experiment then - Clone this repo and make changes and push to your repository @@ -289,7 +398,8 @@ otbpkgs.url = "github:{userName}/{repoName}?ref={branchName}"; ### How to build OTB locally ```markdown 1) clone this repo -2) nix build #.otb +2) nix build #.otb +or # nix build #.otb-all --> this will build OTB with all the remote modules 3) ./result/bin/otbcli_BandMathX -help ``` @@ -312,7 +422,13 @@ Nix should be installed and flakes should be enabled. ## TODO - [X] Build a Nix Docker for the OTB package [Linux AMD64 and Linux ARM64] - - [ ] Build OTB with [remote modules]((https://www.orfeo-toolbox.org/CookBook/RemoteModules.html)) (OTBTF, TimeSeriesGapFilling, etc) + - [X] Build OTB with [remote modules]((https://www.orfeo-toolbox.org/CookBook/RemoteModules.html)) (OTBTF, TimeSeriesGapFilling, etc) + - [ ] Build OTB with Tensorflow for OTBTF (OTBTF remote modules doesn't have Tensorflow) + +## NOTE: +- The `OTBTF` build with `Nix` still doesn't have the `TF` applications such as `TensorflowModelServe` +`ImageClassifierFromDeepFeatures` `TensorflowModelTrain` `TrainClassifierFromDeepFeatures` `ImageClassifierFromDeepFeatures` +as the `TF` building from source via Nix is yet to be done by me. #### **NOTE: This repo is currently experimental, please open an issue in case you encounter any.** \ No newline at end of file