From 03be9e1f7c22d24c0b8d1db2abe65bd053371527 Mon Sep 17 00:00:00 2001 From: daspk Date: Wed, 31 Jul 2024 22:37:48 +0700 Subject: [PATCH] [ADD] Add docker support and CI for OTB project * Introduced `docker.nix` to build Docker images for OTB with Python support. * Updated the flake configuration to include `nix2container` and specify Docker image build. * Enhanced the README with Docker build instructions and added a CI workflow to validate builds on both `amd64` and `arm64` architectures. * Additional minor updates include modifications to the `.gitignore` and the creation of a Makefile for Docker image builds. --- .github/workflows/ci.yml | 33 ++++++++++ .gitignore | 1 + Makefile | 5 ++ README.md | 129 ++++++++++++++++++++++++++++++++++++++- docker.nix | 86 ++++++++++++++++++++++++++ flake.lock | 88 +++++++++++++++++++++++--- flake.nix | 60 ++++++++++++++---- pkgs/otb/default.nix | 31 +++------- 8 files changed, 390 insertions(+), 43 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 Makefile create mode 100644 docker.nix diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..cb5c95e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,33 @@ +name: CI + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + machine: + - host: amd64 + platform: x86_64-linux + - host: arm64 + platform: aarch64-linux + steps: + - uses: actions/checkout@v4 + - if: matrix.machine.platform == 'aarch64-linux' + uses: docker/setup-qemu-action@v3 + - uses: DeterminateSystems/nix-installer-action@main + with: + extra-conf: | + fallback = true + http-connections = 128 + max-substitution-jobs = 128 + extra-platforms = aarch64-linux + - name: Build system + run: | + nix build --system ${{ matrix.machine.platform }} . diff --git a/.gitignore b/.gitignore index 55bdd6a..20b9deb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .idea/ .DS_Store .direnv +/.venv/ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..13b8276 --- /dev/null +++ b/Makefile @@ -0,0 +1,5 @@ +build_docker_x86_64: + nix run .\#otb-docker-x86_64.copyToDockerDaemon + +#build_docker_arch64: +# nix run .\#otb-docker-aarch64.copyToDockerDaemon \ No newline at end of file diff --git a/README.md b/README.md index 822b27e..a4f3eb0 100644 --- a/README.md +++ b/README.md @@ -153,6 +153,129 @@ Here is an example of how to create an `flake.nix` with all the above python pac } ``` +## Docker +- Docker Image for OTB: + - Linux AMD64 + - Linux ARM64: To build OTB for `linux-aarach-64` there are 3 options: + - The easiest way would be to build natively with ARM64 (Still needs to be tested) + - Complie using an emulator (Tested with GitHub Action and it builds fine) + - We can also via nix cross compiler (Still needs to be Tested) + +One can build a docker image for OTB with python support (default). +``` +1) Clone this repo +2) make build_docker_x86_64 +2) docker run -it --rm otb +``` +Example: How build a docker image based on `OTB`, `pyotb`, `gdal` and `rasterio` +without cloning this repo. + +1) Create a local directory +2) Create a `flake.nix` as shown below which already has the required python packages and as +it points to this github directory so we don't need to clone. +3) Copy the contents in [docker.nix](docker.nix) file and put it in the local directory. + +Local directory should basically contain 2 files (optional copy the Makefile): +```bash +flake.nix +docker.nix +``` + +4) Run command as mentioned below: + +``` +1) nix run .\#otb-docker-x86_64.copyToDockerDaemon +2) docker run -it --rm otb +``` + +Example of `flake.nix` with `OTB`, `Gdal`, `Pyotb` and `Rasterio`: +```nix +{ + description = "A flake for Orfeo Toolbox"; + + inputs = { + nixpkgs.url = "github:nixos/nixpkgs/24.05"; + nixpkgs-unstable.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + otbpkgs.url = "github:daspk04/otb-nix"; + nix2container.url = "github:nlewo/nix2container"; + }; + + outputs = { + self, + nixpkgs, + nixpkgs-unstable, + nix2container, + flake-utils, + otbpkgs, + }: + flake-utils.lib.eachDefaultSystem ( + system: let + pkgs = import nixpkgs {inherit system;}; + nix2containerPkgs = nix2container.packages.${system}; + python = pkgs.python312; + pyPkgs = python.pkgs; + otb = otbpkgs.packages.${system}.otb.override { + python3 = python; + enablePython = true; + }; + + otbPath = with pkgs; pkgs.lib.makeLibraryPath [otb]; + + pyotb = pyPkgs.buildPythonPackage rec { + pname = "pyotb"; + version = "2.0.3.dev2"; + format = "pyproject"; + docheck = false; + + src = builtins.fetchGit { + name = pname; + url = "https://github.com/orfeotoolbox/pyotb.git"; + ref = "refs/tags/${version}"; + rev = "de801eae7e2bd80706801df4a48b23998136a5cd"; + }; + + nativeBuildInputs = with pyPkgs; [ + setuptools + ]; + + propagatedBuildInputs = with pyPkgs; [ + numpy + ]; + }; + in rec { + packages = { + otb-docker-x86_64 = pkgs.callPackage ./docker.nix { + inherit pkgs otb nix2containerPkgs; + imageName = "otb"; + python3 = python; + extra-python-packages = with pyPkgs; [pyotb gdal rasterio]; + }; + }; + devShells.default = pkgs.mkShell rec { + packages = with pkgs; [ + bashInteractive + pyPkgs.gdal + pypkgs.rasterio + pyPkgs.python + pyPkgs.venvShellHook + pyotb + otb + ]; + venvDir = "./.venv"; + + postShellHook = '' + export PYTHONPATH="$PYTHONPATH:${otbPath}/otb/python" + ''; + }; + } + ); +} + +``` + + + ## Develop - In case one needs to develop or experiment then - Clone this repo and make changes and push to your repository @@ -180,10 +303,14 @@ Nix should be installed and flakes should be enabled. - https://www.tweag.io/blog/2020-05-25-flakes/ - How to install direnv - https://github.com/nix-community/nix-direnv?tab=readme-ov-file#installation +- How to cross compile in Nix: + - https://thewagner.net/blog/2023/11/20/building-nix-packages-for-the-raspberry-pi-with-github-actions/ + - https://lgug2z.com/articles/building-and-privately-caching-x86-and-aarch64-nixos-systems-on-github-actions/ + ## TODO - - [ ] Build a Nix Docker for the OTB package + - [X] Build a Nix Docker for the OTB package [Linux AMD64 and Linux ARM64] - [ ] Build OTB with [remote modules]((https://www.orfeo-toolbox.org/CookBook/RemoteModules.html)) (OTBTF, TimeSeriesGapFilling, etc) diff --git a/docker.nix b/docker.nix new file mode 100644 index 0000000..915563d --- /dev/null +++ b/docker.nix @@ -0,0 +1,86 @@ +{ + nix2containerPkgs, + python3, + otb, + img-name ? "otb", + img-tag ? "latest", + extra-packages ? [], + extra-python-packages ? [], + pkgs, + ... +}: let + py-env = ( + python3.withPackages (pp: + with pp; + [numpy] + ++ extra-python-packages) + ); + + py-env-sitepackages = "${py-env}/${py-env.sitePackages}"; + py-env-bin = "${py-env}/bin"; + otbLibPath = with pkgs; pkgs.lib.makeLibraryPath [otb]; + otbBinPath = with pkgs; pkgs.lib.makeBinPath otb.propagatedBuildInputs; +in + nix2containerPkgs.nix2container.buildImage rec { + name = img-name; + tag = img-tag; + + copyToRoot = pkgs.buildEnv { + name = "root"; + paths = with pkgs.dockerTools; + with pkgs; + [ + # Base OS + ## GNU + coreutils-full + findutils + bashInteractive + gnugrep + gnused + which + + ## Networking + cacert + caCertificates + fakeNss + shadowSetup + iana-etc + + # Conveniences + git + neovim-unwrapped + zsh + + # Library + otb + py-env + ] + ++ extra-packages; + + pathsToLink = ["/bin" "/etc" "/var" "/run" "/tmp" "/lib"]; + postBuild = '' + mkdir -p $out/tmp + mkdir -p $out/etc + cat < $out/etc/zshrc + autoload -U compinit && compinit + autoload -U promptinit && promptinit && prompt suse && setopt prompt_sp + autoload -U colors && colors + export PS1=$'%{\033[31m%}[nix-shell:%{\033[32m%}%~%{\033[31m%}]%{\033[0m%}$ '; + HEREDOC + ''; + }; + + config = { + Cmd = ["/bin/env" "zsh"]; + Env = [ + "LANG=C.UTF-8" + "LC_ALL=C.UTF-8" + "LC_CTYPE=C.UTF-8" + "EDITOR=nvim" + "PYTHONPATH=${py-env-sitepackages}:${otbLibPath}/otb/python" + "PATH=${py-env-bin}:${otbBinPath}:/bin" + "TMPDIR=/tmp" + ]; + }; + } + diff --git a/flake.lock b/flake.lock index cfe85f8..d37cb1d 100644 --- a/flake.lock +++ b/flake.lock @@ -18,29 +18,65 @@ "type": "github" } }, + "flake-utils_2": { + "inputs": { + "systems": "systems_2" + }, + "locked": { + "lastModified": 1710146030, + "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nix2container": { + "inputs": { + "flake-utils": "flake-utils_2", + "nixpkgs": "nixpkgs" + }, + "locked": { + "lastModified": 1720642556, + "narHash": "sha256-qsnqk13UmREKmRT7c8hEnz26X3GFFyIQrqx4EaRc1Is=", + "owner": "nlewo", + "repo": "nix2container", + "rev": "3853e5caf9ad24103b13aa6e0e8bcebb47649fe4", + "type": "github" + }, + "original": { + "owner": "nlewo", + "repo": "nix2container", + "type": "github" + } + }, "nixpkgs": { "locked": { - "lastModified": 1717179513, - "narHash": "sha256-vboIEwIQojofItm2xGCdZCzW96U85l9nDW3ifMuAIdM=", - "owner": "nixos", + "lastModified": 1712920918, + "narHash": "sha256-1yxFvUcJfUphK9V91KufIQom7gCsztza0H4Rz2VCWUU=", + "owner": "NixOS", "repo": "nixpkgs", - "rev": "63dacb46bf939521bdc93981b4cbb7ecb58427a0", + "rev": "92323443a56f4e9fc4e4b712e3119f66d0969297", "type": "github" }, "original": { - "owner": "nixos", - "ref": "24.05", + "owner": "NixOS", "repo": "nixpkgs", "type": "github" } }, "nixpkgs-unstable": { "locked": { - "lastModified": 1720149507, - "narHash": "sha256-OFJoD1jjxzFlY1tAsehEWVA88DbU5smzDPQuu9SmnXY=", + "lastModified": 1722141560, + "narHash": "sha256-Ul3rIdesWaiW56PS/Ak3UlJdkwBrD4UcagCmXZR9Z7Y=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "d9c0b9d611277e42e6db055636ba0409c59db6d2", + "rev": "038fb464fcfa79b4f08131b07f2d8c9a6bcc4160", "type": "github" }, "original": { @@ -50,10 +86,27 @@ "type": "github" } }, + "nixpkgs_2": { + "locked": { + "lastModified": 1717179513, + "narHash": "sha256-vboIEwIQojofItm2xGCdZCzW96U85l9nDW3ifMuAIdM=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "63dacb46bf939521bdc93981b4cbb7ecb58427a0", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "24.05", + "repo": "nixpkgs", + "type": "github" + } + }, "root": { "inputs": { "flake-utils": "flake-utils", - "nixpkgs": "nixpkgs", + "nix2container": "nix2container", + "nixpkgs": "nixpkgs_2", "nixpkgs-unstable": "nixpkgs-unstable" } }, @@ -71,6 +124,21 @@ "repo": "default", "type": "github" } + }, + "systems_2": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } } }, "root": "root", diff --git a/flake.nix b/flake.nix index 8b5f0c4..23d7be9 100644 --- a/flake.nix +++ b/flake.nix @@ -5,34 +5,72 @@ nixpkgs.url = "github:nixos/nixpkgs/24.05"; nixpkgs-unstable.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; flake-utils.url = "github:numtide/flake-utils"; + nix2container.url = "github:nlewo/nix2container"; }; outputs = { self, nixpkgs, nixpkgs-unstable, + nix2container, flake-utils, }: flake-utils.lib.eachDefaultSystem ( system: let - pkgs = import nixpkgs {inherit system;}; + pkgs = import nixpkgs { + inherit system; + }; python = pkgs.python312; # we fix python version to 3.12 here for the OTB + nix2containerPkgs = nix2container.packages.${system}; + # The minimal version of gdal and doesn't have arrow support but helps in leaner build + # we can mix as per requiremnt which will compile gdal from source + # https://github.com/NixOS/nixpkgs/blob/8c50662509100d53229d4be607f1a3a31157fa12/pkgs/development/libraries/gdal/default.nix#L7 +# gdal = pkgs.gdalMinimal.override {python3 = python;}; +# gdal = pkgs.gdalMinimal.override {python3 = python; +# useArrow = true; +# useHDF = true; +# useNetCDF = true;}; + gdal = pkgs.gdal; # gdal full version + pyPkgs = python.pkgs; in rec { - packages.shark = pkgs.callPackage ./pkgs/shark/. {inherit system;}; - packages.itk_4_13 = pkgs.callPackage ./pkgs/itk_4_13_3/. {inherit system;}; - packages.otb = pkgs.callPackage ./pkgs/otb/. { - inherit system; - shark = packages.shark; - itk_4_13 = packages.itk_4_13; - python3 = python; # build otb with fixed python version - enablePython = true; + packages = { + shark = pkgs.callPackage ./pkgs/shark/. {inherit system;}; + itk_4_13 = pkgs.callPackage ./pkgs/itk_4_13_3/. {inherit system;}; + otb = pkgs.callPackage ./pkgs/otb/. { + inherit system; + shark = packages.shark; + itk_4_13 = packages.itk_4_13; + gdal = gdal; + python3 = python; # build otb with fixed python version + enablePython = true; + }; + + otb-docker-x86_64 = pkgs.callPackage ./docker.nix { + inherit pkgs nix2containerPkgs; + img-name = "otb"; + img-tag = "latest"; + otb = packages.otb; + python3 = python; + extra-python-packages = with pyPkgs; [packages.otb.propagatedBuildInputs]; + }; + + # otb-aarch64 = pkgs.pkgsCross.aarch64-multiplatform.callPackage ./pkgs/otb/. { + # inherit system; + # shark = packages.shark; + # itk_4_13 = packages.itk_4_13; + # python3 = python; # build otb with fixed python version + # enablePython = true; + # }; + + default = packages.otb; }; - packages.default = packages.otb; devShells.default = pkgs.mkShell rec { packages = with pkgs; [ bashInteractive - python + pyPkgs.python + pyPkgs.venvShellHook ]; + venvDir = "./.venv"; }; } ); diff --git a/pkgs/otb/default.nix b/pkgs/otb/default.nix index 7613aab..a3b80c6 100644 --- a/pkgs/otb/default.nix +++ b/pkgs/otb/default.nix @@ -121,31 +121,20 @@ in "-DBUILD_TESTING=ON" ]; - # todo: check if these contains all the required packages for another package to be build against OTB (such remote modules) ? propagatedBuildInputs = - [ - boost - curl - gdal - itk_4_13 - libgeotiff - libsvm - muparser - muparserx - opencv - perl - shark - swig - tinyxml - ] - ++ optionals enablePython [ - python3 - ] - ++ optionals enablePython pythonInputs; + [] + ++ pythonInputs; doInstallCheck = false; - pythonPath = optionals enablePython pythonInputs; + computed_PATH = lib.makeBinPath propagatedBuildInputs; + + # Make PATH available to subprocesses + makeWrapperArgs = [ + "--prefix PATH : ${computed_PATH}" + ]; + + # pythonPath = optionals enablePython pythonInputs ++ ["$out/lib/otb/python"]; # wrap the otbcli with the environment variable postInstall = ''