Skip to content

Commit

Permalink
[ADD] Add docker support and CI for OTB project
Browse files Browse the repository at this point in the history
* Introduced `docker.nix` to build Docker images for OTB with Python support.

* Updated the flake configuration to include `nix2container` and specify Docker image build.

* Enhanced the README with Docker build instructions and added a CI workflow to validate builds on both `amd64` and `arm64` architectures.

* Additional minor updates include modifications to the `.gitignore` and the creation of a Makefile for Docker image builds.
  • Loading branch information
daspk04 committed Jul 31, 2024
1 parent f0d570d commit 03be9e1
Show file tree
Hide file tree
Showing 8 changed files with 390 additions and 43 deletions.
33 changes: 33 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
name: CI

on:
pull_request:
push:
branches: [main]
workflow_dispatch:

jobs:
build:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
machine:
- host: amd64
platform: x86_64-linux
- host: arm64
platform: aarch64-linux
steps:
- uses: actions/checkout@v4
- if: matrix.machine.platform == 'aarch64-linux'
uses: docker/setup-qemu-action@v3
- uses: DeterminateSystems/nix-installer-action@main
with:
extra-conf: |
fallback = true
http-connections = 128
max-substitution-jobs = 128
extra-platforms = aarch64-linux
- name: Build system
run: |
nix build --system ${{ matrix.machine.platform }} .
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
.idea/
.DS_Store
.direnv
/.venv/
5 changes: 5 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
build_docker_x86_64:
nix run .\#otb-docker-x86_64.copyToDockerDaemon

#build_docker_arch64:
# nix run .\#otb-docker-aarch64.copyToDockerDaemon
129 changes: 128 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,129 @@ Here is an example of how to create an `flake.nix` with all the above python pac
}
```

## Docker
- Docker Image for OTB:
- Linux AMD64
- Linux ARM64: To build OTB for `linux-aarach-64` there are 3 options:
- The easiest way would be to build natively with ARM64 (Still needs to be tested)
- Complie using an emulator (Tested with GitHub Action and it builds fine)
- We can also via nix cross compiler (Still needs to be Tested)

One can build a docker image for OTB with python support (default).
```
1) Clone this repo
2) make build_docker_x86_64
2) docker run -it --rm otb
```
Example: How build a docker image based on `OTB`, `pyotb`, `gdal` and `rasterio`
without cloning this repo.

1) Create a local directory
2) Create a `flake.nix` as shown below which already has the required python packages and as
it points to this github directory so we don't need to clone.
3) Copy the contents in [docker.nix](docker.nix) file and put it in the local directory.

Local directory should basically contain 2 files (optional copy the Makefile):
```bash
flake.nix
docker.nix
```

4) Run command as mentioned below:

```
1) nix run .\#otb-docker-x86_64.copyToDockerDaemon
2) docker run -it --rm otb
```

Example of `flake.nix` with `OTB`, `Gdal`, `Pyotb` and `Rasterio`:
```nix
{
description = "A flake for Orfeo Toolbox";
inputs = {
nixpkgs.url = "github:nixos/nixpkgs/24.05";
nixpkgs-unstable.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
flake-utils.url = "github:numtide/flake-utils";
otbpkgs.url = "github:daspk04/otb-nix";
nix2container.url = "github:nlewo/nix2container";
};
outputs = {
self,
nixpkgs,
nixpkgs-unstable,
nix2container,
flake-utils,
otbpkgs,
}:
flake-utils.lib.eachDefaultSystem (
system: let
pkgs = import nixpkgs {inherit system;};
nix2containerPkgs = nix2container.packages.${system};
python = pkgs.python312;
pyPkgs = python.pkgs;
otb = otbpkgs.packages.${system}.otb.override {
python3 = python;
enablePython = true;
};
otbPath = with pkgs; pkgs.lib.makeLibraryPath [otb];
pyotb = pyPkgs.buildPythonPackage rec {
pname = "pyotb";
version = "2.0.3.dev2";
format = "pyproject";
docheck = false;
src = builtins.fetchGit {
name = pname;
url = "https://github.com/orfeotoolbox/pyotb.git";
ref = "refs/tags/${version}";
rev = "de801eae7e2bd80706801df4a48b23998136a5cd";
};
nativeBuildInputs = with pyPkgs; [
setuptools
];
propagatedBuildInputs = with pyPkgs; [
numpy
];
};
in rec {
packages = {
otb-docker-x86_64 = pkgs.callPackage ./docker.nix {
inherit pkgs otb nix2containerPkgs;
imageName = "otb";
python3 = python;
extra-python-packages = with pyPkgs; [pyotb gdal rasterio];
};
};
devShells.default = pkgs.mkShell rec {
packages = with pkgs; [
bashInteractive
pyPkgs.gdal
pypkgs.rasterio
pyPkgs.python
pyPkgs.venvShellHook
pyotb
otb
];
venvDir = "./.venv";
postShellHook = ''
export PYTHONPATH="$PYTHONPATH:${otbPath}/otb/python"
'';
};
}
);
}
```



## Develop
- In case one needs to develop or experiment then
- Clone this repo and make changes and push to your repository
Expand Down Expand Up @@ -180,10 +303,14 @@ Nix should be installed and flakes should be enabled.
- https://www.tweag.io/blog/2020-05-25-flakes/
- How to install direnv
- https://github.com/nix-community/nix-direnv?tab=readme-ov-file#installation
- How to cross compile in Nix:
- https://thewagner.net/blog/2023/11/20/building-nix-packages-for-the-raspberry-pi-with-github-actions/
- https://lgug2z.com/articles/building-and-privately-caching-x86-and-aarch64-nixos-systems-on-github-actions/



## TODO
- [ ] Build a Nix Docker for the OTB package
- [X] Build a Nix Docker for the OTB package [Linux AMD64 and Linux ARM64]
- [ ] Build OTB with [remote modules]((https://www.orfeo-toolbox.org/CookBook/RemoteModules.html)) (OTBTF, TimeSeriesGapFilling, etc)


Expand Down
86 changes: 86 additions & 0 deletions docker.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
{
nix2containerPkgs,
python3,
otb,
img-name ? "otb",
img-tag ? "latest",
extra-packages ? [],
extra-python-packages ? [],
pkgs,
...
}: let
py-env = (
python3.withPackages (pp:
with pp;
[numpy]
++ extra-python-packages)
);

py-env-sitepackages = "${py-env}/${py-env.sitePackages}";
py-env-bin = "${py-env}/bin";
otbLibPath = with pkgs; pkgs.lib.makeLibraryPath [otb];
otbBinPath = with pkgs; pkgs.lib.makeBinPath otb.propagatedBuildInputs;
in
nix2containerPkgs.nix2container.buildImage rec {
name = img-name;
tag = img-tag;

copyToRoot = pkgs.buildEnv {
name = "root";
paths = with pkgs.dockerTools;
with pkgs;
[
# Base OS
## GNU
coreutils-full
findutils
bashInteractive
gnugrep
gnused
which

## Networking
cacert
caCertificates
fakeNss
shadowSetup
iana-etc

# Conveniences
git
neovim-unwrapped
zsh

# Library
otb
py-env
]
++ extra-packages;

pathsToLink = ["/bin" "/etc" "/var" "/run" "/tmp" "/lib"];
postBuild = ''
mkdir -p $out/tmp
mkdir -p $out/etc
cat <<HEREDOC > $out/etc/zshrc
autoload -U compinit && compinit
autoload -U promptinit && promptinit && prompt suse && setopt prompt_sp
autoload -U colors && colors
export PS1=$'%{\033[31m%}[nix-shell:%{\033[32m%}%~%{\033[31m%}]%{\033[0m%}$ ';
HEREDOC
'';
};

config = {
Cmd = ["/bin/env" "zsh"];
Env = [
"LANG=C.UTF-8"
"LC_ALL=C.UTF-8"
"LC_CTYPE=C.UTF-8"
"EDITOR=nvim"
"PYTHONPATH=${py-env-sitepackages}:${otbLibPath}/otb/python"
"PATH=${py-env-bin}:${otbBinPath}:/bin"
"TMPDIR=/tmp"
];
};
}

Loading

0 comments on commit 03be9e1

Please sign in to comment.