#236: Use pyzoltan to call Zoltan as a comparison algorithm in LBAF

.github/workflows/package-installation.yml

@@ -0,0 +1,53 @@
+name: Package Install
+on: push
+concurrency:
+  group: CI-${{ github.head_ref }}
+  cancel-in-progress: true
+
+jobs:
+  Package-Install:
+    continue-on-error: false
+    strategy:
+      max-parallel: 10
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ['3.8', '3.9']
+    runs-on: ${{ matrix.os }}
+    env:
+      home_dir: /home/runner/work/LB-analysis-framework/LB-analysis-framework
+      zoltan_dir: /home/runner/work/LB-analysis-framework/zoltan
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Install system packages
+        run: |
+          sudo apt-get update && sudo apt-get upgrade -y
+          sudo apt-get install -y git xvfb openmpi-bin libopenmpi-dev
+
+      - name: Python Setup
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          cd ${{ env.home_dir }}
+          python -m pip install --upgrade pip
+          pip install -r requirements-${{ matrix.python-version }}.txt
+
+      - name: Install PyZoltan
+        run: |
+          mkdir ${{ env.zoltan_dir }}
+          cd ${{ env.zoltan_dir }}
+          git clone https://github.com/pypr/pyzoltan.git
+          cd pyzoltan
+          /bin/sh build_zoltan.sh ${{ env.zoltan_dir }}
+          export ZOLTAN=${{ env.zoltan_dir }}
+          pip install -r ${{ env.zoltan_dir }}/pyzoltan/requirements.txt
+          pip install pyzoltan --no-build-isolation
+
+      - name: Importing Packages
+        run: |
+          python3 ${{ env.home_dir }}/scripts/test_imports.py

README.md

@@ -34,6 +34,42 @@ Requirements are divided into `LBAF dependencies` and `LBAF testing`.
 
 `LBAF testing` are needed for testing purposes.
 
+### Installing PyZoltan on MacOS/Linux
+
+Open MPI is needed to be installed. For Ubuntu based distributions packages `openmpi-bin libopenmpi-dev` have to be installed.
+
+One needs to clone PyZoltan repository:
+```shell
+cd <some-directory>
+git clone https://github.com/pypr/pyzoltan.git
+```
+
+One needs to build and link PyZoltan. Easiest way is to use the script provided in PyZoltan repository:
+```shell
+cd <some-directory>/pyzoltan
+# INSTALL_PREFIX is an ABSOLUTE path to the directory, where Zoltan will be installed
+./build_zoltan.sh INSTALL_PREFIX
+# One needs to export ZOLTAN as an environment variable
+export ZOLTAN=$INSTALL_PREFIX
+```
+
+`ZOLTAN` environment variable is very important, and it's used when installing `PyZoltan`
+
+One needs to install `PyZoltan` requirements with:
+```shell
+pip install -r <some-directory>/pyzoltan/requirements.txt
+```
+
+Last step is to install `PyZoltan` itself:
+```shell
+pip install pyzoltan --no-build-isolation
+```
+
+One could check if `PyZoltan` is correctly installed by trying to import zoltan:
+```Python3
+from pyzoltan.core import zoltan
+```
+
 ## Configuration file
 
 LBAF run base of configuration file which could be find here:
@@ -160,38 +196,47 @@ python src/lbaf/Utils/lbsVTDataExtractor.py
 
 [//]: # (## Getting Started with Docker)
 
-[//]: # (### Example use:)
+Replace `<in_dir>` with path to existing directory which will be mapped with `/lbaf/in` in container
 
-[//]: # ()
-[//]: # (Replace `<in_dir>` with path to existing directory which will be mapped with `/lbaf/in` in container)
+Replace `<out_dir>` with path to existing directory which will be mapped with `/lbaf/out` in container
 
-[//]: # ()
-[//]: # (Replace `<out_dir>` with path to existing directory which will be mapped with `/lbaf/out` in container)
+Put `conf.yaml` inside mapped `<in_dir>` and set `output_dir` in `conf.yaml` to `/lbaf/out`
 
-[//]: # (```shell)
+When using sample data from repository set `data_stem` in `conf.yaml` to e.g. `/lbaf/data/synthetic_lb_data/data`
 
-[//]: # (docker run -it -v "<out_dir>:/lbaf/out" -v "<in_dir>:/lbaf/in" nganalytics/lbaf "python src/Applications/NodeGossiper.py -l /lbaf/data/vt_example_lb_stats/stats -x 4 -y 2 -z 1 -s 0 -f 4 -k 4 -i 4 -c 1 -e" "/bin/bash")
+When using other data put the data inside mapped `<in_dir>` and set `data_stem` in `conf.yaml` to e.g. `/lbaf/in/<your_data_dir>/<data_name_prefix>`
 
-[//]: # (```)
+#### Building locally (otherwise docker image will be pulled from dockerhub):
+```shell
+cd <main_repository_directory>
+docker build -t nganalytics/lbaf:latest . -f lbaf.Dockerfile
+```
+
+#### Running:
+```shell
+docker run -it -v "<out_dir>:/lbaf/out" -v "<in_dir>:/lbaf/in" nganalytics/lbaf "python /lbaf/src/lbaf/Applications/LBAF_app.py --config=/lbaf/in/conf.yaml" "/bin/bash"
+# in order to exit container
+exit
+```
 
-[//]: # (### Example use explained:)
+### Example use explained:
 
-[//]: # (- container starts with interactive mode &#40;stdout visible&#41;)
+- container starts with interactive mode (stdout visible)
 
-[//]: # (- two volumes are mounted&#40;data exchange between host and container possible&#41;:)
+- two volumes are mounted(data exchange between host and container possible):
 
-[//]: # (  - directory `<in_dir>` on the host and `/lbaf/in` is mount inside container)
+  - directory `<in_dir>` on the host and `/lbaf/in` is mount inside container
 
-[//]: # (  - directory `<out_dir>` on the host and `/lbaf/out` is mount inside container)
+  - directory `<out_dir>` on the host and `/lbaf/out` is mount inside container
 
-[//]: # (- docker image `nganalytics/lbaf`)
+- docker image `nganalytics/lbaf`
 
-[//]: # (- commands executed inside container:)
+- commands executed inside container:
 
-[//]: # (  - sample LBAF usage:)
+  - sample LBAF usage:
 
-[//]: # (    ```"python src/Applications/NodeGossiper.py -l /lbaf/data/vt_example_lb_stats/stats -x 4 -y 2 -z 1 -s 0 -f 4 -k 4 -i 4 -c 1 -e"```)
+    ```"python /lbaf/src/lbaf/Applications/LBAF_app.py --config=/lbaf/in/conf.yaml"```
 
-[//]: # (  - command to stay inside container, after above command is completed:)
+  - command to stay inside container, after above command is completed:
 
 [//]: # (    ```"/bin/bash"```)

lbaf.Dockerfile

@@ -1,26 +1,35 @@
 FROM python:3.8-slim-buster
+ARG INSTALL_PREFIX=/home/install
+ARG ZOLTAN=$INSTALL_PREFIX
+ARG ZOLTAN_INCLUDE=/usr/include/trilinos
+ARG ZOLTAN_LIBRARY=/usr/lib/x86_64-linux-gnu
+ARG USE_TRILINOS=1
 
 RUN apt-get update && apt-get upgrade -y
 RUN apt-get install -y git xvfb
+RUN apt-get install -y openmpi-bin libopenmpi-dev libtrilinos-zoltan-dev build-essential
+
+COPY requirements-3.8.txt /lbaf/requirements.txt
+
+WORKDIR /home
+RUN git clone https://github.com/pypr/pyzoltan.git
+WORKDIR /home/pyzoltan
+RUN sh build_zoltan.sh $INSTALL_PREFIX
+
+WORKDIR /lbaf
+RUN python -m venv venv
+RUN venv/bin/pip install --upgrade pip
+RUN venv/bin/pip install -r /home/pyzoltan/requirements.txt && venv/bin/pip install pyzoltan --no-build-isolation && venv/bin/pip install -r /lbaf/requirements.txt
+
+ENV PYTHONPATH /lbaf/venv/lib/python3.8/site-packages:/lbaf:/lbaf/src
+ENV DISPLAY :99.0
 
 COPY src /lbaf/src
 COPY data /lbaf/data
-COPY requirements.txt /lbaf/requirements.txt
 COPY scripts/entrypoint.sh /lbaf/entrypoint.sh
-COPY scripts/test_lbaf.py /lbaf/tests/test_lbaf.py
-COPY scripts/run_tests.sh /lbaf/tests/run_tests.sh
-COPY scripts/test_config/conf.yaml /lbaf/src/Applications/conf.yaml
 
 WORKDIR /lbaf
 RUN mkdir /lbaf/in /lbaf/out
-RUN /usr/local/bin/python -m pip install --upgrade pip && pip install virtualenv
-RUN virtualenv --python /usr/local/bin/python3.8 venv
-RUN /bin/sh venv/bin/activate && pip install -r requirements.txt
-
-ENV PYTHONPATH /lbaf/venv/lib/python3.8/site-packages:/lbaf:/lbaf/src
-ENV DISPLAY :99.0
 
 RUN ["chmod", "+x", "/lbaf/entrypoint.sh"]
-RUN ["chmod", "+x", "/lbaf/tests/run_tests.sh"]
-RUN ["/bin/sh", "/lbaf/tests/run_tests.sh"]
 ENTRYPOINT ["/lbaf/entrypoint.sh"]

scripts/entrypoint.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-/bin/sh /lbaf/venv/bin/activate
+/bin/sh venv/bin/activate
 Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 &
 for var in "$@"
 do

scripts/test_imports.py

@@ -0,0 +1,54 @@
+try:
+    import vtk
+except ImportError as e:
+    print(f'VTK was not imported: {e}')
+print('==> Package VTK imported successfully!')
+
+try:
+    import numpy
+except ImportError as e:
+    print(f'numpy was not imported: {e}')
+print('==> Package numpy imported successfully!')
+
+try:
+    import yaml
+except ImportError as e:
+    print(f'yaml was not imported: {e}')
+print('==> Package yaml imported successfully!')
+
+try:
+    import brotli
+except ImportError as e:
+    print(f'brotli was not imported: {e}')
+print('==> Package brotli imported successfully!')
+
+try:
+    import schema
+except ImportError as e:
+    print(f'schema was not imported: {e}')
+print('==> Package schema imported successfully!')
+
+try:
+    import sklearn
+except ImportError as e:
+    print(f'sklearn was not imported: {e}')
+print('==> Package sklearn imported successfully!')
+
+try:
+    import colorama
+except ImportError as e:
+    print(f'colorama was not imported: {e}')
+print('==> Package colorama imported successfully!')
+
+try:
+    import mpi4py
+except ImportError as e:
+    print(f'mpi4py was not imported: {e}')
+print('==> Package mpi4py imported successfully!')
+
+try:
+    from pyzoltan.core import zoltan
+except ImportError as e:
+    print(f'pyzoltan was not imported: {e}')
+print('==> Package pyzoltan imported successfully!')
+