diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index e08c1ac..7a27d80 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -1,7 +1,7 @@ # This workflow will install Python dependencies, run tests and lint with a single version of Python name: Test -on: [push, pull_request] +on: [push, pull_request, workflow_dispatch] jobs: build: @@ -10,35 +10,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/cache@v3 - id: cache - with: - path: ~/miniconda - key: ubuntu-conda-${{ hashFiles('environment.yml') }} - - name: Install Conda - if: steps.cache.outputs.cache-hit != 'true' + - name: Test with pytest in run: | - wget https://repo.continuum.io/miniconda/Miniconda3-py37_4.11.0-Linux-x86_64.sh - bash Miniconda3-py37_4.11.0-Linux-x86_64.sh -bfp ~/miniconda - rm Miniconda3-py37_4.11.0-Linux-x86_64.sh - source ~/miniconda/etc/profile.d/conda.sh - conda init - conda install -c conda-forge -y mamba - mamba clean -qafy - mamba env update -f environment.yml - - name: Install dependencies - run: | - ~/miniconda/bin/python -m pip install --upgrade pip - ~/miniconda/bin/python -m pip install flake8 pytest - - name: Install package - run: | - ~/miniconda/bin/python -m pip install ./elephant-core - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - ~/miniconda/bin/flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - ~/miniconda/bin/flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with pytest - run: | - ~/miniconda/bin/pytest + make build + make test diff --git a/Dockerfile b/Dockerfile index 43314cb..812cae1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM nvidia/cuda:11.3.1-runtime-ubuntu20.04 +FROM pytorch/pytorch:2.3.1-cuda12.1-cudnn8-runtime # Modified from https://github.com/tiangolo/uwsgi-nginx-flask-docker (Apache license) LABEL maintainer="Ko Sugawara " @@ -21,23 +21,6 @@ RUN set -x \ ENV PATH /opt/conda/bin:$PATH SHELL ["/bin/bash", "-c"] -# Install Python modules -COPY ./environment.yml /tmp/environment.yml -RUN sed -i '/.\/elephant-core/d' /tmp/environment.yml \ - && curl -OL https://repo.continuum.io/miniconda/Miniconda3-py37_4.11.0-Linux-x86_64.sh \ - && bash Miniconda3-py37_4.11.0-Linux-x86_64.sh -bfp /opt/conda \ - && rm Miniconda3-py37_4.11.0-Linux-x86_64.sh \ - && . /opt/conda/etc/profile.d/conda.sh \ - && conda init \ - && echo "conda activate base" >> ~/.bashrc \ - && conda install -c conda-forge -y mamba=0.19.1 \ - && mamba clean -qafy \ - && mamba env update -f /tmp/environment.yml \ - && mamba clean -qafy \ - && rm -rf /tmp/elephant-core \ - && rm /tmp/environment.yml -RUN mamba install jupyter - # Install and set up RabbbitMQ COPY docker/install-rabbitmq.sh /tmp/install-rabbitmq.sh RUN chmod +x /tmp/install-rabbitmq.sh && /tmp/install-rabbitmq.sh && rm /tmp/install-rabbitmq.sh @@ -56,6 +39,9 @@ RUN groupadd -r nginx && useradd -r -g nginx nginx RUN ln -sf /dev/stdout /var/log/nginx/access.log \ && ln -sf /dev/stderr /var/log/nginx/error.log +# install pypi packages +COPY requirements.txt /tmp/requirements.txt +RUN python -m pip install -r /tmp/requirements.txt && rm /tmp/requirements.txt # RUN pip install memory_profiler line_profiler # RUN pip install --no-deps stardist==0.8.3 csbdeep==0.7.2 numba==0.56.0 llvmlite==0.39.0 natsort==8.1.0 diff --git a/Dockerfile-test b/Dockerfile-test index a266ee2..9a96b15 100644 --- a/Dockerfile-test +++ b/Dockerfile-test @@ -1,4 +1,4 @@ -FROM elephant-server:0.5.6 +FROM elephant-server:0.6.0 LABEL maintainer="Ko Sugawara " diff --git a/Makefile b/Makefile index 615b36d..6ed1461 100755 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ help: ELEPHANT_GPU?=all ELEPHANT_WORKSPACE?=${PWD}/workspace -ELEPHANT_IMAGE_NAME?=elephant-server:0.5.6 +ELEPHANT_IMAGE_NAME?=elephant-server:0.6.0 ELEPHANT_NVIDIA_GID?=$$(ls -n /dev/nvidia0 2>/dev/null | awk '{print $$4}') ELEPHANT_DOCKER?=docker ELEPHANT_RABBITMQ_NODENAME?=rabbit@localhost @@ -82,7 +82,7 @@ notebook: warmup test: $(ELEPHANT_DOCKER) build -t $(ELEPHANT_IMAGE_NAME)-test -f Dockerfile-test . && $(ELEPHANT_DOCKER) image prune -f - $(ELEPHANT_DOCKER) run -it --rm $(ELEPHANT_IMAGE_NAME)-test + $(ELEPHANT_DOCKER) run --rm $(ELEPHANT_IMAGE_NAME)-test apptainer-build: apptainer build --fakeroot elephant.sif elephant.def diff --git a/docker/supervisord.conf b/docker/supervisord.conf index d811569..87012aa 100644 --- a/docker/supervisord.conf +++ b/docker/supervisord.conf @@ -5,7 +5,7 @@ file=/var/run/supervisor%(ENV_ELEPHANT_BATCH_ID)s.sock nodaemon=true [program:uwsgi] -command=/opt/conda/bin/uwsgi --ini /etc/uwsgi/uwsgi.ini +command=uwsgi --ini /etc/uwsgi/uwsgi.ini stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 stderr_logfile=/dev/stderr diff --git a/elephant-core/elephant/version.py b/elephant-core/elephant/version.py index aee4e40..1b796b8 100644 --- a/elephant-core/elephant/version.py +++ b/elephant-core/elephant/version.py @@ -24,4 +24,4 @@ # ============================================================================== """Version definition.""" -__version__ = "0.5.6" +__version__ = "0.6.0" diff --git a/elephant.def b/elephant.def index ec13bea..10b9ae0 100644 --- a/elephant.def +++ b/elephant.def @@ -1,12 +1,12 @@ Bootstrap: docker -From: nvidia/cuda:11.3.1-runtime-ubuntu20.04 +From: pytorch/pytorch:2.3.1-cuda12.1-cudnn8-runtime %files ./docker /docker ./script /opt/elephant/script ./app /app ./elephant-core /src/elephant-core - ./environment.yml /src/environment.yml + ./requirements.txt /src/requirements.txt %environment export UWSGI_UID=$(id -u) @@ -38,22 +38,13 @@ From: nvidia/cuda:11.3.1-runtime-ubuntu20.04 apt-get clean && \ rm -rf /var/lib/apt/lists/* - # Install Python modules - sed -i 's/.\/elephant-core/\/src\/elephant-core/g' /src/environment.yml \ - && curl -OL https://repo.continuum.io/miniconda/Miniconda3-py37_4.11.0-Linux-x86_64.sh \ - && bash Miniconda3-py37_4.11.0-Linux-x86_64.sh -bfp /opt/conda \ - && rm Miniconda3-py37_4.11.0-Linux-x86_64.sh \ - && . /opt/conda/etc/profile.d/conda.sh \ - && conda init \ - && echo "conda activate base" >> ~/.bashrc \ - && conda install -c conda-forge -y mamba==0.19.1 \ - && mamba clean -qafy \ - && mamba env update -f /src/environment.yml \ - && mamba clean -qafy \ - && rm -rf /src/elephant-core \ - && rm /src/environment.yml + # install pypi packages + python -m pip install -r /src/requirements.txt && rm /src/requirements.txt + python -m pip install /src/elephant-core && rm -r /src/elephant-core # Install and set up RabbbitMQ + # Install Python modules + sed -i 's/sudo //g' /docker/install-rabbitmq.sh chmod +x /docker/install-rabbitmq.sh && /docker/install-rabbitmq.sh echo "rabbitmq-server \$@" > /rabbitmq.sh chmod +x /rabbitmq.sh diff --git a/elephant_server.ipynb b/elephant_server.ipynb index f90ad63..c497b74 100644 --- a/elephant_server.ipynb +++ b/elephant_server.ipynb @@ -39,7 +39,7 @@ }, "outputs": [], "source": [ - "%env ELEPHANT_SERVER_VERSION = v0.5.6" + "%env ELEPHANT_SERVER_VERSION = v0.6.0" ] }, { diff --git a/environment.yml b/environment.yml deleted file mode 100644 index f6c658e..0000000 --- a/environment.yml +++ /dev/null @@ -1,270 +0,0 @@ -name: base -channels: - - pytorch - - nvidia - - main - - conda-forge - - defaults -dependencies: - - _libgcc_mutex=0.1=conda_forge - - _openmp_mutex=4.5=1_llvm - - abseil-cpp=20200923.3=h2531618_0 - - absl-py=0.15.0=pyhd3eb1b0_0 - - aiohttp=3.8.1=py37h7f8727e_0 - - aiosignal=1.2.0=pyhd3eb1b0_0 - - amqp=2.6.1=py_0 - - aniso8601=9.0.1=pyhd8ed1ab_0 - - asciitree=0.3.3=py_2 - - astor=0.8.1=py37h06a4308_0 - - astunparse=1.6.3=py_0 - - async-timeout=4.0.1=pyhd3eb1b0_0 - - asynctest=0.13.0=py_0 - - attrs=21.2.0=pyhd3eb1b0_0 - - backcall=0.2.0=pyhd3eb1b0_0 - - beautifulsoup4=4.10.0=pyh06a4308_0 - - billiard=3.6.4.0=py37h27cfd23_0 - - blas=1.0=mkl - - blinker=1.4=py37h06a4308_0 - - blosc=1.21.0=h9c3ff4c_0 - - brotli=1.0.9=he6710b0_2 - - brotlipy=0.7.0=py37h27cfd23_1003 - - brunsli=0.1=h2531618_0 - - bzip2=1.0.8=h7b6447c_0 - - c-ares=1.18.1=h7f98852_0 - - ca-certificates=2021.10.26=h06a4308_2 - - cachetools=4.2.2=pyhd3eb1b0_0 - - celery=5.0.5=pyhd3eb1b0_0 - - certifi=2021.10.8=py37h06a4308_2 - - cffi=1.14.6=py37h400218f_0 - - chardet=4.0.0=py37h06a4308_1003 - - charls=2.2.0=h2531618_0 - - charset-normalizer=2.0.4=pyhd3eb1b0_0 - - click=8.0.3=pyhd3eb1b0_0 - - click-didyoumean=0.0.3=pyhd3eb1b0_0 - - click-plugins=1.1.1=pyhd3eb1b0_0 - - click-repl=0.2.0=pyhd3eb1b0_0 - - cloudpickle=2.0.0=pyhd3eb1b0_0 - - conda=4.11.0=py37h89c1867_0 - - conda-build=3.21.5=py37h06a4308_0 - - conda-package-handling=1.7.3=py37h27cfd23_1 - - coverage=5.5=py37h27cfd23_2 - - cryptography=3.4.8=py37hd23ed53_0 - - cudatoolkit=11.3.1=ha36c431_9 - - cycler=0.11.0=pyhd3eb1b0_0 - - cython=0.29.25=py37hdbfa776_0 - - cytoolz=0.11.0=py37h7b6447c_0 - - dask-core=2021.10.0=pyhd3eb1b0_0 - - dataclasses=0.8=pyh6d0b6a4_7 - - decorator=5.1.0=pyhd3eb1b0_0 - - fasteners=0.16.3=pyhd3eb1b0_0 - - ffmpeg=4.3=hf484d3e_0 - - filelock=3.0.12=pyhd3eb1b0_1 - - flask=1.1.2=pyhd3eb1b0_0 - - flask-redis=0.4.0=py_0 - - flask-restx=0.5.1=pyhd8ed1ab_0 - - fonttools=4.25.0=pyhd3eb1b0_0 - - freetype=2.10.4=h5ab3b9f_0 - - frozenlist=1.2.0=py37h7f8727e_0 - - fsspec=2021.10.1=pyhd3eb1b0_0 - - future=0.18.2=py37_1 - - gast=0.3.3=py_0 - - giflib=5.2.1=h7b6447c_0 - - glob2=0.7=pyhd3eb1b0_0 - - gmp=6.2.1=h2531618_2 - - gnutls=3.6.15=he1e5248_0 - - google-auth=1.33.0=pyhd3eb1b0_0 - - google-auth-oauthlib=0.4.4=pyhd3eb1b0_0 - - google-pasta=0.2.0=pyhd3eb1b0_0 - - greenlet=1.1.1=py37h295c915_0 - - grpc-cpp=1.36.4=hf89561c_1 - - grpcio=1.36.1=py37h2157cd5_1 - - h5py=2.10.0=py37hd6299e0_1 - - hdf5=1.10.6=hb1b8bf9_0 - - icu=58.2=he6710b0_3 - - idna=2.10=pyhd3eb1b0_0 - - imagecodecs=2021.3.31=py37hd0c323f_1 - - imageio=2.9.0=pyhd3eb1b0_0 - - importlib-metadata=4.8.2=py37h06a4308_0 - - importlib_metadata=4.8.2=hd3eb1b0_0 - - importlib_resources=5.4.0=pyhd8ed1ab_0 - - intel-openmp=2021.3.0=h06a4308_3350 - - ipython=7.27.0=py37hb070fc8_0 - - itsdangerous=2.0.1=pyhd3eb1b0_0 - - jansson=2.14=h7f98852_0 - - jbig=2.1=h7f98852_2003 - - jedi=0.18.0=py37h06a4308_1 - - jinja2=2.11.3=pyhd3eb1b0_0 - - joblib=1.1.0=pyhd3eb1b0_0 - - jpeg=9d=h7f8727e_0 - - jsonschema=4.4.0=pyhd8ed1ab_0 - - jxrlib=1.1=h7b6447c_2 - - keras-preprocessing=1.1.2=pyhd3eb1b0_0 - - kiwisolver=1.3.1=py37h2531618_0 - - kombu=4.6.11=py37_0 - - krb5=1.19.2=hcc1bbae_0 - - lame=3.100=h7b6447c_0 - - lcms2=2.12=h3be6417_0 - - ld_impl_linux-64=2.35.1=h7274673_9 - - lerc=2.2.1=h9c3ff4c_0 - - libaec=1.0.4=he6710b0_1 - - libarchive=3.5.2=hccf745f_1 - - libcurl=7.80.0=h0b77cf5_0 - - libdeflate=1.7=h7f98852_5 - - libedit=3.1.20191231=he28a2e2_2 - - libev=4.33=h516909a_1 - - libffi=3.3=he6710b0_2 - - libgcc-ng=11.2.0=h1d223b6_11 - - libgfortran-ng=7.5.0=ha8ba4b0_17 - - libgfortran4=7.5.0=ha8ba4b0_17 - - libiconv=1.16=h516909a_0 - - libidn2=2.3.2=h7f8727e_0 - - liblief=0.10.1=he6710b0_0 - - libmamba=0.19.1=h3985d26_0 - - libmambapy=0.19.1=py37hab1676c_0 - - libnghttp2=1.46.0=hce63b2e_0 - - libpng=1.6.37=hbc83047_0 - - libprotobuf=3.15.8=h780b84a_1 - - libsolv=0.7.19=h780b84a_5 - - libssh2=1.10.0=ha56f1ee_2 - - libstdcxx-ng=11.2.0=he4da1e4_11 - - libtasn1=4.16.0=h27cfd23_0 - - libtiff=4.3.0=hf544144_1 - - libunistring=0.9.10=h27cfd23_0 - - libuv=1.40.0=h7b6447c_0 - - libwebp=1.2.0=h89dd481_0 - - libwebp-base=1.2.0=h27cfd23_0 - - libxml2=2.9.12=h03d6c58_0 - - libzopfli=1.0.3=he6710b0_0 - - llvm-openmp=12.0.1=h4bd325d_1 - - locket=0.2.1=py37h06a4308_1 - - lz4-c=1.9.3=h295c915_1 - - lzo=2.10=h516909a_1000 - - mamba=0.19.1=py37h6dacc13_0 - - markdown=3.3.4=py37h06a4308_0 - - markupsafe=2.0.1=py37h27cfd23_0 - - matplotlib-base=3.4.3=py37hbbc1b5f_0 - - matplotlib-inline=0.1.2=pyhd3eb1b0_2 - - mkl=2020.2=256 - - mkl-service=2.3.0=py37he8ac12f_0 - - mkl_fft=1.3.0=py37h54f3939_0 - - mkl_random=1.1.1=py37h0573a6f_0 - - msgpack-python=1.0.2=py37hff7bd54_1 - - multidict=5.1.0=py37h27cfd23_2 - - munkres=1.1.4=py_0 - - ncurses=6.2=he6710b0_1 - - nettle=3.7.3=hbbd107a_1 - - networkx=2.6.3=pyhd3eb1b0_0 - - numcodecs=0.8.0=py37h2531618_0 - - numpy=1.19.2=py37h54aff64_0 - - numpy-base=1.19.2=py37hfa32c7d_0 - - oauthlib=3.1.1=pyhd3eb1b0_0 - - olefile=0.46=py37_0 - - openh264=2.1.0=hd408876_0 - - openjpeg=2.4.0=h3ad879b_0 - - openssl=1.1.1m=h7f8727e_0 - - opt_einsum=3.3.0=pyhd3eb1b0_1 - - packaging=21.3=pyhd3eb1b0_0 - - parso=0.8.2=pyhd3eb1b0_0 - - partd=1.2.0=pyhd3eb1b0_0 - - patchelf=0.13=h295c915_0 - - pcre=8.45=h295c915_0 - - pexpect=4.8.0=pyhd3eb1b0_3 - - pickleshare=0.7.5=pyhd3eb1b0_1003 - - pika=1.1.0=pyh9f0ad1d_1 - - pillow=8.4.0=py37h5aabda8_0 - - pip=21.0.1=py37h06a4308_0 - - pkginfo=1.7.1=py37h06a4308_0 - - prompt-toolkit=3.0.20=pyhd3eb1b0_0 - - prompt_toolkit=3.0.20=hd3eb1b0_0 - - protobuf=3.15.8=py37hcd2ae1e_0 - - psutil=5.8.0=py37h27cfd23_1 - - ptyprocess=0.7.0=pyhd3eb1b0_2 - - py-lief=0.10.1=py37h403a769_0 - - pyasn1=0.4.8=pyhd3eb1b0_0 - - pyasn1-modules=0.2.8=py_0 - - pybind11-abi=4=hd8ed1ab_3 - - pycosat=0.6.3=py37h27cfd23_0 - - pycparser=2.20=py_2 - - pygments=2.10.0=pyhd3eb1b0_0 - - pyjwt=2.1.0=py37h06a4308_0 - - pyopenssl=20.0.1=pyhd3eb1b0_1 - - pyparsing=3.0.4=pyhd3eb1b0_0 - - pyrsistent=0.18.1=py37h5e8e339_0 - - pysocks=1.7.1=py37_1 - - python=3.7.11=h12debd9_0 - - python-dateutil=2.8.2=pyhd3eb1b0_0 - - python-flatbuffers=1.12=pyhd3eb1b0_0 - - python-libarchive-c=2.9=pyhd3eb1b0_1 - - python_abi=3.7=2_cp37m - - pytorch=1.10.0=py3.7_cuda11.3_cudnn8.2.0_0 - - pytorch-mutex=1.0=cuda - - pytz=2021.3=pyhd3eb1b0_0 - - pywavelets=1.1.1=py37h7b6447c_2 - - pyyaml=5.3.1=py37h7b6447c_0 - - re2=2021.04.01=h9c3ff4c_0 - - readline=8.1=h27cfd23_0 - - redis-py=3.5.3=pyhd3eb1b0_0 - - reproc=14.2.3=h7f98852_0 - - reproc-cpp=14.2.3=h9c3ff4c_0 - - requests=2.25.1=pyhd3eb1b0_0 - - requests-oauthlib=1.3.0=py_0 - - ripgrep=12.1.1=0 - - rsa=4.7.2=pyhd3eb1b0_1 - - ruamel_yaml=0.15.87=py37h7b6447c_0 - - scikit-image=0.17.2=py37hdf5156a_0 - - scikit-learn=0.23.1=py37h423224d_0 - - scipy=1.4.1=py37h0b6359f_0 - - setuptools=58.0.4=py37h06a4308_0 - - six=1.16.0=pyhd3eb1b0_0 - - snappy=1.1.8=he6710b0_0 - - soupsieve=2.2.1=pyhd3eb1b0_0 - - sqlalchemy=1.4.27=py37h7f8727e_0 - - sqlite=3.36.0=hc218d9a_0 - - tensorboard=2.4.0=pyhc547734_0 - - tensorboard-plugin-wit=1.6.0=py_0 - - tensorboardx=2.1=py_0 - - tensorflow=2.4.0=py37h89c1867_0 - - tensorflow-base=2.4.0=py37h00a14e9_0 - - tensorflow-estimator=2.4.0=pyh9656e83_0 - - termcolor=1.1.0=py37h06a4308_1 - - threadpoolctl=2.2.0=pyh0d69192_0 - - tifffile=2021.4.8=pyhd3eb1b0_2 - - tk=8.6.10=hbc83047_0 - - toolz=0.11.2=pyhd3eb1b0_0 - - torchtext=0.11.0=py37 - - torchvision=0.11.0=py37_cu113 - - tornado=6.1=py37h27cfd23_0 - - tqdm=4.48.2=py_0 - - traitlets=5.1.0=pyhd3eb1b0_0 - - typing-extensions=3.10.0.2=hd3eb1b0_0 - - typing_extensions=3.10.0.2=pyh06a4308_0 - - tzdata=2021a=h52ac0ba_0 - - urllib3=1.26.6=pyhd3eb1b0_1 - - uwsgi=2.0.18=py37hd004684_2 - - vine=1.3.0=pyhd3eb1b0_0 - - wcwidth=0.2.5=pyhd3eb1b0_0 - - werkzeug=2.0.2=pyhd3eb1b0_0 - - wheel=0.36.2=pyhd3eb1b0_0 - - wrapt=1.13.3=py37h7f8727e_2 - - xz=5.2.5=h7b6447c_0 - - yaml=0.1.7=had09818_2 - - yaml-cpp=0.6.3=he1b5a44_4 - - yarl=1.6.3=py37h27cfd23_0 - - zarr=2.11.0=pyhd8ed1ab_0 - - zfp=0.5.5=h2531618_6 - - zipp=3.6.0=pyhd3eb1b0_0 - - zlib=1.2.11 - - zstd=1.5.0=ha95c52a_0 - - pip: - - csbdeep==0.7.2 - - dnspython==2.1.0 - - ./elephant-core - - line-profiler==4.1.2 - - llvmlite==0.39.0 - - memory-profiler==0.61.0 - - natsort==8.1.0 - - numba==0.56.0 - - nvsmi==0.4.2 - - python-etcd==0.4.5 - - stardist==0.8.3 - - torchelastic==0.2.0 diff --git a/requirements.txt b/requirements.txt index c0d90d6..433ce7f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,15 @@ -h5py==2.10.0 -flask==1.1.2 -flask-redis==0.4.0 -pika==1.1.0 -torch==1.4.0 -scikit-learn==0.23.1 -scikit-image==0.17.2 -scipy==1.4.1 -tensorboardX==2.1 -tqdm==4.48.2 -uwsgi==2.0.18 -zarr==2.4.0 \ No newline at end of file +celery~=5.4.0 +Flask~=3.0.3 +flask-restx~=1.3.0 +flask-redis~=0.4.0 +zarr~=2.18.2 +filelock~=3.13.1 +nvsmi~=0.4.2 +psutil~=5.9.0 +scikit-image~=0.24.0 +pika~=1.3.2 +redis~=5.0.8 +h5py~=3.11.0 +tqdm~=4.66.4 +tensorflow~=2.17.0 +pyuwsgi~=2.0.26 \ No newline at end of file