Merge branch 'elizabeth/update-python-and-dependencies' into liezl/lo…

…wer-bound-albumentations
talmolab · Oct 10, 2024 · 9c17232 · 9c17232
2 parents 3b7a14e + 0904681
commit 9c17232
Show file tree

Hide file tree

Showing 24 changed files with 279 additions and 232 deletions.
diff --git a/.conda/bld.bat b/.conda/bld.bat
@@ -12,3 +12,14 @@ pip install --no-cache-dir -r .\requirements.txt
 @REM Install sleap itself. This does not install the requirements, but will list which 
 @REM requirements are missing (see "install_requires") when user attempts to install.
 python setup.py install --single-version-externally-managed --record=record.txt
+
+@REM Copied from https://docs.conda.io/projects/conda-build/en/latest/resources/activate-scripts.html
+setlocal EnableDelayedExpansion
+:: Copy the [de]activate scripts to %PREFIX%\etc\conda\[de]activate.d.
+:: This will allow them to be run on environment activation.
+for %%F in (activate deactivate) DO (
+    if not exist %PREFIX%\etc\conda\%%F.d mkdir %PREFIX%\etc\conda\%%F.d
+    copy %RECIPE_DIR%\%%F.bat %PREFIX%\etc\conda\%%F.d\%PKG_NAME%_%%F.bat
+    :: Copy unix shell activation scripts, needed by Windows Bash users
+    copy %RECIPE_DIR%\%%F.sh %PREFIX%\etc\conda\%%F.d\%PKG_NAME%_%%F.sh
+)
diff --git a/.conda/condarc.yaml b/.conda/condarc.yaml
@@ -1,6 +1,5 @@
 channels:
+  - sleap-deps
   - conda-forge
   - nvidia
-  - https://conda.anaconda.org/sleap/label/dev
-  - sleap
   - anaconda
diff --git a/.conda/meta.yaml b/.conda/meta.yaml
@@ -20,8 +20,8 @@ build:
 
 requirements:
   host:
-    - conda-forge::python ==3.7.12  # Run into _MAX_WINDOWS_WORKERS not found if <
-    - numpy >=1.19.5,<1.23.0  # Linux likes anaconda, windows likes conda-forge
+    - conda-forge::python >=3.10.0,<3.11.0
+    - numpy >=1.16.5,<2.0 # SciPy 1.7 requires numpy >=1.16.5
     - conda-forge::cudatoolkit ==11.3.1
     - conda-forge::cudnn=8.2.1
     - nvidia::cuda-nvcc=11.3
@@ -30,71 +30,76 @@ requirements:
 
     # Only the packages above are required to build, but listing them all ensures no 
     # unnecessary pypi packages are installed via the build script (bld.bat, build.sh)
-    - conda-forge::attrs ==21.4.0
-    - conda-forge::cattrs ==1.1.1
-    - conda-forge::h5py ==3.7.0
-    - conda-forge::imageio-ffmpeg # Required for imageio to read/write videos with ffmpeg
+    - conda-forge::attrs
+    - conda-forge::cattrs
+    - conda-forge::h5py
+    - conda-forge::imageio
+    - conda-forge::imageio-ffmpeg
     - conda-forge::jsmin
-    - conda-forge::jsonpickle ==1.2
+    - conda-forge::jsonpickle <=1.5.0 # 1.5.1 breaks compatibility with v1 encoded files
     - conda-forge::networkx
-    - conda-forge::opencv <4.9.0
+    - conda-forge::opencv
     - conda-forge::pandas
-    - conda-forge::pillow >=8.3.2
+    - conda-forge::pillow
     - conda-forge::psutil
     - conda-forge::pykalman
-    - conda-forge::pyside2 >=5.12,<5.14
+    - conda-forge::pyside6 >=6.5.0 # 6.4 conda package uses problematic QtCharts
     - conda-forge::python-rapidjson
     - conda-forge::pyyaml
     - conda-forge::pyzmq
     - conda-forge::qtpy >=2.0.1
     - conda-forge::rich
-    - conda-forge::scipy >=1.4.1,<=1.9.0
+    - conda-forge::scipy >=1.7.0 # Python 3.10 requires scipy >=1.7
     - conda-forge::scikit-image
-    - conda-forge::scikit-learn ==1.0
-    - conda-forge::scikit-video
+    - conda-forge::scikit-learn >=1.0.0 # Python 3.10 requires scikit-learn >=1.0
     - conda-forge::seaborn
+    - sleap-deps::tensorflow ==2.9.2
+    - conda-forge::tensorflow-hub
     - conda-forge::qudida
     - conda-forge::albumentations >=1.4.15 # Handle Nan keypoints
     - conda-forge::ndx-pose <0.2.0
-    - conda-forge::importlib-metadata ==4.11.4
+    - conda-forge::importlib-metadata
   run:
-    - conda-forge::python ==3.7.12  # Run into _MAX_WINDOWS_WORKERS not found if <
-    - conda-forge::attrs ==21.4.0
-    - conda-forge::cattrs ==1.1.1
-    - conda-forge::imageio-ffmpeg # Required for imageio to read/write videos with ffmpeg
+    - conda-forge::python >=3.10.0,<3.11.0
+    - numpy >=1.16.5,<2.0 # SciPy 1.7 requires numpy >=1.16.5
+    - conda-forge::attrs
+    - conda-forge::cattrs
     - conda-forge::cudatoolkit ==11.3.1
     - conda-forge::cudnn=8.2.1
     - nvidia::cuda-nvcc=11.3
-    - conda-forge::h5py ==3.7.0
+    - conda-forge::h5py
+    - conda-forge::imageio
+    - conda-forge::imageio-ffmpeg
     - conda-forge::jsmin
-    - conda-forge::jsonpickle ==1.2
+    - conda-forge::jsonpickle <=1.5.0 # 1.5.1 breaks compatibility with v1 encoded files
     - conda-forge::networkx
-    - numpy >=1.19.5,<1.23.0  # Linux likes anaconda, windows likes conda-forge
-    - conda-forge::opencv <4.9.0
+    - conda-forge::opencv
     - conda-forge::pandas
-    - conda-forge::pillow >=8.3.2
+    - conda-forge::pillow
     - conda-forge::psutil
     - conda-forge::pykalman
-    - conda-forge::pyside2 >=5.12,<5.14  # To ensure works correctly with QtPy.
+    - conda-forge::pyside6 >=6.5.0 # 6.4 conda package uses problematic QtCharts
     - conda-forge::python-rapidjson
     - conda-forge::pyyaml
     - conda-forge::pyzmq
     - conda-forge::qtpy >=2.0.1
     - conda-forge::rich
-    - conda-forge::scipy >=1.4.1,<=1.9.0
+    - conda-forge::scipy >=1.7.0 # Python 3.10 requires scipy >=1.7
     - conda-forge::scikit-image
-    - conda-forge::scikit-learn ==1.0
-    - conda-forge::scikit-video
+    - conda-forge::scikit-learn >=1.0.0 # Python 3.10 requires scikit-learn >=1.0
     - conda-forge::seaborn
-    - sleap/label/dev::tensorflow ==2.7.0  # TODO: Switch to main label when updated
-    - conda-forge::tensorflow-hub <0.14.0  # Causes pynwb conflicts on linux GH-1446
+    - sleap-deps::tensorflow ==2.9.2
+    - conda-forge::tensorflow-hub
     - conda-forge::qudida
     - conda-forge::albumentations >=1.4.15 # Handle Nan keypoints
     - conda-forge::ndx-pose <0.2.0
-    - conda-forge::importlib-metadata ==4.11.4
+    - conda-forge::importlib-metadata
 
+test:
+  requires:
+    - pip
+    - libopencv *=headless*  # [linux] https://github.com/conda-forge/opencv-feedstock/issues/401#issuecomment-2028099423
 # This no longer works so we have moved it to the build workflow
 # https://github.com/talmolab/sleap/pull/1744
-# test:
 #   imports:
-#     - sleap
+#     - sleap
diff --git a/.conda/sleap_activate.bat b/.conda/sleap_activate.bat
@@ -0,0 +1,6 @@
+@REM Remember the old library path for when we deactivate
+set SLEAP_OLD_XLA_FLAGS=%XLA_FLAGS%
+set SLEAP_OLD_NO_ALBUMENTATIONS_UPDATE=%NO_ALBUMENTATIONS_UPDATE%
+@REM Help XLA find CUDA
+set XLA_FLAGS=--xla_gpu_cuda_data_dir=%CONDA_PREFIX%
+set NO_ALBUMENTATIONS_UPDATE=1
diff --git a/.conda/sleap_activate.sh b/.conda/sleap_activate.sh
@@ -1,6 +1,11 @@
 #!/bin/sh
 
-# Remember the old library path for when we deactivate
+# Remember the old variables for when we deactivate
 export SLEAP_OLD_LD_LIBRARY_PATH=$LD_LIBRARY_PATH
+export SLEAP_OLD_XLA_FLAGS=$XLA_FLAGS
 # Help CUDA find GPUs!
-export LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH
+export LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH
+# Help XLA find CUDA
+export XLA_FLAGS=--xla_gpu_cuda_data_dir=$CONDA_PREFIX
+# Disable annoying albumentations message
+export NO_ALBUMENTATIONS_UPDATE=1
diff --git a/.conda/sleap_deactivate.bat b/.conda/sleap_deactivate.bat
@@ -0,0 +1,3 @@
+@REM Reset to the old variables when deactivating the environment
+set XLA_FLAGS=%SLEAP_OLD_XLA_FLAGS%
+set NO_ALBUMENTATIONS_UPDATE=%SLEAP_OLD_NO_ALBUMENTATIONS_UPDATE%
diff --git a/.conda/sleap_deactivate.sh b/.conda/sleap_deactivate.sh
@@ -1,4 +1,6 @@
 #!/bin/sh
 
-# Reset to the old library path for when deactivating the environment
-export LD_LIBRARY_PATH=$SLEAP_OLD_LD_LIBRARY_PATH
+# Reset to the old variables when deactivating the environment
+export LD_LIBRARY_PATH=$SLEAP_OLD_LD_LIBRARY_PATH
+export XLA_FLAGS=$SLEAP_OLD_XLA_FLAGS
+export NO_ALBUMENTATIONS_UPDATE=$SLEAP_OLD_NO_ALBUMENTATIONS_UPDATE
diff --git a/.conda_mac/README.md b/.conda_mac/README.md
@@ -1,4 +1,4 @@
-This folder defines the conda package build for Apple silicon Macs. Until there are aarm64 runners, we have to run this manually on Apple M1 silicon.
+This folder defines the conda package build for Apple silicon Macs.
 
 To build, first go to the base repo directory and install the build environment:
 

diff --git a/.conda_mac/build.sh b/.conda_mac/build.sh
@@ -8,4 +8,12 @@ export PIP_IGNORE_INSTALLED=False
 
 pip install --no-cache-dir -r requirements.txt
 
-python setup.py install --single-version-externally-managed --record=record.txt
+python setup.py install --single-version-externally-managed --record=record.txt
+
+# Copy the activate scripts to $PREFIX/etc/conda/activate.d.
+# This will allow them to be run on environment activation.
+for CHANGE in "activate" "deactivate"
+do
+    mkdir -p "${PREFIX}/etc/conda/${CHANGE}.d"
+    cp "${RECIPE_DIR}/${PKG_NAME}_${CHANGE}.sh" "${PREFIX}/etc/conda/${CHANGE}.d/${PKG_NAME}_${CHANGE}.sh"
+done
diff --git a/.conda_mac/condarc.yaml b/.conda_mac/condarc.yaml
@@ -1,5 +1,6 @@
 # https://github.com/github/roadmap/issues/528
 
 channels:
+  - sleap-deps
   - conda-forge
-  - anaconda
+  - anaconda
diff --git a/.conda_mac/meta.yaml b/.conda_mac/meta.yaml
@@ -23,74 +23,69 @@ source:
 
 requirements:
   host:  
-    - conda-forge::python >=3.9.0, <3.10.0
-    - anaconda::numpy >=1.19.5,<1.23.0
+    - conda-forge::python >=3.10.0,<3.11.0
+    - conda-forge::numpy >=1.16.5 # Scipy 1.7.0 requires numpy >=1.16.5
     - conda-forge::setuptools
     - conda-forge::packaging
     - conda-forge::pip
 
     # Only the packages above are required to build, but listing them all ensures no 
     # unnecessary pypi packages are installed via the build script (bld.bat, build.sh)
-    - conda-forge::attrs >=21.2.0
-    - conda-forge::cattrs ==1.1.1
+    - conda-forge::attrs
+    - conda-forge::cattrs
     - conda-forge::h5py
-    - conda-forge::imageio-ffmpeg # Required for imageio to read/write videos with ffmpeg
+    - conda-forge::imageio
+    - conda-forge::imageio-ffmpeg
     - conda-forge::jsmin
-    - conda-forge::jsonpickle ==1.2
-    - conda-forge::keras <2.10.0,>=2.9.0rc0  # Required by tensorflow-macos
-    - conda-forge::networkx <3.3
+    - conda-forge::jsonpickle <=1.5.0 # 1.5.1 breaks compatibility with v1 encoded files
+    - conda-forge::networkx
     - conda-forge::opencv
     - conda-forge::pandas
     - conda-forge::pillow
     - conda-forge::psutil
     - conda-forge::pykalman
-    - conda-forge::pyside2 >=5.12
+    - conda-forge::pyside6 >=6.5.0 # 6.4 conda package uses problematic QtCharts
     - conda-forge::python-rapidjson
     - conda-forge::pyyaml
     - conda-forge::pyzmq
     - conda-forge::qtpy >=2.0.1
     - conda-forge::rich
-    - conda-forge::scipy >=1.4.1,<=1.9.0
-    - conda-forge::scikit-image
-    - conda-forge::scikit-learn ==1.0
-    - conda-forge::scikit-video
+    - conda-forge::scipy >=1.7.0 # Python 3.10 is compatible with scipy >=1.7.0
+    - conda-forge::scikit-image >=0.21.0 # 0.20.0 requires python_abi 3.8.* *_cp38
+    - conda-forge::scikit-learn >=1.0.0 # Python 3.10 needs scikit-learn >= 1.0
     - conda-forge::seaborn
+    - sleap-deps::tensorflow ==2.12.0
     - conda-forge::qudida
     - conda-forge::albumentations >=1.4.15 # Handle Nan keypoints
     - conda-forge::ndx-pose <0.2.0
 
   run:
-    - conda-forge::python >=3.9.0, <3.10.0
-    - conda-forge::attrs >=21.2.0
-    - conda-forge::cattrs ==1.1.1
+    - conda-forge::python >=3.10.0,<3.11.0
+    - conda-forge::attrs
+    - conda-forge::cattrs
     - conda-forge::h5py
-    - conda-forge::imageio-ffmpeg # Required for imageio to read/write videos with ffmpeg
+    - conda-forge::imageio
+    - conda-forge::imageio-ffmpeg
     - conda-forge::jsmin
-    - conda-forge::jsonpickle ==1.2
-    - conda-forge::keras <2.10.0,>=2.9.0rc0  # Required by tensorflow-macos
-    - conda-forge::networkx <3.3
-    - anaconda::numpy >=1.19.5,<1.23.0
+    - conda-forge::jsonpickle <=1.5.0 # 1.5.1 breaks compatibility with v1 encoded files
+    - conda-forge::networkx
+    - conda-forge::numpy >=1.16.5 # Scipy 1.7.0 requires numpy >=1.16.5
     - conda-forge::opencv
     - conda-forge::pandas
     - conda-forge::pillow
     - conda-forge::psutil
     - conda-forge::pykalman
-    - conda-forge::pyside2 >=5.12
+    - conda-forge::pyside6 >=6.5.0 # 6.4 conda package uses problematic QtCharts
     - conda-forge::python-rapidjson
     - conda-forge::pyyaml
     - conda-forge::pyzmq
     - conda-forge::qtpy >=2.0.1
     - conda-forge::rich
-    - conda-forge::scipy >=1.4.1,<=1.9.0
-    - conda-forge::scikit-image
-    - conda-forge::scikit-learn ==1.0
-    - conda-forge::scikit-video
+    - conda-forge::scipy >=1.7.0 # Python 3.10 is compatible with scipy >=1.7.0
+    - conda-forge::scikit-image >=0.21.0 # 0.20.0 requires python_abi 3.8.* *_cp38
+    - conda-forge::scikit-learn >=1.0.0 # Python 3.10 needs scikit-learn >= 1.0
     - conda-forge::seaborn
-    # - conda-forge::tensorflow-hub  # pulls in tensorflow cpu from conda-forge
+    - sleap-deps::tensorflow ==2.12.0
     - conda-forge::qudida
     - conda-forge::albumentations >=1.4.15 # Handle Nan keypoints
     - conda-forge::ndx-pose <0.2.0
-
-# test:
-#   imports:
-#     - sleap
diff --git a/.conda_mac/sleap_activate.sh b/.conda_mac/sleap_activate.sh
@@ -0,0 +1,4 @@
+#!/bin/sh
+
+# Disable annoying albumentations message
+export NO_ALBUMENTATIONS_UPDATE=1
diff --git a/.conda_mac/sleap_deactivate.sh b/.conda_mac/sleap_deactivate.sh
@@ -0,0 +1,4 @@
+#!/bin/sh
+
+# Reset to the old variables when deactivating the environment
+export NO_ALBUMENTATIONS_UPDATE=$SLEAP_OLD_NO_ALBUMENTATIONS_UPDATE