Use HDBSCAN package pin to 0.8.38 (#5906)

HBSCAN has a [breaking change](scikit-learn-contrib/hdbscan@aa99a71) in `main` branch, which cause wheel nightly tests to fail as cuML picks up HDBSCAN from `main` branch HEAD. As HDBSCAN also releases on pypi now, we can install release packages instead of installing HDBSCAN from git source.

This PR also updates HBDSCAN to v0.8.38.

Authors:
  - Divye Gala (https://github.com/divyegala)

Approvers:
  - Jake Awe (https://github.com/AyodeAwe)
  - Dante Gama Dessavre (https://github.com/dantegd)
  - https://github.com/jakirkham

URL: #5906

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -22,7 +22,7 @@ dependencies:
 - doxygen=1.9.1
 - gcc_linux-64=11.*
 - graphviz
-- hdbscan<=0.8.30
+- hdbscan>=0.8.38,<0.8.39
 - hypothesis>=6.0,<7
 - ipykernel
 - ipython

conda/environments/all_cuda-125_arch-x86_64.yaml

@@ -24,7 +24,7 @@ dependencies:
 - doxygen=1.9.1
 - gcc_linux-64=11.*
 - graphviz
-- hdbscan<=0.8.30
+- hdbscan>=0.8.38,<0.8.39
 - hypothesis>=6.0,<7
 - ipykernel
 - ipython

conda/recipes/cuml-cpu/meta.yaml

@@ -34,7 +34,7 @@ requirements:
     - numpy>=1.23,<2.0a0
     - pandas
     - scikit-learn=1.2
-    - hdbscan<=0.8.30
+    - hdbscan>=0.8.38,<0.8.39
     - umap-learn=0.5.3
     - nvtx
 

dependencies.yaml

@@ -177,7 +177,7 @@ dependencies:
     common:
       - output_types: [conda, requirements, pyproject]
         packages:
-          - cython>=3.0.0
+          - &cython cython>=3.0.0
           - &treelite treelite==4.3.0
       - output_types: conda
         packages:
@@ -486,7 +486,9 @@ dependencies:
     common:
       - output_types: [conda, requirements, pyproject]
         packages:
+          - *cython
           - dask-ml
+          - hdbscan>=0.8.38,<0.8.39
           - hypothesis>=6.0,<7
           - nltk
           - numpydoc
@@ -505,14 +507,9 @@ dependencies:
           - pip
           - pip:
               - dask-glm==0.3.0
-          # TODO: remove pin once a release that includes fixes for the error
-          # is released: https://github.com/rapidsai/cuml/issues/5514
-          - hdbscan<=0.8.30
       - output_types: pyproject
         packages:
           - dask-glm==0.3.0
-            # TODO: Can we stop pulling from the master branch now that there was a release in October?
-          - hdbscan @ git+https://github.com/scikit-learn-contrib/hdbscan.git@master
   test_notebooks:
     common:
       - output_types: [conda, requirements]

python/cuml/pyproject.toml

@@ -109,9 +109,10 @@ classifiers = [
 
 [project.optional-dependencies]
 test = [
+    "cython>=3.0.0",
     "dask-glm==0.3.0",
     "dask-ml",
-    "hdbscan @ git+https://github.com/scikit-learn-contrib/hdbscan.git@master",
+    "hdbscan>=0.8.38,<0.8.39",
     "hypothesis>=6.0,<7",
     "nltk",
     "numpydoc",