Merge branch-24.10 into branch-25.02

.devcontainer/docker-compose.yml

@@ -18,7 +18,7 @@ services:
   triton:
     container_name: morpheus-triton
     runtime: nvidia
-    image: nvcr.io/nvidia/tritonserver:23.06-py3
+    image: nvcr.io/nvidia/tritonserver:24.09-py3
     command: tritonserver --model-repository=/models/triton-model-repo --exit-on-error=false ${TRITON_MODEL_ARGS}
     ports:
       - 8000:8000

.github/workflows/ci_pipe.yml

@@ -47,7 +47,7 @@ on:
 env:
   CHANGE_TARGET: "${{ github.base_ref }}"
   CUDA_PATH: "/usr/local/cuda/"
-  CUDA_VER: "12.1"
+  CUDA_VER: "12.5"
   GH_TOKEN: "${{ github.token }}"
   GIT_COMMIT: "${{ github.sha }}"
   MORPHEUS_ROOT: "${{ github.workspace }}/morpheus"

.github/workflows/pr.yaml

@@ -95,8 +95,8 @@ jobs:
       # Upload morpheus conda packages only for non PR branches. Use 'main' for main branch and 'dev' for all other branches
       conda_upload_label: ${{ !fromJSON(needs.prepare.outputs.is_pr) && (fromJSON(needs.prepare.outputs.is_main_branch) && 'main' || 'dev') || '' }}
       base_container: rapidsai/ci-conda:cuda12.5.1-ubuntu22.04-py3.10
-      container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-build-241004
-      test_container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-test-241004
+      container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-build-241024
+      test_container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-test-241024
     secrets:
       CONDA_TOKEN: ${{ secrets.CONDA_TOKEN }}
       NGC_API_KEY: ${{ secrets.NGC_API_KEY }}

ci/iwyu/mappings.imp

@@ -8,6 +8,7 @@
 { "include": [ "\"mrc/protos/tensor_meta_data.pb.h\"", private, "<mrc/protos/tensor_meta_data.pb.h>", "public" ] },
 
 # stdlib
+{ "include": [ "<bits/chrono.h>", private, "<chrono>", "public" ] },
 { "include": [ "<bits/cxxabi_forced.h>", private, "<mutex>", "public" ] },
 { "include": [ "<bits/cxxabi_forced.h>", private, "<vector>", "public" ] },
 { "include": [ "<bits/this_thread_sleep.h>", private, "<thread>", "public" ] },
@@ -29,6 +30,8 @@
 { "include": ["\"cuda_runtime_api.h\"", "private", "<cuda_runtime.h>", "public"] },
 { "include": ["<driver_types.h>", "private", "<cuda_runtime.h>", "public"] },
 { "include": ["\"driver_types.h\"", "private", "<cuda_runtime.h>", "public"] },
+{ "include": ["\"cuda/__memory_resource/properties.h\"", "private", "<cuda/memory_resource>", "public"] },
+{ "include": ["\"cuda/__memory_resource/resource_ref.h\"", "private", "<cuda/memory_resource>", "public"] },
 
 # gtest
 { "include": ["@<gtest/gtest-.*>", "private", "<gtest/gtest.h>", "public"] },
@@ -47,7 +50,7 @@
 { "include": [ "<google/protobuf/repeated_ptr_field.h>", private, "<google/protobuf/repeated_field.h>", "public" ] },
 
 # pybind11
-{ "include": [ "<pybind11/detail/common.h>", "private", "<pybind11/pytypes.h>", "public" ] },
+{ "include": [ "@<pybind11/detail/.*>", "private", "<pybind11/pybind11.h>", "public" ] },
 { "include": [ "<pybind11/cast.h>", "private", "<pybind11/pybind11.h>", "public" ] },
 
 # rxcpp

ci/runner/Dockerfile

@@ -35,12 +35,6 @@ SHELL ["/bin/bash",  "-c"]
 # Create conda environment
 COPY ./dependencies.yaml /tmp/conda/
 
-RUN <<EOF
-rapids-mamba-retry install -y \
-  "rapids-dependency-file-generator>=1.13.*"
-conda clean -aipty
-EOF
-
 # ============ build ==================
 FROM base as build
 

ci/scripts/github/test.sh

@@ -21,12 +21,7 @@ source ${WORKSPACE}/ci/scripts/github/morpheus_env.sh
 source ${WORKSPACE}/ci/scripts/github/cmake_all.sh
 /usr/bin/nvidia-smi
 
-rapids-dependency-file-generator \
-  --output conda \
-  --file-key all \
-  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee "${WORKSPACE_TMP}/env.yaml"
-
-update_conda_env "${WORKSPACE_TMP}/env.yaml"
+update_conda_env "${WORKSPACE}/conda/environments/all_cuda-125_arch-x86_64.yaml"
 
 log_toolchain
 

ci/scripts/run_ci_local.sh

@@ -58,7 +58,7 @@ GIT_BRANCH=$(git branch --show-current)
 GIT_COMMIT=$(git log -n 1 --pretty=format:%H)
 
 LOCAL_CI_TMP=${LOCAL_CI_TMP:-${MORPHEUS_ROOT}/.tmp/local_ci_tmp}
-CONTAINER_VER=${CONTAINER_VER:-241004}
+CONTAINER_VER=${CONTAINER_VER:-241024}
 CUDA_VER=${CUDA_VER:-12.5}
 CUDA_FULL_VER=${CUDA_FULL_VER:-12.5.1}
 DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""}

conda/environments/all_cuda-125_arch-x86_64.yaml

@@ -15,7 +15,7 @@ dependencies:
 - automake=1.16.5
 - beautifulsoup4=4.12
 - benchmark=1.8.3
-- boto3
+- boto3=1.35
 - breathe=4.35.0
 - c-ares=1.32
 - ccache
@@ -88,6 +88,7 @@ dependencies:
 - pydantic
 - pylibcudf=24.10
 - pylint=3.0.3
+- pynvml=11.4
 - pypdf=3.17.4
 - pypdfium2=4.30
 - pytest-asyncio
@@ -104,10 +105,9 @@ dependencies:
 - requests
 - requests-cache=1.1
 - requests-toolbelt=1.0
-- s3fs
+- s3fs=2024.10
 - scikit-build=0.17.6
 - scikit-learn=1.3.2
-- sentence-transformers=2.7
 - sphinx
 - sphinx_rtd_theme
 - sqlalchemy<2.0
@@ -140,5 +140,7 @@ dependencies:
   - nemollm==0.3.5
   - pymilvus==2.3.6
   - pytest-kafka==0.6.0
+  - python-logging-loki
+  - sentence-transformers==2.7
   - torch==2.4.0+cu124
 name: all_cuda-125_arch-x86_64

conda/environments/dev_cuda-125_arch-x86_64.yaml

@@ -73,6 +73,7 @@ dependencies:
 - pydantic
 - pylibcudf=24.10
 - pylint=3.0.3
+- pynvml=11.4
 - pypdfium2=4.30
 - pytest-asyncio
 - pytest-benchmark=4.0

conda/environments/examples_cuda-125_arch-x86_64.yaml

@@ -13,7 +13,7 @@ dependencies:
 - appdirs
 - arxiv=1.4
 - beautifulsoup4=4.12
-- boto3
+- boto3=1.35
 - click>=8
 - cudf=24.10
 - cuml=24.10.*
@@ -53,9 +53,8 @@ dependencies:
 - requests
 - requests-cache=1.1
 - requests-toolbelt=1.0
-- s3fs
+- s3fs=2024.10
 - scikit-learn=1.3.2
-- sentence-transformers=2.7
 - sqlalchemy<2.0
 - tqdm=4
 - transformers=4.36.2
@@ -78,5 +77,7 @@ dependencies:
   - milvus==2.3.5
   - nemollm==0.3.5
   - pymilvus==2.3.6
+  - python-logging-loki
+  - sentence-transformers==2.7
   - torch==2.4.0+cu124
 name: examples_cuda-125_arch-x86_64

conda/environments/model-utils_cuda-125_arch-x86_64.yaml

@@ -14,10 +14,13 @@ dependencies:
 - matplotlib
 - onnx
 - pandas
+- pip
 - python=3.10
 - scikit-learn=1.3.2
 - seaborn
 - seqeval=1.2.2
 - transformers=4.36.2
 - xgboost
+- pip:
+  - tensorrt-cu12
 name: model-utils_cuda-125_arch-x86_64

dependencies.yaml

@@ -198,6 +198,18 @@ files:
       - morpheus_core_pip
       - morpheus_llm_pip
 
+  # dfp example conda dependencies
+  dfp_example:
+    output: conda
+    matrix:
+      cuda: ["12.5"]
+      arch: [x86_64]
+    output: conda
+    conda_dir: examples/digital_fingerprinting/production/conda/environments
+    includes:
+      - example-dfp-container
+      - example-dfp-prod
+
 channels:
   - conda-forge
   - huggingface
@@ -314,6 +326,7 @@ dependencies:
           - include-what-you-use=0.20
           - isort
           - pylint=3.0.3
+          - pynvml=11.4
           - vale=3.7
           - vale-styles-microsoft
           - vale-styles-write-good
@@ -429,10 +442,24 @@ dependencies:
       - output_types: [conda]
         packages:
           - *nodejs
-          - boto3
+          - boto3=1.35
           - kfp
           - papermill=2.4.0
-          - s3fs
+          - s3fs=2024.10
+          - pip
+          - pip:
+            - python-logging-loki
+
+  # packages needed in the DFP example container
+  example-dfp-container:
+    common:
+      - output_types: [conda]
+        packages:
+          - morpheus-dfp=24.10
+          - tini=0.19
+          - pip:
+            - --extra-index-url https://download.pytorch.org/whl/cu124
+            - torch==2.4.0+cu124
 
   example-gnn:
     common:
@@ -469,14 +496,14 @@ dependencies:
           - *pypdfium2
           - *python-docx
           - requests-toolbelt=1.0 # Transitive dep needed by nemollm, specified here to ensure we get a compatible version
-          - sentence-transformers=2.7
           - pip
           - pip:
             - langchain==0.1.16
             - langchain-nvidia-ai-endpoints==0.0.11
             - faiss-cpu
             - google-search-results==2.4
             - nemollm==0.3.5
+            - sentence-transformers==2.7 # using pip now instead of conda to avoid install of pytorch cpu
 
   model-training-tuning:
     common:
@@ -492,6 +519,9 @@ dependencies:
           - seaborn
           - seqeval=1.2.2
           - xgboost
+          - pip
+          - pip:
+            - tensorrt-cu12
 
   cve-mitigation:
     common:

docker/Dockerfile

@@ -347,7 +347,7 @@ RUN --mount=type=bind,source=.,target=/opt/host_repo \
     git clone file:///opt/host_repo/${MORPHEUS_ROOT_HOST} /tmp/morpheus_repo &&\
     cd /tmp/morpheus_repo &&\
     git lfs install &&\
-    ./scripts/fetch_data.py fetch datasets examples
+    ./scripts/fetch_data.py fetch examples
 
 # ============ Stage: runtime ============
 # Setup container for runtime environment
@@ -360,7 +360,6 @@ COPY --from=git_clone "/tmp/morpheus_repo/conda/environments/*.yaml" "./conda/en
 COPY --from=git_clone "/tmp/morpheus_repo/docker" "./docker"
 COPY --from=build_docs "/workspace/${MORPHEUS_ROOT_HOST}/build/docs/html" "./docs"
 COPY --from=git_clone "/tmp/morpheus_repo/examples" "./examples"
-COPY --from=git_clone "/tmp/morpheus_repo/models/datasets" "./models/datasets"
 COPY --from=git_clone "/tmp/morpheus_repo/scripts" "./scripts"
 COPY --from=git_clone "/tmp/morpheus_repo/*.md" "./"
 COPY --from=git_clone "/tmp/morpheus_repo/LICENSE" "./"