[WIP] Migrate away from centos7

.dockerignore

@@ -9,3 +9,19 @@ llm/llama.cpp
 test_data
 llm/build
 llama/build
+# Docker config
+docker-compose.yaml
+.dockerignore
+Dockerfile
+# docs
+docs
+CONTRIBUTING.md
+README.md
+SECURITY.md
+LICENSE
+# GitHub CI config
+.github
+# cached models
+data
+stable-diff/v2-1_768-ema-pruned.safetensors
+stable-diff/v2-1_768-ema-pruned (1).safetensors

.gitignore

@@ -15,4 +15,5 @@ build/*/*/*
 !build/**/placeholder
 llama/build
 __debug_bin*
-llama/vendor
+llama/vendor
+data

Dockerfile

@@ -1,6 +1,7 @@
+# syntax=docker.io/docker/dockerfile:1.7-labs
 ARG GOLANG_VERSION=1.22.5
 ARG CMAKE_VERSION=3.22.1
-ARG CUDA_VERSION_11=11.3.1
+ARG CUDA_VERSION_11=11.7.1
 ARG CUDA_V11_ARCHITECTURES="50;52;53;60;61;62;70;72;75;80;86"
 ARG CUDA_VERSION_12=12.4.0
 ARG CUDA_V12_ARCHITECTURES="60;61;62;70;72;75;80;86;87;89;90;90a"
@@ -12,11 +13,11 @@ COPY .git .git
 COPY .gitmodules .gitmodules
 COPY llm llm
 
-FROM --platform=linux/amd64 nvidia/cuda:$CUDA_VERSION_11-devel-centos7 AS cuda-11-build-amd64
+FROM --platform=linux/amd64 nvidia/cuda:$CUDA_VERSION_11-devel-rockylinux8 AS cuda-11-build-amd64
 ARG CMAKE_VERSION
 COPY ./scripts/rh_linux_deps.sh /
 RUN CMAKE_VERSION=${CMAKE_VERSION} sh /rh_linux_deps.sh
-ENV PATH=/opt/rh/devtoolset-10/root/usr/bin:$PATH
+ENV PATH=/opt/rh/gcc-toolset-11/root/usr/bin:$PATH
 COPY --from=llm-code / /go/src/github.com/ollama/ollama/
 WORKDIR /go/src/github.com/ollama/ollama/llm/generate
 ARG CGO_CFLAGS
@@ -29,11 +30,11 @@ RUN --mount=type=cache,target=/root/.ccache \
     CUDA_VARIANT="_v11" \
     bash gen_linux.sh
 
-FROM --platform=linux/amd64 nvidia/cuda:$CUDA_VERSION_12-devel-centos7 AS cuda-12-build-amd64
+FROM --platform=linux/amd64 nvidia/cuda:$CUDA_VERSION_12-devel-rockylinux8 AS cuda-12-build-amd64
 ARG CMAKE_VERSION
 COPY ./scripts/rh_linux_deps.sh /
 RUN CMAKE_VERSION=${CMAKE_VERSION} sh /rh_linux_deps.sh
-ENV PATH=/opt/rh/devtoolset-10/root/usr/bin:$PATH
+ENV PATH=/opt/rh/gcc-toolset-11/root/usr/bin:$PATH
 COPY --from=llm-code / /go/src/github.com/ollama/ollama/
 WORKDIR /go/src/github.com/ollama/ollama/llm/generate
 ARG CGO_CFLAGS
@@ -51,7 +52,7 @@ FROM --platform=linux/arm64 nvidia/cuda:$CUDA_VERSION_11-devel-rockylinux8 AS cu
 ARG CMAKE_VERSION
 COPY ./scripts/rh_linux_deps.sh /
 RUN CMAKE_VERSION=${CMAKE_VERSION} sh /rh_linux_deps.sh
-ENV PATH=/opt/rh/gcc-toolset-10/root/usr/bin:$PATH
+ENV PATH=/opt/rh/gcc-toolset-11/root/usr/bin:$PATH
 COPY --from=llm-code / /go/src/github.com/ollama/ollama/
 WORKDIR /go/src/github.com/ollama/ollama/llm/generate
 ARG CGO_CFLAGS
@@ -67,7 +68,7 @@ FROM --platform=linux/arm64 nvidia/cuda:$CUDA_VERSION_12-devel-rockylinux8 AS cu
 ARG CMAKE_VERSION
 COPY ./scripts/rh_linux_deps.sh /
 RUN CMAKE_VERSION=${CMAKE_VERSION} sh /rh_linux_deps.sh
-ENV PATH=/opt/rh/gcc-toolset-10/root/usr/bin:$PATH
+ENV PATH=/opt/rh/gcc-toolset-11/root/usr/bin:$PATH
 COPY --from=llm-code / /go/src/github.com/ollama/ollama/
 WORKDIR /go/src/github.com/ollama/ollama/llm/generate
 ARG CGO_CFLAGS
@@ -81,14 +82,114 @@ RUN --mount=type=cache,target=/root/.ccache \
     OLLAMA_CUSTOM_CUDA_DEFS="-DGGML_CUDA_USE_GRAPHS=on" \
     bash gen_linux.sh
 
+FROM --platform=linux/amd64 rockylinux:8 as rocm-dev-rockylinux
+# note this is an exact copy of the official rocm almalinux image at https://github.com/ROCm/ROCm-docker/blob/db86386c24eeb45f5d3ba73564b00cc66566e537/dev/Dockerfile-almalinux-8-complete
+# with 2 changes:
+#  - use gcc 11 instead of 9
+#  - base on rockylinux instead of almalinux
 
-FROM --platform=linux/amd64 rocm/dev-centos-7:${ROCM_VERSION}-complete AS rocm-build-amd64
+# todo find some way to reuse top level arg for this
+ARG ROCM_VERSION=6.1.2
+ARG AMDGPU_VERSION=${ROCM_VERSION}
+
+# Base
+RUN yum -y install git java-1.8.0-openjdk python39; yum clean all
+
+# Enable epel-release repositories
+RUN dnf install -y 'dnf-command(config-manager)'
+RUN dnf config-manager --set-enabled powertools
+RUN dnf install -y epel-release
+
+# Install required base build and packaging commands for ROCm
+RUN yum -y install \
+    ca-certificates \
+    bc \
+    bridge-utils \
+    cmake \
+    cmake3 \
+    dkms \
+    doxygen \
+    dpkg \
+    dpkg-dev \
+    dpkg-perl \
+    elfutils-libelf-devel \
+    expect \
+    file \
+    python3-devel \
+    python3-pip \
+    gettext \
+    gcc-c++ \
+    libgcc \
+    lzma \
+    glibc.i686 \
+    ncurses \
+    ncurses-base \
+    ncurses-libs \
+    numactl-devel \
+    numactl-libs \
+    libssh \
+    libunwind-devel \
+    libunwind \
+    llvm \
+    llvm-libs \
+    make \
+    openssl \
+    openssl-libs \
+    openssh \
+    openssh-clients \
+    pciutils \
+    pciutils-devel \
+    pciutils-libs \
+    perl \
+    pkgconfig \
+    qemu-kvm \
+    re2c \
+    kmod \
+    rpm \
+    rpm-build \
+    subversion \
+    wget
+
+# Enable the epel repository for fakeroot
+RUN yum install -y fakeroot
+RUN yum clean all
+
+# todo here we should prob go to 10 for consistency with the rest of everything. or maybe 11
+# Install devtoolset 11
+RUN yum install -y gcc-toolset-11
+RUN yum install -y gcc-toolset-11-libatomic-devel gcc-toolset-11-elfutils-libelf-devel
+
+# Install ROCm repo paths
+RUN echo -e "[ROCm]\nname=ROCm\nbaseurl=https://repo.radeon.com/rocm/rhel8/$ROCM_VERSION/main\nenabled=1\ngpgcheck=0\npriority=50" >> /etc/yum.repos.d/rocm.repo
+RUN echo -e "[amdgpu]\nname=amdgpu\nbaseurl=https://repo.radeon.com/amdgpu/$AMDGPU_VERSION/rhel/8.9/main/x86_64\nenabled=1\ngpgcheck=0\npriority=50" >> /etc/yum.repos.d/amdgpu.repo
+
+# Install versioned ROCm packages eg. rocm-dev6.1.0 to avoid issues with "yum update" pulling really old rocm-dev packages from epel
+# COPY scripts/install_versioned_rocm.sh install_versioned_rocm.sh
+# RUN bash install_versioned_rocm.sh ${ROCM_VERSION}
+# RUN rm install_versioned_rocm.sh
+
+RUN yum install -y rocm-dev rocm-libs
+
+# Set ENV to enable devtoolset9 by default
+ENV PATH=/opt/rh/gcc-toolset-11/root/usr/bin:/opt/rocm/bin:${PATH:+:${PATH}}
+ENV MANPATH=/opt/rh/gcc-toolset-11/root/usr/share/man:${MANPATH}
+ENV INFOPATH=/opt/rh/gcc-toolset-11/root/usr/share/info:${INFOPATH:+:${INFOPATH}}
+ENV PCP_DIR=/opt/rh/gcc-toolset-11/root
+ENV PERL5LIB=/opt/rh/gcc-toolset-11/root/usr/lib64/perl5/vendor_perl
+ENV LD_LIBRARY_PATH=/opt/rocm/lib:/usr/local/lib:/opt/rh/gcc-toolset-11/root/lib:/opt/rh/gcc-toolset-11/root/lib64:${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}
+
+# ENV PYTHONPATH=/opt/rh/gcc-toolset-11/root/
+
+ENV LDFLAGS="-Wl,-rpath=/opt/rh/gcc-toolset-11/root/usr/lib64 -Wl,-rpath=/opt/rh/gcc-toolset-11/root/usr/lib"
+
+
+FROM rocm-dev-rockylinux AS rocm-build-amd64
 ARG CMAKE_VERSION
 COPY ./scripts/rh_linux_deps.sh /
 RUN CMAKE_VERSION=${CMAKE_VERSION} sh /rh_linux_deps.sh
-ENV PATH=/opt/rh/devtoolset-10/root/usr/bin:$PATH
+ENV PATH=/opt/rh/gcc-toolset-11/root/usr/bin:$PATH
 ENV LIBRARY_PATH=/opt/amdgpu/lib64
-COPY --from=llm-code / /go/src/github.com/ollama/ollama/
+COPY --link --from=llm-code / /go/src/github.com/ollama/ollama/
 WORKDIR /go/src/github.com/ollama/ollama/llm/generate
 ARG CGO_CFLAGS
 ARG AMDGPU_TARGETS
@@ -98,12 +199,12 @@ RUN --mount=type=cache,target=/root/.ccache \
 RUN mkdir -p ../../dist/linux-amd64-rocm/lib/ollama && \
     (cd /opt/rocm/lib && tar cf - rocblas/library) | (cd ../../dist/linux-amd64-rocm/lib/ollama && tar xf - )
 
-FROM --platform=linux/amd64 centos:7 AS cpu-builder-amd64
+FROM --platform=linux/amd64 rockylinux:8 AS cpu-builder-amd64
 ARG CMAKE_VERSION
 ARG GOLANG_VERSION
 COPY ./scripts/rh_linux_deps.sh /
 RUN CMAKE_VERSION=${CMAKE_VERSION} GOLANG_VERSION=${GOLANG_VERSION} sh /rh_linux_deps.sh
-ENV PATH=/opt/rh/devtoolset-10/root/usr/bin:$PATH
+ENV PATH=/opt/rh/gcc-toolset-11/root/usr/bin:$PATH
 COPY --from=llm-code / /go/src/github.com/ollama/ollama/
 ARG OLLAMA_CUSTOM_CPU_DEFS
 ARG CGO_CFLAGS
@@ -125,7 +226,7 @@ ARG CMAKE_VERSION
 ARG GOLANG_VERSION
 COPY ./scripts/rh_linux_deps.sh /
 RUN CMAKE_VERSION=${CMAKE_VERSION} GOLANG_VERSION=${GOLANG_VERSION} sh /rh_linux_deps.sh
-ENV PATH=/opt/rh/gcc-toolset-10/root/usr/bin:$PATH
+ENV PATH=/opt/rh/gcc-toolset-11/root/usr/bin:$PATH
 COPY --from=llm-code / /go/src/github.com/ollama/ollama/
 ARG OLLAMA_CUSTOM_CPU_DEFS
 ARG CGO_CFLAGS
@@ -185,7 +286,11 @@ FROM dist-$TARGETARCH as dist
 # Optimized container images do not cary nested payloads
 FROM --platform=linux/amd64 cpu-builder-amd64 AS container-build-amd64
 WORKDIR /go/src/github.com/ollama/ollama
-COPY . .
+# Download the Go module dependencies
+COPY go.mod go.sum ./
+RUN --mount=type=cache,target=/go/pkg/mod \
+    go mod download
+COPY --exclude=Dockerfile --exclude=docker-compose.yaml --exclude=data . .
 ARG GOFLAGS
 ARG CGO_CFLAGS
 RUN --mount=type=cache,target=/root/.ccache \
@@ -238,6 +343,25 @@ ENV OLLAMA_HOST=0.0.0.0
 ENTRYPOINT ["/bin/ollama"]
 CMD ["serve"]
 
+FROM --platform=linux/amd64 runtime-rocm AS runtime-rocm-rootless
+ARG RENDER_GROUP_ID
+RUN echo "render:x:${RENDER_GROUP_ID}:root" >> /etc/group
+
+FROM --platform=linux/amd64 rocm/pytorch:rocm6.2.3_ubuntu22.04_py3.10_pytorch_release_2.3.0 as  stable-diffusion-webui
+ENV DEBIAN_FRONTEND=noninteractive
+WORKDIR /opt/webui
+RUN apt-get update && \
+    apt install -y git-all && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+RUN git clone https://github.com/AUTOMATIC1111/stable-diffusion-webui && \
+    cd stable-diffusion-webui && \
+    python -m pip install --upgrade pip wheel
+
+RUN sed -i 's/input:x:106:/input:x:106:root/'  /etc/group
+WORKDIR /opt/webui/stable-diffusion-webui
+COPY stable-diff/sd3_medium.safetensors models/Stable-diffusion/sd3_medium.safetensors
+CMD REQS_FILE='requirements_versions.txt' python launch.py --precision full --no-half --api --listen
+
 FROM runtime-$TARGETARCH
 EXPOSE 11434
 ENV OLLAMA_HOST=0.0.0.0

docker-compose.yaml

@@ -0,0 +1,55 @@
+services:
+  runtime-rocm-rootless:
+    image: ollama/ollama:rocm
+    profiles:
+      - template
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: runtime-rocm-rootless
+      args:
+        RENDER_GROUP_ID: ${RENDER_GROUP_ID:-106}
+    devices:
+      - "/dev/kfd:/dev/kfd:rwm"
+      - "/dev/dri:/dev/dri:rwm"
+    # tty: true
+    # entrypoint:
+    #  -  /bin/bash
+    ipc: host
+    shm_size: 8GB
+    cap_add:
+      - SYS_PTRACE
+    #  privileged: true
+    security_opt:
+      - seccomp=unconfined
+
+    networks:
+      - ollama
+
+  ollama:
+    volumes:
+      - ./data:/root/.ollama
+    ports:
+      - 11434:11434
+    extends:
+     service: runtime-rocm-rootless
+    profiles:
+     - runtime
+
+  stable-diffusion-webui:
+    image: ollama/stable-diffusion-webui:rocm
+    extends:
+      service: runtime-rocm-rootless
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: stable-diffusion-webui
+    profiles:
+      - runtime
+    ports:
+      - 7860:7860
+#    tty: true
+#    command: /bin/bash
+
+networks:
+  ollama: {}

llama/README.md

@@ -171,4 +171,4 @@ In your `./vendor/` directory, create a branch, and cherry-pick the new commit t
 
 Commit the changes in the ollama repo and submit a PR to Ollama, which will include the vendored code update with your change, along with the patches.
 
-After your PR upstream is merged, follow the **Updating Base Commit** instructions above, however first remove your patch before running `apply-patches` since the new base commit contains your change already.
+After your PR upstream is merged, follow the **Updating Base Commit** instructions above, however first remove your patch before running `apply-patches` since the new base commit contains your change already.

scripts/rh_linux_deps.sh

@@ -32,19 +32,9 @@ if grep -i "centos" /etc/system-release >/dev/null; then
     fi
     dnf install -y devtoolset-10-gcc devtoolset-10-gcc-c++ pigz findutils
 elif grep -i "rocky" /etc/system-release >/dev/null; then
-    # Temporary workaround until rocky 8 AppStream ships GCC 10.4 (10.3 is incompatible with NVCC)
-    cat << EOF > /etc/yum.repos.d/Rocky-Vault.repo
-[vault]
-name=Rocky Vault
-baseurl=https://dl.rockylinux.org/vault/rocky/8.5/AppStream/\$basearch/os/
-gpgcheck=1
-enabled=1
-countme=1
-gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-rockyofficial
-EOF
     dnf install -y git \
-        gcc-toolset-10-gcc-10.2.1-8.2.el8 \
-        gcc-toolset-10-gcc-c++-10.2.1-8.2.el8 \
+        gcc-toolset-11-gcc \
+        gcc-toolset-11-gcc-c++ \
         findutils \
         yum-utils \
         pigz