Ubuntu 18.04ベースのDockerイメージ自動ビルドを追加 (#98)

* docker: add BASE_IMAGE arg * build-docker: support BASE_IMAGE arg * build-docker: add ubuntu18.04 images * fix duplicated docker image tags * Makefile: build-linux-docker-ubuntu18.04 * linux: sort workflow job matrix * linux docker: upgrade to VOICEVOX CORE 0.6.0 * linux docker: overwrite gpu so with cpu so * linux docker: rewrite setup.py to use libcore_cpu for cpu image * remove unused core shared object (prevent ldconfig error) * swap build arg order (core library name -> core example version) * revert swap * fix yaml string * fix yaml string * fix matrix * fix matrix * reduce diff * revert matrix build.yml * on push master * split workaround layer * follow voicevox core 0.6.0 * Makefile: follow #114 * reset workaround: rename libcore_cpu to libcore * rm -f instead rm * reorder entrypoint.sh dump to fast debug rebuild * logic to determine old glibc is used or not * replace glibc version check logic with build arg flag * add use_glibc_229_workaround build-arg to workflow * workflow: commonize voicevox_core_version, voicevox_core_example_version * add os in matrix * add tag to matrix * commonize os * fix matrix for build.yml * fix matrix for build.yml * rm os in include * add tag in matrix * use cpu core for cpu-ubuntu18.04 * rename GLIBC_229_WORKAROUND to GLIBC_231_WORKAROUND * fix indent * remove redundant versions * replace retry logic (unsupported feature for old gnu parallel) * fix indent * revert ldconfig location
VOICEVOX · Sep 23, 2021 · 45ac35f · 45ac35f
1 parent 0ad3d57
commit 45ac35f
Show file tree

Hide file tree

Showing 3 changed files with 103 additions and 23 deletions.
diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml
@@ -24,9 +24,12 @@ jobs:
           - cpu-ubuntu20.04
           - nvidia
           - nvidia-ubuntu20.04
+          - cpu-ubuntu18.04
+          - nvidia-ubuntu18.04
         voicevox_core_version: [0.6.0]
         voicevox_core_example_version: [0.6.0]
         include:
+          # Ubuntu 20.04
           - tag: ''
             target: runtime-env
             base_image: ubuntu:focal
@@ -57,6 +60,21 @@ jobs:
             base_runtime_image: nvidia/cuda:11.4.1-cudnn8-runtime-ubuntu20.04
             voicevox_core_library_name: core
             libtorch_url: https://download.pytorch.org/libtorch/cu111/libtorch-cxx11-abi-shared-with-deps-1.9.0%2Bcu111.zip
+          # Ubuntu 18.04
+          - tag: cpu-ubuntu18.04
+            target: runtime-env
+            base_image: ubuntu:bionic
+            base_runtime_image: ubuntu:bionic
+            voicevox_core_library_name: core_cpu
+            libtorch_url: https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-1.9.0%2Bcpu.zip
+            use_glibc_231_workaround: 1
+          - tag: nvidia-ubuntu18.04
+            target: runtime-nvidia-env
+            base_image: ubuntu:bionic
+            base_runtime_image: nvidia/cuda:11.4.1-cudnn8-runtime-ubuntu18.04
+            voicevox_core_library_name: core
+            libtorch_url: https://download.pytorch.org/libtorch/cu111/libtorch-cxx11-abi-shared-with-deps-1.9.0%2Bcu111.zip
+            use_glibc_231_workaround: 1
 
     steps:
       - uses: actions/checkout@v2
@@ -94,6 +112,7 @@ jobs:
             VOICEVOX_CORE_EXAMPLE_VERSION=${{ matrix.voicevox_core_example_version }}
             VOICEVOX_CORE_LIBRARY_NAME=${{ matrix.voicevox_core_library_name }}
             LIBTORCH_URL=${{ matrix.libtorch_url }}
+            USE_GLIBC_231_WORKAROUND=${{ matrix.use_glibc_231_workaround || '0' }}
           target: ${{ matrix.target }}
           push: true
           tags: |

diff --git a/Dockerfile b/Dockerfile
@@ -145,7 +145,6 @@ WORKDIR /opt/voicevox_engine
 # libsndfile1: soundfile shared object
 # ca-certificates: pyopenjtalk dictionary download
 # build-essential: pyopenjtalk local build
-# parallel: retry download pyopenjtalk dictionary
 RUN <<EOF
     apt-get update
     apt-get install -y \
@@ -154,7 +153,6 @@ RUN <<EOF
         libsndfile1 \
         ca-certificates \
         build-essential \
-        parallel \
         gosu
     apt-get clean
     rm -rf /var/lib/apt/lists/*
@@ -186,15 +184,7 @@ ADD ./requirements.txt /tmp/
 ADD ./voicevox_engine /opt/voicevox_engine/voicevox_engine
 ADD ./run.py ./check_tts.py ./VERSION.txt ./LICENSE ./LGPL_LICENSE /opt/voicevox_engine/
 
-# Create container start shell
-COPY <<EOF /entrypoint.sh
-#!/bin/bash
-cat /opt/voicevox_core/README.txt > /dev/stderr
-exec "\$@"
-EOF
-
 RUN <<EOF
-    chmod +x /entrypoint.sh
     # Create a general user
     useradd --create-home user
     # Update ld
@@ -211,14 +201,42 @@ RUN <<EOF
     # Install voicevox_core
     cd /opt/voicevox_core_example/example/python
     gosu user pip3 install .
+EOF
 
-    # FIXME: remove first execution delay
-    # try 5 times, delay 5 seconds before each execution.
-    # if all tries are failed, `docker build` will be failed.
+# Keep layer cache above if dict download failed in local build
+RUN <<EOF
+    set -eux
 
     # Download openjtalk dictionary
-    parallel --retries 5 --delay 5 --ungroup \
-      gosu user python3 -c "import pyopenjtalk; pyopenjtalk._lazy_init()"
+    # try 5 times, sleep 5 seconds before retry
+    for i in $(seq 5); do
+        EXIT_CODE=0
+        gosu user /opt/python/bin/python3 -c "import pyopenjtalk; pyopenjtalk._lazy_init()" || EXIT_CODE=$?
+        if [ "$EXIT_CODE" = "0" ]; then
+            break
+        fi
+        sleep 5
+    done
+
+    if [ "$EXIT_CODE" != "0" ]; then
+        exit "$EXIT_CODE"
+    fi
+EOF
+
+# Create container start shell
+ARG USE_GLIBC_231_WORKAROUND=0
+COPY --chmod=775 <<EOF /entrypoint.sh
+#!/bin/bash
+cat /opt/voicevox_core/README.txt > /dev/stderr
+
+# Workaround: ldconfig fail to load LibTorch if glibc < 2.31.
+# For isolating problems and simplifing script, use flag USE_GLIBC_231_WORKAROUND
+# instead of implementing version check logic.
+if [ "${USE_GLIBC_231_WORKAROUND}" = "1" ]; then
+    export LD_LIBRARY_PATH="/opt/libtorch/lib:\${LD_LIBRARY_PATH:-}"
+fi
+
+exec "\$@"
 EOF
 
 ENTRYPOINT [ "/entrypoint.sh"  ]

diff --git a/Makefile b/Makefile
@@ -1,7 +1,8 @@
 CMD=
 
-.PHONY: build-linux-docker-ubuntu
-build-linux-docker-ubuntu:
+# Ubuntu 20.04
+.PHONY: build-linux-docker-ubuntu20.04
+build-linux-docker-ubuntu20.04:
 	docker buildx build . \
 		-t hiroshiba/voicevox_engine:cpu-ubuntu20.04-latest \
 		--target runtime-env \
@@ -11,14 +12,14 @@ build-linux-docker-ubuntu:
 		--build-arg LIBTORCH_URL=https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-1.9.0%2Bcpu.zip \
 		--build-arg VOICEVOX_CORE_LIBRARY_NAME=core_cpu
 
-.PHONY: run-linux-docker-ubuntu
-run-linux-docker-ubuntu:
+.PHONY: run-linux-docker-ubuntu20.04
+run-linux-docker-ubuntu20.04:
 	docker run --rm -it \
 		-p '127.0.0.1:50021:50021' \
 		hiroshiba/voicevox_engine:cpu-ubuntu20.04-latest $(CMD)
 
-.PHONY: build-linux-docker-nvidia
-build-linux-docker-nvidia:
+.PHONY: build-linux-docker-nvidia-ubuntu20.04
+build-linux-docker-nvidia-ubuntu20.04:
 	docker buildx build . \
 		-t hiroshiba/voicevox_engine:nvidia-ubuntu20.04-latest \
 		--target runtime-nvidia-env \
@@ -28,13 +29,54 @@ build-linux-docker-nvidia:
 		--build-arg LIBTORCH_URL=https://download.pytorch.org/libtorch/cu111/libtorch-cxx11-abi-shared-with-deps-1.9.0%2Bcu111.zip \
 		--build-arg VOICEVOX_CORE_LIBRARY_NAME=core
 
-.PHONY: run-linux-docker-nvidia
-run-linux-docker-nvidia:
+.PHONY: run-linux-docker-nvidia-ubuntu20.04
+run-linux-docker-nvidia-ubuntu20.04:
 	docker run --rm -it \
 		--gpus all \
 		-p '127.0.0.1:50021:50021' \
 		hiroshiba/voicevox_engine:nvidia-ubuntu20.04-latest $(CMD)
 
+
+# Ubuntu 18.04
+.PHONY: build-linux-docker-ubuntu18.04
+build-linux-docker-ubuntu18.04:
+	docker buildx build . \
+		-t hiroshiba/voicevox_engine:cpu-ubuntu18.04-latest \
+		--target runtime-env \
+		--progress plain \
+		--build-arg BASE_IMAGE=ubuntu:bionic \
+		--build-arg BASE_RUNTIME_IMAGE=ubuntu:bionic \
+		--build-arg LIBTORCH_URL=https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-1.9.0%2Bcpu.zip \
+		--build-arg VOICEVOX_CORE_LIBRARY_NAME=core_cpu \
+		--build-arg USE_GLIBC_231_WORKAROUND=1
+
+.PHONY: run-linux-docker-ubuntu18.04
+run-linux-docker-ubuntu18.04:
+	docker run --rm -it \
+		-p '127.0.0.1:50021:50021' \
+		hiroshiba/voicevox_engine:cpu-ubuntu18.04-latest $(CMD)
+
+.PHONY: build-linux-docker-nvidia-ubuntu18.04
+build-linux-docker-nvidia-ubuntu18.04:
+	docker buildx build . \
+		-t hiroshiba/voicevox_engine:nvidia-ubuntu18.04-latest \
+		--target runtime-nvidia-env \
+		--progress plain \
+		--build-arg BASE_IMAGE=ubuntu:bionic \
+		--build-arg BASE_RUNTIME_IMAGE=nvidia/cuda:11.4.1-cudnn8-runtime-ubuntu18.04 \
+		--build-arg LIBTORCH_URL=https://download.pytorch.org/libtorch/cu111/libtorch-cxx11-abi-shared-with-deps-1.9.0%2Bcu111.zip \
+		--build-arg VOICEVOX_CORE_LIBRARY_NAME=core \
+		--build-arg USE_GLIBC_231_WORKAROUND=1
+
+.PHONY: run-linux-docker-nvidia-ubuntu18.04
+run-linux-docker-nvidia-ubuntu18.04:
+	docker run --rm -it \
+		--gpus all \
+		-p '127.0.0.1:50021:50021' \
+		hiroshiba/voicevox_engine:nvidia-ubuntu18.04-latest $(CMD)
+
+
+# Python env for test
 .PHONY: build-linux-docker-compile-python-env
 build-linux-docker-compile-python-env:
 	docker buildx build . \
@@ -48,6 +90,7 @@ run-linux-docker-compile-python-env:
 	docker run --rm -it \
 		hiroshiba/voicevox_engine:compile-python-env $(CMD)
 
+
 # Build linux binary in Docker
 .PHONY: build-linux-docker-build
 build-linux-docker-build: