Update dockerfiles (IntelLabs#81)

* Update dockerfiles; Checkin dockerfiles now only copy vdms folders; remove maven; centralize dependencies in /dependencies folder * Correct zlib and tiledb paths * Remove coverage file check * Change coverage script path * Change coverage script path * cd before coverage script * Add maven build arg to avoid target failure
cwlacewe · Jan 26, 2023 · 726b642 · 726b642
1 parent 0321712
commit 726b642
Show file tree

Hide file tree

Showing 4 changed files with 104 additions and 90 deletions.
diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
@@ -73,6 +73,9 @@ jobs:
           docker stop $(docker ps -aqf "name=${{ matrix.container_name }}") || true
           docker rm $(docker ps -aqf "name=${{ matrix.container_name }}") || true
 
+          # docker build -f docker/check-in/Dockerfile -t ${{ matrix.container_tag }} .
+
+          # REMOVE MAVEN AFTER MERGE
           docker build --build-arg MAVEN_OPTS=${{ secrets.MAVEN_OPTS }} \
             -f docker/check-in/Dockerfile -t ${{ matrix.container_tag }} .
 
@@ -86,25 +89,25 @@ jobs:
           echo "${{ matrix.container_name }}"
 
           # Make sure CPP coverage script is available
-          if [ ! -f docker/check-in/run_coverage_cpp.sh ]; then
-            docker exec ${{ matrix.container_name }} bash -c "touch /run_coverage_cpp.sh && echo 'cd /vdms/tests && chmod +x run_tests.sh && ./run_tests.sh' >> /run_coverage_cpp.sh && chmod +x /run_coverage_cpp.sh && mkdir -p /vdms/tests/coverage_report"
-            docker exec ${{ matrix.container_name }} bash -c "echo 'gcovr -k --root /vdms -e /vdms/src/pmgd -e /vdms/build/CMakeFiles -f "/vdms/client/.*\.cc" -f "/vdms/ext/.*\.cc" -f "/vdms/src/.*\.cc" -f src/SearchExpression.cc --exclude-unreachable-branches --xml-pretty --xml=/vdms/tests/coverage_report/c_coverage_report.xml --txt=/vdms/tests/coverage_report/c_coverage_report.txt' >> /run_coverage_cpp.sh  && echo "echo 'DONE'" >> /run_coverage_cpp.sh"
-            docker exec ${{ matrix.container_name }} bash -c "echo 'cat /vdms/tests/coverage_report/c_coverage_report.txt' >> /run_coverage_cpp.sh"
-          fi
-
-          # Make sure Python coverage script is available
-          if [ ! -f docker/check-in/run_coverage_py.sh ]; then
-            docker exec ${{ matrix.container_name }} bash -c "touch /run_coverage_py.sh && echo 'cd /vdms/tests/python && ./run_python_tests.sh' >> /run_coverage_py.sh && chmod +x /run_coverage_py.sh && mkdir -p /vdms/tests/coverage_report"
-            docker exec ${{ matrix.container_name }} bash -c "echo 'python -m coverage report > /vdms/tests/coverage_report/py_coverage_report.txt' >> /run_coverage_py.sh  && echo "echo 'DONE'" >> /run_coverage_py.sh"
-            docker exec ${{ matrix.container_name }} bash -c "echo 'cat /vdms/tests/coverage_report/py_coverage_report.txt' >> /run_coverage_py.sh"
-          fi
-
-          docker exec ${{ matrix.container_name }} bash -c "./run_coverage_cpp.sh"
+          # if [ ! -f docker/check-in/run_coverage_cpp.sh ]; then
+          #   docker exec ${{ matrix.container_name }} bash -c "touch /run_coverage_cpp.sh && echo 'cd /vdms/tests && chmod +x run_tests.sh && ./run_tests.sh' >> /run_coverage_cpp.sh && chmod +x /run_coverage_cpp.sh && mkdir -p /vdms/tests/coverage_report"
+          #   docker exec ${{ matrix.container_name }} bash -c "echo 'gcovr -k --root /vdms -e /vdms/src/pmgd -e /vdms/build/CMakeFiles -f "/vdms/client/.*\.cc" -f "/vdms/ext/.*\.cc" -f "/vdms/src/.*\.cc" -f src/SearchExpression.cc --exclude-unreachable-branches --xml-pretty --xml=/vdms/tests/coverage_report/c_coverage_report.xml --txt=/vdms/tests/coverage_report/c_coverage_report.txt' >> /run_coverage_cpp.sh  && echo "echo 'DONE'" >> /run_coverage_cpp.sh"
+          #   docker exec ${{ matrix.container_name }} bash -c "echo 'cat /vdms/tests/coverage_report/c_coverage_report.txt' >> /run_coverage_cpp.sh"
+          # fi
+
+          # # Make sure Python coverage script is available
+          # if [ ! -f docker/check-in/run_coverage_py.sh ]; then
+          #   docker exec ${{ matrix.container_name }} bash -c "touch /run_coverage_py.sh && echo 'cd /vdms/tests/python && ./run_python_tests.sh' >> /run_coverage_py.sh && chmod +x /run_coverage_py.sh && mkdir -p /vdms/tests/coverage_report"
+          #   docker exec ${{ matrix.container_name }} bash -c "echo 'python -m coverage report > /vdms/tests/coverage_report/py_coverage_report.txt' >> /run_coverage_py.sh  && echo "echo 'DONE'" >> /run_coverage_py.sh"
+          #   docker exec ${{ matrix.container_name }} bash -c "echo 'cat /vdms/tests/coverage_report/py_coverage_report.txt' >> /run_coverage_py.sh"
+          # fi
+
+          docker exec ${{ matrix.container_name }} bash -c "cd / && ./run_coverage_cpp.sh"
           docker cp $(docker ps -a | grep ${{ matrix.container_name }} | awk '{print $1}'):/vdms/tests/coverage_report/c_coverage_report.txt coverage/c_coverage_report_target.txt
           docker cp $(docker ps -a | grep ${{ matrix.container_name }} | awk '{print $1}'):/vdms/tests/coverage_report/c_coverage_report.xml coverage/c_coverage_report_target.xml
           echo "coverage_value_cpp=$(cat coverage/c_coverage_report_target.xml | grep -oP 'coverage line-rate="([-+]?\d*\.\d+|\d+)"' | grep -oP "[-+]?\d*\.\d+|\d+" | awk '{print $1*100}')" >> $GITHUB_ENV
 
-          docker exec ${{ matrix.container_name }} bash -c "./run_coverage_py.sh"
+          docker exec ${{ matrix.container_name }} bash -c "cd / && ./run_coverage_py.sh"
           docker cp $(docker ps -a | grep ${{ matrix.container_name }} | awk '{print $1}'):/vdms/tests/coverage_report/py_coverage_report.txt coverage/py_coverage_report_target.txt || true
           docker cp $(docker ps -a | grep ${{ matrix.container_name }} | awk '{print $1}'):/vdms/tests/coverage_report/py_coverage_report.xml coverage/py_coverage_report_target.xml || true
           if [ ! -f coverage/py_coverage_report_target.xml ] && [ "${{ matrix.coverage_type }}" == "Target" ]; then
@@ -126,7 +129,7 @@ jobs:
           if [[ -z $coverage_value_cpp ]]
           then
             exit 1
-          fi          
+          fi
           echo "${{ matrix.coverage_type }} CPP Coverage: ${coverage_value_cpp}"
           echo "${{ matrix.output_cpp_name }}=${coverage_value_cpp}" >> $GITHUB_OUTPUT
 

diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile
@@ -4,7 +4,6 @@
 ARG UBUNTU_VERSION=20.04
 ARG UBUNTU_NAME=focal
 ARG BUILD_THREADS=-j16
-ARG MAVEN_OPTS='-Dhttps.nonProxyHosts="localhost|127.0.0.1"'
 
 #1
 FROM ubuntu:${UBUNTU_VERSION}
@@ -13,25 +12,25 @@ FROM ubuntu:${UBUNTU_VERSION}
 ARG UBUNTU_VERSION
 ARG UBUNTU_NAME
 ARG MAVEN_OPTS
-WORKDIR /
 
 # Install Packages
-RUN apt-get update && apt-get -y install --no-install-recommends software-properties-common && \
+RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common && \
     add-apt-repository "deb http://security.ubuntu.com/ubuntu ${UBUNTU_NAME}-security main" && \
-    apt-get -y install --no-install-recommends apt-transport-https autoconf automake bison build-essential \
+    apt-get install -y --no-install-recommends apt-transport-https autoconf automake bison build-essential \
     bzip2 ca-certificates curl ed flex g++ git gnupg-agent javacc libarchive-tools \
     libatlas-base-dev libavcodec-dev libavformat-dev libboost-all-dev libbz2-dev \
     libc-ares-dev libdc1394-22-dev libgflags-dev libgoogle-glog-dev libgtest-dev \
     libgtk-3-dev libgtk2.0-dev libhdf5-serial-dev libjpeg-dev libjpeg8-dev libjsoncpp-dev \
     libleveldb-dev liblmdb-dev liblz4-dev libopenblas-dev libopenmpi-dev \
     libpng-dev librdkafka-dev libsnappy-dev libssl-dev libswscale-dev libtbb-dev \
-    libtbb2 libtiff-dev libtiff5-dev libtool maven mpich openjdk-11-jdk-headless \
+    libtbb2 libtiff-dev libtiff5-dev libtool mpich openjdk-11-jdk-headless \
     pkg-config python3-dev python3-pip unzip && \
     apt-get clean && rm -rf /var/lib/apt/lists/* && \
     update-alternatives --install /usr/bin/python python /usr/bin/python3 1 && \
     pip3 install --no-cache-dir "numpy>=1.23.2"
 
 # Pull and Install Dependencies
+WORKDIR /dependencies
 RUN git clone --branch v3.21.2 https://github.com/Kitware/CMake.git && \
     git clone --branch v4.0.2 https://github.com/swig/swig.git && \
     git clone --branch v1.7.1 https://github.com/facebookresearch/faiss.git && \
@@ -40,32 +39,30 @@ RUN git clone --branch v3.21.2 https://github.com/Kitware/CMake.git && \
     git clone --branch 4.5.3 https://github.com/opencv/opencv.git && \
     git clone --branch v0.6 https://github.com/tristanpenman/valijson.git && \
     curl -L -o /usr/share/java/json-simple-1.1.1.jar https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/json-simple/json-simple-1.1.1.jar && \
-    curl -L -o /1.3.1.tar.gz https://github.com/TileDB-Inc/TileDB/archive/refs/tags/1.3.1.tar.gz && \
-    curl -L -o /zlib-1.2.13.tar.gz http://zlib.net/zlib-1.2.13.tar.gz && \
-    cd /CMake && ./bootstrap && make ${BUILD_THREADS} && make install && \
-    cd /swig && ./autogen.sh && ./configure && make ${BUILD_THREADS} && make install && \
-    cd /faiss && mkdir build && cd build && cmake -DFAISS_ENABLE_GPU=OFF .. && make ${BUILD_THREADS} && make install && \
-    cd /FLINNG && mkdir build && cd build && cmake .. && make ${BUILD_THREADS} && make install && \
-    cd /grpc && git submodule update --init --recursive && pip3 install --no-cache-dir -r requirements.txt && GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip3 install --no-cache-dir . && \
+    curl -L -o /dependencies/1.3.1.tar.gz https://github.com/TileDB-Inc/TileDB/archive/refs/tags/1.3.1.tar.gz && \
+    curl -L -o /dependencies/zlib-1.2.13.tar.gz http://zlib.net/zlib-1.2.13.tar.gz && \
+    cd /dependencies/CMake && ./bootstrap && make ${BUILD_THREADS} && make install && \
+    cd /dependencies/swig && ./autogen.sh && ./configure && make ${BUILD_THREADS} && make install && \
+    cd /dependencies/faiss && mkdir build && cd build && cmake -DFAISS_ENABLE_GPU=OFF .. && make ${BUILD_THREADS} && make install && \
+    cd /dependencies/FLINNG && mkdir build && cd build && cmake .. && make ${BUILD_THREADS} && make install && \
+    cd /dependencies/grpc && git submodule update --init --recursive && pip3 install --no-cache-dir -r requirements.txt && GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip3 install --no-cache-dir . && \
         cd tools/distrib/python/grpcio_tools && python ../make_grpcio_tools.py && GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip3 install --no-cache-dir . && \
-        cd /grpc/third_party/protobuf/cmake && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \
+        cd /dependencies/grpc/third_party/protobuf/cmake && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \
         cd ../../../abseil-cpp && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \
         cd ../../re2/ && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \
         cd ../../zlib/ && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \
-        cd /grpc/cmake && mkdir build && cd build && cmake -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_ABSL_PROVIDER=package \
+        cd /dependencies/grpc/cmake && mkdir build && cd build && cmake -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_ABSL_PROVIDER=package \
             -DgRPC_CARES_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package \
             -DgRPC_RE2_PROVIDER=package -DgRPC_SSL_PROVIDER=package \
             -DgRPC_ZLIB_PROVIDER=package -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE ../.. && make ${BUILD_THREADS} && make install && \
-    cd /opencv && mkdir build && cd build && cmake -D BUILD_PERF_TESTS=OFF -D BUILD_TESTS=OFF .. && make ${BUILD_THREADS} && make install && \
-    cd / && tar -xvzf zlib-1.2.13.tar.gz && cd /zlib-1.2.13 && ./configure && make ${BUILD_THREADS} && make install && \
-    cd / && tar -xvf /1.3.1.tar.gz && cd TileDB-1.3.1 && mkdir build && cd build && \
+    cd /dependencies/opencv && mkdir build && cd build && cmake -D BUILD_PERF_TESTS=OFF -D BUILD_TESTS=OFF .. && make ${BUILD_THREADS} && make install && \
+    cd /dependencies/ && tar -xvzf zlib-1.2.13.tar.gz && cd zlib-1.2.13 && ./configure && make ${BUILD_THREADS} && make install && \
+    cd /dependencies/ && tar -xvf 1.3.1.tar.gz && cd TileDB-1.3.1 && mkdir build && cd build && \
         ../bootstrap --prefix=/usr/local/ && make ${BUILD_THREADS} && make install-tiledb && \
     cd /usr/src/gtest && cmake . && make ${BUILD_THREADS} && mv lib/libgtest* /usr/lib/ && \
-    ln -s /grpc/third_party/protobuf/cmake/build/protoc /grpc/third_party/protobuf/src/protoc && \
-    cd /grpc/third_party/protobuf/java/core && mvn package && \
-        cp "$(ls target/protobuf-java*.jar)" /usr/share/java/protobuf.jar && \
-    cd /valijson && cp -r include/* /usr/local/include/ && \
-    rm -rf /CMake /swig /faiss /FLINNG /grpc /opencv /zlib-1.2.13.tar /zlib-1.2.13 /1.3.1.tar.gz /TileDB-1.3.1 /valijson
+    cd /dependencies/valijson && cp -r include/* /usr/local/include/ && \
+    rm -rf /dependencies
+
 
 # VDMS
 RUN git clone https://github.com/IntelLabs/vdms.git && cd vdms && \