diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 58356ecc..6ff3aa13 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -73,6 +73,9 @@ jobs: docker stop $(docker ps -aqf "name=${{ matrix.container_name }}") || true docker rm $(docker ps -aqf "name=${{ matrix.container_name }}") || true + # docker build -f docker/check-in/Dockerfile -t ${{ matrix.container_tag }} . + + # REMOVE MAVEN AFTER MERGE docker build --build-arg MAVEN_OPTS=${{ secrets.MAVEN_OPTS }} \ -f docker/check-in/Dockerfile -t ${{ matrix.container_tag }} . @@ -86,25 +89,25 @@ jobs: echo "${{ matrix.container_name }}" # Make sure CPP coverage script is available - if [ ! -f docker/check-in/run_coverage_cpp.sh ]; then - docker exec ${{ matrix.container_name }} bash -c "touch /run_coverage_cpp.sh && echo 'cd /vdms/tests && chmod +x run_tests.sh && ./run_tests.sh' >> /run_coverage_cpp.sh && chmod +x /run_coverage_cpp.sh && mkdir -p /vdms/tests/coverage_report" - docker exec ${{ matrix.container_name }} bash -c "echo 'gcovr -k --root /vdms -e /vdms/src/pmgd -e /vdms/build/CMakeFiles -f "/vdms/client/.*\.cc" -f "/vdms/ext/.*\.cc" -f "/vdms/src/.*\.cc" -f src/SearchExpression.cc --exclude-unreachable-branches --xml-pretty --xml=/vdms/tests/coverage_report/c_coverage_report.xml --txt=/vdms/tests/coverage_report/c_coverage_report.txt' >> /run_coverage_cpp.sh && echo "echo 'DONE'" >> /run_coverage_cpp.sh" - docker exec ${{ matrix.container_name }} bash -c "echo 'cat /vdms/tests/coverage_report/c_coverage_report.txt' >> /run_coverage_cpp.sh" - fi - - # Make sure Python coverage script is available - if [ ! -f docker/check-in/run_coverage_py.sh ]; then - docker exec ${{ matrix.container_name }} bash -c "touch /run_coverage_py.sh && echo 'cd /vdms/tests/python && ./run_python_tests.sh' >> /run_coverage_py.sh && chmod +x /run_coverage_py.sh && mkdir -p /vdms/tests/coverage_report" - docker exec ${{ matrix.container_name }} bash -c "echo 'python -m coverage report > /vdms/tests/coverage_report/py_coverage_report.txt' >> /run_coverage_py.sh && echo "echo 'DONE'" >> /run_coverage_py.sh" - docker exec ${{ matrix.container_name }} bash -c "echo 'cat /vdms/tests/coverage_report/py_coverage_report.txt' >> /run_coverage_py.sh" - fi - - docker exec ${{ matrix.container_name }} bash -c "./run_coverage_cpp.sh" + # if [ ! -f docker/check-in/run_coverage_cpp.sh ]; then + # docker exec ${{ matrix.container_name }} bash -c "touch /run_coverage_cpp.sh && echo 'cd /vdms/tests && chmod +x run_tests.sh && ./run_tests.sh' >> /run_coverage_cpp.sh && chmod +x /run_coverage_cpp.sh && mkdir -p /vdms/tests/coverage_report" + # docker exec ${{ matrix.container_name }} bash -c "echo 'gcovr -k --root /vdms -e /vdms/src/pmgd -e /vdms/build/CMakeFiles -f "/vdms/client/.*\.cc" -f "/vdms/ext/.*\.cc" -f "/vdms/src/.*\.cc" -f src/SearchExpression.cc --exclude-unreachable-branches --xml-pretty --xml=/vdms/tests/coverage_report/c_coverage_report.xml --txt=/vdms/tests/coverage_report/c_coverage_report.txt' >> /run_coverage_cpp.sh && echo "echo 'DONE'" >> /run_coverage_cpp.sh" + # docker exec ${{ matrix.container_name }} bash -c "echo 'cat /vdms/tests/coverage_report/c_coverage_report.txt' >> /run_coverage_cpp.sh" + # fi + + # # Make sure Python coverage script is available + # if [ ! -f docker/check-in/run_coverage_py.sh ]; then + # docker exec ${{ matrix.container_name }} bash -c "touch /run_coverage_py.sh && echo 'cd /vdms/tests/python && ./run_python_tests.sh' >> /run_coverage_py.sh && chmod +x /run_coverage_py.sh && mkdir -p /vdms/tests/coverage_report" + # docker exec ${{ matrix.container_name }} bash -c "echo 'python -m coverage report > /vdms/tests/coverage_report/py_coverage_report.txt' >> /run_coverage_py.sh && echo "echo 'DONE'" >> /run_coverage_py.sh" + # docker exec ${{ matrix.container_name }} bash -c "echo 'cat /vdms/tests/coverage_report/py_coverage_report.txt' >> /run_coverage_py.sh" + # fi + + docker exec ${{ matrix.container_name }} bash -c "cd / && ./run_coverage_cpp.sh" docker cp $(docker ps -a | grep ${{ matrix.container_name }} | awk '{print $1}'):/vdms/tests/coverage_report/c_coverage_report.txt coverage/c_coverage_report_target.txt docker cp $(docker ps -a | grep ${{ matrix.container_name }} | awk '{print $1}'):/vdms/tests/coverage_report/c_coverage_report.xml coverage/c_coverage_report_target.xml echo "coverage_value_cpp=$(cat coverage/c_coverage_report_target.xml | grep -oP 'coverage line-rate="([-+]?\d*\.\d+|\d+)"' | grep -oP "[-+]?\d*\.\d+|\d+" | awk '{print $1*100}')" >> $GITHUB_ENV - docker exec ${{ matrix.container_name }} bash -c "./run_coverage_py.sh" + docker exec ${{ matrix.container_name }} bash -c "cd / && ./run_coverage_py.sh" docker cp $(docker ps -a | grep ${{ matrix.container_name }} | awk '{print $1}'):/vdms/tests/coverage_report/py_coverage_report.txt coverage/py_coverage_report_target.txt || true docker cp $(docker ps -a | grep ${{ matrix.container_name }} | awk '{print $1}'):/vdms/tests/coverage_report/py_coverage_report.xml coverage/py_coverage_report_target.xml || true if [ ! -f coverage/py_coverage_report_target.xml ] && [ "${{ matrix.coverage_type }}" == "Target" ]; then @@ -126,7 +129,7 @@ jobs: if [[ -z $coverage_value_cpp ]] then exit 1 - fi + fi echo "${{ matrix.coverage_type }} CPP Coverage: ${coverage_value_cpp}" echo "${{ matrix.output_cpp_name }}=${coverage_value_cpp}" >> $GITHUB_OUTPUT diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile index 5016b6d6..81b3a674 100644 --- a/docker/base/Dockerfile +++ b/docker/base/Dockerfile @@ -4,7 +4,6 @@ ARG UBUNTU_VERSION=20.04 ARG UBUNTU_NAME=focal ARG BUILD_THREADS=-j16 -ARG MAVEN_OPTS='-Dhttps.nonProxyHosts="localhost|127.0.0.1"' #1 FROM ubuntu:${UBUNTU_VERSION} @@ -13,25 +12,25 @@ FROM ubuntu:${UBUNTU_VERSION} ARG UBUNTU_VERSION ARG UBUNTU_NAME ARG MAVEN_OPTS -WORKDIR / # Install Packages -RUN apt-get update && apt-get -y install --no-install-recommends software-properties-common && \ +RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common && \ add-apt-repository "deb http://security.ubuntu.com/ubuntu ${UBUNTU_NAME}-security main" && \ - apt-get -y install --no-install-recommends apt-transport-https autoconf automake bison build-essential \ + apt-get install -y --no-install-recommends apt-transport-https autoconf automake bison build-essential \ bzip2 ca-certificates curl ed flex g++ git gnupg-agent javacc libarchive-tools \ libatlas-base-dev libavcodec-dev libavformat-dev libboost-all-dev libbz2-dev \ libc-ares-dev libdc1394-22-dev libgflags-dev libgoogle-glog-dev libgtest-dev \ libgtk-3-dev libgtk2.0-dev libhdf5-serial-dev libjpeg-dev libjpeg8-dev libjsoncpp-dev \ libleveldb-dev liblmdb-dev liblz4-dev libopenblas-dev libopenmpi-dev \ libpng-dev librdkafka-dev libsnappy-dev libssl-dev libswscale-dev libtbb-dev \ - libtbb2 libtiff-dev libtiff5-dev libtool maven mpich openjdk-11-jdk-headless \ + libtbb2 libtiff-dev libtiff5-dev libtool mpich openjdk-11-jdk-headless \ pkg-config python3-dev python3-pip unzip && \ apt-get clean && rm -rf /var/lib/apt/lists/* && \ update-alternatives --install /usr/bin/python python /usr/bin/python3 1 && \ pip3 install --no-cache-dir "numpy>=1.23.2" # Pull and Install Dependencies +WORKDIR /dependencies RUN git clone --branch v3.21.2 https://github.com/Kitware/CMake.git && \ git clone --branch v4.0.2 https://github.com/swig/swig.git && \ git clone --branch v1.7.1 https://github.com/facebookresearch/faiss.git && \ @@ -40,32 +39,30 @@ RUN git clone --branch v3.21.2 https://github.com/Kitware/CMake.git && \ git clone --branch 4.5.3 https://github.com/opencv/opencv.git && \ git clone --branch v0.6 https://github.com/tristanpenman/valijson.git && \ curl -L -o /usr/share/java/json-simple-1.1.1.jar https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/json-simple/json-simple-1.1.1.jar && \ - curl -L -o /1.3.1.tar.gz https://github.com/TileDB-Inc/TileDB/archive/refs/tags/1.3.1.tar.gz && \ - curl -L -o /zlib-1.2.13.tar.gz http://zlib.net/zlib-1.2.13.tar.gz && \ - cd /CMake && ./bootstrap && make ${BUILD_THREADS} && make install && \ - cd /swig && ./autogen.sh && ./configure && make ${BUILD_THREADS} && make install && \ - cd /faiss && mkdir build && cd build && cmake -DFAISS_ENABLE_GPU=OFF .. && make ${BUILD_THREADS} && make install && \ - cd /FLINNG && mkdir build && cd build && cmake .. && make ${BUILD_THREADS} && make install && \ - cd /grpc && git submodule update --init --recursive && pip3 install --no-cache-dir -r requirements.txt && GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip3 install --no-cache-dir . && \ + curl -L -o /dependencies/1.3.1.tar.gz https://github.com/TileDB-Inc/TileDB/archive/refs/tags/1.3.1.tar.gz && \ + curl -L -o /dependencies/zlib-1.2.13.tar.gz http://zlib.net/zlib-1.2.13.tar.gz && \ + cd /dependencies/CMake && ./bootstrap && make ${BUILD_THREADS} && make install && \ + cd /dependencies/swig && ./autogen.sh && ./configure && make ${BUILD_THREADS} && make install && \ + cd /dependencies/faiss && mkdir build && cd build && cmake -DFAISS_ENABLE_GPU=OFF .. && make ${BUILD_THREADS} && make install && \ + cd /dependencies/FLINNG && mkdir build && cd build && cmake .. && make ${BUILD_THREADS} && make install && \ + cd /dependencies/grpc && git submodule update --init --recursive && pip3 install --no-cache-dir -r requirements.txt && GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip3 install --no-cache-dir . && \ cd tools/distrib/python/grpcio_tools && python ../make_grpcio_tools.py && GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip3 install --no-cache-dir . && \ - cd /grpc/third_party/protobuf/cmake && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \ + cd /dependencies/grpc/third_party/protobuf/cmake && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \ cd ../../../abseil-cpp && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \ cd ../../re2/ && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \ cd ../../zlib/ && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \ - cd /grpc/cmake && mkdir build && cd build && cmake -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_ABSL_PROVIDER=package \ + cd /dependencies/grpc/cmake && mkdir build && cd build && cmake -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_ABSL_PROVIDER=package \ -DgRPC_CARES_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package \ -DgRPC_RE2_PROVIDER=package -DgRPC_SSL_PROVIDER=package \ -DgRPC_ZLIB_PROVIDER=package -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE ../.. && make ${BUILD_THREADS} && make install && \ - cd /opencv && mkdir build && cd build && cmake -D BUILD_PERF_TESTS=OFF -D BUILD_TESTS=OFF .. && make ${BUILD_THREADS} && make install && \ - cd / && tar -xvzf zlib-1.2.13.tar.gz && cd /zlib-1.2.13 && ./configure && make ${BUILD_THREADS} && make install && \ - cd / && tar -xvf /1.3.1.tar.gz && cd TileDB-1.3.1 && mkdir build && cd build && \ + cd /dependencies/opencv && mkdir build && cd build && cmake -D BUILD_PERF_TESTS=OFF -D BUILD_TESTS=OFF .. && make ${BUILD_THREADS} && make install && \ + cd /dependencies/ && tar -xvzf zlib-1.2.13.tar.gz && cd zlib-1.2.13 && ./configure && make ${BUILD_THREADS} && make install && \ + cd /dependencies/ && tar -xvf 1.3.1.tar.gz && cd TileDB-1.3.1 && mkdir build && cd build && \ ../bootstrap --prefix=/usr/local/ && make ${BUILD_THREADS} && make install-tiledb && \ cd /usr/src/gtest && cmake . && make ${BUILD_THREADS} && mv lib/libgtest* /usr/lib/ && \ - ln -s /grpc/third_party/protobuf/cmake/build/protoc /grpc/third_party/protobuf/src/protoc && \ - cd /grpc/third_party/protobuf/java/core && mvn package && \ - cp "$(ls target/protobuf-java*.jar)" /usr/share/java/protobuf.jar && \ - cd /valijson && cp -r include/* /usr/local/include/ && \ - rm -rf /CMake /swig /faiss /FLINNG /grpc /opencv /zlib-1.2.13.tar /zlib-1.2.13 /1.3.1.tar.gz /TileDB-1.3.1 /valijson + cd /dependencies/valijson && cp -r include/* /usr/local/include/ && \ + rm -rf /dependencies + # VDMS RUN git clone https://github.com/IntelLabs/vdms.git && cd vdms && \ diff --git a/docker/check-in/Dockerfile b/docker/check-in/Dockerfile index c6e6cd1c..e24bcaf8 100644 --- a/docker/check-in/Dockerfile +++ b/docker/check-in/Dockerfile @@ -4,7 +4,6 @@ ARG UBUNTU_VERSION=20.04 ARG UBUNTU_NAME=focal ARG BUILD_THREADS=-j16 -ARG MAVEN_OPTS='-Dhttps.nonProxyHosts="localhost|127.0.0.1"' #1 FROM ubuntu:${UBUNTU_VERSION} @@ -13,25 +12,25 @@ FROM ubuntu:${UBUNTU_VERSION} ARG UBUNTU_VERSION ARG UBUNTU_NAME ARG MAVEN_OPTS -WORKDIR / # Install Packages -RUN apt-get update && apt-get -y install --no-install-recommends software-properties-common && \ +RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common && \ add-apt-repository "deb http://security.ubuntu.com/ubuntu ${UBUNTU_NAME}-security main" && \ - apt-get -y install --no-install-recommends apt-transport-https autoconf automake bison build-essential \ + apt-get install -y --no-install-recommends apt-transport-https autoconf automake bison build-essential \ bzip2 ca-certificates curl ed flex g++ git gnupg-agent javacc libarchive-tools \ libatlas-base-dev libavcodec-dev libavformat-dev libboost-all-dev libbz2-dev \ libc-ares-dev libdc1394-22-dev libgflags-dev libgoogle-glog-dev libgtest-dev \ libgtk-3-dev libgtk2.0-dev libhdf5-serial-dev libjpeg-dev libjpeg8-dev libjsoncpp-dev \ libleveldb-dev liblmdb-dev liblz4-dev libopenblas-dev libopenmpi-dev \ libpng-dev librdkafka-dev libsnappy-dev libssl-dev libswscale-dev libtbb-dev \ - libtbb2 libtiff-dev libtiff5-dev libtool maven mpich openjdk-11-jdk-headless \ + libtbb2 libtiff-dev libtiff5-dev libtool mpich openjdk-11-jdk-headless \ pkg-config python3-dev python3-pip unzip lcov && \ apt-get clean && rm -rf /var/lib/apt/lists/* && \ update-alternatives --install /usr/bin/python python /usr/bin/python3 1 && \ pip3 install --no-cache-dir "numpy>=1.23.2" gcovr # Pull and Install Dependencies +WORKDIR /dependencies RUN git clone --branch v3.21.2 https://github.com/Kitware/CMake.git && \ git clone --branch v4.0.2 https://github.com/swig/swig.git && \ git clone --branch v1.7.1 https://github.com/facebookresearch/faiss.git && \ @@ -40,39 +39,47 @@ RUN git clone --branch v3.21.2 https://github.com/Kitware/CMake.git && \ git clone --branch 4.5.3 https://github.com/opencv/opencv.git && \ git clone --branch v0.6 https://github.com/tristanpenman/valijson.git && \ curl -L -o /usr/share/java/json-simple-1.1.1.jar https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/json-simple/json-simple-1.1.1.jar && \ - curl -L -o /1.3.1.tar.gz https://github.com/TileDB-Inc/TileDB/archive/refs/tags/1.3.1.tar.gz && \ - curl -L -o /zlib-1.2.13.tar.gz http://zlib.net/zlib-1.2.13.tar.gz && \ - cd /CMake && ./bootstrap && make ${BUILD_THREADS} && make install && \ - cd /swig && ./autogen.sh && ./configure && make ${BUILD_THREADS} && make install && \ - cd /faiss && mkdir build && cd build && cmake -DFAISS_ENABLE_GPU=OFF .. && make ${BUILD_THREADS} && make install && \ - cd /FLINNG && mkdir build && cd build && cmake .. && make ${BUILD_THREADS} && make install && \ - cd /grpc && git submodule update --init --recursive && pip3 install --no-cache-dir -r requirements.txt && GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip3 install --no-cache-dir . && \ + curl -L -o /dependencies/1.3.1.tar.gz https://github.com/TileDB-Inc/TileDB/archive/refs/tags/1.3.1.tar.gz && \ + curl -L -o /dependencies/zlib-1.2.13.tar.gz http://zlib.net/zlib-1.2.13.tar.gz && \ + cd /dependencies/CMake && ./bootstrap && make ${BUILD_THREADS} && make install && \ + cd /dependencies/swig && ./autogen.sh && ./configure && make ${BUILD_THREADS} && make install && \ + cd /dependencies/faiss && mkdir build && cd build && cmake -DFAISS_ENABLE_GPU=OFF .. && make ${BUILD_THREADS} && make install && \ + cd /dependencies/FLINNG && mkdir build && cd build && cmake .. && make ${BUILD_THREADS} && make install && \ + cd /dependencies/grpc && git submodule update --init --recursive && pip3 install --no-cache-dir -r requirements.txt && GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip3 install --no-cache-dir . && \ cd tools/distrib/python/grpcio_tools && python ../make_grpcio_tools.py && GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip3 install --no-cache-dir . && \ - cd /grpc/third_party/protobuf/cmake && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \ + cd /dependencies/grpc/third_party/protobuf/cmake && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \ cd ../../../abseil-cpp && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \ cd ../../re2/ && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \ cd ../../zlib/ && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \ - cd /grpc/cmake && mkdir build && cd build && cmake -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_ABSL_PROVIDER=package \ + cd /dependencies/grpc/cmake && mkdir build && cd build && cmake -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_ABSL_PROVIDER=package \ -DgRPC_CARES_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package \ -DgRPC_RE2_PROVIDER=package -DgRPC_SSL_PROVIDER=package \ -DgRPC_ZLIB_PROVIDER=package -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE ../.. && make ${BUILD_THREADS} && make install && \ - cd /opencv && mkdir build && cd build && cmake -D BUILD_PERF_TESTS=OFF -D BUILD_TESTS=OFF .. && make ${BUILD_THREADS} && make install && \ - cd / && tar -xvzf zlib-1.2.13.tar.gz && cd /zlib-1.2.13 && ./configure && make ${BUILD_THREADS} && make install && \ - cd / && tar -xvf /1.3.1.tar.gz && cd TileDB-1.3.1 && mkdir build && cd build && \ + cd /dependencies/opencv && mkdir build && cd build && cmake -D BUILD_PERF_TESTS=OFF -D BUILD_TESTS=OFF .. && make ${BUILD_THREADS} && make install && \ + cd /dependencies/ && tar -xvzf zlib-1.2.13.tar.gz && cd zlib-1.2.13 && ./configure && make ${BUILD_THREADS} && make install && \ + cd /dependencies/ && tar -xvf 1.3.1.tar.gz && cd TileDB-1.3.1 && mkdir build && cd build && \ ../bootstrap --prefix=/usr/local/ && make ${BUILD_THREADS} && make install-tiledb && \ cd /usr/src/gtest && cmake . && make ${BUILD_THREADS} && mv lib/libgtest* /usr/lib/ && \ - ln -s /grpc/third_party/protobuf/cmake/build/protoc /grpc/third_party/protobuf/src/protoc && \ - cd /grpc/third_party/protobuf/java/core && mvn package && \ - cp "$(ls target/protobuf-java*.jar)" /usr/share/java/protobuf.jar && \ - cd /valijson && cp -r include/* /usr/local/include/ && \ - rm -rf /CMake /swig /faiss /FLINNG /grpc /opencv /zlib-1.2.13.tar /zlib-1.2.13 /1.3.1.tar.gz /TileDB-1.3.1 /valijson + cd /dependencies/valijson && cp -r include/* /usr/local/include/ && \ + rm -rf /dependencies + # VDMS -COPY . /vdms -COPY docker/check-in/run_coverage_cpp.sh /run_coverage_cpp.sh -COPY docker/check-in/run_coverage_py.sh /run_coverage_py.sh -RUN [ -d /vdms/build ]; rm -rf /vdms/build -RUN cd /vdms && [ -d /vdms/.git ]; git submodule update --init --recursive && mkdir build && \ +COPY ./.git /vdms/.git +COPY ./client /vdms/client +COPY ./distributed /vdms/distributed +COPY ./ext /vdms/ext +COPY ./include /vdms/include +COPY ./src /vdms/src +COPY ./tests /vdms/tests +COPY ./utils /vdms/utils +COPY ./CMakeLists.txt /vdms/ +COPY ./config-vdms.json /vdms/ +COPY ./docker/check-in/run_coverage_cpp.sh / +COPY ./docker/check-in/run_coverage_py.sh / +WORKDIR /vdms + +RUN cd /vdms && git submodule update --init --recursive && mkdir build && \ cd build && cmake -DCODE_COVERAGE=ON .. && make ${BUILD_THREADS} && \ cp /vdms/config-vdms.json /vdms/build/ && \ mkdir -p /vdms/tests/coverage_report && \ diff --git a/docker/check-in/Dockerfile.base b/docker/check-in/Dockerfile.base index d5ad2469..0f0c23ea 100644 --- a/docker/check-in/Dockerfile.base +++ b/docker/check-in/Dockerfile.base @@ -4,7 +4,6 @@ ARG UBUNTU_VERSION=20.04 ARG UBUNTU_NAME=focal ARG BUILD_THREADS=-j16 -ARG MAVEN_OPTS='-Dhttps.nonProxyHosts="localhost|127.0.0.1"' #1 FROM ubuntu:${UBUNTU_VERSION} @@ -13,25 +12,25 @@ FROM ubuntu:${UBUNTU_VERSION} ARG UBUNTU_VERSION ARG UBUNTU_NAME ARG MAVEN_OPTS -WORKDIR / # Install Packages -RUN apt-get update && apt-get -y install --no-install-recommends software-properties-common && \ +RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common && \ add-apt-repository "deb http://security.ubuntu.com/ubuntu ${UBUNTU_NAME}-security main" && \ - apt-get -y install --no-install-recommends apt-transport-https autoconf automake bison build-essential \ + apt-get install -y --no-install-recommends apt-transport-https autoconf automake bison build-essential \ bzip2 ca-certificates curl ed flex g++ git gnupg-agent javacc libarchive-tools \ libatlas-base-dev libavcodec-dev libavformat-dev libboost-all-dev libbz2-dev \ libc-ares-dev libdc1394-22-dev libgflags-dev libgoogle-glog-dev libgtest-dev \ libgtk-3-dev libgtk2.0-dev libhdf5-serial-dev libjpeg-dev libjpeg8-dev libjsoncpp-dev \ libleveldb-dev liblmdb-dev liblz4-dev libopenblas-dev libopenmpi-dev \ libpng-dev librdkafka-dev libsnappy-dev libssl-dev libswscale-dev libtbb-dev \ - libtbb2 libtiff-dev libtiff5-dev libtool maven mpich openjdk-11-jdk-headless \ + libtbb2 libtiff-dev libtiff5-dev libtool mpich openjdk-11-jdk-headless \ pkg-config python3-dev python3-pip unzip && \ apt-get clean && rm -rf /var/lib/apt/lists/* && \ update-alternatives --install /usr/bin/python python /usr/bin/python3 1 && \ pip3 install --no-cache-dir "numpy>=1.23.2" # Pull and Install Dependencies +WORKDIR /dependencies RUN git clone --branch v3.21.2 https://github.com/Kitware/CMake.git && \ git clone --branch v4.0.2 https://github.com/swig/swig.git && \ git clone --branch v1.7.1 https://github.com/facebookresearch/faiss.git && \ @@ -40,37 +39,45 @@ RUN git clone --branch v3.21.2 https://github.com/Kitware/CMake.git && \ git clone --branch 4.5.3 https://github.com/opencv/opencv.git && \ git clone --branch v0.6 https://github.com/tristanpenman/valijson.git && \ curl -L -o /usr/share/java/json-simple-1.1.1.jar https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/json-simple/json-simple-1.1.1.jar && \ - curl -L -o /1.3.1.tar.gz https://github.com/TileDB-Inc/TileDB/archive/refs/tags/1.3.1.tar.gz && \ - curl -L -o /zlib-1.2.13.tar.gz http://zlib.net/zlib-1.2.13.tar.gz && \ - cd /CMake && ./bootstrap && make ${BUILD_THREADS} && make install && \ - cd /swig && ./autogen.sh && ./configure && make ${BUILD_THREADS} && make install && \ - cd /faiss && mkdir build && cd build && cmake -DFAISS_ENABLE_GPU=OFF .. && make ${BUILD_THREADS} && make install && \ - cd /FLINNG && mkdir build && cd build && cmake .. && make ${BUILD_THREADS} && make install && \ - cd /grpc && git submodule update --init --recursive && pip3 install --no-cache-dir -r requirements.txt && GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip3 install --no-cache-dir . && \ + curl -L -o /dependencies/1.3.1.tar.gz https://github.com/TileDB-Inc/TileDB/archive/refs/tags/1.3.1.tar.gz && \ + curl -L -o /dependencies/zlib-1.2.13.tar.gz http://zlib.net/zlib-1.2.13.tar.gz && \ + cd /dependencies/CMake && ./bootstrap && make ${BUILD_THREADS} && make install && \ + cd /dependencies/swig && ./autogen.sh && ./configure && make ${BUILD_THREADS} && make install && \ + cd /dependencies/faiss && mkdir build && cd build && cmake -DFAISS_ENABLE_GPU=OFF .. && make ${BUILD_THREADS} && make install && \ + cd /dependencies/FLINNG && mkdir build && cd build && cmake .. && make ${BUILD_THREADS} && make install && \ + cd /dependencies/grpc && git submodule update --init --recursive && pip3 install --no-cache-dir -r requirements.txt && GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip3 install --no-cache-dir . && \ cd tools/distrib/python/grpcio_tools && python ../make_grpcio_tools.py && GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip3 install --no-cache-dir . && \ - cd /grpc/third_party/protobuf/cmake && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \ + cd /dependencies/grpc/third_party/protobuf/cmake && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \ cd ../../../abseil-cpp && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \ cd ../../re2/ && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \ cd ../../zlib/ && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \ - cd /grpc/cmake && mkdir build && cd build && cmake -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_ABSL_PROVIDER=package \ + cd /dependencies/grpc/cmake && mkdir build && cd build && cmake -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_ABSL_PROVIDER=package \ -DgRPC_CARES_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package \ -DgRPC_RE2_PROVIDER=package -DgRPC_SSL_PROVIDER=package \ -DgRPC_ZLIB_PROVIDER=package -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE ../.. && make ${BUILD_THREADS} && make install && \ - cd /opencv && mkdir build && cd build && cmake -D BUILD_PERF_TESTS=OFF -D BUILD_TESTS=OFF .. && make ${BUILD_THREADS} && make install && \ - cd / && tar -xvzf zlib-1.2.13.tar.gz && cd /zlib-1.2.13 && ./configure && make ${BUILD_THREADS} && make install && \ - cd / && tar -xvf /1.3.1.tar.gz && cd TileDB-1.3.1 && mkdir build && cd build && \ + cd /dependencies/opencv && mkdir build && cd build && cmake -D BUILD_PERF_TESTS=OFF -D BUILD_TESTS=OFF .. && make ${BUILD_THREADS} && make install && \ + cd /dependencies/ && tar -xvzf zlib-1.2.13.tar.gz && cd zlib-1.2.13 && ./configure && make ${BUILD_THREADS} && make install && \ + cd /dependencies/ && tar -xvf 1.3.1.tar.gz && cd TileDB-1.3.1 && mkdir build && cd build && \ ../bootstrap --prefix=/usr/local/ && make ${BUILD_THREADS} && make install-tiledb && \ cd /usr/src/gtest && cmake . && make ${BUILD_THREADS} && mv lib/libgtest* /usr/lib/ && \ - ln -s /grpc/third_party/protobuf/cmake/build/protoc /grpc/third_party/protobuf/src/protoc && \ - cd /grpc/third_party/protobuf/java/core && mvn package && \ - cp "$(ls target/protobuf-java*.jar)" /usr/share/java/protobuf.jar && \ - cd /valijson && cp -r include/* /usr/local/include/ && \ - rm -rf /CMake /swig /faiss /FLINNG /grpc /opencv /zlib-1.2.13.tar /zlib-1.2.13 /1.3.1.tar.gz /TileDB-1.3.1 /valijson + cd /dependencies/valijson && cp -r include/* /usr/local/include/ && \ + rm -rf /dependencies + # VDMS -COPY . /vdms -RUN [ -d /vdms/build ]; rm -rf /vdms/build && \ - cd /vdms && git submodule update --init --recursive && mkdir build && \ +COPY ./.git /vdms/.git +COPY ./client /vdms/client +COPY ./distributed /vdms/distributed +COPY ./ext /vdms/ext +COPY ./include /vdms/include +COPY ./src /vdms/src +COPY ./tests /vdms/tests +COPY ./utils /vdms/utils +COPY ./CMakeLists.txt /vdms/ +COPY ./config-vdms.json /vdms/ +WORKDIR /vdms + +RUN cd /vdms && git submodule update --init --recursive && mkdir build && \ cd build && cmake .. && make ${BUILD_THREADS} && \ cp /vdms/config-vdms.json /vdms/build/ && \ echo '#!/bin/bash' > /start.sh && echo 'cd /vdms/build' >> /start.sh && \