From 45de5d9c3a614adb209d3b0d06b2a1ade038030c Mon Sep 17 00:00:00 2001 From: "Chaunte W. Lacewell" Date: Fri, 14 Jul 2023 12:25:29 -0700 Subject: [PATCH] Update dockerfiles to Debian base image (#120) --- .github/requirements.txt | 32 ++-- .github/workflows/auto-formatter.sh | 2 +- .github/workflows/pull_requests.yml | 29 ++-- .github/workflows/sdl_req.yml | 97 ++++++------ client/python/vdms/queryMessage_pb2.py | 124 ++++++--------- docker/base/Dockerfile | 116 +++++++------- docker/check-in/Dockerfile | 167 +++++++++++--------- docker/check-in/Dockerfile.base | 90 ----------- remote_function/requirements.txt | 2 +- tests/python/run_python_aws_tests.sh | 2 +- tests/python/run_python_tests.sh | 2 +- tests/remote_function_test/requirements.txt | 2 +- tests/run_aws_tests.sh | 2 +- tests/run_tests.sh | 2 +- tests/udf_test/requirements.txt | 2 +- user_defined_operations/requirements.txt | 2 +- 16 files changed, 287 insertions(+), 386 deletions(-) delete mode 100644 docker/check-in/Dockerfile.base diff --git a/.github/requirements.txt b/.github/requirements.txt index bbf10a7f..68847109 100644 --- a/.github/requirements.txt +++ b/.github/requirements.txt @@ -1,16 +1,24 @@ -certifi==2019.11.28 -chardet==3.0.4 -coverage==7.2.3 -Cython==0.29.34 +blinker==1.6.2 +click==8.1.5 dbus-python==1.2.16 +Flask==2.3.2 grpcio==1.40.0 grpcio-tools==1.40.0 -idna==2.8 -numpy==1.24.2 +importlib-metadata==6.8.0 +imutils==0.5.4 +itsdangerous==2.1.2 +Jinja2==3.1.2 +MarkupSafe==2.1.3 +numpy==1.25.1 +opencv-python==4.5.5.64 protobuf==3.20.3 -PyGObject==3.36.0 -python-apt==2.0.1+ubuntu0.20.4.1 -requests==2.22.0 -requests-unixsocket==0.2.0 -six==1.14.0 -urllib3==1.25.8 +pycurl==7.43.0.6 +PyGObject==3.38.0 +python-apt==2.2.1 +pyzmq==25.1.0 +scipy==1.11.1 +six==1.16.0 +sk-video==1.1.10 +Werkzeug==2.3.6 +zipp==3.16.1 +zmq==0.0.0 diff --git a/.github/workflows/auto-formatter.sh b/.github/workflows/auto-formatter.sh index 16258b17..d425ff46 100755 --- a/.github/workflows/auto-formatter.sh +++ b/.github/workflows/auto-formatter.sh @@ -35,4 +35,4 @@ find "${REPO_DIR}" -type f -not -path "${REPO_DIR}/src/pmgd/*" \ # Run Linter on Python Code check_package python 'black>=23.1.0' -black ${REPO_DIR}/ +black ${REPO_DIR}/ --exclude="client/python/vdms/queryMessage_pb2.py" diff --git a/.github/workflows/pull_requests.yml b/.github/workflows/pull_requests.yml index 2003940e..4cc72173 100644 --- a/.github/workflows/pull_requests.yml +++ b/.github/workflows/pull_requests.yml @@ -64,12 +64,8 @@ jobs: fetch-depth: 0 - if: matrix.coverage_type == 'Source' - name: Format C++ Code (clang-format) - run: find "${PWD}" -type f -not -path "${PWD}/src/pmgd/*" -not -path "${PWD}/build/*" -regex '.*\.\(cc\|cpp\|h\|hpp\)' | xargs clang-format -i || true - - - if: matrix.coverage_type == 'Source' - name: Format Python Code (black code) - uses: DataDog/action-py-black-formatter@v2.5 + name: Format C++ Code (clang-format) and Python (black code) + run: ./.github/workflows/auto-formatter.sh - if: matrix.coverage_type == 'Source' name: Check for modified files @@ -79,15 +75,13 @@ jobs: echo "modify_source=$(if git diff-index --quiet HEAD --; then echo "false"; else echo "true"; fi)" >> $GITHUB_OUTPUT echo "added_modified=$(git diff --name-only --diff-filter=ACMRT ${{ github.event.pull_request.base.sha }} HEAD -- . ':!.github' ':!docker'| xargs)" >> $GITHUB_OUTPUT - - name: Build and Run Docker Container + - name: Build Docker Container run: | set -x - docker stop $(docker ps -aqf "name=${{ matrix.container_name }}") || true - docker rm $(docker ps -aqf "name=${{ matrix.container_name }}") || true + docker stop $(docker ps -aqf "name=${{ matrix.container_name }}") | xargs docker rm || true - docker build --rm -f docker/check-in/Dockerfile -t ${{ matrix.container_tag }} . - # docker run --rm -d -v ${PWD}:/local_repo --name ${{ matrix.container_name }} ${{ matrix.container_tag }} + docker build --rm --build-arg="BUILD_COVERAGE=on" --build-arg="BUILD_COVERITY=on" -f docker/check-in/Dockerfile -t ${{ matrix.container_tag }} . - if: matrix.coverage_type == 'Source' && steps.git_check.outputs.added_modified uses: ./.github/actions/coverity-incremental-scan @@ -105,11 +99,10 @@ jobs: run: | set -x mkdir -p coverage - echo "${{ matrix.container_name }}" + docker run --rm -d -v ${PWD}:/local_repo --name ${{ matrix.container_name }} \ --env AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \ - --env AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \ - ${{ matrix.container_tag }} + --env AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} ${{ matrix.container_tag }} docker exec ${{ matrix.container_name }} bash -c "cd / && ./run_coverage_cpp.sh && cd / && ./run_coverage_py.sh" @@ -148,7 +141,8 @@ jobs: run: | rm -rf ${GITHUB_WORKSPACE}/.git* ${GITHUB_ACTION_REPOSITORY} || true rm -rf /tmp/tmp-* ${{ env.ARTIFACT_DIR }} ${GITHUB_WORKSPACE}/* || true - docker ps -aqf "name=${{ matrix.container_name }}" | xargs docker stop || true + + docker stop $(docker ps -aqf "name=${{ matrix.container_name }}") | xargs docker rm || true docker rmi $(docker images | grep '' | awk '{print $3}') || true compare_coverage: @@ -209,10 +203,7 @@ jobs: token: ${{ secrets.FACELESS_TOKEN || github.token }} - if: needs.coverage_job.outputs.modify_source == 'true' - run: find "${PWD}" -type f -not -path "${PWD}/src/pmgd/*" -not -path "${PWD}/build/*" -regex '.*\.\(cc\|cpp\|h\|hpp\)' | xargs clang-format -i || true - - - if: needs.coverage_job.outputs.modify_source == 'true' - uses: DataDog/action-py-black-formatter@v2.5 + run: ./.github/workflows/auto-formatter.sh # Update Code and Push (Should be last steps of workflow since it changes commit) - if: needs.coverage_job.outputs.modify_source == 'true' diff --git a/.github/workflows/sdl_req.yml b/.github/workflows/sdl_req.yml index c080ed4d..c126c26a 100644 --- a/.github/workflows/sdl_req.yml +++ b/.github/workflows/sdl_req.yml @@ -1,19 +1,19 @@ -# Uses docker/check-in/Dockerfile.base -# Dockerfile.base -> Same as docker/base/Dockerfile but builds VDMS with local changes instead of external repo +# Uses docker/check-in/Dockerfile without coverage or coverity +# Same as docker/base/Dockerfile but builds VDMS with local changes instead of external repo name: SDL Requirements using Docker Image -# Controls when the action will run. Triggers the workflow on push or pull request +# Controls when the action will run. Triggers the workflow on push or pull request (for testing) # events but only for the master and develop branch +on: + push: + branches: + - develop # on: # pull_request: # types: [ opened, edited, synchronize, reopened ] # branches: # - develop # - master -on: - push: - branches: - - develop concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -23,20 +23,42 @@ concurrency: env: ARTIFACT_DIR: SDL_artifacts DOCKER_ARTIFACT_DIR: Docker_artifacts - CHECKIN_DOCKERFILE: docker/check-in/Dockerfile.base - SNYK_TOKEN: ${{ secrets.SNYK_TOKEN}} - SNYK_API: ${{ secrets.SNYK_API}} + CHECKIN_DOCKERFILE: docker/check-in/Dockerfile # CHECKOUT_REF: ${{ github.event.pull_request.head.sha }} FACELESS_USERNAME: ${{ secrets.FACELESS_NAME}} - COVERITY_DOCKERFILE: docker/check-in/Dockerfile.coverity FACELESS_AUTHKEY: ${{ secrets.FACELESS_AUTHKEY}} COVERITYSTREAM: ${{ secrets.COVERITYSTREAM}} COVERITYSERVER: ${{ secrets.COVERITYSERVER }} jobs: + delete: + name: Remove old artifacts + runs-on: + group: intellabs-vdms-runners + labels: vdms-check-in + steps: + - uses: actions/github-script@v6 + id: artifact + with: + # Delete all artifacts + script: | + const res = await github.rest.actions.listArtifactsForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + }) + + res.data.artifacts + .forEach(({ id }) => { + github.rest.actions.deleteArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: id, + }) + }) + # RUN HADOLINT & BANDIT; NO DOCKER BUILD NEEDED - # Check format of Dockerfile we will release (docker/base/Dockerfile) Hadolint: + # Check format of Dockerfile we will release (docker/base/Dockerfile) name: Haskell Dockerfile Linter needs: delete runs-on: @@ -104,31 +126,8 @@ jobs: rm -rf ${GITHUB_WORKSPACE}/.git* ${GITHUB_ACTION_PATH} || true rm -rf /tmp/tmp-* ${GITHUB_WORKSPACE}/* || true - # BUILD LATEST CODE AS DOCKER IMAGE; USED WITH SNYK, CIS, & BDBA JOBS - delete: - name: Remove old artifacts - runs-on: - group: intellabs-vdms-runners - labels: vdms-check-in - steps: - - uses: actions/github-script@v6 - id: artifact - with: - # Delete all artifacts - script: | - const res = await github.rest.actions.listArtifactsForRepo({ - owner: context.repo.owner, - repo: context.repo.repo, - }) - - res.data.artifacts - .forEach(({ id }) => { - github.rest.actions.deleteArtifact({ - owner: context.repo.owner, - repo: context.repo.repo, - artifact_id: id, - }) - }) + # BUILD LATEST CODE AS DOCKER IMAGE + # USED WITH TRIVY, CIS, & BDBA JOBS BuildLatest: # This job builds docker container for later use name: Build Latest Docker @@ -145,7 +144,8 @@ jobs: - run: mkdir -p ${{ env.DOCKER_ARTIFACT_DIR }} - name: Build Docker Container run: | - docker build --rm -f ${{ env.CHECKIN_DOCKERFILE}} -t vdms:latest . + docker build --rm --build-arg="BUILD_COVERAGE=off" --build-arg="BUILD_COVERITY=off" \ + -f ${{ env.CHECKIN_DOCKERFILE}} -t vdms:latest . docker save -o ${{ env.DOCKER_ARTIFACT_DIR }}/vdms_latest.tar vdms:latest - name: Upload Docker Image Artifact if: success() @@ -168,8 +168,6 @@ jobs: labels: vdms-check-in name: BDBA needs: BuildLatest - # container: - # image: python:3.8-slim steps: - name: Download Docker Image uses: actions/download-artifact@v3 @@ -241,6 +239,7 @@ jobs: - name: Get Docker Image SBOM run: | docker sbom --format spdx-tag-value --output ${{ env.ARTIFACT_DIR }}/sbom_docker_CT36.txt vdms:latest + python3 ${GITHUB_WORKSPACE}/docker/check-in/spdx2csv.py -i ${{ env.ARTIFACT_DIR }}/sbom_docker_CT36.txt \ -o ${{ env.ARTIFACT_DIR }}/vdms_sbom_docker_CT36.csv @@ -262,7 +261,6 @@ jobs: - name: Cleanup if: always() run: | - docker stop snyk_py && docker rm snyk_py || true rm -rf ${GITHUB_WORKSPACE}/.git* ${GITHUB_ACTION_PATH} || true rm -rf /tmp/tmp-* ${{ env.DOCKER_ARTIFACT_DIR }} ${GITHUB_WORKSPACE}/* || true @@ -324,7 +322,7 @@ jobs: # cf. https://github.com/actions/upload-artifact/issues/256 if: always() run: | - docker stop vdms_test-CIS && docker rm vdms_test-CIS + docker stop vdms_test-CIS && docker rm vdms_test-CIS || true docker rmi $(docker images | grep '' | awk '{print $3}') || true rm -rf ${GITHUB_WORKSPACE}/.git* ${GITHUB_ACTION_PATH} || true rm -rf /tmp/tmp-* ${{ env.DOCKER_ARTIFACT_DIR }} ${GITHUB_WORKSPACE}/* || true @@ -343,16 +341,9 @@ jobs: # ref: ${{ env.CHECKOUT_REF }} - name: Build Docker Container with Coverity run: | - cp ${{ env.CHECKIN_DOCKERFILE}} ${{ env.COVERITY_DOCKERFILE}} - sed -i -e 's|CMD \["/start.sh"]|RUN mkdir /coverity \&\& cd /coverity \&\& \\|g' ${{ env.COVERITY_DOCKERFILE}} - echo " curl -L -o cov-analysis-linux64-2023.3.0.sh https://ubit-artifactory-or.intel.com/artifactory/coverity-or-local/Enterprise/cov-analysis-linux64-2023.3.0.sh && chmod +x cov-analysis-linux64-2023.3.0.sh && \\" >> ${{ env.COVERITY_DOCKERFILE}} - echo " curl -L -o license.dat https://ubit-artifactory-or.intel.com/artifactory/coverity-or-local/Enterprise/license.dat && \\" >> ${{ env.COVERITY_DOCKERFILE}} - echo " ./cov-analysis-linux64-2023.3.0.sh -q --installation.dir=/opt/coverity/analysis/ \\ - --license.agreement=agree --license.region=0 --license.type.choice=0 \\ - --license.cov.path=/coverity/license.dat --component.sdk=false --component.skip.documentation=true" >> ${{ env.COVERITY_DOCKERFILE}} - echo "ENV PATH /opt/coverity/analysis/bin:$PATH" >> ${{ env.COVERITY_DOCKERFILE}} - echo 'CMD ["/start.sh"]' >> ${{ env.COVERITY_DOCKERFILE}} - docker build --rm -f ${{ env.COVERITY_DOCKERFILE}} -t vdms:coverity . + docker build --rm --build-arg="BUILD_COVERAGE=off" --build-arg="BUILD_COVERITY=on" \ + -f ${{ env.CHECKIN_DOCKERFILE}} -t vdms:coverity . + - name: Run Coverity with GCC env: DOCKER_PROXY_RUN_ARGS: "--env HTTPS_PROXY=$HTTPS_PROXY \ @@ -384,7 +375,7 @@ jobs: # cf. https://github.com/actions/upload-artifact/issues/256 if: always() run: | - docker stop vdms_test-Coverity || true + docker stop vdms_test-Coverity && docker rm vdms_test-Coverity || true docker rmi $(docker images | grep '' | awk '{print $3}') || true rm -rf ${GITHUB_WORKSPACE}/.git* ${GITHUB_ACTION_PATH} || true rm -rf /tmp/tmp-* ${{ env.DOCKER_ARTIFACT_DIR }} ${GITHUB_WORKSPACE}/* || true diff --git a/client/python/vdms/queryMessage_pb2.py b/client/python/vdms/queryMessage_pb2.py index b5135380..79134502 100644 --- a/client/python/vdms/queryMessage_pb2.py +++ b/client/python/vdms/queryMessage_pb2.py @@ -6,93 +6,71 @@ from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database - # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() + + DESCRIPTOR = _descriptor.FileDescriptor( - name="queryMessage.proto", - package="VDMS.protobufs", - syntax="proto3", - serialized_options=None, - create_key=_descriptor._internal_create_key, - serialized_pb=b'\n\x12queryMessage.proto\x12\x0eVDMS.protobufs"+\n\x0cqueryMessage\x12\x0c\n\x04json\x18\x01 \x01(\t\x12\r\n\x05\x62lobs\x18\x02 \x03(\x0c\x62\x06proto3', + name='queryMessage.proto', + package='VDMS.protobufs', + syntax='proto3', + serialized_options=None, + create_key=_descriptor._internal_create_key, + serialized_pb=b'\n\x12queryMessage.proto\x12\x0eVDMS.protobufs\"+\n\x0cqueryMessage\x12\x0c\n\x04json\x18\x01 \x01(\t\x12\r\n\x05\x62lobs\x18\x02 \x03(\x0c\x62\x06proto3' ) + + _QUERYMESSAGE = _descriptor.Descriptor( - name="queryMessage", - full_name="VDMS.protobufs.queryMessage", - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name="json", - full_name="VDMS.protobufs.queryMessage.json", - index=0, - number=1, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=b"".decode("utf-8"), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - _descriptor.FieldDescriptor( - name="blobs", - full_name="VDMS.protobufs.queryMessage.blobs", - index=1, - number=2, - type=12, - cpp_type=9, - label=3, - has_default_value=False, - default_value=[], - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - ], - extensions=[], - nested_types=[], - enum_types=[], - serialized_options=None, - is_extendable=False, - syntax="proto3", - extension_ranges=[], - oneofs=[], - serialized_start=38, - serialized_end=81, + name='queryMessage', + full_name='VDMS.protobufs.queryMessage', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='json', full_name='VDMS.protobufs.queryMessage.json', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='blobs', full_name='VDMS.protobufs.queryMessage.blobs', index=1, + number=2, type=12, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=38, + serialized_end=81, ) -DESCRIPTOR.message_types_by_name["queryMessage"] = _QUERYMESSAGE +DESCRIPTOR.message_types_by_name['queryMessage'] = _QUERYMESSAGE _sym_db.RegisterFileDescriptor(DESCRIPTOR) -queryMessage = _reflection.GeneratedProtocolMessageType( - "queryMessage", - (_message.Message,), - { - "DESCRIPTOR": _QUERYMESSAGE, - "__module__": "queryMessage_pb2" - # @@protoc_insertion_point(class_scope:VDMS.protobufs.queryMessage) - }, -) +queryMessage = _reflection.GeneratedProtocolMessageType('queryMessage', (_message.Message,), { + 'DESCRIPTOR' : _QUERYMESSAGE, + '__module__' : 'queryMessage_pb2' + # @@protoc_insertion_point(class_scope:VDMS.protobufs.queryMessage) + }) _sym_db.RegisterMessage(queryMessage) diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile index 13f26ba7..c1ca2955 100644 --- a/docker/base/Dockerfile +++ b/docker/base/Dockerfile @@ -1,78 +1,76 @@ -#Copyright (C) 2021 Intel Corporation +#Copyright (C) 2023 Intel Corporation #SPDX-License-Identifier: MIT -ARG UBUNTU_VERSION=20.04 -ARG UBUNTU_NAME=focal -ARG BUILD_THREADS=-j16 +ARG BASE_VERSION=11.7-slim +ARG BUILD_THREADS="-j16" -#1 -FROM ubuntu:${UBUNTU_VERSION} +FROM debian:${BASE_VERSION} # Dockerfile limitations force a repetition of global args -ARG UBUNTU_VERSION -ARG UBUNTU_NAME +ARG BUILD_THREADS # Install Packages -RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common && \ - add-apt-repository "deb http://security.ubuntu.com/ubuntu ${UBUNTU_NAME}-security main" && \ - apt-get install -y --no-install-recommends apt-transport-https autoconf automake bison build-essential \ - bzip2 ca-certificates curl=7.68.0-1ubuntu2.18 ed flex g++ git gnupg-agent javacc libarchive-tools \ - libatlas-base-dev libavcodec-dev libavformat-dev libboost-all-dev libbz2-dev \ - libc-ares-dev libdc1394-22-dev libgflags-dev libgoogle-glog-dev libgtest-dev \ - libgtk-3-dev libgtk2.0-dev libhdf5-serial-dev libjpeg-dev libjpeg8-dev libjsoncpp-dev \ - libleveldb-dev liblmdb-dev liblz4-dev libopenblas-dev libopenmpi-dev \ - libpng-dev librdkafka-dev libsnappy-dev libssl-dev libswscale-dev libtbb-dev \ - libtbb2 libtiff-dev libtiff5-dev libtool mpich openjdk-11-jdk-headless \ - libcurl4-openssl-dev libzmq3-dev uuid-dev zlib1g-dev libpulse-dev \ - pkg-config python3-dev python3-pip unzip && \ +RUN apt-get update && apt-get install -y --no-install-suggests --no-install-recommends \ + apt-transport-https autoconf automake bison build-essential bzip2 ca-certificates \ + curl ed flex g++-9 gcc-9 git gnupg-agent javacc libarchive-tools libatlas-base-dev \ + libavcodec-dev libavformat-dev libboost-all-dev libbz2-dev libc-ares-dev libcurl4-openssl-dev \ + libdc1394-22-dev libgflags-dev libgoogle-glog-dev libgtest-dev libgtk-3-dev libgtk2.0-dev \ + libhdf5-dev libjpeg-dev libjpeg62-turbo-dev libjsoncpp-dev libleveldb-dev liblmdb-dev \ + liblz4-dev libopenblas-dev libopenmpi-dev libpng-dev librdkafka-dev libsnappy-dev libssl-dev \ + libswscale-dev libtbb-dev libtbb2 libtiff-dev libtiff5-dev libtool libzmq3-dev mpich \ + openjdk-11-jdk-headless pkg-config procps python3-dev python3-pip software-properties-common \ + swig unzip uuid-dev && \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 1 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 1 && \ apt-get clean && rm -rf /var/lib/apt/lists/* && \ - update-alternatives --install /usr/bin/python python /usr/bin/python3 1 && \ - pip3 install --no-cache-dir "numpy>=1.23.2" "setuptools>=65.5.1" + ln -s /usr/bin/python3 /usr/bin/python && \ + pip install --no-cache-dir "numpy>=1.25.1" "grpcio==1.40.0" "grpcio-tools==1.40.0" # Pull and Install Dependencies +ENV CMAKE_VERSION="v3.26.4" \ + PROTOBUF_VERSION="3.20.3" \ + OPENCV_VERSION="4.5.5" \ + FAISS_VERSION="v1.7.3" \ + VALIJSON_VERSION="v0.6" \ + AWS_SDK_VERSION="1.11.0" \ + TILEDB_VERSION="2.14.1" + WORKDIR /dependencies -RUN git clone --branch v3.21.2 https://github.com/Kitware/CMake.git && \ - git clone --branch v4.0.2 https://github.com/swig/swig.git && \ - git clone --branch v1.7.1 https://github.com/facebookresearch/faiss.git && \ - git clone https://github.com/tonyzhang617/FLINNG.git && \ - git clone --recurse-submodules -b v1.40.0 https://github.com/grpc/grpc.git && \ - git clone --branch 4.5.3 https://github.com/opencv/opencv.git && \ - git clone --branch v0.6 https://github.com/tristanpenman/valijson.git && \ - git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp && \ - curl -L -o /usr/share/java/json-simple-1.1.1.jar https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/json-simple/json-simple-1.1.1.jar && \ - curl -L -o /dependencies/2.14.0.tar.gz https://github.com/TileDB-Inc/TileDB/archive/refs/tags/2.14.0.tar.gz && \ - curl -L -o /dependencies/zlib-1.2.13.tar.gz http://zlib.net/zlib-1.2.13.tar.gz && \ - cd /dependencies/CMake && ./bootstrap && make ${BUILD_THREADS} && make install && \ - cd /dependencies/swig && ./autogen.sh && ./configure && make ${BUILD_THREADS} && make install && \ - cd /dependencies/faiss && mkdir build && cd build && cmake -DFAISS_ENABLE_GPU=OFF .. && make ${BUILD_THREADS} && make install && \ - cd /dependencies/FLINNG && mkdir build && cd build && cmake .. && make ${BUILD_THREADS} && make install && \ - cd /dependencies/grpc && pip3 install --no-cache-dir -r requirements.txt && GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip3 install --no-cache-dir . && \ - cd tools/distrib/python/grpcio_tools && python ../make_grpcio_tools.py && GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip3 install --no-cache-dir . && \ - cd /dependencies/grpc/third_party/zlib && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \ - cd /dependencies/grpc/third_party/protobuf/cmake && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE -Dprotobuf_BUILD_TESTS=OFF .. && make ${BUILD_THREADS} && make install && \ - cd ../../../abseil-cpp && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \ - cd ../../re2/ && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \ - cd /dependencies/grpc/cmake && mkdir build && cd build && cmake -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_ABSL_PROVIDER=package \ - -DgRPC_CARES_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package \ - -DgRPC_RE2_PROVIDER=package -DgRPC_SSL_PROVIDER=package \ - -DgRPC_ZLIB_PROVIDER=package -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE ../.. && make ${BUILD_THREADS} && make install && \ - cd /dependencies/opencv && mkdir build && cd build && cmake -D BUILD_PERF_TESTS=OFF -D BUILD_TESTS=OFF .. && make ${BUILD_THREADS} && make install && \ - cd /dependencies/ && tar -xvzf zlib-1.2.13.tar.gz && cd zlib-1.2.13 && ./configure && make ${BUILD_THREADS} && make install && \ - cd /dependencies/ && tar -xvf 2.14.0.tar.gz && cd TileDB-2.14.0 && mkdir build && cd build && \ - ../bootstrap --prefix=/usr/local/ && make ${BUILD_THREADS} && make install-tiledb && \ +RUN git clone --branch ${CMAKE_VERSION} https://github.com/Kitware/CMake.git && \ + cd CMake && ./bootstrap && make ${BUILD_THREADS} && make install && \ cd /usr/src/gtest && cmake . && make ${BUILD_THREADS} && mv lib/libgtest* /usr/lib/ && \ - cd /dependencies/valijson && cp -r include/* /usr/local/include/ && \ - cd /dependencies/aws-sdk-cpp && git checkout 276ee83080fcc521d41d456dbbe61d49392ddf77 && cd .. && mkdir aws_sdk_build && cd aws_sdk_build && \ - cmake ../aws-sdk-cpp -DCMAKE_BUILD_TYPE=Debug -DCMAKE_PREFIX_PATH=/usr/local/ -DCMAKE_INSTALL_PREFIX=/usr/local/ -DBUILD_ONLY="s3" \ - -DCUSTOM_MEMORY_MANAGEMENT=OFF && make && make install && \ - rm -rf /dependencies + git clone --branch ${FAISS_VERSION} https://github.com/facebookresearch/faiss.git /dependencies/faiss && \ + cd /dependencies/faiss && mkdir build && cd build && cmake -DFAISS_ENABLE_GPU=OFF .. && \ + make ${BUILD_THREADS} && make install && cd /dependencies/ && \ + git clone https://github.com/tonyzhang617/FLINNG.git && \ + cd /dependencies/FLINNG && mkdir build && cd build && cmake .. && make ${BUILD_THREADS} && make install && cd /dependencies && \ + curl -L -o /dependencies/${PROTOBUF_VERSION}.tar.gz \ + https://github.com/protocolbuffers/protobuf/archive/refs/tags/v${PROTOBUF_VERSION}.tar.gz && \ + cd /dependencies/ && tar -xvf ${PROTOBUF_VERSION}.tar.gz && \ + cd protobuf-${PROTOBUF_VERSION} && ./autogen.sh && ./configure && make -j$(nproc) && \ + make install && ldconfig && cd /dependencies && \ + git clone --branch ${OPENCV_VERSION} https://github.com/opencv/opencv.git && \ + cd opencv && mkdir build && cd build && cmake -D BUILD_PERF_TESTS=OFF -D BUILD_TESTS=OFF .. && \ + make ${BUILD_THREADS} && make install && cd /dependencies/ && \ + git clone --branch ${VALIJSON_VERSION} https://github.com/tristanpenman/valijson.git && \ + cd valijson && cp -r include/* /usr/local/include/ && cd /dependencies && \ + curl -L -o /dependencies/${TILEDB_VERSION}.tar.gz \ + https://github.com/TileDB-Inc/TileDB/archive/refs/tags/${TILEDB_VERSION}.tar.gz && \ + cd /dependencies/ && tar -xvf ${TILEDB_VERSION}.tar.gz && cd TileDB-${TILEDB_VERSION} && \ + mkdir build && cd build && ../bootstrap --prefix=/usr/local/ && make ${BUILD_THREADS} && \ + make install-tiledb && cd /dependencies && \ + git clone -b ${AWS_SDK_VERSION} --recurse-submodules https://github.com/aws/aws-sdk-cpp && \ + mkdir -p aws-sdk-cpp/build && cd aws-sdk-cpp/build && \ + cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_PREFIX_PATH=/usr/local/ -DCMAKE_INSTALL_PREFIX=/usr/local/ \ + -DBUILD_ONLY="s3" -DCUSTOM_MEMORY_MANAGEMENT=OFF && \ + make ${BUILD_THREADS} && make install && \ + rm -rf /dependencies /usr/local/share/doc /usr/local/share/man # VDMS WORKDIR /vdms -RUN git clone https://github.com/IntelLabs/vdms.git /vdms && cd /vdms && \ - git checkout develop && git submodule update --init --recursive && \ - mkdir build && cd build && cmake .. && make ${BUILD_THREADS} && \ +RUN git clone -b develop --recurse-submodules https://github.com/IntelLabs/vdms.git /vdms && \ + mkdir -p /vdms/build && cd /vdms/build && cmake .. && make ${BUILD_THREADS} && \ cp /vdms/config-vdms.json /vdms/build/ && \ echo '#!/bin/bash' > /start.sh && echo 'cd /vdms/build' >> /start.sh && \ echo './vdms' >> /start.sh && chmod 755 /start.sh diff --git a/docker/check-in/Dockerfile b/docker/check-in/Dockerfile index 158ea0c7..e4266b0b 100644 --- a/docker/check-in/Dockerfile +++ b/docker/check-in/Dockerfile @@ -1,83 +1,75 @@ -#Copyright (C) 2021 Intel Corporation +#Copyright (C) 2023 Intel Corporation #SPDX-License-Identifier: MIT -ARG UBUNTU_VERSION=20.04 -ARG UBUNTU_NAME=focal -ARG BUILD_THREADS=-j16 +ARG BASE_VERSION=11.7-slim +ARG BUILD_THREADS="-j16" -#1 -FROM ubuntu:${UBUNTU_VERSION} +FROM debian:${BASE_VERSION} # Dockerfile limitations force a repetition of global args -ARG UBUNTU_VERSION -ARG UBUNTU_NAME +ARG BUILD_THREADS # Install Packages -RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common && \ - add-apt-repository "deb http://security.ubuntu.com/ubuntu ${UBUNTU_NAME}-security main" && \ - apt-get install -y --no-install-recommends apt-transport-https autoconf automake bison build-essential \ - bzip2 ca-certificates curl=7.68.0-1ubuntu2.18 ed flex g++ git gnupg-agent javacc libarchive-tools \ - libatlas-base-dev libavcodec-dev libavformat-dev libboost-all-dev libbz2-dev \ - libc-ares-dev libdc1394-22-dev libgflags-dev libgoogle-glog-dev libgtest-dev \ - libgtk-3-dev libgtk2.0-dev libhdf5-serial-dev libjpeg-dev libjpeg8-dev libjsoncpp-dev \ - libleveldb-dev liblmdb-dev liblz4-dev libopenblas-dev libopenmpi-dev \ - libpng-dev librdkafka-dev libsnappy-dev libssl-dev libswscale-dev libtbb-dev \ - libtbb2 libtiff-dev libtiff5-dev libtool mpich openjdk-11-jdk-headless \ - libcurl4-openssl-dev libzmq3-dev uuid-dev zlib1g-dev libpulse-dev \ - pkg-config python3-dev python3-pip unzip lcov gdb && \ +RUN apt-get update && apt-get install -y --no-install-suggests --no-install-recommends \ + apt-transport-https autoconf automake bison build-essential bzip2 ca-certificates \ + curl ed flex g++-9 gcc-9 git gnupg-agent javacc libarchive-tools libatlas-base-dev \ + libavcodec-dev libavformat-dev libboost-all-dev libbz2-dev libc-ares-dev libcurl4-openssl-dev \ + libdc1394-22-dev libgflags-dev libgoogle-glog-dev libgtest-dev libgtk-3-dev libgtk2.0-dev \ + libhdf5-dev libjpeg-dev libjpeg62-turbo-dev libjsoncpp-dev libleveldb-dev liblmdb-dev \ + liblz4-dev libopenblas-dev libopenmpi-dev libpng-dev librdkafka-dev libsnappy-dev libssl-dev \ + libswscale-dev libtbb-dev libtbb2 libtiff-dev libtiff5-dev libtool libzmq3-dev mpich \ + openjdk-11-jdk-headless pkg-config procps python3-dev python3-pip software-properties-common \ + swig unzip uuid-dev && \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 1 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 1 && \ apt-get clean && rm -rf /var/lib/apt/lists/* && \ - update-alternatives --install /usr/bin/python python /usr/bin/python3 1 && \ - pip3 install --no-cache-dir "numpy>=1.23.2" "setuptools>=65.5.1" "gcovr>=5.2" + ln -s /usr/bin/python3 /usr/bin/python && \ + pip install --no-cache-dir "numpy>=1.25.1" "grpcio==1.40.0" "grpcio-tools==1.40.0" # Pull and Install Dependencies +ENV CMAKE_VERSION="v3.26.4" \ + PROTOBUF_VERSION="3.20.3" \ + OPENCV_VERSION="4.5.5" \ + FAISS_VERSION="v1.7.3" \ + VALIJSON_VERSION="v0.6" \ + AWS_SDK_VERSION="1.11.0" \ + TILEDB_VERSION="2.14.1" + WORKDIR /dependencies -RUN git clone --branch v3.21.2 https://github.com/Kitware/CMake.git && \ - git clone --branch v4.0.2 https://github.com/swig/swig.git && \ - git clone --branch v1.7.1 https://github.com/facebookresearch/faiss.git && \ - git clone https://github.com/tonyzhang617/FLINNG.git && \ - git clone --recurse-submodules -b v1.40.0 https://github.com/grpc/grpc.git && \ - git clone --branch 4.5.3 https://github.com/opencv/opencv.git && \ - git clone --branch v0.6 https://github.com/tristanpenman/valijson.git && \ - git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp && \ - curl -L -o /usr/share/java/json-simple-1.1.1.jar https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/json-simple/json-simple-1.1.1.jar && \ - curl -L -o /dependencies/2.14.0.tar.gz https://github.com/TileDB-Inc/TileDB/archive/refs/tags/2.14.0.tar.gz && \ - curl -L -o /dependencies/zlib-1.2.13.tar.gz http://zlib.net/zlib-1.2.13.tar.gz && \ - cd /dependencies/CMake && ./bootstrap && make ${BUILD_THREADS} && make install && \ - cd /dependencies/swig && ./autogen.sh && ./configure && make ${BUILD_THREADS} && make install && \ - cd /dependencies/faiss && mkdir build && cd build && cmake -DFAISS_ENABLE_GPU=OFF .. && make ${BUILD_THREADS} && make install && \ - cd /dependencies/FLINNG && mkdir build && cd build && cmake .. && make ${BUILD_THREADS} && make install && \ - cd /dependencies/grpc && pip3 install --no-cache-dir -r requirements.txt && GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip3 install --no-cache-dir . && \ - cd tools/distrib/python/grpcio_tools && python ../make_grpcio_tools.py && GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip3 install --no-cache-dir . && \ - cd /dependencies/grpc/third_party/zlib && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \ - cd /dependencies/grpc/third_party/protobuf/cmake && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE -Dprotobuf_BUILD_TESTS=OFF .. && make ${BUILD_THREADS} && make install && \ - cd ../../../abseil-cpp && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \ - cd ../../re2/ && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \ - cd /dependencies/grpc/cmake && mkdir build && cd build && cmake -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_ABSL_PROVIDER=package \ - -DgRPC_CARES_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package \ - -DgRPC_RE2_PROVIDER=package -DgRPC_SSL_PROVIDER=package \ - -DgRPC_ZLIB_PROVIDER=package -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE ../.. && make ${BUILD_THREADS} && make install && \ - cd /dependencies/opencv && mkdir build && cd build && cmake -D BUILD_PERF_TESTS=OFF -D BUILD_TESTS=OFF .. && make ${BUILD_THREADS} && make install && \ - cd /dependencies/ && tar -xvzf zlib-1.2.13.tar.gz && cd zlib-1.2.13 && ./configure && make ${BUILD_THREADS} && make install && \ - cd /dependencies/ && tar -xvf 2.14.0.tar.gz && cd TileDB-2.14.0 && mkdir build && cd build && \ - ../bootstrap --prefix=/usr/local/ && make ${BUILD_THREADS} && make install-tiledb && \ +RUN git clone --branch ${CMAKE_VERSION} https://github.com/Kitware/CMake.git && \ + cd CMake && ./bootstrap && make ${BUILD_THREADS} && make install && \ cd /usr/src/gtest && cmake . && make ${BUILD_THREADS} && mv lib/libgtest* /usr/lib/ && \ - cd /dependencies/valijson && cp -r include/* /usr/local/include/ && \ - cd /dependencies/aws-sdk-cpp && git checkout 276ee83080fcc521d41d456dbbe61d49392ddf77 && cd .. && mkdir aws_sdk_build && cd aws_sdk_build && \ - cmake ../aws-sdk-cpp -DCMAKE_BUILD_TYPE=Debug -DCMAKE_PREFIX_PATH=/usr/local/ -DCMAKE_INSTALL_PREFIX=/usr/local/ -DBUILD_ONLY="s3" \ - -DCUSTOM_MEMORY_MANAGEMENT=OFF && make && make install && \ - rm -rf /dependencies + git clone --branch ${FAISS_VERSION} https://github.com/facebookresearch/faiss.git /dependencies/faiss && \ + cd /dependencies/faiss && mkdir build && cd build && cmake -DFAISS_ENABLE_GPU=OFF .. && \ + make ${BUILD_THREADS} && make install && cd /dependencies/ && \ + git clone https://github.com/tonyzhang617/FLINNG.git && \ + cd /dependencies/FLINNG && mkdir build && cd build && cmake .. && make ${BUILD_THREADS} && make install && cd /dependencies && \ + curl -L -o /dependencies/${PROTOBUF_VERSION}.tar.gz \ + https://github.com/protocolbuffers/protobuf/archive/refs/tags/v${PROTOBUF_VERSION}.tar.gz && \ + cd /dependencies/ && tar -xvf ${PROTOBUF_VERSION}.tar.gz && \ + cd protobuf-${PROTOBUF_VERSION} && ./autogen.sh && ./configure && make -j$(nproc) && \ + make install && ldconfig && cd /dependencies && \ + git clone --branch ${OPENCV_VERSION} https://github.com/opencv/opencv.git && \ + cd opencv && mkdir build && cd build && cmake -D BUILD_PERF_TESTS=OFF -D BUILD_TESTS=OFF .. && \ + make ${BUILD_THREADS} && make install && cd /dependencies/ && \ + git clone --branch ${VALIJSON_VERSION} https://github.com/tristanpenman/valijson.git && \ + cd valijson && cp -r include/* /usr/local/include/ && cd /dependencies && \ + curl -L -o /dependencies/${TILEDB_VERSION}.tar.gz \ + https://github.com/TileDB-Inc/TileDB/archive/refs/tags/${TILEDB_VERSION}.tar.gz && \ + cd /dependencies/ && tar -xvf ${TILEDB_VERSION}.tar.gz && cd TileDB-${TILEDB_VERSION} && \ + mkdir build && cd build && ../bootstrap --prefix=/usr/local/ && make ${BUILD_THREADS} && \ + make install-tiledb && cd /dependencies && \ + git clone -b ${AWS_SDK_VERSION} --recurse-submodules https://github.com/aws/aws-sdk-cpp && \ + mkdir -p aws-sdk-cpp/build && cd aws-sdk-cpp/build && \ + cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_PREFIX_PATH=/usr/local/ -DCMAKE_INSTALL_PREFIX=/usr/local/ \ + -DBUILD_ONLY="s3" -DCUSTOM_MEMORY_MANAGEMENT=OFF && \ + make ${BUILD_THREADS} && make install && \ + rm -rf /dependencies /usr/local/share/doc /usr/local/share/man -# COVERITY -RUN mkdir -p /coverity /coverity-results && cd /coverity && \ - curl -L -o cov-analysis-linux64-2023.3.0.sh https://ubit-artifactory-or.intel.com/artifactory/coverity-or-local/Enterprise/cov-analysis-linux64-2023.3.0.sh && \ - chmod +x cov-analysis-linux64-2023.3.0.sh && \ - curl -L -o license.dat https://ubit-artifactory-or.intel.com/artifactory/coverity-or-local/Enterprise/license.dat && \ - ./cov-analysis-linux64-2023.3.0.sh -q --installation.dir=/opt/coverity/analysis/ \ - --license.agreement=agree --license.region=0 --license.type.choice=0 \ - --license.cov.path=/coverity/license.dat --component.sdk=false --component.skip.documentation=true -ENV PATH /opt/coverity/analysis/bin:$PATH # VDMS +ARG BUILD_COVERAGE="on" +ARG BUILD_COVERITY="on" COPY ./.git /vdms/.git COPY ./client /vdms/client COPY ./distributed /vdms/distributed @@ -90,15 +82,48 @@ COPY ./CMakeLists.txt /vdms/ COPY ./config-vdms.json /vdms/ COPY ./docker/check-in/run_coverage_cpp.sh / COPY ./docker/check-in/run_coverage_py.sh / -WORKDIR /vdms +ENV PATH /opt/coverity/analysis/bin:$PATH + +# COVERITY & MINIO for S3 Testing +WORKDIR /coverity +RUN if [ "${BUILD_COVERITY}" = "on" ]; then \ + mkdir -p /coverity /opt/coverity ; \ + curl -L -o /coverity/cov-analysis-linux64-2023.3.0.sh https://ubit-artifactory-or.intel.com/artifactory/coverity-or-local/Enterprise/cov-analysis-linux64-2023.3.0.sh ; \ + curl -L -o /coverity/license.dat https://ubit-artifactory-or.intel.com/artifactory/coverity-or-local/Enterprise/license.dat ; \ + chmod +x /coverity/cov-analysis-linux64-2023.3.0.sh ; \ + ./cov-analysis-linux64-2023.3.0.sh -q \ + --installation.dir=/opt/coverity/analysis \ + --license.agreement=agree \ + --license.region=0 \ + --license.type.choice=0 \ + --license.cov.path=/coverity/license.dat \ + --component.sdk=false \ + --component.skip.documentation=true; \ + rm /coverity/cov-analysis-linux64-2023.3.0.sh ; \ + fi + +RUN if [ "${BUILD_COVERAGE}" = "on" ]; then \ + apt-get update ; \ + apt-get install -y --no-install-suggests --no-install-recommends gdb ; \ + apt-get clean ; \ + rm -rf /var/lib/apt/lists/* ; \ + pip3 install --no-cache-dir "gcovr>=6.0" "coverage>=7.2.7" ; \ + curl -L -o /vdms/minio https://dl.min.io/server/minio/release/linux-amd64/minio ; \ + chmod +x /vdms/minio ; \ + mkdir -p /vdms/minio_files/minio-bucket ; \ + mkdir -p /vdms/tests/coverage_report ; \ + chmod +x /run_coverage_*.sh ; \ + else \ + rm -rf /run_coverage_*.sh ; \ + fi -RUN cd /vdms && curl -L -o minio https://dl.min.io/server/minio/release/linux-amd64/minio && \ - chmod +x minio && mkdir -p minio_files/minio-bucket && \ + +WORKDIR /vdms +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN upperCoverage=$(echo ${BUILD_COVERAGE} | tr '[:lower:]' '[:upper:]') && echo ${upperCoverage} && \ git submodule update --init --recursive && mkdir build && \ - cd build && cmake -DCODE_COVERAGE=ON .. && make ${BUILD_THREADS} && \ + cd build && cmake -DCODE_COVERAGE=${upperCoverage} .. && make ${BUILD_THREADS} && \ cp /vdms/config-vdms.json /vdms/build/ && \ - mkdir -p /vdms/tests/coverage_report && \ - chmod +x /run_coverage_cpp.sh && chmod +x /run_coverage_py.sh && \ echo '#!/bin/bash' > /start.sh && echo 'cd /vdms/build' >> /start.sh && \ echo './vdms' >> /start.sh && chmod 755 /start.sh diff --git a/docker/check-in/Dockerfile.base b/docker/check-in/Dockerfile.base deleted file mode 100644 index e1a67b64..00000000 --- a/docker/check-in/Dockerfile.base +++ /dev/null @@ -1,90 +0,0 @@ -#Copyright (C) 2021 Intel Corporation -#SPDX-License-Identifier: MIT - -ARG UBUNTU_VERSION=20.04 -ARG UBUNTU_NAME=focal -ARG BUILD_THREADS=-j16 - -#1 -FROM ubuntu:${UBUNTU_VERSION} - -# Dockerfile limitations force a repetition of global args -ARG UBUNTU_VERSION -ARG UBUNTU_NAME - -# Install Packages -RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common && \ - add-apt-repository "deb http://security.ubuntu.com/ubuntu ${UBUNTU_NAME}-security main" && \ - apt-get install -y --no-install-recommends apt-transport-https autoconf automake bison build-essential \ - bzip2 ca-certificates curl=7.68.0-1ubuntu2.18 ed flex g++ git gnupg-agent javacc libarchive-tools \ - libatlas-base-dev libavcodec-dev libavformat-dev libboost-all-dev libbz2-dev \ - libc-ares-dev libdc1394-22-dev libgflags-dev libgoogle-glog-dev libgtest-dev \ - libgtk-3-dev libgtk2.0-dev libhdf5-serial-dev libjpeg-dev libjpeg8-dev libjsoncpp-dev \ - libleveldb-dev liblmdb-dev liblz4-dev libopenblas-dev libopenmpi-dev \ - libpng-dev librdkafka-dev libsnappy-dev libssl-dev libswscale-dev libtbb-dev \ - libtbb2 libtiff-dev libtiff5-dev libtool mpich openjdk-11-jdk-headless \ - libcurl4-openssl-dev libzmq3-dev uuid-dev zlib1g-dev libpulse-dev \ - pkg-config python3-dev python3-pip unzip && \ - apt-get clean && rm -rf /var/lib/apt/lists/* && \ - update-alternatives --install /usr/bin/python python /usr/bin/python3 1 && \ - pip3 install --no-cache-dir "numpy>=1.23.2" "setuptools>=65.5.1" - -# Pull and Install Dependencies -WORKDIR /dependencies -RUN git clone --branch v3.21.2 https://github.com/Kitware/CMake.git && \ - git clone --branch v4.0.2 https://github.com/swig/swig.git && \ - git clone --branch v1.7.1 https://github.com/facebookresearch/faiss.git && \ - git clone https://github.com/tonyzhang617/FLINNG.git && \ - git clone --recurse-submodules -b v1.40.0 https://github.com/grpc/grpc.git && \ - git clone --branch 4.5.3 https://github.com/opencv/opencv.git && \ - git clone --branch v0.6 https://github.com/tristanpenman/valijson.git && \ - git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp && \ - curl -L -o /usr/share/java/json-simple-1.1.1.jar https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/json-simple/json-simple-1.1.1.jar && \ - curl -L -o /dependencies/2.14.0.tar.gz https://github.com/TileDB-Inc/TileDB/archive/refs/tags/2.14.0.tar.gz && \ - curl -L -o /dependencies/zlib-1.2.13.tar.gz http://zlib.net/zlib-1.2.13.tar.gz && \ - cd /dependencies/CMake && ./bootstrap && make ${BUILD_THREADS} && make install && \ - cd /dependencies/swig && ./autogen.sh && ./configure && make ${BUILD_THREADS} && make install && \ - cd /dependencies/faiss && mkdir build && cd build && cmake -DFAISS_ENABLE_GPU=OFF .. && make ${BUILD_THREADS} && make install && \ - cd /dependencies/FLINNG && mkdir build && cd build && cmake .. && make ${BUILD_THREADS} && make install && \ - cd /dependencies/grpc && pip3 install --no-cache-dir -r requirements.txt && GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip3 install --no-cache-dir . && \ - cd tools/distrib/python/grpcio_tools && python ../make_grpcio_tools.py && GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip3 install --no-cache-dir . && \ - cd /dependencies/grpc/third_party/zlib && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \ - cd /dependencies/grpc/third_party/protobuf/cmake && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE -Dprotobuf_BUILD_TESTS=OFF .. && make ${BUILD_THREADS} && make install && \ - cd ../../../abseil-cpp && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \ - cd ../../re2/ && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \ - cd /dependencies/grpc/cmake && mkdir build && cd build && cmake -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_ABSL_PROVIDER=package \ - -DgRPC_CARES_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package \ - -DgRPC_RE2_PROVIDER=package -DgRPC_SSL_PROVIDER=package \ - -DgRPC_ZLIB_PROVIDER=package -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE ../.. && make ${BUILD_THREADS} && make install && \ - cd /dependencies/opencv && mkdir build && cd build && cmake -D BUILD_PERF_TESTS=OFF -D BUILD_TESTS=OFF .. && make ${BUILD_THREADS} && make install && \ - cd /dependencies/ && tar -xvzf zlib-1.2.13.tar.gz && cd zlib-1.2.13 && ./configure && make ${BUILD_THREADS} && make install && \ - cd /dependencies/ && tar -xvf 2.14.0.tar.gz && cd TileDB-2.14.0 && mkdir build && cd build && \ - ../bootstrap --prefix=/usr/local/ && make ${BUILD_THREADS} && make install-tiledb && \ - cd /usr/src/gtest && cmake . && make ${BUILD_THREADS} && mv lib/libgtest* /usr/lib/ && \ - cd /dependencies/valijson && cp -r include/* /usr/local/include/ && \ - cd /dependencies/aws-sdk-cpp && git checkout 276ee83080fcc521d41d456dbbe61d49392ddf77 && cd .. && mkdir aws_sdk_build && cd aws_sdk_build && \ - cmake ../aws-sdk-cpp -DCMAKE_BUILD_TYPE=Debug -DCMAKE_PREFIX_PATH=/usr/local/ -DCMAKE_INSTALL_PREFIX=/usr/local/ -DBUILD_ONLY="s3" \ - -DCUSTOM_MEMORY_MANAGEMENT=OFF && make && make install && \ - rm -rf /dependencies - - -# VDMS -COPY ./.git /vdms/.git -COPY ./client /vdms/client -COPY ./distributed /vdms/distributed -COPY ./ext /vdms/ext -COPY ./include /vdms/include -COPY ./src /vdms/src -COPY ./tests /vdms/tests -COPY ./utils /vdms/utils -COPY ./CMakeLists.txt /vdms/ -COPY ./config-vdms.json /vdms/ -WORKDIR /vdms - -RUN cd /vdms && git submodule update --init --recursive && mkdir build && \ - cd build && cmake .. && make ${BUILD_THREADS} && \ - cp /vdms/config-vdms.json /vdms/build/ && \ - echo '#!/bin/bash' > /start.sh && echo 'cd /vdms/build' >> /start.sh && \ - echo './vdms' >> /start.sh && chmod 755 /start.sh - -CMD ["/start.sh"] diff --git a/remote_function/requirements.txt b/remote_function/requirements.txt index c553f584..89b80f95 100644 --- a/remote_function/requirements.txt +++ b/remote_function/requirements.txt @@ -1,4 +1,4 @@ -opencv-python +opencv-python==4.5.5.64 flask numpy sk-video diff --git a/tests/python/run_python_aws_tests.sh b/tests/python/run_python_aws_tests.sh index c0c4ac40..2f5ec6f4 100755 --- a/tests/python/run_python_aws_tests.sh +++ b/tests/python/run_python_aws_tests.sh @@ -52,4 +52,4 @@ echo 'Running Python AWS S3 tests...' python3 -m coverage run --include="../../*" --omit="../*" -m unittest discover --pattern=Test*.py -v rm -rf test_db log.log screen.log -kill -9 $py_unittest_pid $py_minio_pid \ No newline at end of file +kill -9 $py_unittest_pid $py_minio_pid || true \ No newline at end of file diff --git a/tests/python/run_python_tests.sh b/tests/python/run_python_tests.sh index a73bf486..62037443 100755 --- a/tests/python/run_python_tests.sh +++ b/tests/python/run_python_tests.sh @@ -46,4 +46,4 @@ echo 'Running Python tests...' python3 -m coverage run --include="../../*" --omit="../*" -m unittest discover --pattern=Test*.py -v rm -rf test_db log.log screen.log -kill -9 $py_unittest_pid +kill -9 $py_unittest_pid || true diff --git a/tests/remote_function_test/requirements.txt b/tests/remote_function_test/requirements.txt index c553f584..89b80f95 100644 --- a/tests/remote_function_test/requirements.txt +++ b/tests/remote_function_test/requirements.txt @@ -1,4 +1,4 @@ -opencv-python +opencv-python==4.5.5.64 flask numpy sk-video diff --git a/tests/run_aws_tests.sh b/tests/run_aws_tests.sh index ed4e4706..9546a022 100755 --- a/tests/run_aws_tests.sh +++ b/tests/run_aws_tests.sh @@ -16,4 +16,4 @@ sleep 2 echo 'Running C++ tests...' ./../build/tests/unit_tests --gtest_filter=RemoteConnectionTest.* -kill -9 $cpp_unittest_pid $py_minio_pid +kill -9 $py_minio_pid || true diff --git a/tests/run_tests.sh b/tests/run_tests.sh index d502c9c7..41933ae7 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -39,4 +39,4 @@ echo 'Running C++ tests...' pkill -9 -f udf_server.py pkill -9 -f udf_local.py -kill -9 $cpp_unittest_pid $client_test_pid +kill -9 $cpp_unittest_pid $client_test_pid || true diff --git a/tests/udf_test/requirements.txt b/tests/udf_test/requirements.txt index 04df877a..5ce1a8b4 100644 --- a/tests/udf_test/requirements.txt +++ b/tests/udf_test/requirements.txt @@ -1,2 +1,2 @@ -opencv-python +opencv-python==4.5.5.64 zmq \ No newline at end of file diff --git a/user_defined_operations/requirements.txt b/user_defined_operations/requirements.txt index 04df877a..5ce1a8b4 100644 --- a/user_defined_operations/requirements.txt +++ b/user_defined_operations/requirements.txt @@ -1,2 +1,2 @@ -opencv-python +opencv-python==4.5.5.64 zmq \ No newline at end of file