diff --git a/.github/workflows/cpp_coverage_source.yml b/.github/workflows/cpp_coverage.yml similarity index 88% rename from .github/workflows/cpp_coverage_source.yml rename to .github/workflows/cpp_coverage.yml index c9de034f..a96fc8f5 100644 --- a/.github/workflows/cpp_coverage_source.yml +++ b/.github/workflows/cpp_coverage.yml @@ -17,8 +17,6 @@ env: jobs: coverage_job: name: Coverage Test - - # Specify runner job will run on runs-on: group: intellabs-generic-runners labels: vdms-check-in @@ -70,7 +68,7 @@ jobs: docker stop $(docker ps -aqf "name=${{ matrix.container_name }}") || true docker rm $(docker ps -aqf "name=${{ matrix.container_name }}") || true - docker build --build-arg MAVEN_OPTS='-Dhttps.proxyHost=proxy-chain.intel.com -Dhttps.proxyPort=912 -Dhttps.nonProxyHosts="localhost|127.0.0.1"' \ + docker build --build-arg MAVEN_OPTS=${{ secrets.MAVEN_OPTS }} \ -f docker/check-in/Dockerfile -t ${{ matrix.container_tag }} . docker run -d --name ${{ matrix.container_name }} ${{ matrix.container_tag }} @@ -92,17 +90,12 @@ jobs: docker exec ${{ matrix.container_name }} bash -c "./run_coverage.sh" docker cp $(docker ps -a | grep ${{ matrix.container_name }} | awk '{print $1}'):/vdms/tests/coverage_report/c_coverage_report.txt coverage/c_coverage_report_target.txt - # report="$(> $GITHUB_ENV - docker cp $(docker ps -a | grep ${{ matrix.container_name }} | awk '{print $1}'):/vdms/tests/coverage_report/c_coverage_report.xml coverage/c_coverage_report_target.xml echo "coverage_value=$(cat coverage/c_coverage_report_target.xml | grep -oP 'coverage line-rate="([-+]?\d*\.\d+|\d+)"' | grep -oP "[-+]?\d*\.\d+|\d+" | awk '{print $1*100}')" >> $GITHUB_ENV docker ps -aqf "name=${{ matrix.container_name }}" | xargs docker stop docker ps -aqf "name=${{ matrix.container_name }}" | xargs docker rm + docker rmi $(docker images | grep '' | awk '{print $3}') || true - name: Report ${{ matrix.coverage_type }} Coverage id: report_coverage @@ -113,24 +106,19 @@ jobs: exit 1 fi echo "${{ matrix.coverage_type }} Coverage: ${coverage_value}" - echo "::set-output name=${{ matrix.output_name }}::${coverage_value}" - # echo "::set-output name=${{ matrix.report_name }}::${coverage_report}" + echo "${{ matrix.output_name }}=${coverage_value}" >> $GITHUB_OUTPUT compare_coverage: name: Compare Reported Coverage - - # Specify runner job will run on runs-on: group: intellabs-generic-runners labels: vdms-check-in - needs: coverage_job steps: - name: Comment Coverage if: (github.event_name == 'pull_request') uses: actions/github-script@v3 with: - # \n\n\nTarget Report: ${{ needs.coverage_job.outputs.target_coverage_report }}\n\n\nSource Report: ${{ needs.coverage_job.outputs.source_coverage_report }}' script: | github.issues.createComment({ issue_number: ${{ github.event.number }}, diff --git a/.github/workflows/sdl_req.yml b/.github/workflows/sdl_req.yml new file mode 100644 index 00000000..721717a6 --- /dev/null +++ b/.github/workflows/sdl_req.yml @@ -0,0 +1,242 @@ +# Uses docker/check-in/Dockerfile.base +# Dockerfile.base -> Same as docker/base/Dockerfile but builds VDMS with local changes instead of external repo +name: SDL Requirements using Docker Image + +# Controls when the action will run. Triggers the workflow on push or pull request +# events but only for the master and develop branch +# on: +# pull_request: +# types: [ opened, edited, synchronize, reopened ] +# branches: +# - develop +# - master +on: + push: + branches: + - develop + + +# Environment variables +env: + ARTIFACT_DIR: SDL_artifacts + DOCKER_ARTIFACT_DIR: Docker_artifacts + NEW_BASE_DOCKERFILE: docker/check-in/Dockerfile.base + SNYK_TOKEN: ${{ secrets.SNYK_TOKEN}} + SNYK_API: ${{ secrets.SNYK_API}} + # CHECKOUT_REF: ${{ github.event.pull_request.head.sha }} + +jobs: + Build: + # This job builds docker container for later use + name: Build Docker + runs-on: + group: intellabs-generic-runners + labels: vdms-check-in + steps: + - name: Checkout Branch + uses: actions/checkout@v3 + with: + submodules: true + # ref: ${{ env.CHECKOUT_REF }} + - run: mkdir -p ${{ env.DOCKER_ARTIFACT_DIR }} + - name: Build Docker Container + run: | + docker build --build-arg MAVEN_OPTS=${{ secrets.MAVEN_OPTS }} -f ${{ env.NEW_BASE_DOCKERFILE}} -t vdms:latest . + docker save -o ${{ env.DOCKER_ARTIFACT_DIR }}/image.tar vdms:latest + - name: Upload Docker Image Artifact + uses: actions/upload-artifact@v3 + with: + name: image.tar + path: ${{ env.DOCKER_ARTIFACT_DIR }}/image.tar + retention-days: 1 + + Hadolint: + # This job check formatting of Dockerfile + name: Haskell Dockerfile Linter + runs-on: + group: intellabs-generic-runners + labels: vdms-check-in + steps: + - name: Checkout Branch + uses: actions/checkout@v3 + with: + submodules: true + # ref: ${{ env.CHECKOUT_REF }} + - run: mkdir -p ${{ env.ARTIFACT_DIR }} + - name: Run Hadolint Docker Container + id: get_hadolint + run: | + set -x + docker run --rm -i hadolint/hadolint:latest < ${{ env.NEW_BASE_DOCKERFILE}} 2>&1 | tee ${{ env.ARTIFACT_DIR }}/hadolint_output.txt + output=$(cat ${{ env.ARTIFACT_DIR }}/hadolint_output.txt | awk '{print $2}' | sort -u) + + echo "hadolint_output<> $GITHUB_ENV + echo "$output" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + - name: Print Hadolint Results in Job Summary + shell: bash + run: | + set -x + echo "### Hadolint Returned Rule Codes" > $GITHUB_STEP_SUMMARY + echo "${{ env.hadolint_output }}" >> $GITHUB_STEP_SUMMARY + - name: Upload Hadolint Artifact + uses: actions/upload-artifact@v3 + with: + name: sdl-artifacts + path: ${{ env.ARTIFACT_DIR }}/hadolint_output.txt + + Snyk: + # This job runs Snyk for Vulnerabilities and extract list of dependencies + name: Snyk Scan for Vulnerabilities + needs: Build + runs-on: + group: intellabs-generic-runners + labels: vdms-check-in + steps: + - name: Checkout Branch + uses: actions/checkout@v3 + with: + submodules: true + # ref: ${{ env.CHECKOUT_REF }} + - run: | + export no_proxy+=',snyk.devtools.intel.com' + export NO_PROXY+=',snyk.devtools.intel.com' + export DOCKER_PROXY_RUN_ARGS="\ + --env HTTPS_PROXY=$HTTPS_PROXY \ + --env https_proxy=$https_proxy \ + --env HTTP_PROXY=$HTTP_PROXY \ + --env http_proxy=$http_proxy \ + --env NO_PROXY=$NO_PROXY \ + --env no_proxy=$no_proxy" + mkdir -p ${{ env.ARTIFACT_DIR }} + - name: Download docker image + uses: actions/download-artifact@v3 + with: + name: image.tar + path: ${{ env.DOCKER_ARTIFACT_DIR }} + - name: Load Docker Image + run: | + docker load -i ${{ env.DOCKER_ARTIFACT_DIR }}/image.tar + - name: Run Snyk Docker Image Scan + env: + PROJ_NAME: 'EVS/vdms' + run: | + docker run --rm -i $DOCKER_PROXY_RUN_ARGS --env SNYK_TOKEN=${{ env.SNYK_TOKEN}} --env SNYK_API=${{ env.SNYK_API}} --env SNYK_DISABLE_ANALYTICS=1 \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v ${PWD}:/vdms/ \ + snyk/snyk:docker snyk container test -d vdms:latest --file=/vdms/${{ env.NEW_BASE_DOCKERFILE}} --exclude-base-image-vulns --project-name="$PROJ_NAME" > snyk.log || true && \ + mv snyk.log ${{ env.ARTIFACT_DIR }}/docker_snyk_scan.log + + output_checks=$(cat ${{ env.ARTIFACT_DIR }}/docker_snyk_scan.log | grep "Tested ") + + echo "snyk_image_results<> $GITHUB_ENV + echo "$output_checks" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + + - name: Get Python Environment requirements.txt & Run Snyk Python Scan + env: + PROJ_NAME: 'EVS/vdms-python' + run: | + docker run --rm -i vdms:latest bash -c "pip3 freeze -l" | tee requirements.txt + docker run --rm -i $DOCKER_PROXY_RUN_ARGS --env SNYK_TOKEN=${{ env.SNYK_TOKEN}} --env SNYK_API=${{ env.SNYK_API}} --env SNYK_DISABLE_ANALYTICS=1 --env COMMAND="pip install -r /app/requirements.txt" \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v ${PWD}:/app/ \ + snyk/snyk:python-3.8 snyk test -d --file=/app/requirements.txt --package-manager=pip --exclude-base-image-vulns --project-name="$PROJ_NAME" > docker_snyk_python_scan.log || true && \ + mv docker_snyk_python_scan.log ${{ env.ARTIFACT_DIR }}/docker_snyk_python_scan.log + + output_checks=$(cat ${{ env.ARTIFACT_DIR }}/docker_snyk_python_scan.log | grep "Tested ") + + echo "snyk_python_results<> $GITHUB_ENV + echo "$output_checks" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + + - name: Get SBOM (Dependencies) + run: | + curl -sSfL https://raw.githubusercontent.com/docker/sbom-cli-plugin/main/install.sh | sh -s -- + docker sbom --format spdx-tag-value --output sbom_vdms_docker.txt vdms:latest + docker sbom --format spdx-tag-value --output sbom_ubuntuBase_docker.txt ubuntu:20.04 + + python3 docker/check-in/spdx2csv.py -i sbom_vdms_docker.txt -o ${{ env.ARTIFACT_DIR }}/sbom_vdms_docker.csv + python3 docker/check-in/spdx2csv.py -i sbom_ubuntuBase_docker.txt -o ${{ env.ARTIFACT_DIR }}/sbom_ubuntuBase_docker.csv + rm sbom_vdms_docker.txt sbom_ubuntuBase_docker.txt + + diff ${{ env.ARTIFACT_DIR }}/sbom_ubuntuBase_docker.csv ${{ env.ARTIFACT_DIR }}/sbom_vdms_docker.csv | grep ">" | cut -d" " -f2 > ${{ env.ARTIFACT_DIR }}/sbom_onlyVDMS.csv + sed -i '1s/^/Package,Version,License,Package Supplier,SPDXID\n/' ${{ env.ARTIFACT_DIR }}/sbom_onlyVDMS.csv + - name: Upload SNYK & Dependency Artifacts + uses: actions/upload-artifact@v3 + with: + name: sdl-artifacts + path: ${{ env.ARTIFACT_DIR }} + - name: Print SNYK Results in Job Summary + shell: bash + run: | + echo "### SNYK Results" > $GITHUB_STEP_SUMMARY + echo "Docker Scan :point_right:${{ env.snyk_image_results }}" >> $GITHUB_STEP_SUMMARY + echo "Python 3.8 Scan :point_right:${{ env.snyk_python_results }}" >> $GITHUB_STEP_SUMMARY + + CIS: + # This job runs CIS Docker Benchmark + name: CIS Docker Benchmark + needs: Build + runs-on: + group: intellabs-generic-runners + labels: vdms-check-in + steps: + - name: Checkout Branch + uses: actions/checkout@v3 + with: + submodules: true + # ref: ${{ env.CHECKOUT_REF }} + - name: Download Docker Image + uses: actions/download-artifact@v3 + with: + name: image.tar + path: ${{ env.DOCKER_ARTIFACT_DIR }} + - name: Load Docker Image + run: | + docker stop vdms_test-CIS || true + docker rm vdms_test-CIS || true + docker load -i ${{ env.DOCKER_ARTIFACT_DIR }}/image.tar + - name: Run Benchmark + id: run_CIS + run: | + set -x + mkdir -p ${{ env.ARTIFACT_DIR }} + git clone https://github.com/docker/docker-bench-security.git + cd docker-bench-security + + # docker container run --net=host -d --name vdms_test vdms:latest + docker container run --net=host -d \ + --security-opt=no-new-privileges \ + --health-cmd='cd /vdms/build && ./vdms || exit 1' \ + --restart on-failure:5 \ + --name vdms_test-CIS vdms:latest + + mkdir -p ${{ env.ARTIFACT_DIR }} + sh docker-bench-security.sh -c container_runtime -i vdms_test-CIS -l cis_output.txt + cd .. + mv docker-bench-security/cis_output.txt ${{ env.ARTIFACT_DIR }}/cis_output.txt + + docker stop vdms_test-CIS && docker rm vdms_test-CIS + docker rmi $(docker images | grep '' | awk '{print $3}') || true + output_checks=$(cat ${{ env.ARTIFACT_DIR }}/cis_output.txt | grep "Checks:" | sed 's/^.*Checks/Checks/') + output_score=$(cat ${{ env.ARTIFACT_DIR }}/cis_output.txt | grep "Score:" | sed 's/^.*Score/Score/') + + echo "cis_output_checks<> $GITHUB_ENV + echo "$output_checks" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + + echo "cis_output_score<> $GITHUB_ENV + echo "$output_score" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + - name: Upload CIS Artifact + uses: actions/upload-artifact@v3 + with: + name: sdl-artifacts + path: ${{ env.ARTIFACT_DIR }}/cis_output.txt + - name: Print CIS Results in Job Summary + shell: bash + run: | + echo "### CIS Docker Results" > $GITHUB_STEP_SUMMARY + echo "${{ env.cis_output_checks }}" >> $GITHUB_STEP_SUMMARY + echo "${{ env.cis_output_score }}" >> $GITHUB_STEP_SUMMARY diff --git a/INSTALL.md b/INSTALL.md index 6439df9d..1c913715 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -4,10 +4,10 @@ Here is the detailed process of installation of VDMS dependencies. ## Dependencies Here we will install the Ubuntu 20.04 packages. ```bash -apt-get update -apt-get -y install software-properties-common -add-apt-repository "deb http://security.ubuntu.com/ubuntu focal-security main" -apt-get -y install apt-transport-https autoconf automake bison build-essential \ +sudo apt-get update +sudo apt-get -y install --no-install-recommends software-properties-common +sudo add-apt-repository "deb http://security.ubuntu.com/ubuntu focal-security main" +sudo apt-get -y install --no-install-recommends apt-transport-https autoconf automake bison build-essential \ bzip2 ca-certificates curl ed flex g++ git gnupg-agent javacc libarchive-tools \ libatlas-base-dev libavcodec-dev libavformat-dev libboost-all-dev libbz2-dev \ libc-ares-dev libdc1394-22-dev libgflags-dev libgoogle-glog-dev libgtest-dev \ @@ -15,113 +15,104 @@ apt-get -y install apt-transport-https autoconf automake bison build-essential \ libleveldb-dev liblmdb-dev liblz4-dev libopenblas-dev libopenmpi-dev \ libpng-dev librdkafka-dev libsnappy-dev libssl-dev libswscale-dev libtbb-dev \ libtbb2 libtiff-dev libtiff5-dev libtool maven mpich openjdk-11-jdk-headless \ - pkg-config python python-dev python3-pip unzip wget -pip3 install numpy + pkg-config python3-dev python3-pip unzip +pip3 install --no-cache-dir "numpy>=1.23.2" ``` ### Clone/Download Dependencies -Here we clone the repositories for grpc v1.40.0, libpng12, Swig v4.0.2, OpenCV 4.5.3, Valijson v0.6, CMake v3.21.2, Faiss v1.7.1, and FLINNG. Then download necesarry files for zlib v1.2.12, Json-simple v1.1.1, and TileDB v1.3.1. -Here we assume `/` is the working directory. This is important when installing the dependencies. +Here we clone the repositories for grpc v1.40.0, libpng12, Swig v4.0.2, OpenCV 4.5.3, Valijson v0.6, CMake v3.21.2, Faiss v1.7.1, and FLINNG. Then download necesarry files for zlib v1.2.13, Json-simple v1.1.1, and TileDB v1.3.1. +Here we assume `$VDMS_DEP_DIR` is the working directory for installing dependencies and `python` is Python 3. ```bash -git clone --branch v1.40.0 https://github.com/grpc/grpc.git && \ -git clone --branch v4.0.2 https://github.com/swig/swig.git && \ -git clone --branch 4.5.3 https://github.com/opencv/opencv.git && \ -git clone --branch v0.6 https://github.com/tristanpenman/valijson.git && \ +cd $VDMS_DEP_DIR git clone --branch v3.21.2 https://github.com/Kitware/CMake.git && \ +git clone --branch v4.0.2 https://github.com/swig/swig.git && \ git clone --branch v1.7.1 https://github.com/facebookresearch/faiss.git && \ -git clone https://github.com/tonyzhang617/FLINNG.git +git clone https://github.com/tonyzhang617/FLINNG.git && \ +git clone --branch v1.40.0 https://github.com/grpc/grpc.git && \ +git clone --branch 4.5.3 https://github.com/opencv/opencv.git && \ +git clone --branch v0.6 https://github.com/tristanpenman/valijson.git -curl http://zlib.net/zlib-1.2.12.tar.gz -o zlib-1.2.12.tar.gz && \ -curl https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/json-simple/json-simple-1.1.1.jar \ - -o /usr/share/java/json-simple-1.1.1.jar && \ -wget https://github.com/TileDB-Inc/TileDB/archive/1.3.1.tar.gz +sudo curl -L -o /usr/share/java/json-simple-1.1.1.jar https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/json-simple/json-simple-1.1.1.jar && \ +sudo curl -L -o 1.3.1.tar.gz https://github.com/TileDB-Inc/TileDB/archive/refs/tags/1.3.1.tar.gz && \ +sudo curl -L -o zlib-1.2.13.tar.gz http://zlib.net/zlib-1.2.13.tar.gz ``` ### Install Dependencies These instructions assume you have full permissions to your system. -If needed, use `sudo` where necessary. +If running as root, remove `sudo` where necessary. + #### CMAKE ```bash -cd /CMake && ./bootstrap -make -j && make install +cd $VDMS_DEP_DIR/CMake && ./bootstrap +make -j && sudo make install ``` ### Swig ```bash -cd /swig +cd $VDMS_DEP_DIR/swig ./autogen.sh && ./configure -make -j && make install +make -j && sudo make install ``` ### Faiss ```bash -cd /faiss +cd $VDMS_DEP_DIR/faiss mkdir build && cd build cmake -DFAISS_ENABLE_GPU=OFF .. -make -j && make install +make -j && sudo make install ``` ### FLINNG ```bash -cd /FLINNG +cd $VDMS_DEP_DIR/FLINNG mkdir build && cd build cmake .. -make -j && make install +make -j && sudo make install ``` ### grpc ```bash -cd /grpc && git submodule update --init --recursive -cd third_party/protobuf/cmake && mkdir build && cd build +cd $VDMS_DEP_DIR/grpc && git submodule update --init --recursive +pip3 install --no-cache-dir -r requirements.txt && \ + GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip3 install --no-cache-dir . + +cd tools/distrib/python/grpcio_tools +python ../make_grpcio_tools.py +GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip3 install --no-cache-dir . + +cd ../../../../third_party/protobuf/cmake +mkdir build && cd build cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. -make -j && make install +make -j && sudo make install cd ../../../abseil-cpp && mkdir build && cd build cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. -make -j && make install +make -j && sudo make install cd ../../re2/ && mkdir build && cd build cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. -make -j && make install +make -j && sudo make install cd ../../zlib/ && mkdir build && cd build cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. -make -j && make install +make -j && sudo make install -cd /grpc/cmake && mkdir build && cd build +cd ../../../cmake && mkdir build && cd build cmake -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_ABSL_PROVIDER=package \ -DgRPC_CARES_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package \ -DgRPC_RE2_PROVIDER=package -DgRPC_SSL_PROVIDER=package \ -DgRPC_ZLIB_PROVIDER=package -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE ../.. -make -j && make install -``` - -### Zlib -```bash -cd / && gunzip zlib-1.2.12.tar.gz && tar -xvf zlib-1.2.12.tar -cd zlib-1.2.12 && ./configure -make -j && make install -cd / && rm -rf zlib-1.2.12.tar zlib-1.2.12 -``` - -### gtest -Unfortunately apt doesn't build gtest; -you need to do the following steps to get it to work correctly: -```bash -cd /usr/src/gtest/ -cmake . -make -j -mv lib/libgtest* /usr/lib +make -j && sudo make install ``` ### [OpenCV](https://opencv.org/) Below are instructions for installing ***OpenCV v4.5.3***. It may also work with newer versions of OpenCV. ```bash -cd /opencv +cd $VDMS_DEP_DIR/opencv mkdir build && cd build -cmake -DBUILD_PERF_TESTS=OFF -DBUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. +cmake -DBUILD_PERF_TESTS=OFF -DBUILD_TESTS=OFF .. make -j -make install +sudo make install ``` **Note**: When using videos, and getting the following error: "Unable to stop the stream: Inappropriate ioctl for device", you may need to include more flags when compiling OpenCV. Follow these instructions ([source](https://stackoverflow.com/questions/41200201/opencv-unable-to-stop-the-stream-inappropriate-ioctl-for-device)): @@ -137,25 +128,41 @@ make -j make install ``` +### Zlib +```bash +cd $VDMS_DEP_DIR && tar -xvzf zlib-1.2.13.tar.gz +cd zlib-1.2.13 && ./configure +make -j && sudo make install +``` + ### [TileDB](https://tiledb.io/) VDMS works with ***TileDB v1.3.1.***
The directions below will help you install TileDB v1.3.1 from the source. You can also follow the directions listed [here](https://docs.tiledb.io/en/latest/installation.html). ```bash -cd / && tar xf 1.3.1.tar.gz && rm 1.3.1.tar.gz +cd $VDMS_DEP_DIR && tar -xvf 1.3.1.tar.gz cd TileDB-1.3.1 && mkdir build && cd build ../bootstrap --prefix=/usr/local/ -make -j && make install-tiledb -rm -rf /TileDB-1.3.1 +make -j && sudo make install-tiledb +``` + +### gtest +Unfortunately apt doesn't build gtest; +you need to do the following steps to get it to work correctly: +```bash +cd /usr/src/gtest/ +sudo cmake . +sudo make -j +sudo mv lib/libgtest* /usr/lib ``` ### Maven ```bash -ln -s /grpc/third_party/protobuf/cmake/build/protoc grpc/third_party/protobuf/src/protoc -cd /grpc/third_party/protobuf/java/core +ln -s $VDMS_DEP_DIR/grpc/third_party/protobuf/cmake/build/protoc $VDMS_DEP_DIR/grpc/third_party/protobuf/src/protoc +cd $VDMS_DEP_DIR/grpc/third_party/protobuf/java/core mvn package -cp target/protobuf-java-3.13.0.jar /usr/share/java/protobuf.jar +sudo cp "$(ls target/protobuf-java*.jar)" /usr/share/java/protobuf.jar ``` You may need to change proxy setting for Maven if you are behind a proxy like this example. @@ -178,25 +185,26 @@ Add setting.xml file to ~/.m2 folder ### Valijson This is a headers-only library, no compilation/installation necessary ```bash -cd /valijson -cp -r include/* /usr/local/include +cd $VDMS_DEP_DIR/valijson +sudo cp -r include/* /usr/local/include ``` ## Install VDMS +This version of VDMS treats PMGD as a submodule so both libraries are compiled at one time. After entering the vdms directory, the command `git submodule update --init --recursive` will pull pmgd into the appropriate directory. Furthermore, Cmake is used to compile all directories. ```bash git clone https://github.com/IntelLabs/vdms.git cd vdms && git checkout develop git submodule update --init --recursive +``` + +When compiling on a target without Optane persistent memory, use the following: +```bash mkdir build && cd build cmake .. make -j cp ../config-vdms.json . ``` - -### Compilation -This version of VDMS treats PMGD as a submodule so both libraries are compiled at one time. After entering the vdms directory, the command `git submodule update --init --recursive` will pull pmgd into the appropriate directory. Furthermore, Cmake is used to compile all directories. - When compiling on a target with Optane persistent memory, use the command set: ```bash mkdir build && cd build @@ -204,9 +212,3 @@ cmake -DCMAKE_CXX_FLAGS='-DPM' .. make -j ``` -For systems without Optane, use the command set: -```bash -mkdir build && cd build -cmake .. -make -j -``` diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile index 79590d2d..5016b6d6 100644 --- a/docker/base/Dockerfile +++ b/docker/base/Dockerfile @@ -41,7 +41,7 @@ RUN git clone --branch v3.21.2 https://github.com/Kitware/CMake.git && \ git clone --branch v0.6 https://github.com/tristanpenman/valijson.git && \ curl -L -o /usr/share/java/json-simple-1.1.1.jar https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/json-simple/json-simple-1.1.1.jar && \ curl -L -o /1.3.1.tar.gz https://github.com/TileDB-Inc/TileDB/archive/refs/tags/1.3.1.tar.gz && \ - curl -L -o /zlib-1.2.12.tar.gz http://zlib.net/zlib-1.2.12.tar.gz && \ + curl -L -o /zlib-1.2.13.tar.gz http://zlib.net/zlib-1.2.13.tar.gz && \ cd /CMake && ./bootstrap && make ${BUILD_THREADS} && make install && \ cd /swig && ./autogen.sh && ./configure && make ${BUILD_THREADS} && make install && \ cd /faiss && mkdir build && cd build && cmake -DFAISS_ENABLE_GPU=OFF .. && make ${BUILD_THREADS} && make install && \ @@ -57,7 +57,7 @@ RUN git clone --branch v3.21.2 https://github.com/Kitware/CMake.git && \ -DgRPC_RE2_PROVIDER=package -DgRPC_SSL_PROVIDER=package \ -DgRPC_ZLIB_PROVIDER=package -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE ../.. && make ${BUILD_THREADS} && make install && \ cd /opencv && mkdir build && cd build && cmake -D BUILD_PERF_TESTS=OFF -D BUILD_TESTS=OFF .. && make ${BUILD_THREADS} && make install && \ - cd / && gunzip zlib-1.2.12.tar.gz && tar -xvf /zlib-1.2.12.tar && cd /zlib-1.2.12 && ./configure && make ${BUILD_THREADS} && make install && \ + cd / && tar -xvzf zlib-1.2.13.tar.gz && cd /zlib-1.2.13 && ./configure && make ${BUILD_THREADS} && make install && \ cd / && tar -xvf /1.3.1.tar.gz && cd TileDB-1.3.1 && mkdir build && cd build && \ ../bootstrap --prefix=/usr/local/ && make ${BUILD_THREADS} && make install-tiledb && \ cd /usr/src/gtest && cmake . && make ${BUILD_THREADS} && mv lib/libgtest* /usr/lib/ && \ @@ -65,7 +65,7 @@ RUN git clone --branch v3.21.2 https://github.com/Kitware/CMake.git && \ cd /grpc/third_party/protobuf/java/core && mvn package && \ cp "$(ls target/protobuf-java*.jar)" /usr/share/java/protobuf.jar && \ cd /valijson && cp -r include/* /usr/local/include/ && \ - rm -rf /CMake /swig /faiss /FLINNG /grpc /opencv /zlib-1.2.12.tar /zlib-1.2.12 /1.3.1.tar.gz /TileDB-1.3.1 /valijson + rm -rf /CMake /swig /faiss /FLINNG /grpc /opencv /zlib-1.2.13.tar /zlib-1.2.13 /1.3.1.tar.gz /TileDB-1.3.1 /valijson # VDMS RUN git clone https://github.com/IntelLabs/vdms.git && cd vdms && \ diff --git a/docker/check-in/Dockerfile b/docker/check-in/Dockerfile index 4d494e7f..3e39150f 100644 --- a/docker/check-in/Dockerfile +++ b/docker/check-in/Dockerfile @@ -41,7 +41,7 @@ RUN git clone --branch v3.21.2 https://github.com/Kitware/CMake.git && \ git clone --branch v0.6 https://github.com/tristanpenman/valijson.git && \ curl -L -o /usr/share/java/json-simple-1.1.1.jar https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/json-simple/json-simple-1.1.1.jar && \ curl -L -o /1.3.1.tar.gz https://github.com/TileDB-Inc/TileDB/archive/refs/tags/1.3.1.tar.gz && \ - curl -L -o /zlib-1.2.12.tar.gz http://zlib.net/zlib-1.2.12.tar.gz && \ + curl -L -o /zlib-1.2.13.tar.gz http://zlib.net/zlib-1.2.13.tar.gz && \ cd /CMake && ./bootstrap && make ${BUILD_THREADS} && make install && \ cd /swig && ./autogen.sh && ./configure && make ${BUILD_THREADS} && make install && \ cd /faiss && mkdir build && cd build && cmake -DFAISS_ENABLE_GPU=OFF .. && make ${BUILD_THREADS} && make install && \ @@ -57,7 +57,7 @@ RUN git clone --branch v3.21.2 https://github.com/Kitware/CMake.git && \ -DgRPC_RE2_PROVIDER=package -DgRPC_SSL_PROVIDER=package \ -DgRPC_ZLIB_PROVIDER=package -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE ../.. && make ${BUILD_THREADS} && make install && \ cd /opencv && mkdir build && cd build && cmake -D BUILD_PERF_TESTS=OFF -D BUILD_TESTS=OFF .. && make ${BUILD_THREADS} && make install && \ - cd / && gunzip zlib-1.2.12.tar.gz && tar -xvf /zlib-1.2.12.tar && cd /zlib-1.2.12 && ./configure && make ${BUILD_THREADS} && make install && \ + cd / && tar -xvzf zlib-1.2.13.tar.gz && cd /zlib-1.2.13 && ./configure && make ${BUILD_THREADS} && make install && \ cd / && tar -xvf /1.3.1.tar.gz && cd TileDB-1.3.1 && mkdir build && cd build && \ ../bootstrap --prefix=/usr/local/ && make ${BUILD_THREADS} && make install-tiledb && \ cd /usr/src/gtest && cmake . && make ${BUILD_THREADS} && mv lib/libgtest* /usr/lib/ && \ @@ -65,7 +65,7 @@ RUN git clone --branch v3.21.2 https://github.com/Kitware/CMake.git && \ cd /grpc/third_party/protobuf/java/core && mvn package && \ cp "$(ls target/protobuf-java*.jar)" /usr/share/java/protobuf.jar && \ cd /valijson && cp -r include/* /usr/local/include/ && \ - rm -rf /CMake /swig /faiss /FLINNG /grpc /opencv /zlib-1.2.12.tar /zlib-1.2.12 /1.3.1.tar.gz /TileDB-1.3.1 /valijson + rm -rf /CMake /swig /faiss /FLINNG /grpc /opencv /zlib-1.2.13.tar /zlib-1.2.13 /1.3.1.tar.gz /TileDB-1.3.1 /valijson # VDMS COPY . /vdms diff --git a/docker/check-in/Dockerfile.base b/docker/check-in/Dockerfile.base new file mode 100644 index 00000000..d5ad2469 --- /dev/null +++ b/docker/check-in/Dockerfile.base @@ -0,0 +1,79 @@ +#Copyright (C) 2021 Intel Corporation +#SPDX-License-Identifier: MIT + +ARG UBUNTU_VERSION=20.04 +ARG UBUNTU_NAME=focal +ARG BUILD_THREADS=-j16 +ARG MAVEN_OPTS='-Dhttps.nonProxyHosts="localhost|127.0.0.1"' + +#1 +FROM ubuntu:${UBUNTU_VERSION} + +# Dockerfile limitations force a repetition of global args +ARG UBUNTU_VERSION +ARG UBUNTU_NAME +ARG MAVEN_OPTS +WORKDIR / + +# Install Packages +RUN apt-get update && apt-get -y install --no-install-recommends software-properties-common && \ + add-apt-repository "deb http://security.ubuntu.com/ubuntu ${UBUNTU_NAME}-security main" && \ + apt-get -y install --no-install-recommends apt-transport-https autoconf automake bison build-essential \ + bzip2 ca-certificates curl ed flex g++ git gnupg-agent javacc libarchive-tools \ + libatlas-base-dev libavcodec-dev libavformat-dev libboost-all-dev libbz2-dev \ + libc-ares-dev libdc1394-22-dev libgflags-dev libgoogle-glog-dev libgtest-dev \ + libgtk-3-dev libgtk2.0-dev libhdf5-serial-dev libjpeg-dev libjpeg8-dev libjsoncpp-dev \ + libleveldb-dev liblmdb-dev liblz4-dev libopenblas-dev libopenmpi-dev \ + libpng-dev librdkafka-dev libsnappy-dev libssl-dev libswscale-dev libtbb-dev \ + libtbb2 libtiff-dev libtiff5-dev libtool maven mpich openjdk-11-jdk-headless \ + pkg-config python3-dev python3-pip unzip && \ + apt-get clean && rm -rf /var/lib/apt/lists/* && \ + update-alternatives --install /usr/bin/python python /usr/bin/python3 1 && \ + pip3 install --no-cache-dir "numpy>=1.23.2" + +# Pull and Install Dependencies +RUN git clone --branch v3.21.2 https://github.com/Kitware/CMake.git && \ + git clone --branch v4.0.2 https://github.com/swig/swig.git && \ + git clone --branch v1.7.1 https://github.com/facebookresearch/faiss.git && \ + git clone https://github.com/tonyzhang617/FLINNG.git && \ + git clone --branch v1.40.0 https://github.com/grpc/grpc.git && \ + git clone --branch 4.5.3 https://github.com/opencv/opencv.git && \ + git clone --branch v0.6 https://github.com/tristanpenman/valijson.git && \ + curl -L -o /usr/share/java/json-simple-1.1.1.jar https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/json-simple/json-simple-1.1.1.jar && \ + curl -L -o /1.3.1.tar.gz https://github.com/TileDB-Inc/TileDB/archive/refs/tags/1.3.1.tar.gz && \ + curl -L -o /zlib-1.2.13.tar.gz http://zlib.net/zlib-1.2.13.tar.gz && \ + cd /CMake && ./bootstrap && make ${BUILD_THREADS} && make install && \ + cd /swig && ./autogen.sh && ./configure && make ${BUILD_THREADS} && make install && \ + cd /faiss && mkdir build && cd build && cmake -DFAISS_ENABLE_GPU=OFF .. && make ${BUILD_THREADS} && make install && \ + cd /FLINNG && mkdir build && cd build && cmake .. && make ${BUILD_THREADS} && make install && \ + cd /grpc && git submodule update --init --recursive && pip3 install --no-cache-dir -r requirements.txt && GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip3 install --no-cache-dir . && \ + cd tools/distrib/python/grpcio_tools && python ../make_grpcio_tools.py && GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip3 install --no-cache-dir . && \ + cd /grpc/third_party/protobuf/cmake && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \ + cd ../../../abseil-cpp && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \ + cd ../../re2/ && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \ + cd ../../zlib/ && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \ + cd /grpc/cmake && mkdir build && cd build && cmake -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_ABSL_PROVIDER=package \ + -DgRPC_CARES_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package \ + -DgRPC_RE2_PROVIDER=package -DgRPC_SSL_PROVIDER=package \ + -DgRPC_ZLIB_PROVIDER=package -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE ../.. && make ${BUILD_THREADS} && make install && \ + cd /opencv && mkdir build && cd build && cmake -D BUILD_PERF_TESTS=OFF -D BUILD_TESTS=OFF .. && make ${BUILD_THREADS} && make install && \ + cd / && tar -xvzf zlib-1.2.13.tar.gz && cd /zlib-1.2.13 && ./configure && make ${BUILD_THREADS} && make install && \ + cd / && tar -xvf /1.3.1.tar.gz && cd TileDB-1.3.1 && mkdir build && cd build && \ + ../bootstrap --prefix=/usr/local/ && make ${BUILD_THREADS} && make install-tiledb && \ + cd /usr/src/gtest && cmake . && make ${BUILD_THREADS} && mv lib/libgtest* /usr/lib/ && \ + ln -s /grpc/third_party/protobuf/cmake/build/protoc /grpc/third_party/protobuf/src/protoc && \ + cd /grpc/third_party/protobuf/java/core && mvn package && \ + cp "$(ls target/protobuf-java*.jar)" /usr/share/java/protobuf.jar && \ + cd /valijson && cp -r include/* /usr/local/include/ && \ + rm -rf /CMake /swig /faiss /FLINNG /grpc /opencv /zlib-1.2.13.tar /zlib-1.2.13 /1.3.1.tar.gz /TileDB-1.3.1 /valijson + +# VDMS +COPY . /vdms +RUN [ -d /vdms/build ]; rm -rf /vdms/build && \ + cd /vdms && git submodule update --init --recursive && mkdir build && \ + cd build && cmake .. && make ${BUILD_THREADS} && \ + cp /vdms/config-vdms.json /vdms/build/ && \ + echo '#!/bin/bash' > /start.sh && echo 'cd /vdms/build' >> /start.sh && \ + echo './vdms' >> /start.sh && chmod 755 /start.sh + +CMD ["/start.sh"] diff --git a/docker/check-in/spdx2csv.py b/docker/check-in/spdx2csv.py new file mode 100644 index 00000000..b1f9ac62 --- /dev/null +++ b/docker/check-in/spdx2csv.py @@ -0,0 +1,73 @@ +import csv +import argparse + +header=['Package', 'Version', 'License', 'Package Supplier', 'SPDXID'] + + +def get_parameters(): + obj = argparse.ArgumentParser() + obj.add_argument('-i', type=str, dest='INPUT_FILE', + default='docker/check-in/vdms_docker_sbom.txt', + help='Path to SBOM') + obj.add_argument('-o', type=str, dest='OUTPUT_FILE', + default='docker/check-in/vdms_docker_sbom.csv', + help='Path to output SBOM as CSV') + + params = obj.parse_args() + return params + + +def remove_newline(line): + if "\n" in line: + return line.replace("\n","") + return line + + +def main(args): + output_fh = open(args.OUTPUT_FILE, 'w', newline='', encoding='utf-8') + csv_writer = csv.writer(output_fh) + csv_writer.writerow(header) + + rows = [] + default_val = "" + with open(args.INPUT_FILE, 'r') as fh: + # Skip File info + for line in fh: + if line in ['\n','\r\n']: + break + + # Parse remaining lines + for line in fh: + pkg_str = "##### Package: " + if line.startswith(pkg_str): + package_name = remove_newline(line[len(pkg_str):]) + + ver_str = "PackageVersion: " + if line.startswith(ver_str): + version_num = remove_newline(line[len(ver_str):]) + + lic_str = "PackageLicenseConcluded: " + if line.startswith(lic_str): + license_names = remove_newline(line[len(lic_str):]) + + extref_str = "ExternalRef: PACKAGE_MANAGER purl pkg:" + if line.startswith(extref_str): + package_type = remove_newline(line.split("/")[0].replace(extref_str,"")) + # row = ",".join([package_name, version_num, license_names, package_type, spdxid]) + rows.append([package_name, version_num, license_names, package_type, spdxid]) + package_name, version_num, license_names, package_type, spdxid = default_val, default_val, default_val, default_val, default_val + + spdxid_str = "SPDXID: " + if line.startswith(spdxid_str): + spdxid = remove_newline(line[len(spdxid_str):]) + + # Write rows + csv_writer.writerows(rows) + + # Close output file + output_fh.close() + + +if __name__ == '__main__': + args = get_parameters() + main(args)