diff --git a/.github/workflows/buildAndTest.yml b/.github/workflows/self-hosted-build-and-test.yml similarity index 69% rename from .github/workflows/buildAndTest.yml rename to .github/workflows/self-hosted-build-and-test.yml index c5311f7cdf4..d3deedbf505 100644 --- a/.github/workflows/buildAndTest.yml +++ b/.github/workflows/self-hosted-build-and-test.yml @@ -1,6 +1,6 @@ # This is a basic workflow to help you get started with Actions -name: Build and Test +name: Self-hosted Build and Test # Controls when the action will run. Triggers the workflow on push or pull request # events but only for the master branch @@ -34,18 +34,13 @@ jobs: run: clang --version shell: bash - - name: Get workflow spec hash - id: get-workflow-hash - run: echo "::set-output name=hash::$(md5sum $GITHUB_WORKSPACE/.github/workflows/buildAndTest.yml)" - shell: bash - # Try to fetch LLVM from the cache. - name: Cache LLVM id: cache-llvm uses: actions/cache@v2 with: path: llvm/build - key: ${{ runner.os }}-llvm-${{ steps.get-llvm-hash.outputs.hash }}-${{ steps.get-workflow-hash.outputs.hash }} + key: ${{ runner.os }}-llvm-${{ steps.get-llvm-hash.outputs.hash }} # Build LLVM if we didn't hit in the cache. - name: Rebuild and Install LLVM @@ -60,22 +55,10 @@ jobs: # Build Phism and run its tests. build-phism: - name: Build and Test Phism + name: Build Phism needs: build-llvm runs-on: self-hosted steps: - # - name: Configure Environment - # run: echo "${GITHUB_WORKSPACE}/llvm/install/bin" >> $GITHUB_PATH - # - name: Get dependences - # run: | - # sudo apt-get update -y - # sudo apt-get install -y build-essential libtool autoconf pkg-config flex bison libgmp-dev clang-9 libclang-9-dev texinfo - # - name: Update the LLVM/Clang version to 9 - # run: | - # sudo update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-9 100 - # sudo update-alternatives --install /usr/bin/FileCheck FileCheck /usr/bin/FileCheck-9 100 - - # Clone the Phism repo and its submodules. Do shallow clone to save clone # time. - name: Get Phism @@ -93,18 +76,13 @@ jobs: run: echo "::set-output name=hash::$(git rev-parse @:./llvm)" shell: bash - - name: Get workflow spec hash - id: get-workflow-hash - run: echo "::set-output name=hash::$(md5sum $GITHUB_WORKSPACE/.github/workflows/buildAndTest.yml)" - shell: bash - # Try to fetch LLVM from the cache. - name: Cache LLVM id: cache-llvm uses: actions/cache@v2 with: path: llvm/build - key: ${{ runner.os }}-llvm-${{ steps.get-llvm-hash.outputs.hash }}-${{ steps.get-workflow-hash.outputs.hash }} + key: ${{ runner.os }}-llvm-${{ steps.get-llvm-hash.outputs.hash }} # Build LLVM if we didn't hit in the cache. Even though we build it in # the previous job, there is a low chance that it'll have been evicted by @@ -115,9 +93,8 @@ jobs: ./scripts/build-llvm.sh ci # -------- - # Build and test Phism in both debug and release mode. + # Build Phism in debug mode. # -------- - - name: Build and Test Phism (Assert) + - name: Build Phism run: | ./scripts/build-phism.sh ci - diff --git a/.gitignore b/.gitignore index 6838d7f9651..427bb33c7ff 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,6 @@ build/ # Editor .vscode/ + +# Temporary output +/tmp* diff --git a/Docker/Dockerfile b/Docker/Dockerfile new file mode 100644 index 00000000000..3698e20a082 --- /dev/null +++ b/Docker/Dockerfile @@ -0,0 +1,33 @@ +FROM ubuntu:20.04 +ARG GID +ARG UID +RUN echo "Group ID: $GID" +RUN echo "User ID: $UID" + +USER root +RUN apt-get update +RUN DEBIAN_FRONTEND="noninteractive" apt-get -y install tzdata --assume-yes + +# Install Essential Packages +RUN apt-get update +RUN apt-get install build-essential libtool autoconf pkg-config flex bison libgmp-dev clang-9 libclang-9-dev texinfo cmake vim ninja-build git libncurses5 gcc-multilib --assume-yes + +RUN update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-9 100 +RUN update-alternatives --install /usr/bin/FileCheck FileCheck /usr/bin/FileCheck-9 100 +RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-9 100 +RUN update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-9 100 + +CMD ["bash"] + +RUN apt-get install sudo --assume-yes + +# Add dev-user +RUN groupadd -g $GID dev-user +RUN useradd -r -g $GID -u $UID -m -d /home/dev-user -s /sbin/nologin -c "User" dev-user +RUN echo "dev-user ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers +USER dev-user + +RUN echo 'PATH=$PATH:/workspace/llvm/build/bin:/workspace/build/bin:/tools/Vitis_HLS/2020.2/bin' >> /home/dev-user/.bashrc +RUN echo 'export LIBRARY_PATH=/usr/lib/x86_64-linux-gnu:$LIBRARY_PATH' >> /home/dev-user/.bashrc +RUN echo 'export LD_LIBRARY_PATH=""' >> /home/dev-user/.bashrc +WORKDIR workspace diff --git a/Makefile b/Makefile new file mode 100644 index 00000000000..e5645732c16 --- /dev/null +++ b/Makefile @@ -0,0 +1,66 @@ +user=$(if $(shell id -u),$(shell id -u),9001) +group=$(if $(shell id -g),$(shell id -g),1000) +phism=/workspace +vhls=~/tools/Xilinx/2020.2 + +# Build Phism +build-docker: test-docker + docker run -it -v $(shell pwd):/workspace -v $(vhls):/tools phism20:latest /bin/bash \ + -c "make build_" + echo "Phism has been installed successfully!" + +# Clone submodule and build docker container +test-docker: + git submodule update --init --recursive + (cd Docker; docker build --no-cache --build-arg UID=$(user) --build-arg GID=$(group) . --tag phism20) + +# Enter docker container +shell: + docker run -it -v $(shell pwd):/workspace -v $(vhls):/tools phism20:latest /bin/bash + +# Evaluate polybench (baseline) - need to be used in environment +test-polybench: + ./scripts/pb-flow ./example/polybench -c COSIM 2>&1 | tee phism-test.log + +# Evaluate polybench (polymer) - need to be used in environment +test-polybench-polymer: + ./scripts/pb-flow ./example/polybench -p USE POLYMER -c COSIM 2>&1 | tee phism-test.log + +# Build LLVM and Phism +build_: + set -e # Abort if one of the commands fail + mkdir -p $(phism)/llvm/build + (cd $(phism)/llvm/build; \ + cmake ../llvm \ + -DCMAKE_C_COMPILER=gcc \ + -DCMAKE_CXX_COMPILER=g++ \ + -DLLVM_ENABLE_PROJECTS="mlir;llvm;clang;clang-extra-tools" \ + -DCMAKE_BUILD_TYPE=DEBUG \ + -DLLVM_BUILD_EXAMPLES=OFF \ + -DLLVM_TARGETS_TO_BUILD="host" \ + -DLLVM_OPTIMIZED_TABLEGEN=ON \ + -DLLVM_ENABLE_OCAMLDOC=OFF \ + -DLLVM_ENABLE_BINDINGS=OFF \ + -DLLVM_INSTALL_UTILS=ON \ + -DBUILD_POLYMER=ON \ + -DPLUTO_LIBCLANG_PREFIX=$(shell llvm-config-9 --prefix) \ + -DLLVM_ENABLE_ASSERTIONS=ON; \ + cmake --build . --target all -- -j 32) + mkdir -p $(phism)/build + (cd $(phism)/build; \ + cmake .. \ + -DCMAKE_BUILD_TYPE=Debug \ + -DLLVM_ENABLE_ASSERTIONS=ON \ + -DMLIR_DIR=$(phism)/llvm/build/lib/cmake/mlir/ \ + -DLLVM_DIR=$(phism)/llvm/build/lib/cmake/llvm/ \ + -DCMAKE_C_COMPILER=clang-9 \ + -DCMAKE_CXX_COMPILER=clang++-9 \ + -DLLVM_EXTERNAL_LIT=$(phism)/llvm/build/bin/llvm-lit \ + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON; \ + cmake --build . --target check-phism -- -j 32) + +clean: clean_phism + rm -rf $(phism)/llvm/build + +clean_phism: + rm -rf $(phism)/build diff --git a/README.md b/README.md index 7ef89014f8e..32a1ba57b71 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,19 @@ It should run the Phism regression test in the end. And if all the tests passed, ## Usage -[pb-flow](scripts/pb-flow) provides a CLI utility to test Phism with Polybench examples. You can grab a rough idea about the whole Phism pipeline over there. +### The `pb-flow` script -More usage info will be added later. +[pb-flow](scripts/pb-flow) provides a CLI utility to test Phism with Polybench examples. You can grab a rough idea about the whole Phism pipeline over there. You can use `pb-flow` in the following ways: + +```sh +./scripts/pb-flow example/polybench # Run all polybench synth-only, w/o Polyhedral optimization. +./scripts/pb-flow example/polybench -p # Run all polybench synth-only, w/ Polyhedral optimization. +./scripts/pb-flow example/polybench -c # Run all polybench w/ cosim, w/ Polyhedral optimization. +./scripts/pb-flow example/polybench -pc # Run all polybench w/ cosim, w/o Polyhedral optimization. +``` + +If you attach `-d`, the build effort won't be set to `high`. This can save some time. + +### Using Docker + +This [doc](docs/DOCKER.md) gives an introduction on how to run Phism with docker. diff --git a/docs/DOCKER.md b/docs/DOCKER.md new file mode 100644 index 00000000000..a2b89db231d --- /dev/null +++ b/docs/DOCKER.md @@ -0,0 +1,25 @@ +--- +Author: Jianyi Cheng +--- + +# Build with Docker + +To build a Docker container with Phism installed: +```sh +make build-docker +``` + +To use Phism in the Docker container: +``` +make shell vhls=${YOUR_VITIS_DIR} +``` +PS: To check your `${YOUR_VITIS_DIR}`, you should see the following when run: +``` +$ ls ${YOUR_VITIS_DIR} +DocNav Model_Composer Vitis Vitis_HLS Vivado xic +``` + +For instance, to run Polybench: +``` +make test-polybench +``` diff --git a/llvm b/llvm index c1094cd9bd2..4d85b8e1a15 160000 --- a/llvm +++ b/llvm @@ -1 +1 @@ -Subproject commit c1094cd9bd222c481a6525551c8c1b4b79000d99 +Subproject commit 4d85b8e1a15ec521171dcacdb3e2dad11ecb1590 diff --git a/scripts/build-llvm.sh b/scripts/build-llvm.sh index 0e0e1b8cbee..d5b714ecfc2 100755 --- a/scripts/build-llvm.sh +++ b/scripts/build-llvm.sh @@ -11,6 +11,12 @@ echo "" TARGET="${1:-"local"}" +# If ninja is available, use it. +CMAKE_GENERATOR="Unix Makefiles" +if which ninja &>/dev/null; then + CMAKE_GENERATOR="Ninja" +fi + # The absolute path to the directory of this script. DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" @@ -30,8 +36,10 @@ mkdir -p build cd build # Configure CMake -CC=gcc CXX=g++ cmake ../llvm \ - -DLLVM_ENABLE_PROJECTS="mlir;llvm;clang;clang-extra-tools" \ +export CC=gcc +export CXX=g++ +cmake ../llvm \ + -DLLVM_ENABLE_PROJECTS="mlir;llvm;clang" \ -DCMAKE_BUILD_TYPE=RELEASE \ -DLLVM_BUILD_EXAMPLES=OFF \ -DLLVM_TARGETS_TO_BUILD="host" \ @@ -41,7 +49,8 @@ CC=gcc CXX=g++ cmake ../llvm \ -DLLVM_INSTALL_UTILS=ON \ -DLLVM_ENABLE_ASSERTIONS=ON \ -DBUILD_POLYMER=ON \ - -DPLUTO_LIBCLANG_PREFIX="$(llvm-config --prefix)" + -DPLUTO_LIBCLANG_PREFIX="$(llvm-config --prefix)" \ + -G "${CMAKE_GENERATOR}" # Run building cmake --build . --target all -- -j "$(nproc)" diff --git a/scripts/pb-flow b/scripts/pb-flow index 6c52f1ec62b..d5530074d2d 100755 --- a/scripts/pb-flow +++ b/scripts/pb-flow @@ -11,7 +11,9 @@ set -o nounset # ---------------------- GLOBALS -------------------------------- TIMESTAMP="$(date "+%Y%m%d-%H%M%S")" + DIR="$(cd "$(dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd)" +WORKDIR="${DIR}/../tmp/phism/pb-flow.${TIMESTAMP}" LLVM_DIR="${DIR}/../llvm" LLVM_BINDIR="${LLVM_DIR}/build/bin" LLVM_LIBDIR="${LLVM_DIR}/build/lib" @@ -31,8 +33,29 @@ if [ ! -d "${POLYBENCH_DIR}" ]; then echo "Please pass in a valid polybench directory as #1 arg." exit 1 fi - -ENABLE_POLYMER="${2:-0}" +shift # Remove POLYBENCH_DIR from the argument list. + +# Variables to set. +ENABLE_POLYMER="0" +ENABLE_COSIM="0" +DEBUG="0" + +while getopts ":dpc" opt; do + case ${opt} in + d) + DEBUG="1" + ;; + p) + ENABLE_POLYMER="1" + ;; + c) + ENABLE_COSIM="1" + ;; + \?) + echo "Usage: pb-flow [-p USE POLYMER] [-c COSIM] [-d DEBUG]" + ;; + esac +done # ---------------------- FUNCTIONS ------------------------------ @@ -53,12 +76,25 @@ function compile_c_to_mlir() { mlir-clang "${src_file}" \ -memref-fullrank \ + -D MINI_DATASET \ -I="${LLVM_LIBDIR}/clang/13.0.0/include" \ -I="${POLYBENCH_DIR}/utilities" > "${dst_file}" echo "${dst_file}" } +# Do some preprocessing before extracting top function. +function preprocess_mlir() { + local src_file="${1}" + local dst_file="${src_file%.mlir}.pre.mlir" + + mlir-opt "${src_file}" \ + -sccp \ + -canonicalize > "${dst_file}" + + echo "${dst_file}" +} + # Extract the top function and all the stuff it calls. function extract_top_func_from_mlir() { local src_file="${1}" @@ -83,7 +119,7 @@ function polymer_opt() { -insert-redundant-load \ -extract-scop-stmt \ -pluto-opt \ - -inline 2>/dev/null > "${dst_file}" + 2>"${dst_file%.mlir}.log" > "${dst_file}" else cp "${src_file}" "${dst_file}" fi @@ -113,11 +149,6 @@ function opt_llvm_for_vitis() { local top_func top_func="$(get_top_func "${src_file}")" - # Polymer optimized functions have an _opt suffix. - if [[ ${ENABLE_POLYMER} -eq 1 ]]; then - top_func="${top_func}_opt" - fi - "${LLVM_BINDIR}/opt" "${src_file}" \ -S \ -enable-new-pm=0 \ @@ -145,11 +176,6 @@ function gen_dummy_c() { dst_file="$(dirname "${src_file}")/${src_base}.dummy.c" top_func="$(get_top_func "${src_file}")" - # Polymer optimized functions have an _opt suffix. - if [[ ${ENABLE_POLYMER} -eq 1 ]]; then - top_func="${top_func}_opt" - fi - cat < "${dst_file}" void ${top_func}() {} EOF @@ -157,36 +183,115 @@ EOF echo "${dst_file}" } -# Generate the run_hls.tcl file -function gen_vitis_tcl() { +# Generate run_hls.tcl that passes Phism generated LLVM-IR to Vitis. +# Returns the full file path of run_hls.tcl +function gen_vitis_phism_tcl() { local src_file="${1}" local src_base + local src_dir local dst_file local top_func local dummy_c_src + local config="config_bind -effort high" - src_base="$(basename "$(dirname "${src_file}")")" + # Disable high effort in debug mode. + if [[ ${DEBUG} -eq 1 ]]; then + config="" + fi + + src_dir="$(dirname "${src_file}")" + src_base="$(basename "${src_dir}")" dst_file="$(dirname "${src_file}")/run_hls.tcl" top_func="$(get_top_func "${src_file}")" dummy_c_src="$(gen_dummy_c "${src_file}")" - # Polymer optimized functions have an _opt suffix. - if [[ ${ENABLE_POLYMER} -eq 1 ]]; then - top_func="${top_func}_opt" - fi - -cat < "${dst_file}" + # Synthesis script + cat < "${dst_file}" open_project -reset proj add_files ${dummy_c_src} set_top ${top_func} open_solution -reset solution1 set_part "zynq" -create_clock -period "75MHz" +create_clock -period "100MHz" +${config} + +set ::LLVM_CUSTOM_CMD {\$LLVM_CUSTOM_OPT -no-warn ${src_file} -o \$LLVM_CUSTOM_OUTPUT} + +csynth_design + +exit +EOF + + echo "${dst_file}" +} + +# Generate tb_gen.tcl that generates the test bench. +# Returns the full file path of tb_gen.tcl +function gen_vitis_tbgen_tcl() { + local src_file="${1}" + local src_base + local src_dir + local dst_file + local top_func + local polybench_dir="${WORKDIR}" + local config="config_bind -effort high" + + src_dir="$(dirname "${src_file}")" + src_base="$(basename "${src_dir}")" + dst_file="$(dirname "${src_file}")/tb_gen.tcl" + top_func="$(get_top_func "${src_file}")" + + # Disable high effort in debug mode. + if [[ ${DEBUG} -eq 1 ]]; then + config="" + fi -set ::LLVM_CUSTOM_CMD {\$LLVM_CUSTOM_OPT -mem2reg ${src_file} -o \$LLVM_CUSTOM_OUTPUT} + # Remove 'static' in all functions - the top design cannot be static + sed -i 's/static//g' "${src_dir}/${src_base}.c" + # Testbench generation script / C baseline + # TODO: need to handle these lib and polybench.c properly + cat < "${dst_file}" +open_project -reset tb +add_files {${src_dir}/${src_base}.c} -cflags "-I ${src_dir} -I ${polybench_dir}/utilities -DMINI_DATASET" -csimflags "-I ${src_dir} -I ${polybench_dir}/utilities -DMINI_DATASET" +add_files -tb {${src_dir}/${src_base}.c ${polybench_dir}/utilities/polybench.c} -cflags "-I ${src_dir} -I ${polybench_dir}/utilities -DMINI_DATASET" -csimflags "-I ${src_dir} -I ${polybench_dir}/utilities -DMINI_DATASET" +set_top ${top_func} + +open_solution -reset solution1 +set_part "zynq" +create_clock -period "100MHz" +${config} + +csim_design csynth_design +cosim_design + +exit +EOF + + echo "${dst_file}" +} + +# Generate cosim.tcl file that generates the test bench. +# Returns the full file path of cosim.tcl +function gen_vitis_cosim_tcl() { + local src_file="${1}" + local src_base + local src_dir + local dst_file + + src_dir="$(dirname "${src_file}")" + src_base="$(basename "${src_dir}")" + dst_file="$(dirname "${src_file}")/cosim.tcl" + + cat < "${dst_file}" +open_project tb + +open_solution solution1 + +cosim_design + exit EOF @@ -196,12 +301,54 @@ EOF # Run Vitis. function run_vitis() { local src_file="${1}" + local phism_tcl_file local src_dir src_dir="$(dirname "${src_file}")" + phism_tcl_file="$(gen_vitis_phism_tcl "${src_file}")" cd "${src_dir}" - vitis_hls "${src_file}" &>/dev/null + + # Synthesize for Phism + vitis_hls "${phism_tcl_file}" &> "${src_dir}"/vhls.syn.log + + # Generate test bench + if [[ ${ENABLE_COSIM} -eq 1 ]]; then + local tbgen_tcl_file + local cosim_tcl_file + + tbgen_tcl_file="$(gen_vitis_tbgen_tcl "${src_file}")" + cosim_tcl_file="$(gen_vitis_cosim_tcl "${src_file}")" + + vitis_hls "${tbgen_tcl_file}" &> "${src_dir}"/vhls.tbgen.log + if grep -q "C/RTL co-simulation finished: FAIL" "${src_dir}"/vhls.tbgen.log; then + echo "*** C/RTL co-simulation finished: C BASELINE - FAIL ***" + elif ! grep -q "C/RTL co-simulation finished: PASS" "${src_dir}"/vhls.tbgen.log; then + echo "*** C/RTL co-simulation failed: C BASELINE - UNKNOWN ERROR ***" + fi + + if [ ! -d "${src_dir}"/tb/solution1/sim/verilog/ ]; then + echo "tbgen failed to create the target directory: " + echo " ${src_dir}/tb/solution1/sim/verilog/" + exit 1 + fi + + # Copy the kernel from Phism to the test bench + cp "${src_dir}"/proj/solution1/syn/verilog/*.v* "${src_dir}"/tb/solution1/sim/verilog/ + + # Run co-simulation + vitis_hls "${cosim_tcl_file}" &> "${src_dir}"/vhls.cosim.log + if grep -q "C/RTL co-simulation finished: FAIL" "${src_dir}"/vhls.cosim.log; then + echo "*** C/RTL co-simulation finished: PHISM - FAIL ***" + elif ! grep -q "C/RTL co-simulation finished: PASS" "${src_dir}"/vhls.cosim.log; then + echo "*** C/RTL co-simulation failed: PHISM - UNKNOWN ERROR ***" + fi + + fi + + # Read time from vitis_hls.log + # e.g. $finish called at time : 13920290 ps + # JC: there is a formula to convert this to cycles, but I do not remember now - it should be OK for now local status=$? @@ -220,14 +367,14 @@ function eval_file() { local kern_src_file mlir_src_file="$(compile_c_to_mlir "${src_file}")" - kern_src_file="$(extract_top_func_from_mlir "${mlir_src_file}")" + prep_src_file="$(preprocess_mlir "${mlir_src_file}")" + kern_src_file="$(extract_top_func_from_mlir "${prep_src_file}")" poly_src_file="$(polymer_opt "${kern_src_file}")" llvm_src_file="$(lower_mlir_to_llvm "${poly_src_file}")" vitis_src_file="$(opt_llvm_for_vitis "${llvm_src_file}")" - vitis_tcl_file="$(gen_vitis_tcl "${vitis_src_file}")" local status - status="$(run_vitis "${vitis_tcl_file}")" + status="$(run_vitis "${vitis_src_file}")" if [[ "${status}" = "0" ]]; then echo " SUCCESS" @@ -274,7 +421,6 @@ function copy_source_files() { # Main entry. function main() { local polybench_dir="${1}" - local tmp_dir="/tmp/phism/pb-flow.${TIMESTAMP}" # Welcome messages. echo "" @@ -283,11 +429,11 @@ function main() { echo "Setup:" echo " TIMESTAMP: ${TIMESTAMP}" echo " POLYBENCH_DIR: ${polybench_dir}" - echo " WORKING_DIR: ${tmp_dir}" + echo " WORKING_DIR: ${WORKDIR}" echo "" - copy_source_files "${polybench_dir}" "${tmp_dir}" - eval_dir "${tmp_dir}" + copy_source_files "${polybench_dir}" "${WORKDIR}" + eval_dir "${WORKDIR}" } # ----------------------- MAIN -------------------------