diff --git a/cmake/external/brpc.cmake b/cmake/external/brpc.cmake index f806b30252969..03e8e36f8dcab 100755 --- a/cmake/external/brpc.cmake +++ b/cmake/external/brpc.cmake @@ -13,7 +13,7 @@ # limitations under the License. include(ExternalProject) - +set(OPENSSL_USE_STATIC_LIBS ON) find_package(OpenSSL REQUIRED) message(STATUS "ssl:" ${OPENSSL_SSL_LIBRARY}) diff --git a/tools/coverage/paddle_coverage.sh b/tools/coverage/paddle_coverage.sh index 016e076e7d5ad..81ff6eb173456 100644 --- a/tools/coverage/paddle_coverage.sh +++ b/tools/coverage/paddle_coverage.sh @@ -33,7 +33,7 @@ make install cd /paddle/build -python3.7 ${PADDLE_ROOT}/tools/coverage/gcda_clean.py ${GIT_PR_ID} || exit 101 +python ${PADDLE_ROOT}/tools/coverage/gcda_clean.py ${GIT_PR_ID} || exit 101 lcov --capture -d ./ -o coverage.info --rc lcov_branch_coverage=0 @@ -125,9 +125,9 @@ fi function gen_diff_html_report() { if [ "${GIT_PR_ID}" != "" ]; then - COVERAGE_DIFF_PATTERN="`python3.7 ${PADDLE_ROOT}/tools/coverage/pull_request.py files ${GIT_PR_ID}`" + COVERAGE_DIFF_PATTERN="`python ${PADDLE_ROOT}/tools/coverage/pull_request.py files ${GIT_PR_ID}`" - python3.7 ${PADDLE_ROOT}/tools/coverage/pull_request.py diff ${GIT_PR_ID} > git-diff.out + python ${PADDLE_ROOT}/tools/coverage/pull_request.py diff ${GIT_PR_ID} > git-diff.out fi lcov --extract coverage-full.info \ @@ -135,7 +135,7 @@ function gen_diff_html_report() { -o coverage-diff.info \ --rc lcov_branch_coverage=0 - python3.7 ${PADDLE_ROOT}/tools/coverage/coverage_diff.py coverage-diff.info git-diff.out > coverage-diff.tmp + python ${PADDLE_ROOT}/tools/coverage/coverage_diff.py coverage-diff.info git-diff.out > coverage-diff.tmp mv -f coverage-diff.tmp coverage-diff.info @@ -154,7 +154,7 @@ coverage combine `$(ls python-coverage.data.*)` || NO_PYTHON_COVERAGE_DATA=1 sed -i 's/mnt\/paddle/paddle/g' python-coverage.xml -`$(python3.7 ${PADDLE_ROOT}/tools/coverage/python_coverage.py > python-coverage.info)` || [[ "${NO_PYTHON_COVERAGE_DATA}" == "1" ]] +`$(python ${PADDLE_ROOT}/tools/coverage/python_coverage.py > python-coverage.info)` || [[ "${NO_PYTHON_COVERAGE_DATA}" == "1" ]] # python full html report # @@ -180,9 +180,9 @@ gen_python_full_html_report || true function gen_python_diff_html_report() { if [ "${GIT_PR_ID}" != "" ]; then - COVERAGE_DIFF_PATTERN="`python3.7 ${PADDLE_ROOT}/tools/coverage/pull_request.py files ${GIT_PR_ID}`" + COVERAGE_DIFF_PATTERN="`python ${PADDLE_ROOT}/tools/coverage/pull_request.py files ${GIT_PR_ID}`" - python3.7 ${PADDLE_ROOT}/tools/coverage/pull_request.py diff ${GIT_PR_ID} > python-git-diff.out + python ${PADDLE_ROOT}/tools/coverage/pull_request.py diff ${GIT_PR_ID} > python-git-diff.out fi lcov --extract python-coverage-full.info \ @@ -190,7 +190,7 @@ function gen_python_diff_html_report() { -o python-coverage-diff.info \ --rc lcov_branch_coverage=0 - python3.7 ${PADDLE_ROOT}/tools/coverage/coverage_diff.py python-coverage-diff.info python-git-diff.out > python-coverage-diff.tmp + python ${PADDLE_ROOT}/tools/coverage/coverage_diff.py python-coverage-diff.info python-git-diff.out > python-coverage-diff.tmp mv -f python-coverage-diff.tmp python-coverage-diff.info @@ -208,7 +208,7 @@ gen_python_diff_html_report || true echo "Assert Diff Coverage" -python3.7 ${PADDLE_ROOT}/tools/coverage/coverage_lines.py coverage-diff.info 0.9 || COVERAGE_LINES_ASSERT=1 +python ${PADDLE_ROOT}/tools/coverage/coverage_lines.py coverage-diff.info 0.9 || COVERAGE_LINES_ASSERT=1 echo "Assert Python Diff Coverage" @@ -216,7 +216,7 @@ if [ ${WITH_XPU:-OFF} == "ON" ]; then echo "XPU has no python coverage!" else if [[ "${NO_PYTHON_COVERAGE_DATA}" != "1" ]];then - python3.7 ${PADDLE_ROOT}/tools/coverage/coverage_lines.py python-coverage-diff.info 0.9 || PYTHON_COVERAGE_LINES_ASSERT=1 + python ${PADDLE_ROOT}/tools/coverage/coverage_lines.py python-coverage-diff.info 0.9 || PYTHON_COVERAGE_LINES_ASSERT=1 fi fi diff --git a/tools/dockerfile/Dockerfile.ubuntu20 b/tools/dockerfile/Dockerfile.ubuntu20 index e3c4f0a518fdb..2b7967d16245e 100644 --- a/tools/dockerfile/Dockerfile.ubuntu20 +++ b/tools/dockerfile/Dockerfile.ubuntu20 @@ -57,22 +57,22 @@ WORKDIR /home RUN wget -q https://cmake.org/files/v3.18/cmake-3.18.0-Linux-x86_64.tar.gz && tar -zxvf cmake-3.18.0-Linux-x86_64.tar.gz && rm cmake-3.18.0-Linux-x86_64.tar.gz ENV PATH=/home/cmake-3.18.0-Linux-x86_64/bin:$PATH - RUN apt-get update && \ apt-get install -y python3.7 python3.7-dev python3.7-distutils\ python3.8 python3.8-dev python3.8-distutils \ - python3.9 python3.9-dev python3.9-distutils && \ + python3.9 python3.9-dev python3.9-distutils \ + python3.10 python3.10-dev python3.10-distutils && \ apt-get install python-is-python3 && \ - rm /usr/bin/python && ln -s /usr/bin/python3.7 /usr/bin/python && \ - rm /usr/bin/python3 && ln -s /usr/bin/python3.7 /usr/bin/python3 - + rm /usr/bin/python && ln -s /usr/bin/python3.9 /usr/bin/python && \ + rm /usr/bin/python3 && ln -s /usr/bin/python3.9 /usr/bin/python3 WORKDIR /home RUN wget https://files.pythonhosted.org/packages/a7/e0/30642b9c2df516506d40b563b0cbd080c49c6b3f11a70b4c7a670f13a78b/setuptools-50.3.2.zip && apt-get -y install unzip && unzip setuptools-50.3.2.zip WORKDIR /home/setuptools-50.3.2 RUN python3.9 setup.py build && python3.9 setup.py install && \ python3.8 setup.py build && python3.8 setup.py install && \ - python3.7 setup.py build && python3.7 setup.py install + python3.7 setup.py build && python3.7 setup.py install && \ + python3.10 setup.py build && python3.10 setup.py install WORKDIR /home RUN wget https://files.pythonhosted.org/packages/28/af/2c76c8aa46ccdf7578b83d97a11a2d1858794d4be4a1610ade0d30182e8b/pip-20.0.1.tar.gz && tar -zxvf pip-20.0.1.tar.gz @@ -80,14 +80,12 @@ WORKDIR pip-20.0.1 RUN python3.9 setup.py install && \ python3.8 setup.py install && \ - python3.7 setup.py install + python3.7 setup.py install && \ + python3.10 setup.py install WORKDIR /home RUN rm setuptools-50.3.2.zip pip-20.0.1.tar.gz && \ rm -r setuptools-50.3.2 pip-20.0.1 -RUN rm /usr/local/bin/pip && ln -s /usr/local/bin/pip3.7 /usr/local/bin/pip && \ - rm /usr/local/bin/pip3 && ln -s /usr/local/bin/pip3.7 /usr/local/bin/pip3 - # remove them when apt-get support 2.27 and higher version RUN wget -q https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.gz && \ @@ -95,7 +93,6 @@ RUN wget -q https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.gz && \ cd binutils-2.33.1 && \ ./configure && make -j && make install && cd .. && rm -rf binutils-2.33.1 binutils-2.33.1.tar.gz - # Install Go and glide RUN wget --no-check-certificate -qO- https://paddle-ci.gz.bcebos.com/go1.17.2.linux-amd64.tar.gz | \ tar -xz -C /usr/local && \ @@ -106,7 +103,7 @@ ENV GOROOT=/usr/local/go GOPATH=/root/gopath # should not be in the same line with GOROOT definition, otherwise docker build could not find GOROOT. ENV PATH=${PATH}:${GOROOT}/bin:${GOPATH}/bin # install glide -RUN curl -s -q https://glide.sh/get | sh +RUN apt-get install -y golang-glide # git credential to skip password typing RUN git config --global credential.helper store @@ -114,46 +111,60 @@ RUN git config --global credential.helper store # Fix locales to en_US.UTF-8 RUN localedef -i en_US -f UTF-8 en_US.UTF-8 +#For pre-commit +RUN python3.7 -m pip --no-cache-dir install --upgrade pip && \ + python3.8 -m pip --no-cache-dir install --upgrade pip && \ + python3.9 -m pip --no-cache-dir install --upgrade pip && \ + python3.10 -m pip --no-cache-dir install --upgrade pip + +RUN rm -f /usr/local/bin/pip && ln -s /usr/local/bin/pip3.9 /usr/local/bin/pip && \ + rm -f /usr/local/bin/pip3 && ln -s /usr/local/bin/pip3.9 /usr/local/bin/pip3 + RUN pip3.7 --no-cache-dir install ipython==5.3.0 && \ pip3.7 --no-cache-dir install ipykernel==4.6.0 wheel && \ pip3.8 --no-cache-dir install ipython==5.3.0 && \ pip3.8 --no-cache-dir install ipykernel==4.6.0 wheel && \ pip3.9 --no-cache-dir install ipython==5.3.0 && \ - pip3.9 --no-cache-dir install ipykernel==4.6.0 wheel + pip3.9 --no-cache-dir install ipykernel==4.6.0 wheel && \ + pip3.10 --no-cache-dir install ipython==5.3.0 && \ + pip3.10 --no-cache-dir install ipykernel==4.6.0 wheel #For docstring checker RUN pip3.7 --no-cache-dir install pytest astroid isort && \ pip3.8 --no-cache-dir install pytest astroid isort && \ - pip3.9 --no-cache-dir install pytest astroid isort - -#For pre-commit -RUN pip3.7 --no-cache-dir install --upgrade pip && \ - pip3.8 --no-cache-dir install --upgrade pip && \ - pip3.9 --no-cache-dir install --upgrade pip + pip3.9 --no-cache-dir install pytest astroid isort && \ + pip3.10 --no-cache-dir install pytest astroid isort RUN pip3.7 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \ pip3.8 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \ pip3.9 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \ + pip3.10 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \ pip3.7 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \ pip3.8 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \ - pip3.9 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 + pip3.9 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \ + pip3.10 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 COPY ./python/requirements.txt /root/ +COPY ./python/unittest_py/requirements.txt /home/ + RUN pip3.7 --no-cache-dir install -r /root/requirements.txt && \ + pip3.7 --no-cache-dir install -r /home/requirements.txt && \ pip3.8 --no-cache-dir install -r /root/requirements.txt && \ - pip3.9 --no-cache-dir install -r /root/requirements.txt - -# Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service -#RUN mkdir /var/run/sshd && echo 'root:root' | chpasswd && sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config && sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config -#CMD source ~/.bashrc - -# ccache 3.7.9 -RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \ - tar xf ccache-3.7.9.tar.gz && mkdir /usr/local/ccache-3.7.9 && cd ccache-3.7.9 && \ - ./configure -prefix=/usr/local/ccache-3.7.9 && \ + pip3.8 --no-cache-dir install -r /home/requirements.txt && \ + pip3.9 --no-cache-dir install -r /root/requirements.txt && \ + pip3.9 --no-cache-dir install -r /home/requirements.txt && \ + pip3.10 --no-cache-dir install -r /root/requirements.txt && \ + pip3.10 --no-cache-dir install -r /home/requirements.txt + + +# ccache 4.2.0 +RUN wget -q https://paddle-ci.gz.bcebos.com/ccache-4.8.2.tar.gz && \ + tar xf ccache-4.8.2.tar.gz && mkdir /usr/local/ccache-4.8.2 && cd ccache-4.8.2 && \ + mkdir build && cd build && \ + cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local/ccache-4.8.2 .. && \ make -j8 && make install && \ - ln -s /usr/local/ccache-3.7.9/bin/ccache /usr/local/bin/ccache && \ - cd ../ && rm -rf ccache-3.7.9 ccache-3.7.9.tar.gz + ln -s /usr/local/ccache-4.8.2/bin/ccache /usr/local/bin/ccache && \ + cd ../../ && rm -rf ccache-4.8.2.tar.gz # clang+llvm 3.8.0 RUN wget https://paddle-ci.cdn.bcebos.com/clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz && \ diff --git a/tools/dockerfile/ci_dockerfile.sh b/tools/dockerfile/ci_dockerfile.sh index 83651c781c760..835c1c50f47c7 100644 --- a/tools/dockerfile/ci_dockerfile.sh +++ b/tools/dockerfile/ci_dockerfile.sh @@ -121,30 +121,20 @@ function make_cinn_dockerfile(){ function make_ce_framework_dockcerfile(){ dockerfile_name="Dockerfile.cuda11.2_cudnn8_gcc82_trt8" - sed "s//11.2.0-cudnn8-devel-ubuntu16.04/g" ./Dockerfile.ubuntu >${dockerfile_name} + sed "s##nvidia/cuda:11.8.0-cudnn8-devel-ubuntu20.04#g" ./Dockerfile.ubuntu20 >${dockerfile_name} dockerfile_line=$(wc -l ${dockerfile_name}|awk '{print $1}') + sed -i "s##ENV LD_LIBRARY_PATH=/usr/local/cuda-11.8/targets/x86_64-linux/lib:\$LD_LIBRARY_PATH #g" ${dockerfile_name} + sed -i 's###g' ${dockerfile_name} sed -i "7i RUN chmod 777 /tmp" ${dockerfile_name} sed -i "${dockerfile_line}i RUN wget --no-check-certificate -q https://paddle-edl.bj.bcebos.com/hadoop-2.7.7.tar.gz \&\& \ tar -xzf hadoop-2.7.7.tar.gz && mv hadoop-2.7.7 /usr/local/" ${dockerfile_name} - sed -i "${dockerfile_line}i RUN apt remove git -y \&\& apt install -y zstd pigz libcurl4-openssl-dev gettext ninja-build \&\& wget -q https://paddle-ci.gz.bcebos.com/git-2.17.1.tar.gz \&\& \ - tar -xvf git-2.17.1.tar.gz \&\& \ - cd git-2.17.1 \&\& \ - ./configure --with-openssl --with-curl --prefix=/usr/local \&\& \ - make -j8 \&\& make install " ${dockerfile_name} - sed -i "${dockerfile_line}i RUN pip install wheel \&\& pip3 install PyGithub wheel \&\& pip3.7 install PyGithub distro" ${dockerfile_name} + sed -i "${dockerfile_line}i RUN apt install -y zstd pigz libcurl4-openssl-dev gettext ninja-build" ${dockerfile_name} + sed -i "${dockerfile_line}i RUN pip install wheel distro" ${dockerfile_name} sed -i "s##WORKDIR /usr/bin \\ - COPY tools/dockerfile/build_scripts /build_scripts \\ - RUN bash /build_scripts/install_gcc.sh gcc82 \&\& rm -rf /build_scripts \\ - RUN cp gcc gcc.bak \&\& cp g++ g++.bak \&\& rm gcc \&\& rm g++ \\ - RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/local/bin/gcc \\ - RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/local/bin/g++ \\ - RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/gcc \\ - RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/bin/g++ \\ ENV PATH=/usr/local/gcc-8.2/bin:\$PATH #g" ${dockerfile_name} + sed -i "s#gcc121#gcc82#g" ${dockerfile_name} + sed -i "s#gcc-12.1#gcc-8.2#g" ${dockerfile_name} sed -i 's#RUN bash /build_scripts/install_trt.sh#RUN bash /build_scripts/install_trt.sh trt8531#g' ${dockerfile_name} - sed -i 's#28/af/2c76c8aa46ccdf7578b83d97a11a2d1858794d4be4a1610ade0d30182e8b/pip-20.0.1.tar.gz#b7/2d/ad02de84a4c9fd3b1958dc9fb72764de1aa2605a9d7e943837be6ad82337/pip-21.0.1.tar.gz#g' ${dockerfile_name} - sed -i 's#pip-20.0.1#pip-21.0.1#g' ${dockerfile_name} - sed -i 's#python setup.py install#python3.7 setup.py install#g' ${dockerfile_name} } function make_unbuntu18_cu117_dockerfile(){ @@ -194,9 +184,17 @@ function make_ubuntu18_cu112_dockerfile(){ ENV PATH=/usr/local/gcc-8.2/bin:\$PATH #g" ${dockerfile_name} } +function make_ubuntu_trt7_dockerfile_temp_ues(){ + dockerfile_name="Dockerfile.cuda102_cudnn8_gcc82_ubuntu16" + echo "FROM registry.baidubce.com/paddlepaddle/paddleqa:coverage-ci-temp-use" >> ${dockerfile_name} + echo "RUN wget https://www.openssl.org/source/openssl-1.1.1v.tar.gz && tar -xvf openssl-1.1.1v.tar.gz && cd openssl-1.1.1v && ./config -fPIC --prefix=/usr/local/ssl > /dev/null && make > /dev/null && make install > /dev/null && cd ../ && rm -rf openssl-1.1.1v*" >> ${dockerfile_name} + echo "ENV OPENSSL_ROOT_DIR=/usr/local/ssl" >> ${dockerfile_name} + echo "ENV LD_LIBRARY_PATH=/usr/local/ssl/lib:\g$LD_LIBRARY_PATH" >> ${dockerfile_name} +} + function main() { make_ubuntu_dockerfile - make_ubuntu_trt7_dockerfile + make_ubuntu_trt7_dockerfile_temp_ues make_centos_dockerfile make_cinn_dockerfile make_ce_framework_dockcerfile