Skip to content

Commit

Permalink
fix openssl bug ,test=document_fix (#57147)
Browse files Browse the repository at this point in the history
  • Loading branch information
@risemeup1
risemeup1 authored Sep 26, 2023
1 parent a09a2bb commit ff9d102
Show file tree
Hide file tree
Showing 4 changed files with 70 additions and 61 deletions.
2 changes: 1 addition & 1 deletion cmake/external/brpc.cmake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
# limitations under the License.

include(ExternalProject)

set(OPENSSL_USE_STATIC_LIBS ON)
find_package(OpenSSL REQUIRED)

message(STATUS "ssl:" ${OPENSSL_SSL_LIBRARY})
Expand Down
20 changes: 10 additions & 10 deletions tools/coverage/paddle_coverage.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ make install

cd /paddle/build

python3.7 ${PADDLE_ROOT}/tools/coverage/gcda_clean.py ${GIT_PR_ID} || exit 101
python ${PADDLE_ROOT}/tools/coverage/gcda_clean.py ${GIT_PR_ID} || exit 101

lcov --capture -d ./ -o coverage.info --rc lcov_branch_coverage=0

Expand Down Expand Up @@ -125,17 +125,17 @@ fi
function gen_diff_html_report() {
if [ "${GIT_PR_ID}" != "" ]; then

COVERAGE_DIFF_PATTERN="`python3.7 ${PADDLE_ROOT}/tools/coverage/pull_request.py files ${GIT_PR_ID}`"
COVERAGE_DIFF_PATTERN="`python ${PADDLE_ROOT}/tools/coverage/pull_request.py files ${GIT_PR_ID}`"

python3.7 ${PADDLE_ROOT}/tools/coverage/pull_request.py diff ${GIT_PR_ID} > git-diff.out
python ${PADDLE_ROOT}/tools/coverage/pull_request.py diff ${GIT_PR_ID} > git-diff.out
fi

lcov --extract coverage-full.info \
${COVERAGE_DIFF_PATTERN} \
-o coverage-diff.info \
--rc lcov_branch_coverage=0

python3.7 ${PADDLE_ROOT}/tools/coverage/coverage_diff.py coverage-diff.info git-diff.out > coverage-diff.tmp
python ${PADDLE_ROOT}/tools/coverage/coverage_diff.py coverage-diff.info git-diff.out > coverage-diff.tmp

mv -f coverage-diff.tmp coverage-diff.info

Expand All @@ -154,7 +154,7 @@ coverage combine `$(ls python-coverage.data.*)` || NO_PYTHON_COVERAGE_DATA=1

sed -i 's/mnt\/paddle/paddle/g' python-coverage.xml

`$(python3.7 ${PADDLE_ROOT}/tools/coverage/python_coverage.py > python-coverage.info)` || [[ "${NO_PYTHON_COVERAGE_DATA}" == "1" ]]
`$(python ${PADDLE_ROOT}/tools/coverage/python_coverage.py > python-coverage.info)` || [[ "${NO_PYTHON_COVERAGE_DATA}" == "1" ]]

# python full html report
#
Expand All @@ -180,17 +180,17 @@ gen_python_full_html_report || true

function gen_python_diff_html_report() {
if [ "${GIT_PR_ID}" != "" ]; then
COVERAGE_DIFF_PATTERN="`python3.7 ${PADDLE_ROOT}/tools/coverage/pull_request.py files ${GIT_PR_ID}`"
COVERAGE_DIFF_PATTERN="`python ${PADDLE_ROOT}/tools/coverage/pull_request.py files ${GIT_PR_ID}`"

python3.7 ${PADDLE_ROOT}/tools/coverage/pull_request.py diff ${GIT_PR_ID} > python-git-diff.out
python ${PADDLE_ROOT}/tools/coverage/pull_request.py diff ${GIT_PR_ID} > python-git-diff.out
fi

lcov --extract python-coverage-full.info \
${COVERAGE_DIFF_PATTERN} \
-o python-coverage-diff.info \
--rc lcov_branch_coverage=0

python3.7 ${PADDLE_ROOT}/tools/coverage/coverage_diff.py python-coverage-diff.info python-git-diff.out > python-coverage-diff.tmp
python ${PADDLE_ROOT}/tools/coverage/coverage_diff.py python-coverage-diff.info python-git-diff.out > python-coverage-diff.tmp

mv -f python-coverage-diff.tmp python-coverage-diff.info

Expand All @@ -208,15 +208,15 @@ gen_python_diff_html_report || true

echo "Assert Diff Coverage"

python3.7 ${PADDLE_ROOT}/tools/coverage/coverage_lines.py coverage-diff.info 0.9 || COVERAGE_LINES_ASSERT=1
python ${PADDLE_ROOT}/tools/coverage/coverage_lines.py coverage-diff.info 0.9 || COVERAGE_LINES_ASSERT=1

echo "Assert Python Diff Coverage"

if [ ${WITH_XPU:-OFF} == "ON" ]; then
echo "XPU has no python coverage!"
else
if [[ "${NO_PYTHON_COVERAGE_DATA}" != "1" ]];then
python3.7 ${PADDLE_ROOT}/tools/coverage/coverage_lines.py python-coverage-diff.info 0.9 || PYTHON_COVERAGE_LINES_ASSERT=1
python ${PADDLE_ROOT}/tools/coverage/coverage_lines.py python-coverage-diff.info 0.9 || PYTHON_COVERAGE_LINES_ASSERT=1
fi
fi

Expand Down
75 changes: 43 additions & 32 deletions tools/dockerfile/Dockerfile.ubuntu20
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,45 +57,42 @@ WORKDIR /home
RUN wget -q https://cmake.org/files/v3.18/cmake-3.18.0-Linux-x86_64.tar.gz && tar -zxvf cmake-3.18.0-Linux-x86_64.tar.gz && rm cmake-3.18.0-Linux-x86_64.tar.gz
ENV PATH=/home/cmake-3.18.0-Linux-x86_64/bin:$PATH


RUN apt-get update && \
apt-get install -y python3.7 python3.7-dev python3.7-distutils\
python3.8 python3.8-dev python3.8-distutils \
python3.9 python3.9-dev python3.9-distutils && \
python3.9 python3.9-dev python3.9-distutils \
python3.10 python3.10-dev python3.10-distutils && \
apt-get install python-is-python3 && \
rm /usr/bin/python && ln -s /usr/bin/python3.7 /usr/bin/python && \
rm /usr/bin/python3 && ln -s /usr/bin/python3.7 /usr/bin/python3

rm /usr/bin/python && ln -s /usr/bin/python3.9 /usr/bin/python && \
rm /usr/bin/python3 && ln -s /usr/bin/python3.9 /usr/bin/python3

WORKDIR /home
RUN wget https://files.pythonhosted.org/packages/a7/e0/30642b9c2df516506d40b563b0cbd080c49c6b3f11a70b4c7a670f13a78b/setuptools-50.3.2.zip && apt-get -y install unzip && unzip setuptools-50.3.2.zip
WORKDIR /home/setuptools-50.3.2
RUN python3.9 setup.py build && python3.9 setup.py install && \
python3.8 setup.py build && python3.8 setup.py install && \
python3.7 setup.py build && python3.7 setup.py install
python3.7 setup.py build && python3.7 setup.py install && \
python3.10 setup.py build && python3.10 setup.py install

WORKDIR /home
RUN wget https://files.pythonhosted.org/packages/28/af/2c76c8aa46ccdf7578b83d97a11a2d1858794d4be4a1610ade0d30182e8b/pip-20.0.1.tar.gz && tar -zxvf pip-20.0.1.tar.gz
WORKDIR pip-20.0.1

RUN python3.9 setup.py install && \
python3.8 setup.py install && \
python3.7 setup.py install
python3.7 setup.py install && \
python3.10 setup.py install

WORKDIR /home
RUN rm setuptools-50.3.2.zip pip-20.0.1.tar.gz && \
rm -r setuptools-50.3.2 pip-20.0.1
RUN rm /usr/local/bin/pip && ln -s /usr/local/bin/pip3.7 /usr/local/bin/pip && \
rm /usr/local/bin/pip3 && ln -s /usr/local/bin/pip3.7 /usr/local/bin/pip3


# remove them when apt-get support 2.27 and higher version
RUN wget -q https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.gz && \
tar -xzf binutils-2.33.1.tar.gz && \
cd binutils-2.33.1 && \
./configure && make -j && make install && cd .. && rm -rf binutils-2.33.1 binutils-2.33.1.tar.gz


# Install Go and glide
RUN wget --no-check-certificate -qO- https://paddle-ci.gz.bcebos.com/go1.17.2.linux-amd64.tar.gz | \
tar -xz -C /usr/local && \
Expand All @@ -106,54 +103,68 @@ ENV GOROOT=/usr/local/go GOPATH=/root/gopath
# should not be in the same line with GOROOT definition, otherwise docker build could not find GOROOT.
ENV PATH=${PATH}:${GOROOT}/bin:${GOPATH}/bin
# install glide
RUN curl -s -q https://glide.sh/get | sh
RUN apt-get install -y golang-glide

# git credential to skip password typing
RUN git config --global credential.helper store

# Fix locales to en_US.UTF-8
RUN localedef -i en_US -f UTF-8 en_US.UTF-8

#For pre-commit
RUN python3.7 -m pip --no-cache-dir install --upgrade pip && \
python3.8 -m pip --no-cache-dir install --upgrade pip && \
python3.9 -m pip --no-cache-dir install --upgrade pip && \
python3.10 -m pip --no-cache-dir install --upgrade pip

RUN rm -f /usr/local/bin/pip && ln -s /usr/local/bin/pip3.9 /usr/local/bin/pip && \
rm -f /usr/local/bin/pip3 && ln -s /usr/local/bin/pip3.9 /usr/local/bin/pip3

RUN pip3.7 --no-cache-dir install ipython==5.3.0 && \
pip3.7 --no-cache-dir install ipykernel==4.6.0 wheel && \
pip3.8 --no-cache-dir install ipython==5.3.0 && \
pip3.8 --no-cache-dir install ipykernel==4.6.0 wheel && \
pip3.9 --no-cache-dir install ipython==5.3.0 && \
pip3.9 --no-cache-dir install ipykernel==4.6.0 wheel
pip3.9 --no-cache-dir install ipykernel==4.6.0 wheel && \
pip3.10 --no-cache-dir install ipython==5.3.0 && \
pip3.10 --no-cache-dir install ipykernel==4.6.0 wheel

#For docstring checker
RUN pip3.7 --no-cache-dir install pytest astroid isort && \
pip3.8 --no-cache-dir install pytest astroid isort && \
pip3.9 --no-cache-dir install pytest astroid isort

#For pre-commit
RUN pip3.7 --no-cache-dir install --upgrade pip && \
pip3.8 --no-cache-dir install --upgrade pip && \
pip3.9 --no-cache-dir install --upgrade pip
pip3.9 --no-cache-dir install pytest astroid isort && \
pip3.10 --no-cache-dir install pytest astroid isort

RUN pip3.7 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \
pip3.8 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \
pip3.9 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \
pip3.10 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \
pip3.7 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \
pip3.8 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \
pip3.9 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0
pip3.9 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \
pip3.10 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0

COPY ./python/requirements.txt /root/
COPY ./python/unittest_py/requirements.txt /home/

RUN pip3.7 --no-cache-dir install -r /root/requirements.txt && \
pip3.7 --no-cache-dir install -r /home/requirements.txt && \
pip3.8 --no-cache-dir install -r /root/requirements.txt && \
pip3.9 --no-cache-dir install -r /root/requirements.txt

# Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service
#RUN mkdir /var/run/sshd && echo 'root:root' | chpasswd && sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config && sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config
#CMD source ~/.bashrc

# ccache 3.7.9
RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \
tar xf ccache-3.7.9.tar.gz && mkdir /usr/local/ccache-3.7.9 && cd ccache-3.7.9 && \
./configure -prefix=/usr/local/ccache-3.7.9 && \
pip3.8 --no-cache-dir install -r /home/requirements.txt && \
pip3.9 --no-cache-dir install -r /root/requirements.txt && \
pip3.9 --no-cache-dir install -r /home/requirements.txt && \
pip3.10 --no-cache-dir install -r /root/requirements.txt && \
pip3.10 --no-cache-dir install -r /home/requirements.txt


# ccache 4.2.0
RUN wget -q https://paddle-ci.gz.bcebos.com/ccache-4.8.2.tar.gz && \
tar xf ccache-4.8.2.tar.gz && mkdir /usr/local/ccache-4.8.2 && cd ccache-4.8.2 && \
mkdir build && cd build && \
cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local/ccache-4.8.2 .. && \
make -j8 && make install && \
ln -s /usr/local/ccache-3.7.9/bin/ccache /usr/local/bin/ccache && \
cd ../ && rm -rf ccache-3.7.9 ccache-3.7.9.tar.gz
ln -s /usr/local/ccache-4.8.2/bin/ccache /usr/local/bin/ccache && \
cd ../../ && rm -rf ccache-4.8.2.tar.gz

# clang+llvm 3.8.0
RUN wget https://paddle-ci.cdn.bcebos.com/clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz && \
Expand Down
34 changes: 16 additions & 18 deletions tools/dockerfile/ci_dockerfile.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,30 +121,20 @@ function make_cinn_dockerfile(){

function make_ce_framework_dockcerfile(){
dockerfile_name="Dockerfile.cuda11.2_cudnn8_gcc82_trt8"
sed "s/<baseimg>/11.2.0-cudnn8-devel-ubuntu16.04/g" ./Dockerfile.ubuntu >${dockerfile_name}
sed "s#<baseimg>#nvidia/cuda:11.8.0-cudnn8-devel-ubuntu20.04#g" ./Dockerfile.ubuntu20 >${dockerfile_name}
dockerfile_line=$(wc -l ${dockerfile_name}|awk '{print $1}')
sed -i "s#<setcuda>#ENV LD_LIBRARY_PATH=/usr/local/cuda-11.8/targets/x86_64-linux/lib:\$LD_LIBRARY_PATH #g" ${dockerfile_name}
sed -i 's#<install_cpu_package>##g' ${dockerfile_name}
sed -i "7i RUN chmod 777 /tmp" ${dockerfile_name}
sed -i "${dockerfile_line}i RUN wget --no-check-certificate -q https://paddle-edl.bj.bcebos.com/hadoop-2.7.7.tar.gz \&\& \
tar -xzf hadoop-2.7.7.tar.gz && mv hadoop-2.7.7 /usr/local/" ${dockerfile_name}
sed -i "${dockerfile_line}i RUN apt remove git -y \&\& apt install -y zstd pigz libcurl4-openssl-dev gettext ninja-build \&\& wget -q https://paddle-ci.gz.bcebos.com/git-2.17.1.tar.gz \&\& \
tar -xvf git-2.17.1.tar.gz \&\& \
cd git-2.17.1 \&\& \
./configure --with-openssl --with-curl --prefix=/usr/local \&\& \
make -j8 \&\& make install " ${dockerfile_name}
sed -i "${dockerfile_line}i RUN pip install wheel \&\& pip3 install PyGithub wheel \&\& pip3.7 install PyGithub distro" ${dockerfile_name}
sed -i "${dockerfile_line}i RUN apt install -y zstd pigz libcurl4-openssl-dev gettext ninja-build" ${dockerfile_name}
sed -i "${dockerfile_line}i RUN pip install wheel distro" ${dockerfile_name}
sed -i "s#<install_gcc>#WORKDIR /usr/bin \\
COPY tools/dockerfile/build_scripts /build_scripts \\
RUN bash /build_scripts/install_gcc.sh gcc82 \&\& rm -rf /build_scripts \\
RUN cp gcc gcc.bak \&\& cp g++ g++.bak \&\& rm gcc \&\& rm g++ \\
RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/local/bin/gcc \\
RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/local/bin/g++ \\
RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/gcc \\
RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/bin/g++ \\
ENV PATH=/usr/local/gcc-8.2/bin:\$PATH #g" ${dockerfile_name}
sed -i "s#gcc121#gcc82#g" ${dockerfile_name}
sed -i "s#gcc-12.1#gcc-8.2#g" ${dockerfile_name}
sed -i 's#RUN bash /build_scripts/install_trt.sh#RUN bash /build_scripts/install_trt.sh trt8531#g' ${dockerfile_name}
sed -i 's#28/af/2c76c8aa46ccdf7578b83d97a11a2d1858794d4be4a1610ade0d30182e8b/pip-20.0.1.tar.gz#b7/2d/ad02de84a4c9fd3b1958dc9fb72764de1aa2605a9d7e943837be6ad82337/pip-21.0.1.tar.gz#g' ${dockerfile_name}
sed -i 's#pip-20.0.1#pip-21.0.1#g' ${dockerfile_name}
sed -i 's#python setup.py install#python3.7 setup.py install#g' ${dockerfile_name}
}

function make_unbuntu18_cu117_dockerfile(){
Expand Down Expand Up @@ -194,9 +184,17 @@ function make_ubuntu18_cu112_dockerfile(){
ENV PATH=/usr/local/gcc-8.2/bin:\$PATH #g" ${dockerfile_name}
}

function make_ubuntu_trt7_dockerfile_temp_ues(){
dockerfile_name="Dockerfile.cuda102_cudnn8_gcc82_ubuntu16"
echo "FROM registry.baidubce.com/paddlepaddle/paddleqa:coverage-ci-temp-use" >> ${dockerfile_name}
echo "RUN wget https://www.openssl.org/source/openssl-1.1.1v.tar.gz && tar -xvf openssl-1.1.1v.tar.gz && cd openssl-1.1.1v && ./config -fPIC --prefix=/usr/local/ssl > /dev/null && make > /dev/null && make install > /dev/null && cd ../ && rm -rf openssl-1.1.1v*" >> ${dockerfile_name}
echo "ENV OPENSSL_ROOT_DIR=/usr/local/ssl" >> ${dockerfile_name}
echo "ENV LD_LIBRARY_PATH=/usr/local/ssl/lib:\g$LD_LIBRARY_PATH" >> ${dockerfile_name}
}

function main() {
make_ubuntu_dockerfile
make_ubuntu_trt7_dockerfile
make_ubuntu_trt7_dockerfile_temp_ues
make_centos_dockerfile
make_cinn_dockerfile
make_ce_framework_dockcerfile
Expand Down

0 comments on commit ff9d102

Please sign in to comment.