From 04ac49a73851f350f8334a45dc29aa75808b8ef0 Mon Sep 17 00:00:00 2001 From: Andrey Zhavoronkov Date: Tue, 15 Dec 2020 22:09:16 +0300 Subject: [PATCH] OpenH264 encoder (#2562) * use Cisco openh264 encoder instead libx264 * installation of source code * multistage build * Reduced the size of docker image * updated travis config * set ubuntu 20.04 as build env --- .travis.yml | 6 +- Dockerfile | 139 ++++++++++++++++++--------- Dockerfile.ci | 10 +- cvat/apps/engine/media_extractors.py | 17 ++-- cvat/requirements/base.txt | 4 +- supervisord.conf | 14 +-- 6 files changed, 122 insertions(+), 68 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9450e7ea7f19..207d155ee69c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,5 @@ -language: python - -python: - - '3.5' +language: generic +dist: focal cache: npm: true diff --git a/Dockerfile b/Dockerfile index 844a18c01196..fe79a5eb844b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,52 @@ +FROM ubuntu:20.04 as build-image + +ARG http_proxy +ARG https_proxy +ARG no_proxy +ARG socks_proxy +ARG DJANGO_CONFIGURATION + +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get --no-install-recommends install -yq \ + apache2-dev \ + build-essential \ + curl \ + libldap2-dev \ + libsasl2-dev \ + nasm \ + git \ + pkg-config \ + python3-dev \ + python3-pip \ + python3-venv && \ + rm -rf /var/lib/apt/lists/* + +# Compile Openh264 and FFmpeg +ARG PREFIX=/opt/ffmpeg +ARG PKG_CONFIG_PATH=${PREFIX}/lib/pkgconfig + +ENV FFMPEG_VERSION=4.3.1 \ + OPENH264_VERSION=2.1.1 + +WORKDIR /tmp/openh264 +RUN curl -sL https://github.com/cisco/openh264/archive/v${OPENH264_VERSION}.tar.gz --output openh264-${OPENH264_VERSION}.tar.gz && \ + tar -zx --strip-components=1 -f openh264-${OPENH264_VERSION}.tar.gz && \ + make -j5 && make install PREFIX=${PREFIX} && make clean + +WORKDIR /tmp/ffmpeg +RUN curl -sLO https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.bz2 && \ + tar -jx --strip-components=1 -f ffmpeg-${FFMPEG_VERSION}.tar.bz2 && \ + ./configure --disable-nonfree --disable-gpl --enable-libopenh264 --enable-shared --disable-static --prefix="${PREFIX}" && \ + make -j5 && make install && make distclean + +# Install requirements +RUN python3 -m venv /opt/venv +ENV PATH="/opt/venv/bin:${PATH}" +RUN python3 -m pip install --no-cache-dir -U pip==20.0.1 setuptools==49.6.0 wheel==0.35.1 +COPY cvat/requirements/ /tmp/requirements/ +RUN DATUMARO_HEADLESS=1 python3 -m pip install --no-cache-dir -r /tmp/requirements/${DJANGO_CONFIGURATION}.txt + + FROM ubuntu:20.04 ARG http_proxy @@ -21,59 +70,26 @@ ENV DJANGO_CONFIGURATION=${DJANGO_CONFIGURATION} # Install necessary apt packages RUN apt-get update && \ - apt-get --no-install-recommends install -yq \ - software-properties-common && \ - apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get --no-install-recommends install -yq \ apache2 \ - apache2-dev \ - apt-utils \ - build-essential \ libapache2-mod-xsendfile \ supervisor \ - libavcodec-dev=7:4.2.4-1ubuntu0.1 \ - libavdevice-dev=7:4.2.4-1ubuntu0.1 \ - libavfilter-dev=7:4.2.4-1ubuntu0.1 \ - libavformat-dev=7:4.2.4-1ubuntu0.1 \ - libavutil-dev=7:4.2.4-1ubuntu0.1 \ - libswresample-dev=7:4.2.4-1ubuntu0.1 \ - libswscale-dev=7:4.2.4-1ubuntu0.1 \ - libldap2-dev \ - libsasl2-dev \ - pkg-config \ - python3-dev \ - python3-pip \ + libldap-2.4-2 \ + libsasl2-2 \ + libpython3-dev \ tzdata \ + python3-distutils \ p7zip-full \ git \ git-lfs \ - ssh \ poppler-utils \ + ssh \ curl && \ - python3 -m pip install --no-cache-dir -U pip==20.0.1 setuptools==49.6.0 wheel==0.35.1 && \ ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime && \ dpkg-reconfigure -f noninteractive tzdata && \ rm -rf /var/lib/apt/lists/* && \ echo 'application/wasm wasm' >> /etc/mime.types -# Add a non-root user -ENV USER=${USER} -ENV HOME /home/${USER} -WORKDIR ${HOME} -RUN adduser --shell /bin/bash --disabled-password --gecos "" ${USER} && \ - if [ -z ${socks_proxy} ]; then \ - echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30\"" >> ${HOME}/.bashrc; \ - else \ - echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30 -o ProxyCommand='nc -X 5 -x ${socks_proxy} %h %p'\"" >> ${HOME}/.bashrc; \ - fi - -COPY components /tmp/components - -# Install and initialize CVAT, copy all necessary files -COPY cvat/requirements/ /tmp/requirements/ -COPY supervisord.conf mod_wsgi.conf wait-for-it.sh manage.py ${HOME}/ -RUN python3 -m pip install --no-cache-dir -r /tmp/requirements/${DJANGO_CONFIGURATION}.txt - ARG CLAM_AV ENV CLAM_AV=${CLAM_AV} RUN if [ "$CLAM_AV" = "yes" ]; then \ @@ -87,20 +103,51 @@ RUN if [ "$CLAM_AV" = "yes" ]; then \ rm -rf /var/lib/apt/lists/*; \ fi -COPY ssh ${HOME}/.ssh -COPY utils ${HOME}/utils -COPY cvat/ ${HOME}/cvat -COPY cvat-core/ ${HOME}/cvat-core -COPY cvat-data/ ${HOME}/cvat-data -COPY tests ${HOME}/tests +# Add a non-root user +ENV USER=${USER} +ENV HOME /home/${USER} +RUN adduser --shell /bin/bash --disabled-password --gecos "" ${USER} && \ + if [ -z ${socks_proxy} ]; then \ + echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30\"" >> ${HOME}/.bashrc; \ + else \ + echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30 -o ProxyCommand='nc -X 5 -x ${socks_proxy} %h %p'\"" >> ${HOME}/.bashrc; \ + fi + +ARG INSTALL_SOURCES='no' +WORKDIR ${HOME}/sources +RUN if [ "$INSTALL_SOURCES" = "yes" ]; then \ + sed -Ei 's/^# deb-src /deb-src /' /etc/apt/sources.list && \ + apt-get update && \ + dpkg --get-selections | while read -r line; do \ + package=$(echo "$line" | awk '{print $1}'); \ + mkdir "$package"; \ + ( \ + cd "$package"; \ + apt-get -q --download-only source "$package"; \ + ) \ + done && \ + rm -rf /var/lib/apt/lists/*; \ + fi +COPY --from=build-image /tmp/openh264/openh264*.tar.gz /tmp/ffmpeg/ffmpeg*.tar.bz2 ${HOME}/sources/ + +# Copy python virtual enviroment and FFmpeg binaries from build-image +COPY --from=build-image /opt/venv /opt/venv +ENV PATH="/opt/venv/bin:${PATH}" +COPY --from=build-image /opt/ffmpeg /usr -RUN chown -R ${USER}:${USER} . +# Install and initialize CVAT, copy all necessary files +COPY --chown=${USER} components /tmp/components +COPY --chown=${USER} ssh ${HOME}/.ssh +COPY --chown=${USER} supervisord.conf mod_wsgi.conf wait-for-it.sh manage.py ${HOME}/ +COPY --chown=${USER} cvat/ ${HOME}/cvat +COPY --chown=${USER} utils/ ${HOME}/utils # RUN all commands below as 'django' user USER ${USER} +WORKDIR ${HOME} RUN mkdir data share media keys logs /tmp/supervisord RUN python3 manage.py collectstatic -EXPOSE 8080 8443 +EXPOSE 8080 ENTRYPOINT ["/usr/bin/supervisord"] diff --git a/Dockerfile.ci b/Dockerfile.ci index b51268dd1957..bc642137e17b 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -6,6 +6,7 @@ USER root RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get --no-install-recommends install -yq \ gpg-agent \ + gnupg2 \ apt-utils \ build-essential \ python3-dev \ @@ -20,10 +21,17 @@ RUN apt-get update && \ && \ rm -rf /var/lib/apt/lists/*; -RUN python3 -m pip install --no-cache-dir -r /tmp/requirements/${DJANGO_CONFIGURATION}.txt && \ +COPY cvat/requirements/ /tmp/requirements/ + +RUN DATUMARO_HEADLESS=1 python3 -m pip install --no-cache-dir -r /tmp/requirements/${DJANGO_CONFIGURATION}.txt && \ python3 -m pip install --no-cache-dir coveralls RUN gem install coveralls-lcov +COPY utils ${HOME}/utils +COPY cvat-core ${HOME}/cvat-core +COPY cvat-data ${HOME}/cvat-data +COPY tests ${HOME}/tests + COPY .coveragerc . ENTRYPOINT [] diff --git a/cvat/apps/engine/media_extractors.py b/cvat/apps/engine/media_extractors.py index a83fb6bb5399..7ee7725eafa2 100644 --- a/cvat/apps/engine/media_extractors.py +++ b/cvat/apps/engine/media_extractors.py @@ -348,7 +348,7 @@ def _create_av_container(path, w, h, rate, options, f='mp4'): w += 1 container = av.open(path, 'w',format=f) - video_stream = container.add_stream('libx264', rate=rate) + video_stream = container.add_stream('libopenh264', rate=rate) video_stream.pix_fmt = "yuv420p" video_stream.width = w video_stream.height = h @@ -369,8 +369,10 @@ def save_as_chunk(self, images, chunk_path): h=input_h, rate=self._output_fps, options={ - "crf": str(self._image_quality), - "preset": "ultrafast", + 'profile': 'constrained_baseline', + 'qmin': str(self._image_quality), + 'qmax': str(self._image_quality), + 'rc_mode': 'buffer', }, ) @@ -419,11 +421,10 @@ def save_as_chunk(self, images, chunk_path): h=output_h, rate=self._output_fps, options={ - 'profile': 'baseline', - 'coder': '0', - 'crf': str(self._image_quality), - 'wpredp': '0', - 'flags': '-loop' + 'profile': 'constrained_baseline', + 'qmin': str(self._image_quality), + 'qmax': str(self._image_quality), + 'rc_mode': 'buffer', }, ) diff --git a/cvat/requirements/base.txt b/cvat/requirements/base.txt index 4d9dabf1e336..79329bbb729a 100644 --- a/cvat/requirements/base.txt +++ b/cvat/requirements/base.txt @@ -34,7 +34,7 @@ Shapely==1.7.1 pdf2image==1.14.0 django-rest-auth[with_social]==0.9.5 cython==0.29.21 -opencv-python==4.4.0.42 +opencv-python-headless==4.4.0.42 h5py==2.10.0 django-cors-headers==3.5.0 furl==2.1.0 @@ -44,4 +44,4 @@ tensorflow==2.2.1 # Optional requirement of Datumaro # archives. Don't use as a python module because it has GPL license. patool==1.12 diskcache==5.0.2 -git+https://github.com/openvinotoolkit/datumaro@v0.1.3 \ No newline at end of file +git+https://github.com/openvinotoolkit/datumaro@v0.1.4 diff --git a/supervisord.conf b/supervisord.conf index 2191549d98de..6d9b8ee2f42f 100644 --- a/supervisord.conf +++ b/supervisord.conf @@ -24,26 +24,26 @@ autorestart=true [program:rqworker_default] command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -ic \ - "exec /usr/bin/python3 %(ENV_HOME)s/manage.py rqworker -v 3 default" + "exec python3 %(ENV_HOME)s/manage.py rqworker -v 3 default" environment=SSH_AUTH_SOCK="/tmp/ssh-agent.sock" numprocs=2 process_name=rqworker_default_%(process_num)s [program:rqworker_low] command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -ic \ - "exec /usr/bin/python3 %(ENV_HOME)s/manage.py rqworker -v 3 low" + "exec python3 %(ENV_HOME)s/manage.py rqworker -v 3 low" environment=SSH_AUTH_SOCK="/tmp/ssh-agent.sock" numprocs=1 [program:git_status_updater] command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -ic \ - "/usr/bin/python3 ~/manage.py update_git_states" + "python3 ~/manage.py update_git_states" environment=SSH_AUTH_SOCK="/tmp/ssh-agent.sock" numprocs=1 [program:rqscheduler] command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -ic \ - "/usr/bin/python3 /usr/local/bin/rqscheduler --host %(ENV_CVAT_REDIS_HOST)s -i 30" + "python3 /opt/venv/bin/rqscheduler --host %(ENV_CVAT_REDIS_HOST)s -i 30" environment=SSH_AUTH_SOCK="/tmp/ssh-agent.sock" numprocs=1 @@ -59,9 +59,9 @@ numprocs=1 ; with docker cache. Thus it is necessary to run collectstatic here for such ; apps. command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_POSTGRES_HOST)s:5432 -t 0 -- bash -ic \ - "rm -f /tmp/cvat-server/httpd.pid && /usr/bin/python3 ~/manage.py migrate && \ - /usr/bin/python3 ~/manage.py collectstatic --no-input && \ - exec /usr/bin/python3 $HOME/manage.py runmodwsgi --log-to-terminal --port 8080 \ + "rm -f /tmp/cvat-server/httpd.pid && python3 ~/manage.py migrate && \ + python3 ~/manage.py collectstatic --no-input && \ + exec python3 $HOME/manage.py runmodwsgi --log-to-terminal --port 8080 \ --limit-request-body 1073741824 --log-level INFO --include-file ~/mod_wsgi.conf \ %(ENV_DJANGO_MODWSGI_EXTRA_ARGS)s --locale %(ENV_LC_ALL)s \ --server-root /tmp/cvat-server"