From f1f10e24ac28324e21d97c842b67d394256e47d0 Mon Sep 17 00:00:00 2001 From: Frederic Tausch Date: Thu, 28 Nov 2019 13:04:37 +0100 Subject: [PATCH 01/10] Moved back-end docker file to jessie-slim Switched backend docker file to jessie-slim :) --- Dockerfile | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index cb5da2f40539..866c254b56e5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:16.04 +FROM debian:jessie-slim ARG http_proxy ARG https_proxy @@ -19,29 +19,32 @@ ARG USER ARG DJANGO_CONFIGURATION ENV DJANGO_CONFIGURATION=${DJANGO_CONFIGURATION} +# Install ffmpeg on debian jessie (https://www.deb-multimedia.org/) +# It is not available on official repositories (https://wiki.debian.org/ffmpeg) +RUN echo "deb http://www.deb-multimedia.org jessie main non-free" > /etc/apt/sources.list.d/deb-multimedia.list \ + && apt-get update \ + && apt-get install -y --force-yes deb-multimedia-keyring \ + && apt-get update \ + && apt-get install -y ffmpeg \ + && apt-get install -y gstreamer0.10-ffmpeg + # Install necessary apt packages RUN apt-get update && \ apt-get install -yq \ - python-software-properties \ software-properties-common \ wget && \ - add-apt-repository ppa:mc3man/xerus-media -y && \ - add-apt-repository ppa:mc3man/gstffmpeg-keep -y && \ - apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -yq \ apache2 \ apache2-dev \ libapache2-mod-xsendfile \ supervisor \ - ffmpeg \ - gstreamer0.10-ffmpeg \ libldap2-dev \ libsasl2-dev \ python3-dev \ python3-pip \ tzdata \ unzip \ - unrar \ + unrar-free \ p7zip-full \ vim \ git-core \ @@ -50,8 +53,6 @@ RUN apt-get update && \ pip3 install -U setuptools && \ ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime && \ dpkg-reconfigure -f noninteractive tzdata && \ - add-apt-repository --remove ppa:mc3man/gstffmpeg-keep -y && \ - add-apt-repository --remove ppa:mc3man/xerus-media -y && \ rm -rf /var/lib/apt/lists/* # Add a non-root user @@ -154,20 +155,19 @@ COPY utils ${HOME}/utils COPY cvat/ ${HOME}/cvat COPY cvat-core/ ${HOME}/cvat-core COPY tests ${HOME}/tests -COPY datumaro/ ${HOME}/datumaro - -RUN sed -r "s/^(.*)#.*$/\1/g" ${HOME}/datumaro/requirements.txt | xargs -n 1 -L 1 pip3 install --no-cache-dir # Binary option is necessary to correctly apply the patch on Windows platform. # https://unix.stackexchange.com/questions/239364/how-to-fix-hunk-1-failed-at-1-different-line-endings-message RUN patch --binary -p1 < ${HOME}/cvat/apps/engine/static/engine/js/3rdparty.patch -RUN chown -R ${USER}:${USER} . +RUN chown -R ${USER}:${USER} ${HOME} +RUN chown -R ${USER}:${USER} /var/log/ +RUN chown -R ${USER}:${USER} /tmp/ # RUN all commands below as 'django' user USER ${USER} RUN mkdir data share media keys logs /tmp/supervisord -RUN python3 manage.py collectstatic +RUN python3 manage.py collectstatic || python3 manage.py collectstatic EXPOSE 8080 8443 ENTRYPOINT ["/usr/bin/supervisord"] From f71e38ebca4ebbdefa8cc5625fae9d7d64581d11 Mon Sep 17 00:00:00 2001 From: Frederic Tausch Date: Thu, 28 Nov 2019 13:06:21 +0100 Subject: [PATCH 02/10] Moved from ubuntu to buster slim Moved from ubuntu to buster slim :) --- Dockerfile.ui | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Dockerfile.ui b/Dockerfile.ui index c5939296849e..642d382d9ce1 100644 --- a/Dockerfile.ui +++ b/Dockerfile.ui @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 AS cvat-ui +FROM node:10.16.3-buster-slim AS cvat-ui ARG http_proxy ARG https_proxy @@ -14,10 +14,6 @@ ENV TERM=xterm \ ENV LANG='C.UTF-8' \ LC_ALL='C.UTF-8' -# Install necessary apt packages -RUN apt update && apt install -yq nodejs npm curl && \ - npm install -g n && n 10.16.3 - # Create output directories RUN mkdir /tmp/cvat-ui /tmp/cvat-core From 533d69b473d03a73ebdb3b10b25def231fa74b78 Mon Sep 17 00:00:00 2001 From: Frederic Tausch Date: Thu, 28 Nov 2019 14:43:17 +0100 Subject: [PATCH 03/10] fixed django issue switched to python3 prebuild to fix django issue --- Dockerfile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 866c254b56e5..9a05a8d04bf5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:jessie-slim +FROM python:3.6.9-slim-jessie ARG http_proxy ARG https_proxy @@ -40,8 +40,6 @@ RUN apt-get update && \ supervisor \ libldap2-dev \ libsasl2-dev \ - python3-dev \ - python3-pip \ tzdata \ unzip \ unrar-free \ @@ -116,7 +114,7 @@ RUN if [ "$WITH_TESTS" = "yes" ]; then \ # Install and initialize CVAT, copy all necessary files COPY cvat/requirements/ /tmp/requirements/ COPY supervisord.conf mod_wsgi.conf wait-for-it.sh manage.py ${HOME}/ -RUN pip3 install --no-cache-dir -r /tmp/requirements/${DJANGO_CONFIGURATION}.txt +RUN pip3 install -r /tmp/requirements/${DJANGO_CONFIGURATION}.txt # pycocotools package is impossible to install with its dependencies by one pip install command RUN pip3 install --no-cache-dir pycocotools==2.0.0 From a9d8c984200e91edc65d8af76178347b66df57a2 Mon Sep 17 00:00:00 2001 From: Langhalsdino Date: Thu, 28 Nov 2019 22:35:59 +0100 Subject: [PATCH 04/10] fixed supervised to run in new os --- Dockerfile | 12 ++++++------ cvat/requirements/production.txt | 3 ++- supervisord.conf | 14 +++++++------- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9a05a8d04bf5..d54129c59292 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.6.9-slim-jessie +FROM python:3.6.9-jessie ARG http_proxy ARG https_proxy @@ -37,7 +37,6 @@ RUN apt-get update && \ apache2 \ apache2-dev \ libapache2-mod-xsendfile \ - supervisor \ libldap2-dev \ libsasl2-dev \ tzdata \ @@ -153,13 +152,14 @@ COPY utils ${HOME}/utils COPY cvat/ ${HOME}/cvat COPY cvat-core/ ${HOME}/cvat-core COPY tests ${HOME}/tests +COPY datumaro/ ${HOME}/datumaro + +RUN sed -r "s/^(.*)#.*$/\1/g" ${HOME}/datumaro/requirements.txt | xargs -n 1 -L 1 pip3 install --no-cache-dir # Binary option is necessary to correctly apply the patch on Windows platform. # https://unix.stackexchange.com/questions/239364/how-to-fix-hunk-1-failed-at-1-different-line-endings-message RUN patch --binary -p1 < ${HOME}/cvat/apps/engine/static/engine/js/3rdparty.patch -RUN chown -R ${USER}:${USER} ${HOME} -RUN chown -R ${USER}:${USER} /var/log/ -RUN chown -R ${USER}:${USER} /tmp/ +RUN chown -R ${USER}:${USER} . # RUN all commands below as 'django' user USER ${USER} @@ -168,4 +168,4 @@ RUN mkdir data share media keys logs /tmp/supervisord RUN python3 manage.py collectstatic || python3 manage.py collectstatic EXPOSE 8080 8443 -ENTRYPOINT ["/usr/bin/supervisord"] +ENTRYPOINT ["supervisord"] diff --git a/cvat/requirements/production.txt b/cvat/requirements/production.txt index 340e26bed754..b969cd127050 100644 --- a/cvat/requirements/production.txt +++ b/cvat/requirements/production.txt @@ -1,3 +1,4 @@ -r base.txt psycopg2-binary==2.7.4 -mod-wsgi==4.6.2 \ No newline at end of file +mod-wsgi==4.6.2 +supervisor==4.1.0 diff --git a/supervisord.conf b/supervisord.conf index d5e222d2a99b..3c1485302d55 100644 --- a/supervisord.conf +++ b/supervisord.conf @@ -24,26 +24,26 @@ autorestart=true [program:rqworker_default] command=%(ENV_HOME)s/wait-for-it.sh redis:6379 -t 0 -- bash -ic \ - "exec /usr/bin/python3 %(ENV_HOME)s/manage.py rqworker -v 3 default" + "exec python3 %(ENV_HOME)s/manage.py rqworker -v 3 default" environment=SSH_AUTH_SOCK="/tmp/ssh-agent.sock" numprocs=2 process_name=rqworker_default_%(process_num)s [program:rqworker_low] command=%(ENV_HOME)s/wait-for-it.sh redis:6379 -t 0 -- bash -ic \ - "exec /usr/bin/python3 %(ENV_HOME)s/manage.py rqworker -v 3 low" + "exec python3 %(ENV_HOME)s/manage.py rqworker -v 3 low" environment=SSH_AUTH_SOCK="/tmp/ssh-agent.sock" numprocs=1 [program:git_status_updater] command=%(ENV_HOME)s/wait-for-it.sh redis:6379 -t 0 -- bash -ic \ - "/usr/bin/python3 ~/manage.py update_git_states" + "python3 ~/manage.py update_git_states" environment=SSH_AUTH_SOCK="/tmp/ssh-agent.sock" numprocs=1 [program:rqscheduler] command=%(ENV_HOME)s/wait-for-it.sh redis:6379 -t 0 -- bash -ic \ - "/usr/bin/python3 /usr/local/bin/rqscheduler --host redis -i 30" + "python3 /usr/local/bin/rqscheduler --host redis -i 30" environment=SSH_AUTH_SOCK="/tmp/ssh-agent.sock" numprocs=1 @@ -54,9 +54,9 @@ numprocs=1 ; with docker cache. Thus it is necessary to run collectstatic here for such ; apps. command=%(ENV_HOME)s/wait-for-it.sh db:5432 -t 0 -- bash -ic \ - "/usr/bin/python3 ~/manage.py migrate && \ - /usr/bin/python3 ~/manage.py collectstatic --no-input && \ - exec /usr/bin/python3 $HOME/manage.py runmodwsgi --log-to-terminal --port 8080 \ + "python3 ~/manage.py migrate && \ + python3 ~/manage.py collectstatic --no-input && \ + exec python3 $HOME/manage.py runmodwsgi --log-to-terminal --port 8080 \ --limit-request-body 1073741824 --log-level INFO --include-file ~/mod_wsgi.conf \ %(ENV_DJANGO_MODWSGI_EXTRA_ARGS)s --locale %(ENV_LC_ALL)s" environment=SSH_AUTH_SOCK="/tmp/ssh-agent.sock" From 8fc901b71de09fe99d6e687be29fca89cbeadf3f Mon Sep 17 00:00:00 2001 From: Langhalsdino Date: Thu, 28 Nov 2019 22:50:15 +0100 Subject: [PATCH 05/10] complied to standarts --- Dockerfile | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index d54129c59292..bbe041f05c1b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,10 +23,14 @@ ENV DJANGO_CONFIGURATION=${DJANGO_CONFIGURATION} # It is not available on official repositories (https://wiki.debian.org/ffmpeg) RUN echo "deb http://www.deb-multimedia.org jessie main non-free" > /etc/apt/sources.list.d/deb-multimedia.list \ && apt-get update \ - && apt-get install -y --force-yes deb-multimedia-keyring \ + && apt-get install -yq --force-yes \ + deb-multimedia-keyring \ && apt-get update \ - && apt-get install -y ffmpeg \ - && apt-get install -y gstreamer0.10-ffmpeg + && apt-get install -yq \ + ffmpeg \ + gstreamer0.10-ffmpeg \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* # Install necessary apt packages RUN apt-get update && \ From 543e2da7304ed5ac7bfa16f9dc52fdddc9c67925 Mon Sep 17 00:00:00 2001 From: Langhalsdino Date: Thu, 28 Nov 2019 23:08:44 +0100 Subject: [PATCH 06/10] cleaning up dockerfile --- Dockerfile | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index bbe041f05c1b..1877af3f43a3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,16 +21,16 @@ ENV DJANGO_CONFIGURATION=${DJANGO_CONFIGURATION} # Install ffmpeg on debian jessie (https://www.deb-multimedia.org/) # It is not available on official repositories (https://wiki.debian.org/ffmpeg) -RUN echo "deb http://www.deb-multimedia.org jessie main non-free" > /etc/apt/sources.list.d/deb-multimedia.list \ - && apt-get update \ - && apt-get install -yq --force-yes \ - deb-multimedia-keyring \ - && apt-get update \ - && apt-get install -yq \ +RUN echo "deb http://www.deb-multimedia.org jessie main non-free" > /etc/apt/sources.list.d/deb-multimedia.list && \ + apt-get update && \ + apt-get install -yq --force-yes && \ + deb-multimedia-keyring && \ + apt-get update && \ + apt-get install -yq \ ffmpeg \ - gstreamer0.10-ffmpeg \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* + gstreamer0.10-ffmpeg && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* # Install necessary apt packages RUN apt-get update && \ From b1fa31c943a2446108837c83735d502f8d1cfffd Mon Sep 17 00:00:00 2001 From: Langhalsdino Date: Thu, 28 Nov 2019 23:10:19 +0100 Subject: [PATCH 07/10] moved supervised to base requierments --- cvat/requirements/base.txt | 1 + cvat/requirements/production.txt | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat/requirements/base.txt b/cvat/requirements/base.txt index 3c8c5d52617d..9a4d308fca20 100644 --- a/cvat/requirements/base.txt +++ b/cvat/requirements/base.txt @@ -47,3 +47,4 @@ h5py==2.9.0 imgaug==0.2.9 django-cors-headers==3.0.2 furl==2.0.0 +supervisor==4.1.0 diff --git a/cvat/requirements/production.txt b/cvat/requirements/production.txt index b969cd127050..a7edbe9bbe84 100644 --- a/cvat/requirements/production.txt +++ b/cvat/requirements/production.txt @@ -1,4 +1,3 @@ -r base.txt psycopg2-binary==2.7.4 mod-wsgi==4.6.2 -supervisor==4.1.0 From dce6ee288e7cd4ed9e497e4958aa43890e1dd532 Mon Sep 17 00:00:00 2001 From: Langhalsdino Date: Thu, 28 Nov 2019 23:13:54 +0100 Subject: [PATCH 08/10] fixed apt install --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1877af3f43a3..9afb24ae1124 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ ENV DJANGO_CONFIGURATION=${DJANGO_CONFIGURATION} # It is not available on official repositories (https://wiki.debian.org/ffmpeg) RUN echo "deb http://www.deb-multimedia.org jessie main non-free" > /etc/apt/sources.list.d/deb-multimedia.list && \ apt-get update && \ - apt-get install -yq --force-yes && \ + apt-get install -yq --force-yes \ deb-multimedia-keyring && \ apt-get update && \ apt-get install -yq \ From 1777a93741f1c3ae5603619df936e500bc629e63 Mon Sep 17 00:00:00 2001 From: Langhalsdino Date: Fri, 29 Nov 2019 00:01:17 +0100 Subject: [PATCH 09/10] made apt-get installs consistent --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9afb24ae1124..3ad24b2aaac7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ ENV DJANGO_CONFIGURATION=${DJANGO_CONFIGURATION} # It is not available on official repositories (https://wiki.debian.org/ffmpeg) RUN echo "deb http://www.deb-multimedia.org jessie main non-free" > /etc/apt/sources.list.d/deb-multimedia.list && \ apt-get update && \ - apt-get install -yq --force-yes \ + apt-get install -yq \ deb-multimedia-keyring && \ apt-get update && \ apt-get install -yq \ From abd1a1ff5987b3453e2f3b41c154d7bd33a72d7f Mon Sep 17 00:00:00 2001 From: Langhalsdino Date: Fri, 29 Nov 2019 11:53:44 +0100 Subject: [PATCH 10/10] revert changes --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3ad24b2aaac7..9afb24ae1124 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ ENV DJANGO_CONFIGURATION=${DJANGO_CONFIGURATION} # It is not available on official repositories (https://wiki.debian.org/ffmpeg) RUN echo "deb http://www.deb-multimedia.org jessie main non-free" > /etc/apt/sources.list.d/deb-multimedia.list && \ apt-get update && \ - apt-get install -yq \ + apt-get install -yq --force-yes \ deb-multimedia-keyring && \ apt-get update && \ apt-get install -yq \