From 9e24fe0827faaf451c04ec50da76954e7fb3d77a Mon Sep 17 00:00:00 2001 From: A-312 Date: Fri, 1 Feb 2019 11:16:02 +0100 Subject: [PATCH 01/69] =?UTF-8?q?Travis=20utilise=20le=20m=C3=AAme=20scrip?= =?UTF-8?q?t=20que=20nous?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .travis.yml | 145 ++++---------- Makefile | 5 +- requirements.txt | 2 + scripts/ci_mysql_setup.sh | 49 +++-- scripts/define_variable.sh | 7 + scripts/dependencies/arch.txt | 15 ++ scripts/dependencies/debian.txt | 20 ++ scripts/dependencies/fedora.txt | 18 ++ scripts/dependencies/ubuntu.txt | 20 ++ scripts/install_zds.sh | 333 ++++++++++++++++++++++++++------ scripts/travis_header.sh | 52 +++++ scripts/travis_script.sh | 63 ++++++ 12 files changed, 553 insertions(+), 176 deletions(-) mode change 100644 => 100755 scripts/define_variable.sh create mode 100644 scripts/dependencies/arch.txt create mode 100644 scripts/dependencies/debian.txt create mode 100644 scripts/dependencies/fedora.txt create mode 100644 scripts/dependencies/ubuntu.txt create mode 100755 scripts/travis_header.sh create mode 100755 scripts/travis_script.sh diff --git a/.travis.yml b/.travis.yml index a086b948d1..a67352c2de 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,8 +16,6 @@ addons: - mysql-client-5.6 - libmysqlclient-dev - language-pack-fr - - unzip - - oracle-java9-set-default git: @@ -31,12 +29,21 @@ matrix: env: global: - secure: "azmDZZQZzf88zpbkYpLpxI66vpEVyv+kniW0QdWAt4qlys8P5OcO3VJBR5YT85vlvnjN9b6raWQAL1ymee0WmVdTmzXed8XjZv7t9QXVw7pfezxMKlEftVp/4Cu4wtvbew0ViZXNWV2hNXHzEqlhgnoIOq94i0UzZ7grMrI0xm0=" - matrix: - - ZDS_TEST_JOB="zds.tutorialv2" - - ZDS_TEST_JOB="zds.member zds.utils zds.forum" - - ZDS_TEST_JOB="front zds.mp zds.gallery zds.pages zds.featured zds.notification zds.searchv2" - - ZDS_TEST_JOB="selenium" - - ZDS_TEST_JOB="doc" + + +jobs: + include: + - name: tuto + env: ZDS_TEST_JOB="zds.tutorialv2" + - name: "test1 : member utils forum" + env: ZDS_TEST_JOB="zds.member zds.utils zds.forum" + - name: "test2 : front mp gallery pages featured notification searchv2" + env: ZDS_TEST_JOB="front zds.mp zds.gallery zds.pages zds.featured zds.notification zds.searchv2" + - name: selenium + env: ZDS_TEST_JOB="selenium" + - name: doc + env: ZDS_TEST_JOB="doc" + notifications: webhooks: @@ -62,120 +69,46 @@ cache: before_install: - - . ./scripts/ci_turbo.sh # This script exports environment variables, it must be sourced + - HACK_SphinxBuild=$VIRTUAL_ENV - - source ./scripts/define_variable.sh + - source ./scripts/ci_turbo.sh # This script exports environment variables, it must be sourced - - travis_retry pip install -q coveralls + - source ./scripts/define_variable.sh - - | - # install elasticsearch - if [[ "$ZDS_TEST_JOB" == *"zds.searchv2"* ]]; then - # see https://docs.travis-ci.com/user/database-setup/#Installing-specific-versions-of-ElasticSearch - curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$ZDS_ELASTIC_VERSION.deb \ - && sudo dpkg -i --force-confnew elasticsearch-$ZDS_ELASTIC_VERSION.deb \ - && sudo service elasticsearch start - fi + - source ./scripts/travis_header.sh # Some function for this script - - | - # setup mysql - if [[ "$ZDS_TEST_JOB" == *"zds."* ]] || [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then - # MySQL config - ./scripts/ci_mysql_setup.sh - fi + - ./scripts/ci_mysql_setup.sh -install: - - | - # install webdriver for selenium. - if [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then - wget https://github.com/mozilla/geckodriver/releases/download/v0.23.0/geckodriver-v0.23.0-linux64.tar.gz \ - && mkdir geckodriver \ - && tar -xzf geckodriver-v0.23.0-linux64.tar.gz -C geckodriver \ - && export PATH=$PATH:$PWD/geckodriver \ - && export DISPLAY=:99.0 - fi + - install_geckodriver - - | - if [[ "$ZDS_TEST_JOB" == *"zds.tutorialv2"* ]]; then - # install latex - git clone $ZDS_LATEX_REPO - TEMPLATEDIR=$HOME/.texlive/texmf-local/tex/latex/ - ./latex-template/scripts/install_font.sh \ - && ./latex-template/scripts/install_texlive.sh \ - && export PATH=$HOME/.texlive/bin/x86_64-linux:$PATH \ - && rm -rf $TEMPLATEDIR/latex-template \ - && mkdir -p $TEMPLATEDIR && cp -r ./latex-template $TEMPLATEDIR && texhash - fi + - zds_register_module_for_installation - - | - # install and run zmarkdown - nvm install $ZDS_NODE_VERSION \ - && nvm use $ZDS_NODE_VERSION \ - && npm i -g pm2 zmarkdown \ - && cd "$(nvm which current | sed s/bin/lib/)_modules/zmarkdown/" \ - && npm run server \ - && cd $TRAVIS_BUILD_DIR - - | - # install frontend dependencies - npm i -g yarn \ - && yarn --silent +install: + - sudo apt-get update -qq - - | - # install backend dependencies - if [[ "$ZDS_TEST_JOB" == *"zds."* ]] || [[ "$ZDS_TEST_JOB" == *"selenium"* ]] || [[ "$ZDS_TEST_JOB" == *"doc"* ]]; then - pip install -q -r requirements-dev.txt -r requirements-prod.txt - fi + - ./scripts/install_zds.sh --travis-output $zds_install_argument script: - - | - # lint backend - if [[ "$ZDS_TEST_JOB" == *"zds.gallery"* ]]; then - ./scripts/no_import_zds_settings.sh \ - && flake8 \ - && flake8 --config=zds/settings/.flake8 zds/settings - fi + - source ./$ZDS_VENV/bin/activate - - | - # test and build frontend - if [[ "$ZDS_TEST_JOB" == *"front"* ]]; then - yarn run lint && yarn run build # we need to upload the assets later - fi - - | - # test backend - if [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then - python manage.py makemigrations --dry-run --check \ - && coverage run --source='.' manage.py \ - test -v=2\ - --keepdb \ - --settings zds.settings.ci_test \ - --exclude-tag=front \ - ${ZDS_TEST_JOB/front/} - fi + - ./scripts/travis_script.sh "start_elasticsearch" - - | - # show zmarkdown logs - if [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then - pm2 logs --nostream --raw --lines 1000 - fi + - ./scripts/travis_script.sh "start_latex" - - | - # selenium tests - if [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then - yarn run build - xvfb-run --server-args="-screen 0 1280x720x8" python manage.py \ - test -v=2\ - --settings zds.settings.ci_test \ - --tag=front \ - --keepdb - fi + - ./scripts/travis_script.sh "lint_backend" + + - ./scripts/travis_script.sh "test_backend" + + - ./scripts/travis_script.sh "print_zmarkdown_log" + + - ./scripts/travis_script.sh "selenium_test" + + - source $HACK_SphinxBuild/bin/activate + + - ./scripts/travis_script.sh "build_documentation" - - | - # build documentation - if [[ "$ZDS_TEST_JOB" == *"doc"* ]]; then - make --directory doc html - fi after_success: - du -sh $HOME/.texlive 2>/dev/null | true diff --git a/Makefile b/Makefile index e7674512e6..e37c3aaa0a 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,10 @@ clean: clean-back clean-front ## Clean everything (`clean-back` & `clean-front`) ## ~ Backend install-back: ## Install the Python packages for the backend - pip3 install --upgrade -r requirements-dev.txt + pip install --upgrade -r requirements-dev.txt + +install-back-with-prod: + pip install --upgrade -r requirements-dev.txt -r requirements-prod.txt run-back: zmd-check ## Run the backend server python manage.py runserver diff --git a/requirements.txt b/requirements.txt index b028a1522e..0d032c01ea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ social-auth-app-django==3.1.0 elasticsearch==5.5.2 elasticsearch-dsl==5.4.0 sqlparse==0.2.4 +setuptools==4.6.3 # Explicit dependencies (references in code) Django==2.1.7 @@ -11,6 +12,7 @@ django-model-utils==3.1.2 django-munin==0.2.1 python-memcached==1.59 lxml==4.2.5 +libxml2-python3==2.9.5 factory-boy==2.8.1 pygeoip==0.3.2 pillow==5.0.0 diff --git a/scripts/ci_mysql_setup.sh b/scripts/ci_mysql_setup.sh index 909e17acfa..1d8833a2bb 100755 --- a/scripts/ci_mysql_setup.sh +++ b/scripts/ci_mysql_setup.sh @@ -1,15 +1,38 @@ #!/bin/bash -set -ex - -sudo sed -i'' 's/\[client\]/\[client\]\ndefault-character-set=utf8mb4/' /etc/mysql/my.cnf -sudo sed -i'' 's/\[mysql\]/\[mysql\]\ndefault-character-set=utf8mb4/' /etc/mysql/my.cnf -sudo sed -i'' 's/\[mysqld\]/\[mysqld\]\ninnodb_file_per_table=on\ninnodb_file_format=barracuda\ninnodb_large_prefix=on\ncharacter-set-client-handshake=false\ncharacter-set-server=utf8mb4\ncollation-server=utf8mb4_unicode_ci/' /etc/mysql/my.cnf -sudo /etc/init.d/mysql restart -# Travis should fail as soon as possible -mysql -u root -e "SET GLOBAL sql_mode = 'NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES';" -# Avoid "mysql has gone away" errors -mysql -u root -e "SET GLOBAL wait_timeout = 36000;" -mysql -u root -e "SET GLOBAL max_allowed_packet = 134209536;" -# Create database with the correct charset and collation -mysql -u root -e "CREATE DATABASE zds_test CHARACTER SET = utf8mb4 COLLATE utf8mb4_unicode_ci;" +function print_info { + echo -n -e "\033[0;36m"; + echo "$1"; + echo -n -e "\033[00m"; +} + +if [[ "$ZDS_TEST_JOB" == *"zds."* ]] || [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then + #display print_info + forwho="" + if [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then + forwho="'zds.*' tasks (-> needed for tests)" + + if [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then + forwho="$forwho and selenium" + fi + else + forwho="selenium" + fi + print_info "* Install mysql for $forwho." + + #install + + set -ex + + sudo sed -i'' 's/\[client\]/\[client\]\ndefault-character-set=utf8mb4/' /etc/mysql/my.cnf + sudo sed -i'' 's/\[mysql\]/\[mysql\]\ndefault-character-set=utf8mb4/' /etc/mysql/my.cnf + sudo sed -i'' 's/\[mysqld\]/\[mysqld\]\ninnodb_file_per_table=on\ninnodb_file_format=barracuda\ninnodb_large_prefix=on\ncharacter-set-client-handshake=false\ncharacter-set-server=utf8mb4\ncollation-server=utf8mb4_unicode_ci/' /etc/mysql/my.cnf + sudo /etc/init.d/mysql restart + # Travis should fail as soon as possible + mysql -u root -e "SET GLOBAL sql_mode = 'NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES';" + # Avoid "mysql has gone away" errors + mysql -u root -e "SET GLOBAL wait_timeout = 36000;" + mysql -u root -e "SET GLOBAL max_allowed_packet = 134209536;" + # Create database with the correct charset and collation + mysql -u root -e "CREATE DATABASE zds_test CHARACTER SET = utf8mb4 COLLATE utf8mb4_unicode_ci;" +fi diff --git a/scripts/define_variable.sh b/scripts/define_variable.sh old mode 100644 new mode 100755 index ebec3411bf..8dee2bde19 --- a/scripts/define_variable.sh +++ b/scripts/define_variable.sh @@ -19,3 +19,10 @@ fi if [[ $ZDS_LATEX_REPO == "" ]]; then ZDS_LATEX_REPO="https://github.com/zestedesavoir/latex-template.git" fi + +if [[ $ZDS_JDK_HASH == "" ]]; then + ZDS_JDK_HASH="f51449fcd52f4d52b93a989c5c56ed3c" + # shellcheck disable=SC2034 + ZDS_JDK_VERSION="11.0.2" + ZDS_JDK_REV="+9" +fi diff --git a/scripts/dependencies/arch.txt b/scripts/dependencies/arch.txt new file mode 100644 index 0000000000..87b3dc84d1 --- /dev/null +++ b/scripts/dependencies/arch.txt @@ -0,0 +1,15 @@ +#title=Arch +#desc=Utilisation de pacman +#installcmd=pacman -Syu +git +wget +curl +python +python3-pip +libxslt +zlib +libffi +libjpeg-turbo +freetype2 +base-devel +unzip diff --git a/scripts/dependencies/debian.txt b/scripts/dependencies/debian.txt new file mode 100644 index 0000000000..da902049f1 --- /dev/null +++ b/scripts/dependencies/debian.txt @@ -0,0 +1,20 @@ +#title=Debian +#desc=Utilisation de apt-get / Testé sur : Debian Jessie +#installcmd=apt-get -y install +git +wget +curl +python3-dev +python3-pip +python3-venv +libxslt-dev +libz-dev +libjpeg62-turbo +libjpeg62-turbo-dev +libfreetype6 +libfreetype6-dev +libffi-dev +build-essential +imagemagick +librsvg2-bin +xzdec diff --git a/scripts/dependencies/fedora.txt b/scripts/dependencies/fedora.txt new file mode 100644 index 0000000000..e5ef5db558 --- /dev/null +++ b/scripts/dependencies/fedora.txt @@ -0,0 +1,18 @@ +#title=Fedora +#desc=Utilisation de dnf +#installcmd=dnf -y install +git +wget +curl +python3-devel +libxslt-devel +zlib-devel +libjpeg-turbo-devel +libjpeg-turbo-devel +freetype +freetype-devel +libffi-devel +python3-pip +python-virtualenv +gcc +redhat-rpm-config diff --git a/scripts/dependencies/ubuntu.txt b/scripts/dependencies/ubuntu.txt new file mode 100644 index 0000000000..68cd1ee6b0 --- /dev/null +++ b/scripts/dependencies/ubuntu.txt @@ -0,0 +1,20 @@ +#title=Ubuntu +#desc=Utilisation de apt-get / Testé sur : Ubuntu 14.04 +#installcmd=apt-get -y install +git +wget +curl +python3-dev +python3-pip +python3-venv +libxslt1-dev +zlib1g-dev +libjpeg8 +libjpeg8-dev +libfreetype6 +libfreetype6-dev +libffi-dev +build-essential +imagemagick +librsvg2-bin +xzdec diff --git a/scripts/install_zds.sh b/scripts/install_zds.sh index f7c9166c65..4b943157d2 100755 --- a/scripts/install_zds.sh +++ b/scripts/install_zds.sh @@ -2,6 +2,7 @@ # Install script for the zds-site repository + function _in { # credits: https://stackoverflow.com/a/8574392 local e match="$1" @@ -10,78 +11,226 @@ function _in { return 1 } + function _nvm { - export NVM_DIR="$HOME/.nvm" - [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm + export NVM_DIR="$HOME/.nvm" + [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm +} + +# >>>>> +ZDS_SHOW_TRAVIS_FOLD=0 +if $(_in "--travis-output" $@); then + ZDS_SHOW_TRAVIS_FOLD=1 +fi + +zds_fold_current="" +function zds_fold_start { + if [[ $ZDS_SHOW_TRAVIS_FOLD == 1 ]]; then + if [[ $zds_fold_current == $1 ]]; then # for virtualenv fold + return + fi + + zds_fold_current="$1" + echo "travis_fold:start:install_${zds_fold_current}" + echo -en "\033[0K"; + fi + + print_info "$2" --bold +} + +function zds_fold_end { + if [[ $ZDS_SHOW_TRAVIS_FOLD == 1 ]] && [[ $zds_fold_current =~ "" ]]; then + echo "travis_fold:end:install_${zds_fold_current}" + echo -en "\033[0K"; + zds_fold_current="" + fi } +# <<<<< + -function info { - echo -n -e "\033[0;36m"; +function print_info { + if [[ "$2" == "--bold" ]]; then + echo -en "\033[36;1m"; + else + echo -en "\033[0;36m"; + fi echo "$1"; - echo -n -e "\033[00m"; + echo -en "\033[00m"; } -function error { - echo -n -e "\033[0;31m"; + +function print_error { + echo -en "\033[31;1m"; echo "$1"; - echo -n -e "\033[00m"; + echo -en "\033[0m"; +} + + +# >>>>> +function progressfilt { + local flag=false c count cr=$'\r' nl=$'\n' + while IFS='' read -d '' -rn 1 c + do + if $flag; then + printf '%s' "$c" + else + if [[ $c != $cr && $c != $nl ]]; then + count=0 + else + ((count++)) + if ((count > 1)); then + flag=true + fi + fi + fi + done +} + +# Hack for "-q --show-progress" (at least v1.16) and travis uses (travis uses wget 1.15) +function wget_nv { + wget "$@" --progress=bar:force 2>&1 | progressfilt } +# <<<<< + +# lastSTDERR="" +# Store error of an command, example : `lookafter "sudo apt-get -qq -y install aaaaaaaa"` +# function lookafter { +# lastSTDERR=$($@ 3>&1 1>&2 2>&3 | sudo tee /dev/stderr) +# } # variables LOCAL_DIR="$(cd "$(dirname "$0")" && pwd)" source $LOCAL_DIR/define_variable.sh - -# os-specific package install +# Install packages if ! $(_in "-packages" $@) && ( $(_in "+packages" $@) || $(_in "+base" $@) || $(_in "+full" $@) ); then - info "* [+packages] installing packages (this subcommand will be run as super-user)" - version=$(cat /proc/version) - if [[ "$version" =~ "ubuntu" ]]; then - REALPATH="realpath" - release=$(lsb_release -c) - if [[ "$release" =~ "bionic" ]]; then - REALPATH="coreutils" + zds_fold_start "packages" "* [+packages] installing packages (this subcommand will be run as super-user)" + + if $(_in "--detect-os-version" $@); then + version=$(cat /proc/version) + + if [[ "$version" =~ "ubuntu" ]]; then + filepath="$LOCAL_DIR/dependencies/ubuntu.txt" + elif [[ "$version" =~ "debian" ]]; then + filepath="$LOCAL_DIR/dependencies/debian.txt" + elif [[ "$version" =~ "fedora" ]]; then + filepath="$LOCAL_DIR/dependencies/fedora.txt" + elif [[ "$version" =~ "arch" ]]; then + filepath="$LOCAL_DIR/dependencies/arch.txt" + else + print_error "!! I did not detect your linux version" + print_error "!! Please manually install the packages and run again without `--detect-os-version`" + exit 1 fi - sudo apt-get -y install git python3-dev python3-venv python3-setuptools libxml2-dev python3-lxml libxslt1-dev zlib1g-dev python3-sqlparse libjpeg8 libjpeg8-dev libfreetype6 libfreetype6-dev libffi-dev python3-pip build-essential curl $REALPATH librsvg2-bin imagemagick xzdec - elif [[ "$version" =~ "debian" ]]; then - sudo apt-get -y install git python3-dev python3-venv python3-setuptools libxml2-dev python3-lxml libxslt-dev libz-dev python3-sqlparse libjpeg62-turbo libjpeg62-turbo-dev libfreetype6 libfreetype6-dev libffi-dev python3-pip virtualenv build-essential curl librsvg2-bin imagemagick xzdec - elif [[ "$version" =~ "fedora" ]]; then - sudo dnf -y install git python3-devel python3-setuptools libxml2-devel python3-lxml libxslt-devel zlib-devel python3-sqlparse libjpeg-turbo-devel libjpeg-turbo-devel freetype freetype-devel libffi-devel python3-pip python-virtualenv gcc redhat-rpm-config - elif [[ "$version" =~ "arch" ]]; then - sudo pacman -Syu git wget python python-setuptools python-pip libxml2 python-lxml libxslt zlib python-sqlparse libffi libjpeg-turbo freetype2 base-devel unzip else - error "!! I did not detect your linux version" - error "!! Please manually install the packages and run again with \`-packages\`" - exit 1 - fi + echo -en "\033[33;1m"; + n=1 + arr=() + + for filepath in $LOCAL_DIR/dependencies/*.txt; do + title=$(grep -oP '#title=\K(.*)' $filepath) + desc=$(grep -oP '#desc=\K(.*)' $filepath) + echo "$n. $title - $desc" + arr[n]=$filepath + ((n++)) + done + + echo -en "\033[00m" + echo -n "Choix : " + read -n 1 + echo "" + + filepath=${arr[$REPLY]} + if [[ $filepath == "" ]]; then + print_error "!! You don't pick the right choice." + exit 1 + fi + fi; + + packagingTool_install=$(grep -oP '#installcmd=\K(.*)' $filepath) + print_info "$filepath" + IFS=$'\n' + + for dep in $(cat $filepath); do + if [[ $dep == "#"* ]]; then + continue; + fi + + print_info "sudo $packagingTool_install $dep" + echo " " + eval "sudo $packagingTool_install $dep" + + exVal=$? + if [[ $exVal > 0 && $dep == "python3-venv" ]]; then + print_info "!! We were unable to install virtualenv with apt-get. We try to install virtualenv with pip." + pip install virtualenv + elif [[ $exVal > 0 && ! $(_in "--answer-yes" $@) ]]; then + print_error "\`$dep\` not found, press \`y\` to continue the script." + read -n 1 + echo "" + if [ "$REPLY" == "y" ]; then + print_info "Continue installation" + else + print_error "Abort installation" + exit 1 + fi + elif [[ $exVal > 0 && $(_in "--answer-yes" $@) ]]; then + print_info "Continue installation (auto answer yes)." + else + echo " " + print_info "$dep: success." + echo " " + fi + done + + zds_fold_end fi # virtualenv if ! $(_in "-virtualenv" $@) && ( $(_in "+virtualenv" $@) || $(_in "+base" $@) || $(_in "+full" $@) ); then + zds_fold_start "virtualenv" "* Load virtualenv" + + print_info "* [+virtualenv] installing \`virtualenv 16.2.0\` with pip" + pip install virtualenv==16.2.0 + if [ ! -d $ZDS_VENV ]; then - info "* [+virtualenv] creating virtualenv" - python3 -m venv $ZDS_VENV + print_info "* [+virtualenv] creating virtualenv" + msg=$(python3 -m venv $ZDS_VENV) + if [[ $? != "0" && $msg == *"ensurepip"* ]]; then + echo $msg + print_info "!! Try --without-pip" + python3 -m venv $ZDS_VENV --without-pip + elif [[ $? != "0" ]]; then + echo $msg + fi fi fi if [[ $VIRTUAL_ENV == "" || $(basename $VIRTUAL_ENV) != $ZDS_VENV ]]; then - info "* activating venv \`$ZDS_VENV\`" + zds_fold_start "virtualenv" "* Load virtualenv" + + print_info "* activating venv \`$ZDS_VENV\`" if [ -d $HOME/.nvm ]; then # force nvm activation, in case of _nvm fi - . ./$ZDS_VENV/bin/activate + source ./$ZDS_VENV/bin/activate if [[ $? != "0" ]]; then - error "!! no virtualenv, cannot continue" + print_error "!! no virtualenv, cannot continue" exit 1 fi + + zds_fold_end +else + zds_fold_end fi + # nvm node & yarn if ! $(_in "-node" $@) && ( $(_in "+node" $@) || $(_in "+base" $@) || $(_in "+full" $@) ); then - info "* [+node] installing nvm (v$ZDS_NVM_VERSION) & node (v$ZDS_NODE_VERSION) & yarn" + zds_fold_start "node" "* [+node] installing nvm (v$ZDS_NVM_VERSION) & node (v$ZDS_NODE_VERSION) & yarn" wget -qO- https://raw.githubusercontent.com/creationix/nvm/v${ZDS_NVM_VERSION}/install.sh | bash if [[ $? == 0 ]]; then @@ -103,14 +252,56 @@ if ! $(_in "-node" $@) && ( $(_in "+node" $@) || $(_in "+base" $@) || $(_in "+f echo $ACTIVATE_NVM >> $VIRTUAL_ENV/bin/activate.fish fi else - error "!! Cannot obtain nvm v${ZDS_NVM_VERSION}" + print_error "!! Cannot obtain nvm v${ZDS_NVM_VERSION}" + exit 1 + fi + + zds_fold_end +fi + + +# local jdk +if ! $(_in "-jdk-local" $@) && ( $(_in "+jdk-local" $@) || $(_in "+full" $@) ); then + zds_fold_start "jdk" "* [+jdk-local] installing a local version of JDK (v$ZDS_JDK_VERSION)" + + mkdir -p zdsenv/lib/ + cd zdsenv/lib/ + + if [ -d jdk ]; then # remove previous install + rm -R jdk + fi + + baseURL="https://download.oracle.com/otn-pub/java/jdk/" + foldername="jdk-${ZDS_JDK_VERSION}" + folderPATH="${ZDS_JDK_VERSION}${ZDS_JDK_REV}/${ZDS_JDK_HASH}/${foldername}_linux-x64_bin.tar.gz" + + echo "GET ${baseURL}${folderPATH}" + wget_nv -O ${foldername}.tar.gz --header "Cookie: oraclelicense=accept-securebackup-cookie" ${baseURL}${folderPATH} + + if [[ $? == 0 ]]; then + tar xf ${foldername}.tar.gz + rm ${foldername}.tar.gz + mv ${foldername} jdk + + echo $(./jdk/bin/java --version) + + export PATH="$PATH:$(pwd)/jdk/bin" + export JAVA_HOME="$(pwd)/jdk" + export ES_JAVA_OPTS="-Xms512m -Xmx512m" + else + print_error "!! Cannot get jdk ${JDK_VERSION}" exit 1 fi + cd ../../ + + zds_fold_end fi + # local elasticsearch if ! $(_in "-elastic-local" $@) && ( $(_in "+elastic-local" $@) || $(_in "+full" $@) ); then - info "* [+elastic-local] installing a local version of elasticsearch (v$ZDS_ELASTIC_VERSION)" + zds_fold_start "elasticsearch" "* [+elastic-local] installing a local version of elasticsearch (v$ZDS_ELASTIC_VERSION)" + mkdir -p .local cd .local @@ -118,29 +309,32 @@ if ! $(_in "-elastic-local" $@) && ( $(_in "+elastic-local" $@) || $(_in "+full rm -R elasticsearch fi - wget -q https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ZDS_ELASTIC_VERSION}.zip + wget_nv https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ZDS_ELASTIC_VERSION}.zip if [[ $? == 0 ]]; then - unzip elasticsearch-${ZDS_ELASTIC_VERSION}.zip + unzip -q elasticsearch-${ZDS_ELASTIC_VERSION}.zip rm elasticsearch-${ZDS_ELASTIC_VERSION}.zip mv elasticsearch-${ZDS_ELASTIC_VERSION} elasticsearch # add options to reduce memory consumption - info "#Options added by install_zds.sh" >> elasticsearch/config/jvm.options - info "-Xms512m" >> elasticsearch/config/jvm.options - info "-Xmx512m" >> elasticsearch/config/jvm.options + print_info "#Options added by install_zds.sh" >> elasticsearch/config/jvm.options + print_info "-Xms512m" >> elasticsearch/config/jvm.options + print_info "-Xmx512m" >> elasticsearch/config/jvm.options # symbolic link to elastic start script - ln -s $(realpath elasticsearch/bin/elasticsearch) $VIRTUAL_ENV/bin/ + ln -s elasticsearch/bin/elasticsearch $VIRTUAL_ENV/bin/ else - error "!! Cannot get elasticsearch ${ZDS_ELASTIC_VERSION}" + print_error "!! Cannot get elasticsearch ${ZDS_ELASTIC_VERSION}" exit 1; fi; cd .. + + zds_fold_end fi + # local texlive if ! $(_in "-tex-local" $@) && ( $(_in "+tex-local" $@) || $(_in "+full" $@) ); then - info "* [+tex-local] install texlive" + zds_fold_start "texlive" "* [+tex-local] install texlive" CURRENT=$(pwd) mkdir -p .local @@ -172,7 +366,7 @@ if ! $(_in "-tex-local" $@) && ( $(_in "+tex-local" $@) || $(_in "+full" $@) ); sed -i 's@.texlive@texlive@' texlive.profile # change directory sed -i "s@\$HOME@$LOCAL@" texlive.profile # change destination - wget -q -O install-tl.tar.gz http://mirror.ctan.org/systems/texlive/tlnet/install-tl-unx.tar.gz + wget_nv -O install-tl.tar.gz http://mirror.ctan.org/systems/texlive/tlnet/install-tl-unx.tar.gz if [[ $? == 0 ]]; then if [[ ! -f ./bin/x86_64-linux/tlmgr ]]; then # install texlive tar xzf install-tl.tar.gz @@ -188,25 +382,28 @@ if ! $(_in "-tex-local" $@) && ( $(_in "+tex-local" $@) || $(_in "+full" $@) ); ./bin/x86_64-linux/tlmgr update --self rm -Rf $REPO else - error "!! Cannot download texlive" + print_error "!! Cannot download texlive" exit 1 fi else - error "!! cannot clone repository $ZDS_LATEX_REPO" + print_error "!! cannot clone repository $ZDS_LATEX_REPO" exit 1 fi cd $CURRENT + + zds_fold_end fi + # latex-template in TEXMFHOME. if ! $(_in "-latex-template" $@) && ( $(_in "+latex-template" $@) || $(_in "+full" $@) ); then - info "* [+latex-template] install latex-template (from $ZDS_LATEX_REPO)" + zds_fold_start "latex-template" "* [+latex-template] install latex-template (from $ZDS_LATEX_REPO)" CURRENT=$(pwd) if [[ $(which kpsewhich) == "" ]]; then # no texlive ? - error "!! Cannot find kpsewhich, do you have texlive?" + print_error "!! Cannot find kpsewhich, do you have texlive?" exit 1; fi @@ -223,40 +420,64 @@ if ! $(_in "-latex-template" $@) && ( $(_in "+latex-template" $@) || $(_in "+fu git clone $ZDS_LATEX_REPO if [[ $? != 0 ]]; then - error "!! cannot clone repository $ZDS_LATEX_REPO" + print_error "!! cannot clone repository $ZDS_LATEX_REPO" exit 1 fi cd $CURRENT + + zds_fold_end fi + # install back if ! $(_in "-back" $@) && ( $(_in "+back" $@) || $(_in "+base" $@) || $(_in "+full" $@) ); then - info "* [+back] install back dependencies & migration" - make install-back + zds_fold_start "back" "* [+back] install back dependencies & migration" + + if $(_in "+prod" $@); then + make install-back-with-prod + else + make install-back + fi make migrate-db # migration are required for the instance to run properly anyway + + zds_fold_end fi + # install front if ! $(_in "-front" $@) && ( $(_in "+front" $@) || $(_in "+base" $@) || $(_in "+full" $@) ); then - info "* [+front] install front dependencies & build front" + zds_fold_start "front" "* [+front] install front dependencies & build front" + if [ -d node_modules ]; then # delete previous modules rm -R node_modules fi; make install-front make build-front + + zds_fold_end fi + # zmd if ! $(_in "-zmd" $@) && ( $(_in "+zmd" $@) || $(_in "+base" $@) || $(_in "+full" $@) ); then - info "* [+zmd] install zmarkdown dependencies" + zds_fold_start "zmd" "* [+zmd] install zmarkdown dependencies" + make zmd-install + + zds_fold_end fi + # fixtures if ! $(_in "-data" $@) && ( $(_in "+data" $@) || $(_in "+base" $@) || $(_in "+full" $@) ); then - info "* [+data] fixtures" + zds_fold_start "fixtures" "* [+data] fixtures" + make generate-fixtures + + zds_fold_end fi -info "Done. You can now run instance with \`source $ZDS_VENV/bin/activate\`, and then, \`make zmd-start && make run-back\`" + + +print_info "Done. You can now run instance with \`source $ZDS_VENV/bin/activate\`, and then, \`make zmd-start && make run-back\`" diff --git a/scripts/travis_header.sh b/scripts/travis_header.sh new file mode 100755 index 0000000000..ea21856d01 --- /dev/null +++ b/scripts/travis_header.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +zds_install_argument="" +function zds_register_for_install { + if [[ "$zds_install_argument" == "" ]]; then + zds_install_argument="$1" + else + zds_install_argument="$zds_install_argument $1" + fi +} + +function print_info { + echo -n -e "\033[0;36m"; + echo "$1"; + echo -n -e "\033[00m"; +} + +function install_geckodriver { + print_info "* Install webdriver for selenium." + + if [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then + wget "https://github.com/mozilla/geckodriver/releases/download/v0.23.0/geckodriver-v0.23.0-linux64.tar.gz" \ + && mkdir "geckodriver" \ + && tar -xzf "geckodriver-v0.23.0-linux64.tar.gz" -C "geckodriver" \ + && export PATH="$PATH:$PWD/geckodriver" \ + && export DISPLAY=":99.0" + fi +} + +function zds_register_module_for_installation { + zds_register_for_install "+base +prod" + + # install elastic-local + if [[ "$ZDS_TEST_JOB" == *"zds.searchv2"* ]]; then + print_info "* Register elasticsearch with jdk. It needed to test : zds.searchv2." + zds_register_for_install "+jdk-local +elastic-local" + fi + + # install latex + if [[ "$ZDS_TEST_JOB" == *"zds.tutorialv2"* ]]; then + print_info "* Register latex for zds.tutorialv2." + zds_register_for_install "+tex-local +latex-template" + fi + + # install backend dependencies + if ! ( [[ "$ZDS_TEST_JOB" == *"zds."* ]] || [[ "$ZDS_TEST_JOB" == *"selenium"* ]] || [[ "$ZDS_TEST_JOB" == *"doc"* ]] ); then + print_info "* Don't register back because zds.* task, selenium and doc are not installed." + zds_register_for_install "-back" + fi + + print_info "* Argument for installation : $zds_install_argument" +} diff --git a/scripts/travis_script.sh b/scripts/travis_script.sh new file mode 100755 index 0000000000..c09845df3f --- /dev/null +++ b/scripts/travis_script.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +function print_info { + echo -n -e "\033[0;36m"; + echo "$1"; + echo -n -e "\033[00m"; +} + +# start elastic +if [[ "$1" == "start_elasticsearch" ]] && [[ "$ZDS_TEST_JOB" == *"zds.searchv2"* ]]; then + print_info "* Start elasticsearch as service" + sudo service elasticsearch start +fi + +# start latex +if [[ "$1" == "start_elasticsearch" ]] && [[ "$ZDS_TEST_JOB" == *"zds.tutorialv2"* ]]; then + print_info "* Start texhash" + texhash +fi + +# lint backend +if [[ "$1" == "lint_backend" ]] && [[ "$ZDS_TEST_JOB" == *"zds.gallery"* ]]; then + print_info "* Run lint for backend" + ./scripts/no_import_zds_settings.sh \ + && flake8 \ + && flake8 --config=zds/settings/.flake8 zds/settings +fi + +# test backend +if [[ "$1" == "test_backend" ]] && [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then + print_info "* Run test for backend" + python manage.py makemigrations --dry-run --check \ + && coverage run --source='.' manage.py \ + test -v=2\ + --keepdb \ + --settings zds.settings.ci_test \ + --exclude-tag=front \ + ${1/front/} +fi + +# print zmarkdown log +if [[ "$1" == "print_zmarkdown_log" ]] && [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then + print_info "* Print zmarkdown log" + pm2 logs --nostream --raw --lines 1000 +fi + +# selenium test +if [[ "$1" == "selenium_test" ]] && [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then + print_info "* Run selenium test for frontend" + xvfb-run --server-args="-screen 0 1280x720x8" python manage.py \ + test -v=2\ + --settings zds.settings.ci_test \ + --tag=front \ + --keepdb +fi + +# build documentation +if [[ "$1" == "build_documentation" ]] && [[ "$ZDS_TEST_JOB" == *"doc"* ]]; then + print_info "* Build documentation" + if [[ "$ZDS_TEST_JOB" == *"doc"* ]]; then + make generate-doc + fi +fi From 0a767cac6c5220f8ceb4ba94ec046358ebfd4a2e Mon Sep 17 00:00:00 2001 From: A-312 Date: Sun, 10 Mar 2019 20:49:39 +0100 Subject: [PATCH 02/69] Bypass la selection Humaine de l'OS... --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a67352c2de..a72127dc3b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -87,7 +87,7 @@ before_install: install: - sudo apt-get update -qq - - ./scripts/install_zds.sh --travis-output $zds_install_argument + - ./scripts/install_zds.sh --travis-output --detect-os-version $zds_install_argument script: From 66f069dc3dfa94ea02631a1dd105f246c050e59a Mon Sep 17 00:00:00 2001 From: A-312 Date: Sun, 10 Mar 2019 21:02:52 +0100 Subject: [PATCH 03/69] Sans hack --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a72127dc3b..dae03b6b21 100644 --- a/.travis.yml +++ b/.travis.yml @@ -105,7 +105,7 @@ script: - ./scripts/travis_script.sh "selenium_test" - - source $HACK_SphinxBuild/bin/activate + - #source $HACK_SphinxBuild/bin/activate - ./scripts/travis_script.sh "build_documentation" From 6e7b980058204d63cb6c1e44ad3d63d54d1796ee Mon Sep 17 00:00:00 2001 From: A-312 Date: Sun, 10 Mar 2019 21:06:22 +0100 Subject: [PATCH 04/69] il manque un 0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0d032c01ea..5da28cfb39 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ social-auth-app-django==3.1.0 elasticsearch==5.5.2 elasticsearch-dsl==5.4.0 sqlparse==0.2.4 -setuptools==4.6.3 +setuptools==40.6.3 # Explicit dependencies (references in code) Django==2.1.7 From da710efaef7dc9b1a49b8e009b0bd54a4c2feb99 Mon Sep 17 00:00:00 2001 From: A-312 Date: Sun, 10 Mar 2019 21:39:44 +0100 Subject: [PATCH 05/69] uncomment --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index dae03b6b21..a72127dc3b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -105,7 +105,7 @@ script: - ./scripts/travis_script.sh "selenium_test" - - #source $HACK_SphinxBuild/bin/activate + - source $HACK_SphinxBuild/bin/activate - ./scripts/travis_script.sh "build_documentation" From a9f4614ae6cddb6a02044a1af7501e406033794b Mon Sep 17 00:00:00 2001 From: A-312 Date: Sun, 10 Mar 2019 21:49:20 +0100 Subject: [PATCH 06/69] remet coveralls --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index a72127dc3b..53ad9458d8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -73,6 +73,8 @@ before_install: - source ./scripts/ci_turbo.sh # This script exports environment variables, it must be sourced + - travis_retry pip install -q coveralls + - source ./scripts/define_variable.sh - source ./scripts/travis_header.sh # Some function for this script From e8169741bda497263c2787adad1dfe835b8c1a66 Mon Sep 17 00:00:00 2001 From: A-312 Date: Sun, 10 Mar 2019 22:01:18 +0100 Subject: [PATCH 07/69] Doublon avec ligne 191 "installing `virtualenv 16.2.0`..." --- scripts/install_zds.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/scripts/install_zds.sh b/scripts/install_zds.sh index 4b943157d2..62e0df8aa1 100755 --- a/scripts/install_zds.sh +++ b/scripts/install_zds.sh @@ -161,10 +161,7 @@ if ! $(_in "-packages" $@) && ( $(_in "+packages" $@) || $(_in "+base" $@) || $ eval "sudo $packagingTool_install $dep" exVal=$? - if [[ $exVal > 0 && $dep == "python3-venv" ]]; then - print_info "!! We were unable to install virtualenv with apt-get. We try to install virtualenv with pip." - pip install virtualenv - elif [[ $exVal > 0 && ! $(_in "--answer-yes" $@) ]]; then + if [[ $exVal > 0 && ! $(_in "--answer-yes" $@) ]]; then print_error "\`$dep\` not found, press \`y\` to continue the script." read -n 1 echo "" From 0f42955c6f39d93ed212a5543e1195802135fe69 Mon Sep 17 00:00:00 2001 From: A-312 Date: Sun, 10 Mar 2019 22:17:20 +0100 Subject: [PATCH 08/69] =?UTF-8?q?J'ai=20oubli=C3=A9=20de=20mettre=20--answ?= =?UTF-8?q?er-yes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .travis.yml | 2 +- scripts/install_zds.sh | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 53ad9458d8..c80a7100a8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -89,7 +89,7 @@ before_install: install: - sudo apt-get update -qq - - ./scripts/install_zds.sh --travis-output --detect-os-version $zds_install_argument + - ./scripts/install_zds.sh --answer-yes --travis-output --detect-os-version $zds_install_argument script: diff --git a/scripts/install_zds.sh b/scripts/install_zds.sh index 62e0df8aa1..5023a6102a 100755 --- a/scripts/install_zds.sh +++ b/scripts/install_zds.sh @@ -161,18 +161,20 @@ if ! $(_in "-packages" $@) && ( $(_in "+packages" $@) || $(_in "+base" $@) || $ eval "sudo $packagingTool_install $dep" exVal=$? - if [[ $exVal > 0 && ! $(_in "--answer-yes" $@) ]]; then - print_error "\`$dep\` not found, press \`y\` to continue the script." + if [[ $dep == "python3-venv" ]]; then + print_error "!! We were unable to install virtualenv. Don't panic, we will try with pip." + elif [[ $exVal > 0 && ! $(_in "--answer-yes" $@) ]]; then + print_error "Unable to install \`$dep\`, press \`y\` to continue the script." read -n 1 echo "" if [ "$REPLY" == "y" ]; then - print_info "Continue installation" + print_info "Installation continued" else - print_error "Abort installation" + print_error "Installation aborted" exit 1 fi elif [[ $exVal > 0 && $(_in "--answer-yes" $@) ]]; then - print_info "Continue installation (auto answer yes)." + print_info "Installation continued (auto answer: \`yes\`)." else echo " " print_info "$dep: success." From 27e3cb213b73a7ac7689f5d76a5d6befdd74e906 Mon Sep 17 00:00:00 2001 From: A-312 Date: Sun, 10 Mar 2019 22:19:18 +0100 Subject: [PATCH 09/69] =?UTF-8?q?Affichage=20le=20message=20seulement=20si?= =?UTF-8?q?=20n=C3=A9cessaire?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .travis.yml | 2 +- scripts/install_zds.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c80a7100a8..d4d6b7eb6e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -89,7 +89,7 @@ before_install: install: - sudo apt-get update -qq - - ./scripts/install_zds.sh --answer-yes --travis-output --detect-os-version $zds_install_argument + - ./scripts/install_zds.sh --answer-yes --detect-os-version --travis-output $zds_install_argument script: diff --git a/scripts/install_zds.sh b/scripts/install_zds.sh index 5023a6102a..9559f9e8cf 100755 --- a/scripts/install_zds.sh +++ b/scripts/install_zds.sh @@ -161,7 +161,7 @@ if ! $(_in "-packages" $@) && ( $(_in "+packages" $@) || $(_in "+base" $@) || $ eval "sudo $packagingTool_install $dep" exVal=$? - if [[ $dep == "python3-venv" ]]; then + if [[ $exVal > 0 && $dep == "python3-venv" ]]; then print_error "!! We were unable to install virtualenv. Don't panic, we will try with pip." elif [[ $exVal > 0 && ! $(_in "--answer-yes" $@) ]]; then print_error "Unable to install \`$dep\`, press \`y\` to continue the script." From ba21234b1ef87d00c9a80f383bbf1eec95b942d3 Mon Sep 17 00:00:00 2001 From: A-312 Date: Sun, 10 Mar 2019 22:22:02 +0100 Subject: [PATCH 10/69] =?UTF-8?q?Ajout=20d'un=20saut=20=C3=A0=20la=20ligne?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/install_zds.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/install_zds.sh b/scripts/install_zds.sh index 9559f9e8cf..5505453d88 100755 --- a/scripts/install_zds.sh +++ b/scripts/install_zds.sh @@ -157,8 +157,9 @@ if ! $(_in "-packages" $@) && ( $(_in "+packages" $@) || $(_in "+base" $@) || $ fi print_info "sudo $packagingTool_install $dep" - echo " " + echo "" eval "sudo $packagingTool_install $dep" + echo "" exVal=$? if [[ $exVal > 0 && $dep == "python3-venv" ]]; then @@ -176,10 +177,9 @@ if ! $(_in "-packages" $@) && ( $(_in "+packages" $@) || $(_in "+base" $@) || $ elif [[ $exVal > 0 && $(_in "--answer-yes" $@) ]]; then print_info "Installation continued (auto answer: \`yes\`)." else - echo " " print_info "$dep: success." - echo " " fi + echo "" done zds_fold_end From b87f1f7e27aebdbd2dd6928511e4598de138a141 Mon Sep 17 00:00:00 2001 From: A-312 Date: Sun, 10 Mar 2019 23:06:07 +0100 Subject: [PATCH 11/69] Listen make error and exit 1, if we get one --- scripts/install_zds.sh | 48 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/scripts/install_zds.sh b/scripts/install_zds.sh index 5505453d88..2d2c4f30eb 100755 --- a/scripts/install_zds.sh +++ b/scripts/install_zds.sh @@ -171,7 +171,7 @@ if ! $(_in "-packages" $@) && ( $(_in "+packages" $@) || $(_in "+base" $@) || $ if [ "$REPLY" == "y" ]; then print_info "Installation continued" else - print_error "Installation aborted" + print_error "!! Installation aborted" exit 1 fi elif [[ $exVal > 0 && $(_in "--answer-yes" $@) ]]; then @@ -195,11 +195,11 @@ if ! $(_in "-virtualenv" $@) && ( $(_in "+virtualenv" $@) || $(_in "+base" $@) if [ ! -d $ZDS_VENV ]; then print_info "* [+virtualenv] creating virtualenv" msg=$(python3 -m venv $ZDS_VENV) - if [[ $? != "0" && $msg == *"ensurepip"* ]]; then + if [[ $? != 0 && $msg == *"ensurepip"* ]]; then echo $msg print_info "!! Try --without-pip" python3 -m venv $ZDS_VENV --without-pip - elif [[ $? != "0" ]]; then + elif [[ $? != 0 ]]; then echo $msg fi fi @@ -216,8 +216,8 @@ if [[ $VIRTUAL_ENV == "" || $(basename $VIRTUAL_ENV) != $ZDS_VENV ]]; then source ./$ZDS_VENV/bin/activate - if [[ $? != "0" ]]; then - print_error "!! no virtualenv, cannot continue" + if [[ $? != 0 ]]; then + print_error "!! No virtualenv, cannot continue" exit 1 fi @@ -385,7 +385,7 @@ if ! $(_in "-tex-local" $@) && ( $(_in "+tex-local" $@) || $(_in "+full" $@) ); exit 1 fi else - print_error "!! cannot clone repository $ZDS_LATEX_REPO" + print_error "!! Cannot clone repository $ZDS_LATEX_REPO" exit 1 fi @@ -419,7 +419,7 @@ if ! $(_in "-latex-template" $@) && ( $(_in "+latex-template" $@) || $(_in "+fu git clone $ZDS_LATEX_REPO if [[ $? != 0 ]]; then - print_error "!! cannot clone repository $ZDS_LATEX_REPO" + print_error "!! Cannot clone repository $ZDS_LATEX_REPO" exit 1 fi @@ -435,11 +435,24 @@ if ! $(_in "-back" $@) && ( $(_in "+back" $@) || $(_in "+base" $@) || $(_in "+f if $(_in "+prod" $@); then make install-back-with-prod + exVal=$? else make install-back + exVal=$? fi + + if [[ $exVal != 0 ]]; then + print_error "!! Cannot install back dependencies (use \`-back\` to skip)" + exit 1 + fi + make migrate-db # migration are required for the instance to run properly anyway + if [[ $? != 0 ]]; then + print_error "!! Cannot migrate database after the back installation (use \`-back\` to skip)" + exit 1 + fi + zds_fold_end fi @@ -453,8 +466,19 @@ if ! $(_in "-front" $@) && ( $(_in "+front" $@) || $(_in "+base" $@) || $(_in " fi; make install-front + + if [[ $? != 0 ]]; then + print_error "!! Cannot install-front (use \`-front\` to skip)" + exit 1 + fi + make build-front + if [[ $? != 0 ]]; then + print_error "!! Cannot build-front (use \`-front\` to skip)" + exit 1 + fi + zds_fold_end fi @@ -465,6 +489,11 @@ if ! $(_in "-zmd" $@) && ( $(_in "+zmd" $@) || $(_in "+base" $@) || $(_in "+ful make zmd-install + if [[ $? != 0 ]]; then + print_error "!! Cannot install zmd (use \`-zmd\` to skip)" + exit 1 + fi + zds_fold_end fi @@ -475,6 +504,11 @@ if ! $(_in "-data" $@) && ( $(_in "+data" $@) || $(_in "+base" $@) || $(_in "+f make generate-fixtures + if [[ $? != 0 ]]; then + print_error "!! Cannot generate-fixtures (use \`-data\` to skip)" + exit 1 + fi + zds_fold_end fi From 27f6d1d4e42b93760d503e9e3c60e7f60980fe01 Mon Sep 17 00:00:00 2001 From: A-312 Date: Sun, 10 Mar 2019 23:52:24 +0100 Subject: [PATCH 12/69] =?UTF-8?q?Change=20l'op=C3=A9rateur?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/install_zds.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/install_zds.sh b/scripts/install_zds.sh index 2d2c4f30eb..0a24ac83e1 100755 --- a/scripts/install_zds.sh +++ b/scripts/install_zds.sh @@ -62,7 +62,7 @@ function print_info { function print_error { echo -en "\033[31;1m"; echo "$1"; - echo -en "\033[0m"; + echo -en "\033[00m"; } @@ -162,9 +162,9 @@ if ! $(_in "-packages" $@) && ( $(_in "+packages" $@) || $(_in "+base" $@) || $ echo "" exVal=$? - if [[ $exVal > 0 && $dep == "python3-venv" ]]; then + if [[ $exVal != 0 && $dep == "python3-venv" ]]; then print_error "!! We were unable to install virtualenv. Don't panic, we will try with pip." - elif [[ $exVal > 0 && ! $(_in "--answer-yes" $@) ]]; then + elif [[ $exVal != 0 && ! $(_in "--answer-yes" $@) ]]; then print_error "Unable to install \`$dep\`, press \`y\` to continue the script." read -n 1 echo "" @@ -174,7 +174,7 @@ if ! $(_in "-packages" $@) && ( $(_in "+packages" $@) || $(_in "+base" $@) || $ print_error "!! Installation aborted" exit 1 fi - elif [[ $exVal > 0 && $(_in "--answer-yes" $@) ]]; then + elif [[ $exVal != 0 && $(_in "--answer-yes" $@) ]]; then print_info "Installation continued (auto answer: \`yes\`)." else print_info "$dep: success." From a5a2ca9a02ad4785cc4be7e969056eeb78261197 Mon Sep 17 00:00:00 2001 From: A-312 Date: Mon, 11 Mar 2019 00:10:06 +0100 Subject: [PATCH 13/69] =?UTF-8?q?D=C3=A9marre=20zmd=20avant=20d'installer?= =?UTF-8?q?=20les=20fixtures?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/install_zds.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/scripts/install_zds.sh b/scripts/install_zds.sh index 0a24ac83e1..2022498508 100755 --- a/scripts/install_zds.sh +++ b/scripts/install_zds.sh @@ -502,6 +502,13 @@ fi if ! $(_in "-data" $@) && ( $(_in "+data" $@) || $(_in "+base" $@) || $(_in "+full" $@) ); then zds_fold_start "fixtures" "* [+data] fixtures" + make zmd-start + + if [[ $? != 0 ]]; then + print_error "!! Cannot start zmd to generate-fixtures (use \`-data\` to skip)" + exit 1 + fi + make generate-fixtures if [[ $? != 0 ]]; then @@ -509,6 +516,12 @@ if ! $(_in "-data" $@) && ( $(_in "+data" $@) || $(_in "+base" $@) || $(_in "+f exit 1 fi + make zmd-stop + + if [[ $? != 0 ]]; then + print_error "Warning: Cannot stop zmd" + fi + zds_fold_end fi From 32723c79c1f1cc32f16a5cb07718f5128caee088 Mon Sep 17 00:00:00 2001 From: A-312 Date: Mon, 11 Mar 2019 01:09:22 +0100 Subject: [PATCH 14/69] Positionne correctement la variable --- scripts/install_zds.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/install_zds.sh b/scripts/install_zds.sh index 2022498508..238a533291 100755 --- a/scripts/install_zds.sh +++ b/scripts/install_zds.sh @@ -158,10 +158,9 @@ if ! $(_in "-packages" $@) && ( $(_in "+packages" $@) || $(_in "+base" $@) || $ print_info "sudo $packagingTool_install $dep" echo "" - eval "sudo $packagingTool_install $dep" + eval "sudo $packagingTool_install $dep"; exVal=$? echo "" - exVal=$? if [[ $exVal != 0 && $dep == "python3-venv" ]]; then print_error "!! We were unable to install virtualenv. Don't panic, we will try with pip." elif [[ $exVal != 0 && ! $(_in "--answer-yes" $@) ]]; then From 8aa2937149c5b538f558f09166d33ffe478cbc20 Mon Sep 17 00:00:00 2001 From: A-312 Date: Mon, 11 Mar 2019 19:29:16 +0100 Subject: [PATCH 15/69] Diminue le niveau de log --- zds/tutorialv2/epub_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/zds/tutorialv2/epub_utils.py b/zds/tutorialv2/epub_utils.py index d28a62fe56..ff96a1e279 100644 --- a/zds/tutorialv2/epub_utils.py +++ b/zds/tutorialv2/epub_utils.py @@ -156,8 +156,10 @@ def build_ebook(published_content_entity, working_dir, final_file_path): build_container_xml(meta_inf_dir_path) build_nav_xhtml(ops_dir, published_content_entity, chapters) + zip_logger = logging.getLogger(__name__ + '.zip') + zip_logger.setLevel(logging.WARN) shutil.make_archive(str(final_file_path), format='zip', root_dir=str(Path(working_dir, 'ebook')), - logger=logging.getLogger(__name__)) + logger=zip_logger) shutil.move(str(final_file_path) + '.zip', str(final_file_path)) From b2df49517b3b4d5697d9a4c5ff9139a3cc494ae5 Mon Sep 17 00:00:00 2001 From: A-312 Date: Mon, 11 Mar 2019 19:56:29 +0100 Subject: [PATCH 16/69] =?UTF-8?q?Test=20niveau=20log=20diff=C3=A9rent?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- zds/utils/management/commands/load_fixtures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zds/utils/management/commands/load_fixtures.py b/zds/utils/management/commands/load_fixtures.py index 7d85f38ee9..553d756543 100644 --- a/zds/utils/management/commands/load_fixtures.py +++ b/zds/utils/management/commands/load_fixtures.py @@ -191,7 +191,7 @@ def load_tags(cli, size, fake, *_, **__): title = fake.word() with contextlib.suppress(IntegrityError): tag, created = Tag.objects.get_or_create(title=title.lower()) - logging.getLogger(cli.__class__.__name__).debug('tag=%s is_new=%s', tag, created) + #logging.getLogger(cli.__class__.__name__).debug('tag=%s is_new=%s', tag, created) sys.stdout.write(' Tag {}/{} \r'.format(i + 1, nb_tags)) sys.stdout.flush() tps2 = time.time() From 7302a4b1989b8b7eb60c1f12d3bef29e2145963f Mon Sep 17 00:00:00 2001 From: A-312 Date: Mon, 11 Mar 2019 20:03:03 +0100 Subject: [PATCH 17/69] revert --- zds/utils/management/commands/load_fixtures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zds/utils/management/commands/load_fixtures.py b/zds/utils/management/commands/load_fixtures.py index 553d756543..7d85f38ee9 100644 --- a/zds/utils/management/commands/load_fixtures.py +++ b/zds/utils/management/commands/load_fixtures.py @@ -191,7 +191,7 @@ def load_tags(cli, size, fake, *_, **__): title = fake.word() with contextlib.suppress(IntegrityError): tag, created = Tag.objects.get_or_create(title=title.lower()) - #logging.getLogger(cli.__class__.__name__).debug('tag=%s is_new=%s', tag, created) + logging.getLogger(cli.__class__.__name__).debug('tag=%s is_new=%s', tag, created) sys.stdout.write(' Tag {}/{} \r'.format(i + 1, nb_tags)) sys.stdout.flush() tps2 = time.time() From 98983cd5ded97425008dc9bb6106d0bfe3be483f Mon Sep 17 00:00:00 2001 From: A-312 Date: Mon, 11 Mar 2019 20:46:20 +0100 Subject: [PATCH 18/69] Diminue le niveau de log pour diminuer l'output --- scripts/install_zds.sh | 2 +- zds/settings/ci_test.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/install_zds.sh b/scripts/install_zds.sh index 238a533291..7d783c3093 100755 --- a/scripts/install_zds.sh +++ b/scripts/install_zds.sh @@ -501,7 +501,7 @@ fi if ! $(_in "-data" $@) && ( $(_in "+data" $@) || $(_in "+base" $@) || $(_in "+full" $@) ); then zds_fold_start "fixtures" "* [+data] fixtures" - make zmd-start + npm run server --prefix zmd/node_modules/zmarkdown -- --silent if [[ $? != 0 ]]; then print_error "!! Cannot start zmd to generate-fixtures (use \`-data\` to skip)" diff --git a/zds/settings/ci_test.py b/zds/settings/ci_test.py index 239785e667..394c870129 100644 --- a/zds/settings/ci_test.py +++ b/zds/settings/ci_test.py @@ -19,3 +19,7 @@ for logger in LOGGING['loggers'].values(): logger['level'] = 'ERROR' + +LOGGING['loggers']['zds.utils.templatetags.emarkdown'] = { + 'handlers': [] +} From 26bcf7e7bdca2388efa748c90fc33bf7861b9c4a Mon Sep 17 00:00:00 2001 From: A-312 Date: Mon, 11 Mar 2019 21:27:46 +0100 Subject: [PATCH 19/69] =?UTF-8?q?Nouveau=20syst=C3=A8me=20de=20log=20:)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/install_zds.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/scripts/install_zds.sh b/scripts/install_zds.sh index 7d783c3093..b3e01ef4ef 100755 --- a/scripts/install_zds.sh +++ b/scripts/install_zds.sh @@ -508,9 +508,19 @@ if ! $(_in "-data" $@) && ( $(_in "+data" $@) || $(_in "+base" $@) || $(_in "+f exit 1 fi - make generate-fixtures + python manage.py loaddata fixtures/*.yaml + exVal=$? + python manage.py load_factory_data fixtures/advanced/aide_tuto_media.yaml + exVal=($exVal + $?) - if [[ $? != 0 ]]; then + if $(_in "--travis-output" $@); then + python manage.py load_fixtures --size=low --all --settings zds.settings.ci_test + else + python manage.py load_fixtures --size=low --all + fi + exVal=($exVal + $?) + + if [[ $exVal != 0 ]]; then print_error "!! Cannot generate-fixtures (use \`-data\` to skip)" exit 1 fi From 2fa1fae299ba9a1f6592c63a777dcbe7c40bcdb4 Mon Sep 17 00:00:00 2001 From: A-312 Date: Mon, 11 Mar 2019 21:55:55 +0100 Subject: [PATCH 20/69] =?UTF-8?q?Settings=20personnalis=C3=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/install_zds.sh | 2 +- zds/settings/ci_test.py | 4 ---- zds/settings/travis_fixture.py | 5 +++++ 3 files changed, 6 insertions(+), 5 deletions(-) create mode 100644 zds/settings/travis_fixture.py diff --git a/scripts/install_zds.sh b/scripts/install_zds.sh index b3e01ef4ef..cb9c743d6b 100755 --- a/scripts/install_zds.sh +++ b/scripts/install_zds.sh @@ -514,7 +514,7 @@ if ! $(_in "-data" $@) && ( $(_in "+data" $@) || $(_in "+base" $@) || $(_in "+f exVal=($exVal + $?) if $(_in "--travis-output" $@); then - python manage.py load_fixtures --size=low --all --settings zds.settings.ci_test + python manage.py load_fixtures --size=low --all --settings zds.settings.travis_fixture else python manage.py load_fixtures --size=low --all fi diff --git a/zds/settings/ci_test.py b/zds/settings/ci_test.py index 394c870129..239785e667 100644 --- a/zds/settings/ci_test.py +++ b/zds/settings/ci_test.py @@ -19,7 +19,3 @@ for logger in LOGGING['loggers'].values(): logger['level'] = 'ERROR' - -LOGGING['loggers']['zds.utils.templatetags.emarkdown'] = { - 'handlers': [] -} diff --git a/zds/settings/travis_fixture.py b/zds/settings/travis_fixture.py new file mode 100644 index 0000000000..4e8ca3a9b7 --- /dev/null +++ b/zds/settings/travis_fixture.py @@ -0,0 +1,5 @@ +from .dev import * + +LOGGING['loggers']['zds.utils.templatetags.emarkdown'] = { + 'handlers': [] +} From f5b43294780e550ce197badcf94acddafa2d6a88 Mon Sep 17 00:00:00 2001 From: A-312 Date: Mon, 11 Mar 2019 22:15:37 +0100 Subject: [PATCH 21/69] =?UTF-8?q?Settings=20personnalis=C3=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- zds/settings/travis_fixture.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/zds/settings/travis_fixture.py b/zds/settings/travis_fixture.py index 4e8ca3a9b7..f754fe91cf 100644 --- a/zds/settings/travis_fixture.py +++ b/zds/settings/travis_fixture.py @@ -1,5 +1,10 @@ from .dev import * LOGGING['loggers']['zds.utils.templatetags.emarkdown'] = { - 'handlers': [] + 'loggers': { + 'zds': { + 'level': 'INFO', + 'handlers': ['console'], + }, + }, } From 593d0447251ff149f66c7add903416c7193e3593 Mon Sep 17 00:00:00 2001 From: A-312 Date: Mon, 11 Mar 2019 22:39:53 +0100 Subject: [PATCH 22/69] =?UTF-8?q?Settings=20personnalis=C3=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- zds/settings/travis_fixture.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/zds/settings/travis_fixture.py b/zds/settings/travis_fixture.py index f754fe91cf..93eccc2e07 100644 --- a/zds/settings/travis_fixture.py +++ b/zds/settings/travis_fixture.py @@ -1,10 +1,6 @@ from .dev import * LOGGING['loggers']['zds.utils.templatetags.emarkdown'] = { - 'loggers': { - 'zds': { - 'level': 'INFO', - 'handlers': ['console'], - }, - }, + 'level': 'INFO', + 'handlers': ['console'], } From 8233ad03762ff53cbbfe5bd7877946476d83489d Mon Sep 17 00:00:00 2001 From: A-312 Date: Tue, 12 Mar 2019 00:13:13 +0100 Subject: [PATCH 23/69] =?UTF-8?q?Change=20le=20titre=20de=20la=20cat=C3=A9?= =?UTF-8?q?gorie?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/install_zds.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install_zds.sh b/scripts/install_zds.sh index cb9c743d6b..78bd3c1311 100755 --- a/scripts/install_zds.sh +++ b/scripts/install_zds.sh @@ -186,7 +186,7 @@ fi # virtualenv if ! $(_in "-virtualenv" $@) && ( $(_in "+virtualenv" $@) || $(_in "+base" $@) || $(_in "+full" $@) ); then - zds_fold_start "virtualenv" "* Load virtualenv" + zds_fold_start "virtualenv" "* Install virtualenv" print_info "* [+virtualenv] installing \`virtualenv 16.2.0\` with pip" pip install virtualenv==16.2.0 From d6ce054dbe4bd032664e9d50b7a3464903ba4a0b Mon Sep 17 00:00:00 2001 From: A-312 Date: Tue, 12 Mar 2019 00:19:45 +0100 Subject: [PATCH 24/69] use pip3 --- scripts/install_zds.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/install_zds.sh b/scripts/install_zds.sh index 78bd3c1311..68c2107345 100755 --- a/scripts/install_zds.sh +++ b/scripts/install_zds.sh @@ -162,7 +162,7 @@ if ! $(_in "-packages" $@) && ( $(_in "+packages" $@) || $(_in "+base" $@) || $ echo "" if [[ $exVal != 0 && $dep == "python3-venv" ]]; then - print_error "!! We were unable to install virtualenv. Don't panic, we will try with pip." + print_error "!! We were unable to install virtualenv. Don't panic, we will try with pip3." elif [[ $exVal != 0 && ! $(_in "--answer-yes" $@) ]]; then print_error "Unable to install \`$dep\`, press \`y\` to continue the script." read -n 1 @@ -189,7 +189,7 @@ if ! $(_in "-virtualenv" $@) && ( $(_in "+virtualenv" $@) || $(_in "+base" $@) zds_fold_start "virtualenv" "* Install virtualenv" print_info "* [+virtualenv] installing \`virtualenv 16.2.0\` with pip" - pip install virtualenv==16.2.0 + pip3 install virtualenv==16.2.0 if [ ! -d $ZDS_VENV ]; then print_info "* [+virtualenv] creating virtualenv" From a2bc17d90f882c00b9bb1d087ea3d857e29060c7 Mon Sep 17 00:00:00 2001 From: A-312 Date: Tue, 12 Mar 2019 13:52:11 +0100 Subject: [PATCH 25/69] =?UTF-8?q?unzip=20n'=C3=A9tait=20pas=20install?= =?UTF-8?q?=C3=A9=20pour=20tout=20le=20monde?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/dependencies/arch.txt | 2 +- scripts/dependencies/debian.txt | 1 + scripts/dependencies/fedora.txt | 1 + scripts/dependencies/ubuntu.txt | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/dependencies/arch.txt b/scripts/dependencies/arch.txt index 87b3dc84d1..18c13dc911 100644 --- a/scripts/dependencies/arch.txt +++ b/scripts/dependencies/arch.txt @@ -4,6 +4,7 @@ git wget curl +unzip python python3-pip libxslt @@ -12,4 +13,3 @@ libffi libjpeg-turbo freetype2 base-devel -unzip diff --git a/scripts/dependencies/debian.txt b/scripts/dependencies/debian.txt index da902049f1..1d7ab9e8f3 100644 --- a/scripts/dependencies/debian.txt +++ b/scripts/dependencies/debian.txt @@ -4,6 +4,7 @@ git wget curl +unzip python3-dev python3-pip python3-venv diff --git a/scripts/dependencies/fedora.txt b/scripts/dependencies/fedora.txt index e5ef5db558..1024e2f81a 100644 --- a/scripts/dependencies/fedora.txt +++ b/scripts/dependencies/fedora.txt @@ -4,6 +4,7 @@ git wget curl +unzip python3-devel libxslt-devel zlib-devel diff --git a/scripts/dependencies/ubuntu.txt b/scripts/dependencies/ubuntu.txt index 68cd1ee6b0..edc2f57f27 100644 --- a/scripts/dependencies/ubuntu.txt +++ b/scripts/dependencies/ubuntu.txt @@ -4,6 +4,7 @@ git wget curl +unzip python3-dev python3-pip python3-venv From 3ecd901014cb109222431ccf04d1d5ae769111c9 Mon Sep 17 00:00:00 2001 From: A-312 Date: Tue, 12 Mar 2019 14:41:45 +0100 Subject: [PATCH 26/69] =?UTF-8?q?Am=C3=A9liore=20la=20syntaxe?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/install_zds.sh | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/scripts/install_zds.sh b/scripts/install_zds.sh index 68c2107345..b94b44c06c 100755 --- a/scripts/install_zds.sh +++ b/scripts/install_zds.sh @@ -92,12 +92,6 @@ function wget_nv { } # <<<<< -# lastSTDERR="" -# Store error of an command, example : `lookafter "sudo apt-get -qq -y install aaaaaaaa"` -# function lookafter { -# lastSTDERR=$($@ 3>&1 1>&2 2>&3 | sudo tee /dev/stderr) -# } - # variables LOCAL_DIR="$(cd "$(dirname "$0")" && pwd)" source $LOCAL_DIR/define_variable.sh @@ -119,7 +113,7 @@ if ! $(_in "-packages" $@) && ( $(_in "+packages" $@) || $(_in "+base" $@) || $ filepath="$LOCAL_DIR/dependencies/arch.txt" else print_error "!! I did not detect your linux version" - print_error "!! Please manually install the packages and run again without `--detect-os-version`" + print_error "!! Please manually install the packages and run again without \`--detect-os-version\`" exit 1 fi else @@ -128,30 +122,32 @@ if ! $(_in "-packages" $@) && ( $(_in "+packages" $@) || $(_in "+base" $@) || $ arr=() for filepath in $LOCAL_DIR/dependencies/*.txt; do - title=$(grep -oP '#title=\K(.*)' $filepath) - desc=$(grep -oP '#desc=\K(.*)' $filepath) + title=$(grep -oP '#title=\K(.*)' "$filepath") + desc=$(grep -oP '#desc=\K(.*)' "$filepath") echo "$n. $title - $desc" arr[n]=$filepath ((n++)) done echo -en "\033[00m" - echo -n "Choix : " + + echo -n "Choice : " read -n 1 echo "" - filepath=${arr[$REPLY]} + filepath="${arr[$REPLY]}" if [[ $filepath == "" ]]; then print_error "!! You don't pick the right choice." exit 1 fi - fi; + fi + echo "" - packagingTool_install=$(grep -oP '#installcmd=\K(.*)' $filepath) + packagingTool_install=$(grep -oP '#installcmd=\K(.*)' "$filepath") print_info "$filepath" IFS=$'\n' - for dep in $(cat $filepath); do + for dep in $(cat "$filepath"); do if [[ $dep == "#"* ]]; then continue; fi @@ -165,9 +161,10 @@ if ! $(_in "-packages" $@) && ( $(_in "+packages" $@) || $(_in "+base" $@) || $ print_error "!! We were unable to install virtualenv. Don't panic, we will try with pip3." elif [[ $exVal != 0 && ! $(_in "--answer-yes" $@) ]]; then print_error "Unable to install \`$dep\`, press \`y\` to continue the script." + echo -n "Choice : " read -n 1 echo "" - if [ "$REPLY" == "y" ]; then + if [[ $REPLY == "y" ]]; then print_info "Installation continued" else print_error "!! Installation aborted" From b974ac7f9519f52467dcea02bdc98a6b57653096 Mon Sep 17 00:00:00 2001 From: A-312 Date: Tue, 12 Mar 2019 14:42:18 +0100 Subject: [PATCH 27/69] =?UTF-8?q?Am=C3=A9liore=20la=20gestion=20du=20virtu?= =?UTF-8?q?alenv?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/install_zds.sh | 110 ++++++++++++++++++++++++++++------------- 1 file changed, 77 insertions(+), 33 deletions(-) diff --git a/scripts/install_zds.sh b/scripts/install_zds.sh index b94b44c06c..a58758711c 100755 --- a/scripts/install_zds.sh +++ b/scripts/install_zds.sh @@ -96,6 +96,9 @@ function wget_nv { LOCAL_DIR="$(cd "$(dirname "$0")" && pwd)" source $LOCAL_DIR/define_variable.sh +# zds-site root folder +ZDSSITE_DIR="$(realpath $LOCAL_DIR/../)" + # Install packages if ! $(_in "-packages" $@) && ( $(_in "+packages" $@) || $(_in "+base" $@) || $(_in "+full" $@) ); then zds_fold_start "packages" "* [+packages] installing packages (this subcommand will be run as super-user)" @@ -183,25 +186,54 @@ fi # virtualenv if ! $(_in "-virtualenv" $@) && ( $(_in "+virtualenv" $@) || $(_in "+base" $@) || $(_in "+full" $@) ); then - zds_fold_start "virtualenv" "* Install virtualenv" + zds_fold_start "virtualenv" "* Create virtualenv" - print_info "* [+virtualenv] installing \`virtualenv 16.2.0\` with pip" - pip3 install virtualenv==16.2.0 + if [ ! -f $ZDS_VENV/bin/activate ]; then + if [ -d $ZDS_VENV ]; then + print_info "!! Find corrupted virtualenv folder without bin/activate" + + if $(_in "--answer-yes" $@); then + print_info "remove $ZDS_VENV" + rm -r $ZDS_VENV + else + print_error "We recommanded to delete this folder, press \`y\` to delete this folder" + echo -n "Choice : " + read -n 1 + echo "" + if [[ $REPLY == "y" ]]; then + print_info "remove $ZDS_VENV" + rm -r $ZDS_VENV + else + print_error "!! Cannot continue. Move, rename or delete this folder before retry" + exit 1 + fi + fi + fi + + print_info "* [+virtualenv] installing \`virtualenv 16.2.0\` with pip" + pip3 install virtualenv==16.2.0 - if [ ! -d $ZDS_VENV ]; then print_info "* [+virtualenv] creating virtualenv" - msg=$(python3 -m venv $ZDS_VENV) - if [[ $? != 0 && $msg == *"ensurepip"* ]]; then - echo $msg - print_info "!! Try --without-pip" - python3 -m venv $ZDS_VENV --without-pip - elif [[ $? != 0 ]]; then - echo $msg + err=$(python3 -m venv $ZDS_VENV 3>&1 1>&2 2>&3 | sudo tee /dev/stderr) + if [[ $err != "" ]]; then + exVal=1 + if [[ $err == *"ensurepip"* ]]; then # possible issue on python 3.6 + print_info "!! Trying to create the virtualenv without pip" + python3 -m venv $ZDS_VENV --without-pip + exVal=$? + fi + + if [[ $exVal != 0 ]]; then + print_error "!! Cannot create (use \`-virtualenv\` to skip)" + print_info "You can try to change the path of zdsenv folder before retrying this command" + print_info "with : \`export $ZDS_VENV=../zdsenv\`" + exit 1 + fi fi fi fi -if [[ $VIRTUAL_ENV == "" || $(basename $VIRTUAL_ENV) != $ZDS_VENV ]]; then +if ! $(_in "--force-skip-activating" $@) && [[ ( $VIRTUAL_ENV == "" || $(realpath $VIRTUAL_ENV) != $(realpath $ZDS_VENV) ) ]]; then zds_fold_start "virtualenv" "* Load virtualenv" print_info "* activating venv \`$ZDS_VENV\`" @@ -210,18 +242,30 @@ if [[ $VIRTUAL_ENV == "" || $(basename $VIRTUAL_ENV) != $ZDS_VENV ]]; then _nvm fi - source ./$ZDS_VENV/bin/activate + if [ ! -f $ZDS_VENV/bin/activate ]; then + print_error "!! No virtualenv, cannot continue (use \`--force-skip-activating\` to skip -> NOT RECOMMANDED)" + exit 1 + fi + + source $ZDS_VENV/bin/activate if [[ $? != 0 ]]; then - print_error "!! No virtualenv, cannot continue" + print_error "!! Cannot load virtualenv (use \`--force-skip-activating\` to skip -> NOT RECOMMANDED)" exit 1 fi zds_fold_end else + print_info "!! Add \`$(realpath $ZDS_VENV)\` in your PATH." + + if [ ! -d $ZDS_VENV ]; then + mkdir $ZDS_VENV + fi + zds_fold_end fi +ZDS_ENV=$(realpath $ZDS_VENV) # nvm node & yarn if ! $(_in "-node" $@) && ( $(_in "+node" $@) || $(_in "+base" $@) || $(_in "+full" $@) ); then @@ -239,12 +283,12 @@ if ! $(_in "-node" $@) && ( $(_in "+node" $@) || $(_in "+base" $@) || $(_in "+f npm -g add yarn - if [[ $(grep -c -i "nvm use" $VIRTUAL_ENV/bin/activate) == "0" ]]; then # add nvm activation to venv activate's - ACTIVATE_NVM="nvm use # activate nvm (from install_zds.sh)" + if [[ $(grep -c -i "nvm use" $ZDS_ENV/bin/activate) == "0" ]]; then # add nvm activation to venv activate's + ACTIVATE_NVM="nvm use > /dev/null # activate nvm (from install_zds.sh)" - echo $ACTIVATE_NVM >> $VIRTUAL_ENV/bin/activate - echo $ACTIVATE_NVM >> $VIRTUAL_ENV/bin/activate.csh - echo $ACTIVATE_NVM >> $VIRTUAL_ENV/bin/activate.fish + echo $ACTIVATE_NVM >> $ZDS_ENV/bin/activate + echo $ACTIVATE_NVM >> $ZDS_ENV/bin/activate.csh + echo $ACTIVATE_NVM >> $ZDS_ENV/bin/activate.fish fi else print_error "!! Cannot obtain nvm v${ZDS_NVM_VERSION}" @@ -259,8 +303,8 @@ fi if ! $(_in "-jdk-local" $@) && ( $(_in "+jdk-local" $@) || $(_in "+full" $@) ); then zds_fold_start "jdk" "* [+jdk-local] installing a local version of JDK (v$ZDS_JDK_VERSION)" - mkdir -p zdsenv/lib/ - cd zdsenv/lib/ + mkdir -p $ZDS_VENV/lib/ + cd $ZDS_VENV/lib/ if [ -d jdk ]; then # remove previous install rm -R jdk @@ -287,7 +331,7 @@ if ! $(_in "-jdk-local" $@) && ( $(_in "+jdk-local" $@) || $(_in "+full" $@) ); print_error "!! Cannot get jdk ${JDK_VERSION}" exit 1 fi - cd ../../ + cd $ZDSSITE_DIR zds_fold_end fi @@ -316,12 +360,12 @@ if ! $(_in "-elastic-local" $@) && ( $(_in "+elastic-local" $@) || $(_in "+full print_info "-Xmx512m" >> elasticsearch/config/jvm.options # symbolic link to elastic start script - ln -s elasticsearch/bin/elasticsearch $VIRTUAL_ENV/bin/ + ln -s elasticsearch/bin/elasticsearch $ZDS_ENV/bin/ else print_error "!! Cannot get elasticsearch ${ZDS_ELASTIC_VERSION}" exit 1; fi; - cd .. + cd $ZDSSITE_DIR zds_fold_end fi @@ -331,10 +375,9 @@ fi if ! $(_in "-tex-local" $@) && ( $(_in "+tex-local" $@) || $(_in "+full" $@) ); then zds_fold_start "texlive" "* [+tex-local] install texlive" - CURRENT=$(pwd) mkdir -p .local cd .local - LOCAL=$CURRENT/.local + LOCAL=$ZDSSITE_DIR/.local # clone BASE_TEXLIVE=$LOCAL/texlive @@ -369,7 +412,7 @@ if ! $(_in "-tex-local" $@) && ( $(_in "+tex-local" $@) || $(_in "+full" $@) ); # Symlink the binaries to bin of venv for i in $BASE_TEXLIVE/bin/x86_64-linux/*; do - ln -sf $i $VIRTUAL_ENV/bin/ + ln -sf $i $ZDS_ENV/bin/ done fi @@ -385,7 +428,7 @@ if ! $(_in "-tex-local" $@) && ( $(_in "+tex-local" $@) || $(_in "+full" $@) ); exit 1 fi - cd $CURRENT + cd $ZDSSITE_DIR zds_fold_end fi @@ -395,8 +438,6 @@ fi if ! $(_in "-latex-template" $@) && ( $(_in "+latex-template" $@) || $(_in "+full" $@) ); then zds_fold_start "latex-template" "* [+latex-template] install latex-template (from $ZDS_LATEX_REPO)" - CURRENT=$(pwd) - if [[ $(which kpsewhich) == "" ]]; then # no texlive ? print_error "!! Cannot find kpsewhich, do you have texlive?" exit 1; @@ -419,7 +460,7 @@ if ! $(_in "-latex-template" $@) && ( $(_in "+latex-template" $@) || $(_in "+fu exit 1 fi - cd $CURRENT + cd $ZDSSITE_DIR zds_fold_end fi @@ -531,5 +572,8 @@ if ! $(_in "-data" $@) && ( $(_in "+data" $@) || $(_in "+base" $@) || $(_in "+f zds_fold_end fi - -print_info "Done. You can now run instance with \`source $ZDS_VENV/bin/activate\`, and then, \`make zmd-start && make run-back\`" +if ! $(_in "--force-skip-activating" $@); then + print_info "Done. You can now run instance with \`source $ZDS_VENV/bin/activate\`, and then, \`make zmd-start && make run-back\`" +else + print_info "Done. You can now run instance with \`make zmd-start && make run-back\`" +fi From f899565d0071da10f0afe153f5fcf19005d5c515 Mon Sep 17 00:00:00 2001 From: A-312 Date: Tue, 12 Mar 2019 15:09:37 +0100 Subject: [PATCH 28/69] Supprime le dossier sans confirmation fichier par fichier --- scripts/install_zds.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install_zds.sh b/scripts/install_zds.sh index a58758711c..86ea2e51fa 100755 --- a/scripts/install_zds.sh +++ b/scripts/install_zds.sh @@ -307,7 +307,7 @@ if ! $(_in "-jdk-local" $@) && ( $(_in "+jdk-local" $@) || $(_in "+full" $@) ); cd $ZDS_VENV/lib/ if [ -d jdk ]; then # remove previous install - rm -R jdk + rm -Rf jdk fi baseURL="https://download.oracle.com/otn-pub/java/jdk/" From 7009c194ba798f6ec23f2a8cd9bb5db3a2b48ede Mon Sep 17 00:00:00 2001 From: A-312 Date: Tue, 12 Mar 2019 18:05:17 +0100 Subject: [PATCH 29/69] =?UTF-8?q?Gulp=20n'est=20jamais=20install=C3=A9=20e?= =?UTF-8?q?n=20global=20par=20d=C3=A9faut?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 02d231aa0c..2e7af452f2 100644 --- a/package.json +++ b/package.json @@ -9,11 +9,11 @@ "node": ">=8.0.0" }, "scripts": { - "gulp": "gulp", - "build": "gulp build", - "watch": "gulp watch", - "lint": "gulp js:lint", - "clean": "gulp clean" + "gulp": "node_modules/gulp/bin/gulp.js", + "build": "node_modules/gulp/bin/gulp.js build", + "watch": "node_modules/gulp/bin/gulp.js watch", + "lint": "node_modules/gulp/bin/gulp.js js:lint", + "clean": "node_modules/gulp/bin/gulp.js clean" }, "repository": { "type": "git", From b877a60ac6b06557fd7ec61c5b7a53da1ff271db Mon Sep 17 00:00:00 2001 From: A-312 Date: Tue, 12 Mar 2019 18:06:49 +0100 Subject: [PATCH 30/69] Fix "error: invalid command 'bdist_wheel'" car manquant dans setuptools --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 5da28cfb39..d215e8e4d7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ elasticsearch==5.5.2 elasticsearch-dsl==5.4.0 sqlparse==0.2.4 setuptools==40.6.3 +wheel==0.33.1 # Explicit dependencies (references in code) Django==2.1.7 From d01b7d1a19b1c143a9670e52d688f7fcaadedeb2 Mon Sep 17 00:00:00 2001 From: A-312 Date: Tue, 12 Mar 2019 18:09:27 +0100 Subject: [PATCH 31/69] =?UTF-8?q?Suit=20la=20m=C3=AAme=20norme?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/install_zds.sh | 49 +++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/scripts/install_zds.sh b/scripts/install_zds.sh index 86ea2e51fa..0dea849b0e 100755 --- a/scripts/install_zds.sh +++ b/scripts/install_zds.sh @@ -17,12 +17,14 @@ function _nvm { [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm } -# >>>>> + +## start fold for travis ZDS_SHOW_TRAVIS_FOLD=0 if $(_in "--travis-output" $@); then ZDS_SHOW_TRAVIS_FOLD=1 fi + zds_fold_current="" function zds_fold_start { if [[ $ZDS_SHOW_TRAVIS_FOLD == 1 ]]; then @@ -32,41 +34,42 @@ function zds_fold_start { zds_fold_current="$1" echo "travis_fold:start:install_${zds_fold_current}" - echo -en "\033[0K"; + echo -en "\033[0K" fi print_info "$2" --bold } + function zds_fold_end { if [[ $ZDS_SHOW_TRAVIS_FOLD == 1 ]] && [[ $zds_fold_current =~ "" ]]; then echo "travis_fold:end:install_${zds_fold_current}" - echo -en "\033[0K"; + echo -en "\033[0K" zds_fold_current="" fi } -# <<<<< +## end function print_info { if [[ "$2" == "--bold" ]]; then - echo -en "\033[36;1m"; + echo -en "\033[36;1m" else - echo -en "\033[0;36m"; + echo -en "\033[0;36m" fi - echo "$1"; - echo -en "\033[00m"; + echo "$1" + echo -en "\033[00m" } function print_error { - echo -en "\033[31;1m"; - echo "$1"; - echo -en "\033[00m"; + echo -en "\033[31;1m" + echo "$1" + echo -en "\033[00m" } -# >>>>> +## start quiet mode function progressfilt { local flag=false c count cr=$'\r' nl=$'\n' while IFS='' read -d '' -rn 1 c @@ -86,11 +89,13 @@ function progressfilt { done } + # Hack for "-q --show-progress" (at least v1.16) and travis uses (travis uses wget 1.15) function wget_nv { wget "$@" --progress=bar:force 2>&1 | progressfilt } -# <<<<< +## end + # variables LOCAL_DIR="$(cd "$(dirname "$0")" && pwd)" @@ -307,7 +312,7 @@ if ! $(_in "-jdk-local" $@) && ( $(_in "+jdk-local" $@) || $(_in "+full" $@) ); cd $ZDS_VENV/lib/ if [ -d jdk ]; then # remove previous install - rm -Rf jdk + rm -rf jdk fi baseURL="https://download.oracle.com/otn-pub/java/jdk/" @@ -345,7 +350,7 @@ if ! $(_in "-elastic-local" $@) && ( $(_in "+elastic-local" $@) || $(_in "+full cd .local if [ -d elasticsearch ]; then # remove previous install - rm -R elasticsearch + rm -r elasticsearch fi wget_nv https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ZDS_ELASTIC_VERSION}.zip @@ -363,8 +368,8 @@ if ! $(_in "-elastic-local" $@) && ( $(_in "+elastic-local" $@) || $(_in "+full ln -s elasticsearch/bin/elasticsearch $ZDS_ENV/bin/ else print_error "!! Cannot get elasticsearch ${ZDS_ELASTIC_VERSION}" - exit 1; - fi; + exit 1 + fi cd $ZDSSITE_DIR zds_fold_end @@ -388,7 +393,7 @@ if ! $(_in "-tex-local" $@) && ( $(_in "+tex-local" $@) || $(_in "+full" $@) ); cd $BASE_REPO if [ -d $REPO ]; then # remove previous version of the template - rm -Rf $REPO + rm -rf $REPO fi git clone $ZDS_LATEX_REPO @@ -418,7 +423,7 @@ if ! $(_in "-tex-local" $@) && ( $(_in "+tex-local" $@) || $(_in "+full" $@) ); ./bin/x86_64-linux/tlmgr install $(cat packages) # extra packages ./bin/x86_64-linux/tlmgr update --self - rm -Rf $REPO + rm -rf $REPO else print_error "!! Cannot download texlive" exit 1 @@ -448,7 +453,7 @@ if ! $(_in "-latex-template" $@) && ( $(_in "+latex-template" $@) || $(_in "+fu REPO=$BASE_REPO/latex-template if [ -d $REPO ]; then # remove previous version of the template - rm -Rf $REPO + rm -rf $REPO fi mkdir -p $BASE_REPO @@ -499,8 +504,8 @@ if ! $(_in "-front" $@) && ( $(_in "+front" $@) || $(_in "+base" $@) || $(_in " zds_fold_start "front" "* [+front] install front dependencies & build front" if [ -d node_modules ]; then # delete previous modules - rm -R node_modules - fi; + rm -r node_modules + fi make install-front From 3473e20419340623f68e9a975f2dd10fdf2a3a2f Mon Sep 17 00:00:00 2001 From: A-312 Date: Tue, 12 Mar 2019 18:15:03 +0100 Subject: [PATCH 32/69] =?UTF-8?q?Anticipe=20une=20possible=20erreur=20lors?= =?UTF-8?q?=20de=20prochain=20changement=20J'ai=20mis=20exVal=20=C3=A0=20l?= =?UTF-8?q?a=20fin=20de=20la=20ligne=20pour=20=C3=AAtre=20s=C3=BBr=20que?= =?UTF-8?q?=20personne=20ira=20mettre=20du=20code=20entre=20la=20commande?= =?UTF-8?q?=20souhait=C3=A9e=20et=20`$=3F`.=20Ce=20qui,=20si=20cela=20se?= =?UTF-8?q?=20produit=20casserait=20la=20gestion=20des=20erreurs.=20:(?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/install_zds.sh | 57 ++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/scripts/install_zds.sh b/scripts/install_zds.sh index 0dea849b0e..03f395296b 100755 --- a/scripts/install_zds.sh +++ b/scripts/install_zds.sh @@ -224,14 +224,13 @@ if ! $(_in "-virtualenv" $@) && ( $(_in "+virtualenv" $@) || $(_in "+base" $@) exVal=1 if [[ $err == *"ensurepip"* ]]; then # possible issue on python 3.6 print_info "!! Trying to create the virtualenv without pip" - python3 -m venv $ZDS_VENV --without-pip - exVal=$? + python3 -m venv $ZDS_VENV --without-pip; exVal=$? + fi fi if [[ $exVal != 0 ]]; then print_error "!! Cannot create (use \`-virtualenv\` to skip)" - print_info "You can try to change the path of zdsenv folder before retrying this command" - print_info "with : \`export $ZDS_VENV=../zdsenv\`" + print_info "You can try to change the path of zdsenv folder before retrying this command with \`export ZDS_VENV=../zdsenv\`" exit 1 fi fi @@ -248,14 +247,20 @@ if ! $(_in "--force-skip-activating" $@) && [[ ( $VIRTUAL_ENV == "" || $(realpat fi if [ ! -f $ZDS_VENV/bin/activate ]; then - print_error "!! No virtualenv, cannot continue (use \`--force-skip-activating\` to skip -> NOT RECOMMANDED)" + echo "" + print_error "!! No virtualenv, cannot continue" + print_info " - Install virtualenv with \`+virtualenv\` (recommanded) ;" + echo " - If you don't have other choice, use \`--force-skip-activating\`." exit 1 fi - source $ZDS_VENV/bin/activate + source $ZDS_VENV/bin/activate; exVal=$? - if [[ $? != 0 ]]; then - print_error "!! Cannot load virtualenv (use \`--force-skip-activating\` to skip -> NOT RECOMMANDED)" + if [[ $exVal != 0 ]]; then + echo "" + print_error "!! Cannot load virtualenv" + print_info " - Reinstall virtualenv with \`+virtualenv\` (recommanded) ;" + echo " - If you don't have other choice, use \`--force-skip-activating\`." exit 1 fi @@ -488,9 +493,9 @@ if ! $(_in "-back" $@) && ( $(_in "+back" $@) || $(_in "+base" $@) || $(_in "+f exit 1 fi - make migrate-db # migration are required for the instance to run properly anyway + make migrate-db; exVal=$? # migration are required for the instance to run properly anyway - if [[ $? != 0 ]]; then + if [[ $exVal != 0 ]]; then print_error "!! Cannot migrate database after the back installation (use \`-back\` to skip)" exit 1 fi @@ -507,16 +512,16 @@ if ! $(_in "-front" $@) && ( $(_in "+front" $@) || $(_in "+base" $@) || $(_in " rm -r node_modules fi - make install-front + make install-front; exVal=$? - if [[ $? != 0 ]]; then + if [[ $exVal != 0 ]]; then print_error "!! Cannot install-front (use \`-front\` to skip)" exit 1 fi - make build-front + make build-front; exVal=$? - if [[ $? != 0 ]]; then + if [[ $exVal != 0 ]]; then print_error "!! Cannot build-front (use \`-front\` to skip)" exit 1 fi @@ -529,9 +534,9 @@ fi if ! $(_in "-zmd" $@) && ( $(_in "+zmd" $@) || $(_in "+base" $@) || $(_in "+full" $@) ); then zds_fold_start "zmd" "* [+zmd] install zmarkdown dependencies" - make zmd-install + make zmd-install; exVal=$? - if [[ $? != 0 ]]; then + if [[ $exVal != 0 ]]; then print_error "!! Cannot install zmd (use \`-zmd\` to skip)" exit 1 fi @@ -544,33 +549,31 @@ fi if ! $(_in "-data" $@) && ( $(_in "+data" $@) || $(_in "+base" $@) || $(_in "+full" $@) ); then zds_fold_start "fixtures" "* [+data] fixtures" - npm run server --prefix zmd/node_modules/zmarkdown -- --silent + npm run server --prefix zmd/node_modules/zmarkdown -- --silent; exVal=$? - if [[ $? != 0 ]]; then + if [[ $exVal != 0 ]]; then print_error "!! Cannot start zmd to generate-fixtures (use \`-data\` to skip)" exit 1 fi - python manage.py loaddata fixtures/*.yaml - exVal=$? - python manage.py load_factory_data fixtures/advanced/aide_tuto_media.yaml - exVal=($exVal + $?) + python manage.py loaddata fixtures/*.yaml; exVal=$? + + python manage.py load_factory_data fixtures/advanced/aide_tuto_media.yaml; exVal=($exVal + $?) if $(_in "--travis-output" $@); then - python manage.py load_fixtures --size=low --all --settings zds.settings.travis_fixture + python manage.py load_fixtures --size=low --all --settings zds.settings.travis_fixture; exVal=($exVal + $?) else - python manage.py load_fixtures --size=low --all + python manage.py load_fixtures --size=low --all; exVal=($exVal + $?) fi - exVal=($exVal + $?) if [[ $exVal != 0 ]]; then print_error "!! Cannot generate-fixtures (use \`-data\` to skip)" exit 1 fi - make zmd-stop + make zmd-stop; exVal=$? - if [[ $? != 0 ]]; then + if [[ $exVal != 0 ]]; then print_error "Warning: Cannot stop zmd" fi From d63fabb77582478173b6e3817cc6c2fc4a906197 Mon Sep 17 00:00:00 2001 From: A-312 Date: Tue, 12 Mar 2019 18:16:54 +0100 Subject: [PATCH 33/69] =?UTF-8?q?Support=20pour=20les=20VM=20(+=20dossier?= =?UTF-8?q?=20partag=C3=A9)=20avec=20Windows=20en=20host?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 4 +++ scripts/install_zds.sh | 56 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index e37c3aaa0a..4fb874f65d 100644 --- a/Makefile +++ b/Makefile @@ -48,6 +48,10 @@ clean-back: ## Remove Python bytecode files (*.pyc) install-front: ## Install the Node.js packages for the frontend yarn install +## Fix for Shared folder on VirtualMachine with Windows host +install-front-without-symlink: ## Install the Node.js packages for the frontend + yarn install --no-bin-links + build-front: ## Build the frontend assets (CSS, JS, images) yarn run build diff --git a/scripts/install_zds.sh b/scripts/install_zds.sh index 03f395296b..1a1fe5b354 100755 --- a/scripts/install_zds.sh +++ b/scripts/install_zds.sh @@ -104,6 +104,47 @@ source $LOCAL_DIR/define_variable.sh # zds-site root folder ZDSSITE_DIR="$(realpath $LOCAL_DIR/../)" + +## Fix for Shared folder on VirtualMachine with Windows host +isSharedFolderWithWindowsHost=0 + +function symlink_isdisabled { + fold="." + if [[ $1 != "" ]]; then + fold=$1 + fi + rm -rf "$fold/test_symlink" "$fold/test_symlink2" + + if [[ $exVal != 0 ]]; then + print_error "!!Cannot continue because $ZDSSITE_DIR is read-only." + exit 1 + fi + + ln -s "$fold/test_symlink" "$fold/test_symlink2" 2> /dev/null; local exVal=$? + $(mountpoint -q "$fold" 2> /dev/null); local ismounted=$? + + rm -rf "$fold/test_symlink" "$fold/test_symlink2" + echo "ok" + echo $exVal + echo $ismounted + + if [[ $exVal != 0 && ( $ismounted == 0 || $ismounted == 127 ) ]]; then + return 1 + else + return 0 + fi +} + +if [[ $(symlink_isdisabled "$ZDSSITE_DIR") ]]; then + echo "" + print_info "The symbolic link are disabled in $ZDSSITE_DIR:" + print_info "Enabled: Support for shared folder on VirtualMachine with Windows host" + isSharedFolderWithWindowsHost=1 + echo "" +fi +## end + + # Install packages if ! $(_in "-packages" $@) && ( $(_in "+packages" $@) || $(_in "+base" $@) || $(_in "+full" $@) ); then zds_fold_start "packages" "* [+packages] installing packages (this subcommand will be run as super-user)" @@ -198,7 +239,7 @@ if ! $(_in "-virtualenv" $@) && ( $(_in "+virtualenv" $@) || $(_in "+base" $@) print_info "!! Find corrupted virtualenv folder without bin/activate" if $(_in "--answer-yes" $@); then - print_info "remove $ZDS_VENV" + print_info "remove $(realpath $ZDS_VENV)" rm -r $ZDS_VENV else print_error "We recommanded to delete this folder, press \`y\` to delete this folder" @@ -206,7 +247,7 @@ if ! $(_in "-virtualenv" $@) && ( $(_in "+virtualenv" $@) || $(_in "+base" $@) read -n 1 echo "" if [[ $REPLY == "y" ]]; then - print_info "remove $ZDS_VENV" + print_info "remove $(realpath $ZDS_VENV)" rm -r $ZDS_VENV else print_error "!! Cannot continue. Move, rename or delete this folder before retry" @@ -226,6 +267,11 @@ if ! $(_in "-virtualenv" $@) && ( $(_in "+virtualenv" $@) || $(_in "+base" $@) print_info "!! Trying to create the virtualenv without pip" python3 -m venv $ZDS_VENV --without-pip; exVal=$? fi + + if [[ $exVal != 0 && $isSharedFolderWithWindowsHost && $(symlink_isdisabled "$ZDS_VENV") ]]; then + print_error "!! Symlink are disabled" + print_info "!! \$ZDS_VENV should be move in another folder because the symbolic link are disabled in this folder" + echo "" fi if [[ $exVal != 0 ]]; then @@ -512,7 +558,11 @@ if ! $(_in "-front" $@) && ( $(_in "+front" $@) || $(_in "+base" $@) || $(_in " rm -r node_modules fi - make install-front; exVal=$? + if [[ isSharedFolderWithWindowsHost ]]; then + make install-front-without-symlink; exVal=$? + else + make install-front; exVal=$? + fi if [[ $exVal != 0 ]]; then print_error "!! Cannot install-front (use \`-front\` to skip)" From 8260d9f516c60cd746e40c1bef9d836f06b43e34 Mon Sep 17 00:00:00 2001 From: A-312 Date: Tue, 12 Mar 2019 18:32:02 +0100 Subject: [PATCH 34/69] =?UTF-8?q?Attend=20que=20realpath=20soit=20install?= =?UTF-8?q?=C3=A9=20s'il=20ne=20l'est=20pas?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/dependencies/arch.txt | 1 + scripts/dependencies/debian.txt | 1 + scripts/dependencies/fedora.txt | 1 + scripts/dependencies/ubuntu.txt | 1 + scripts/install_zds.sh | 83 +++++++++++++++++---------------- 5 files changed, 46 insertions(+), 41 deletions(-) diff --git a/scripts/dependencies/arch.txt b/scripts/dependencies/arch.txt index 18c13dc911..d0b6a2d1a8 100644 --- a/scripts/dependencies/arch.txt +++ b/scripts/dependencies/arch.txt @@ -5,6 +5,7 @@ git wget curl unzip +realpath python python3-pip libxslt diff --git a/scripts/dependencies/debian.txt b/scripts/dependencies/debian.txt index 1d7ab9e8f3..7c70779403 100644 --- a/scripts/dependencies/debian.txt +++ b/scripts/dependencies/debian.txt @@ -5,6 +5,7 @@ git wget curl unzip +realpath python3-dev python3-pip python3-venv diff --git a/scripts/dependencies/fedora.txt b/scripts/dependencies/fedora.txt index 1024e2f81a..0dfacc413a 100644 --- a/scripts/dependencies/fedora.txt +++ b/scripts/dependencies/fedora.txt @@ -5,6 +5,7 @@ git wget curl unzip +realpath python3-devel libxslt-devel zlib-devel diff --git a/scripts/dependencies/ubuntu.txt b/scripts/dependencies/ubuntu.txt index edc2f57f27..2c216b5494 100644 --- a/scripts/dependencies/ubuntu.txt +++ b/scripts/dependencies/ubuntu.txt @@ -5,6 +5,7 @@ git wget curl unzip +realpath python3-dev python3-pip python3-venv diff --git a/scripts/install_zds.sh b/scripts/install_zds.sh index 1a1fe5b354..b28900a8ba 100755 --- a/scripts/install_zds.sh +++ b/scripts/install_zds.sh @@ -102,47 +102,7 @@ LOCAL_DIR="$(cd "$(dirname "$0")" && pwd)" source $LOCAL_DIR/define_variable.sh # zds-site root folder -ZDSSITE_DIR="$(realpath $LOCAL_DIR/../)" - - -## Fix for Shared folder on VirtualMachine with Windows host -isSharedFolderWithWindowsHost=0 - -function symlink_isdisabled { - fold="." - if [[ $1 != "" ]]; then - fold=$1 - fi - rm -rf "$fold/test_symlink" "$fold/test_symlink2" - - if [[ $exVal != 0 ]]; then - print_error "!!Cannot continue because $ZDSSITE_DIR is read-only." - exit 1 - fi - - ln -s "$fold/test_symlink" "$fold/test_symlink2" 2> /dev/null; local exVal=$? - $(mountpoint -q "$fold" 2> /dev/null); local ismounted=$? - - rm -rf "$fold/test_symlink" "$fold/test_symlink2" - echo "ok" - echo $exVal - echo $ismounted - - if [[ $exVal != 0 && ( $ismounted == 0 || $ismounted == 127 ) ]]; then - return 1 - else - return 0 - fi -} - -if [[ $(symlink_isdisabled "$ZDSSITE_DIR") ]]; then - echo "" - print_info "The symbolic link are disabled in $ZDSSITE_DIR:" - print_info "Enabled: Support for shared folder on VirtualMachine with Windows host" - isSharedFolderWithWindowsHost=1 - echo "" -fi -## end +ZDSSITE_DIR=$(pwd) # Install packages @@ -230,6 +190,47 @@ if ! $(_in "-packages" $@) && ( $(_in "+packages" $@) || $(_in "+base" $@) || $ zds_fold_end fi + +## Fix for Shared folder on VirtualMachine with Windows host +isSharedFolderWithWindowsHost=0 + +function symlink_isdisabled { + fold="." + if [[ $1 != "" ]]; then + fold=$1 + fi + rm -rf "$fold/test_symlink" "$fold/test_symlink2" + + if [[ $exVal != 0 ]]; then + print_error "!!Cannot continue because $ZDSSITE_DIR is read-only." + exit 1 + fi + + ln -s "$fold/test_symlink" "$fold/test_symlink2" 2> /dev/null; local exVal=$? + $(mountpoint -q "$fold" 2> /dev/null); local ismounted=$? + + rm -rf "$fold/test_symlink" "$fold/test_symlink2" + echo "ok" + echo $exVal + echo $ismounted + + if [[ $exVal != 0 && ( $ismounted == 0 || $ismounted == 127 ) ]]; then + return 1 + else + return 0 + fi +} + +if [[ $(symlink_isdisabled "$ZDSSITE_DIR") ]]; then + echo "" + print_info "The symbolic link are disabled in $ZDSSITE_DIR:" + print_info "Enabled: Support for shared folder on VirtualMachine with Windows host" + isSharedFolderWithWindowsHost=1 + echo "" +fi +## end + + # virtualenv if ! $(_in "-virtualenv" $@) && ( $(_in "+virtualenv" $@) || $(_in "+base" $@) || $(_in "+full" $@) ); then zds_fold_start "virtualenv" "* Create virtualenv" From d8ee521db99996c6c9d26b2a04819d1e7022efc5 Mon Sep 17 00:00:00 2001 From: A-312 Date: Tue, 12 Mar 2019 19:05:45 +0100 Subject: [PATCH 35/69] Undo last commit (for shared folder) --- Makefile | 4 ---- scripts/install_zds.sh | 54 ++---------------------------------------- 2 files changed, 2 insertions(+), 56 deletions(-) diff --git a/Makefile b/Makefile index 4fb874f65d..e37c3aaa0a 100644 --- a/Makefile +++ b/Makefile @@ -48,10 +48,6 @@ clean-back: ## Remove Python bytecode files (*.pyc) install-front: ## Install the Node.js packages for the frontend yarn install -## Fix for Shared folder on VirtualMachine with Windows host -install-front-without-symlink: ## Install the Node.js packages for the frontend - yarn install --no-bin-links - build-front: ## Build the frontend assets (CSS, JS, images) yarn run build diff --git a/scripts/install_zds.sh b/scripts/install_zds.sh index b28900a8ba..fe37b5a57b 100755 --- a/scripts/install_zds.sh +++ b/scripts/install_zds.sh @@ -191,46 +191,6 @@ if ! $(_in "-packages" $@) && ( $(_in "+packages" $@) || $(_in "+base" $@) || $ fi -## Fix for Shared folder on VirtualMachine with Windows host -isSharedFolderWithWindowsHost=0 - -function symlink_isdisabled { - fold="." - if [[ $1 != "" ]]; then - fold=$1 - fi - rm -rf "$fold/test_symlink" "$fold/test_symlink2" - - if [[ $exVal != 0 ]]; then - print_error "!!Cannot continue because $ZDSSITE_DIR is read-only." - exit 1 - fi - - ln -s "$fold/test_symlink" "$fold/test_symlink2" 2> /dev/null; local exVal=$? - $(mountpoint -q "$fold" 2> /dev/null); local ismounted=$? - - rm -rf "$fold/test_symlink" "$fold/test_symlink2" - echo "ok" - echo $exVal - echo $ismounted - - if [[ $exVal != 0 && ( $ismounted == 0 || $ismounted == 127 ) ]]; then - return 1 - else - return 0 - fi -} - -if [[ $(symlink_isdisabled "$ZDSSITE_DIR") ]]; then - echo "" - print_info "The symbolic link are disabled in $ZDSSITE_DIR:" - print_info "Enabled: Support for shared folder on VirtualMachine with Windows host" - isSharedFolderWithWindowsHost=1 - echo "" -fi -## end - - # virtualenv if ! $(_in "-virtualenv" $@) && ( $(_in "+virtualenv" $@) || $(_in "+base" $@) || $(_in "+full" $@) ); then zds_fold_start "virtualenv" "* Create virtualenv" @@ -269,12 +229,6 @@ if ! $(_in "-virtualenv" $@) && ( $(_in "+virtualenv" $@) || $(_in "+base" $@) python3 -m venv $ZDS_VENV --without-pip; exVal=$? fi - if [[ $exVal != 0 && $isSharedFolderWithWindowsHost && $(symlink_isdisabled "$ZDS_VENV") ]]; then - print_error "!! Symlink are disabled" - print_info "!! \$ZDS_VENV should be move in another folder because the symbolic link are disabled in this folder" - echo "" - fi - if [[ $exVal != 0 ]]; then print_error "!! Cannot create (use \`-virtualenv\` to skip)" print_info "You can try to change the path of zdsenv folder before retrying this command with \`export ZDS_VENV=../zdsenv\`" @@ -559,11 +513,7 @@ if ! $(_in "-front" $@) && ( $(_in "+front" $@) || $(_in "+base" $@) || $(_in " rm -r node_modules fi - if [[ isSharedFolderWithWindowsHost ]]; then - make install-front-without-symlink; exVal=$? - else - make install-front; exVal=$? - fi + make install-front; exVal=$? if [[ $exVal != 0 ]]; then print_error "!! Cannot install-front (use \`-front\` to skip)" @@ -600,7 +550,7 @@ fi if ! $(_in "-data" $@) && ( $(_in "+data" $@) || $(_in "+base" $@) || $(_in "+full" $@) ); then zds_fold_start "fixtures" "* [+data] fixtures" - npm run server --prefix zmd/node_modules/zmarkdown -- --silent; exVal=$? + npm run server --prefix ../zmd/node_modules/zmarkdown -- --silent; exVal=$? if [[ $exVal != 0 ]]; then print_error "!! Cannot start zmd to generate-fixtures (use \`-data\` to skip)" From 0864d012cd0f25e40b29bdc5833e16fd6506e1a3 Mon Sep 17 00:00:00 2001 From: A-312 Date: Tue, 12 Mar 2019 22:42:45 +0100 Subject: [PATCH 36/69] j'ai push un changement qui devait rester en local :/ --- scripts/install_zds.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install_zds.sh b/scripts/install_zds.sh index fe37b5a57b..9deb4c5c90 100755 --- a/scripts/install_zds.sh +++ b/scripts/install_zds.sh @@ -550,7 +550,7 @@ fi if ! $(_in "-data" $@) && ( $(_in "+data" $@) || $(_in "+base" $@) || $(_in "+full" $@) ); then zds_fold_start "fixtures" "* [+data] fixtures" - npm run server --prefix ../zmd/node_modules/zmarkdown -- --silent; exVal=$? + npm run server --prefix zmd/node_modules/zmarkdown -- --silent; exVal=$? if [[ $exVal != 0 ]]; then print_error "!! Cannot start zmd to generate-fixtures (use \`-data\` to skip)" From e0c481723476ad8ff7f60915f4920f32a106e05d Mon Sep 17 00:00:00 2001 From: A-312 Date: Sat, 23 Mar 2019 00:28:26 +0100 Subject: [PATCH 37/69] test avec echo --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index d4d6b7eb6e..72d4585af1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -89,6 +89,8 @@ before_install: install: - sudo apt-get update -qq + - echo $zds_install_argument + - ./scripts/install_zds.sh --answer-yes --detect-os-version --travis-output $zds_install_argument From b044dbe47e8b2ad7de7c5c16546d701cdcd41913 Mon Sep 17 00:00:00 2001 From: A-312 Date: Sat, 30 Mar 2019 15:23:15 +0100 Subject: [PATCH 38/69] essaye avec pip sans travis_retry --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 72d4585af1..3641ded82d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -73,7 +73,7 @@ before_install: - source ./scripts/ci_turbo.sh # This script exports environment variables, it must be sourced - - travis_retry pip install -q coveralls + - pip install -q coveralls - source ./scripts/define_variable.sh @@ -89,8 +89,6 @@ before_install: install: - sudo apt-get update -qq - - echo $zds_install_argument - - ./scripts/install_zds.sh --answer-yes --detect-os-version --travis-output $zds_install_argument From a62bbb9f8d46eecef7329adc07fbe4e4eba61ab7 Mon Sep 17 00:00:00 2001 From: A-312 Date: Mon, 1 Apr 2019 09:56:07 +0200 Subject: [PATCH 39/69] Test avec un autre environnement --- .travis.yml | 6 +++++- scripts/travis_script.sh | 8 ++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3641ded82d..b34462f262 100644 --- a/.travis.yml +++ b/.travis.yml @@ -73,7 +73,7 @@ before_install: - source ./scripts/ci_turbo.sh # This script exports environment variables, it must be sourced - - pip install -q coveralls + - travis_retry pip install -q coveralls - source ./scripts/define_variable.sh @@ -89,6 +89,8 @@ before_install: install: - sudo apt-get update -qq + - echo $zds_install_argument + - ./scripts/install_zds.sh --answer-yes --detect-os-version --travis-output $zds_install_argument @@ -109,6 +111,8 @@ script: - source $HACK_SphinxBuild/bin/activate + - ./scripts/travis_script.sh "coverage_backend" + - ./scripts/travis_script.sh "build_documentation" diff --git a/scripts/travis_script.sh b/scripts/travis_script.sh index c09845df3f..28031f71e8 100755 --- a/scripts/travis_script.sh +++ b/scripts/travis_script.sh @@ -29,8 +29,12 @@ fi # test backend if [[ "$1" == "test_backend" ]] && [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then print_info "* Run test for backend" - python manage.py makemigrations --dry-run --check \ - && coverage run --source='.' manage.py \ + python manage.py makemigrations --dry-run --check +fi + +# coverage backend +if [[ "$1" == "coverage_backend" ]] && [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then + coverage run --source='.' manage.py \ test -v=2\ --keepdb \ --settings zds.settings.ci_test \ From 0df31e40b733139d106dbf56510af68764be2fd5 Mon Sep 17 00:00:00 2001 From: A-312 Date: Mon, 1 Apr 2019 11:03:33 +0200 Subject: [PATCH 40/69] Fix coverage --- .travis.yml | 10 +++------- scripts/travis_script.sh | 6 +++++- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index b34462f262..a41f1a4945 100644 --- a/.travis.yml +++ b/.travis.yml @@ -73,7 +73,7 @@ before_install: - source ./scripts/ci_turbo.sh # This script exports environment variables, it must be sourced - - travis_retry pip install -q coveralls + - pip install -q coveralls - source ./scripts/define_variable.sh @@ -89,8 +89,6 @@ before_install: install: - sudo apt-get update -qq - - echo $zds_install_argument - - ./scripts/install_zds.sh --answer-yes --detect-os-version --travis-output $zds_install_argument @@ -105,14 +103,12 @@ script: - ./scripts/travis_script.sh "test_backend" + - ./scripts/travis_script.sh "coverage_backend" + - ./scripts/travis_script.sh "print_zmarkdown_log" - ./scripts/travis_script.sh "selenium_test" - - source $HACK_SphinxBuild/bin/activate - - - ./scripts/travis_script.sh "coverage_backend" - - ./scripts/travis_script.sh "build_documentation" diff --git a/scripts/travis_script.sh b/scripts/travis_script.sh index 28031f71e8..9081d0bd27 100755 --- a/scripts/travis_script.sh +++ b/scripts/travis_script.sh @@ -34,12 +34,14 @@ fi # coverage backend if [[ "$1" == "coverage_backend" ]] && [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then + source $HACK_SphinxBuild/bin/activate coverage run --source='.' manage.py \ test -v=2\ --keepdb \ --settings zds.settings.ci_test \ --exclude-tag=front \ - ${1/front/} + ${ZDS_TEST_JOB/front/} + source ./$ZDS_VENV/bin/activate fi # print zmarkdown log @@ -62,6 +64,8 @@ fi if [[ "$1" == "build_documentation" ]] && [[ "$ZDS_TEST_JOB" == *"doc"* ]]; then print_info "* Build documentation" if [[ "$ZDS_TEST_JOB" == *"doc"* ]]; then + source $HACK_SphinxBuild/bin/activate make generate-doc + source ./$ZDS_VENV/bin/activate fi fi From 5ffc301382cd4e6a73442b1b5b181847d9e50b64 Mon Sep 17 00:00:00 2001 From: A-312 Date: Mon, 1 Apr 2019 13:15:45 +0200 Subject: [PATCH 41/69] beautify l'affichage --- .travis.yml | 36 ++---------------- scripts/ci_mysql_setup.sh | 49 +++++++----------------- scripts/define_function.sh | 55 +++++++++++++++++++++++++++ scripts/define_script.sh | 77 +++++++++++++++++++++++++++++++++++++ scripts/install_zds.sh | 52 +------------------------ scripts/travis_header.sh | 73 ++++++++++++++++++++++++++--------- scripts/travis_script.sh | 78 +++++++++----------------------------- 7 files changed, 221 insertions(+), 199 deletions(-) create mode 100644 scripts/define_function.sh create mode 100644 scripts/define_script.sh diff --git a/.travis.yml b/.travis.yml index a41f1a4945..48dfe54204 100644 --- a/.travis.yml +++ b/.travis.yml @@ -69,47 +69,17 @@ cache: before_install: - - HACK_SphinxBuild=$VIRTUAL_ENV + - $HACK_VIRTUALENV=$VIRTUAL_ENV - - source ./scripts/ci_turbo.sh # This script exports environment variables, it must be sourced - - - pip install -q coveralls - - - source ./scripts/define_variable.sh - - - source ./scripts/travis_header.sh # Some function for this script - - - ./scripts/ci_mysql_setup.sh - - - install_geckodriver - - - zds_register_module_for_installation + - source ./scripts/travis_header.sh # Need to be sourced install: - - sudo apt-get update -qq - - ./scripts/install_zds.sh --answer-yes --detect-os-version --travis-output $zds_install_argument script: - - source ./$ZDS_VENV/bin/activate - - - ./scripts/travis_script.sh "start_elasticsearch" - - - ./scripts/travis_script.sh "start_latex" - - - ./scripts/travis_script.sh "lint_backend" - - - ./scripts/travis_script.sh "test_backend" - - - ./scripts/travis_script.sh "coverage_backend" - - - ./scripts/travis_script.sh "print_zmarkdown_log" - - - ./scripts/travis_script.sh "selenium_test" - - - ./scripts/travis_script.sh "build_documentation" + - ./scripts/travis_script.sh after_success: diff --git a/scripts/ci_mysql_setup.sh b/scripts/ci_mysql_setup.sh index 1d8833a2bb..909e17acfa 100755 --- a/scripts/ci_mysql_setup.sh +++ b/scripts/ci_mysql_setup.sh @@ -1,38 +1,15 @@ #!/bin/bash -function print_info { - echo -n -e "\033[0;36m"; - echo "$1"; - echo -n -e "\033[00m"; -} - -if [[ "$ZDS_TEST_JOB" == *"zds."* ]] || [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then - #display print_info - forwho="" - if [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then - forwho="'zds.*' tasks (-> needed for tests)" - - if [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then - forwho="$forwho and selenium" - fi - else - forwho="selenium" - fi - print_info "* Install mysql for $forwho." - - #install - - set -ex - - sudo sed -i'' 's/\[client\]/\[client\]\ndefault-character-set=utf8mb4/' /etc/mysql/my.cnf - sudo sed -i'' 's/\[mysql\]/\[mysql\]\ndefault-character-set=utf8mb4/' /etc/mysql/my.cnf - sudo sed -i'' 's/\[mysqld\]/\[mysqld\]\ninnodb_file_per_table=on\ninnodb_file_format=barracuda\ninnodb_large_prefix=on\ncharacter-set-client-handshake=false\ncharacter-set-server=utf8mb4\ncollation-server=utf8mb4_unicode_ci/' /etc/mysql/my.cnf - sudo /etc/init.d/mysql restart - # Travis should fail as soon as possible - mysql -u root -e "SET GLOBAL sql_mode = 'NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES';" - # Avoid "mysql has gone away" errors - mysql -u root -e "SET GLOBAL wait_timeout = 36000;" - mysql -u root -e "SET GLOBAL max_allowed_packet = 134209536;" - # Create database with the correct charset and collation - mysql -u root -e "CREATE DATABASE zds_test CHARACTER SET = utf8mb4 COLLATE utf8mb4_unicode_ci;" -fi +set -ex + +sudo sed -i'' 's/\[client\]/\[client\]\ndefault-character-set=utf8mb4/' /etc/mysql/my.cnf +sudo sed -i'' 's/\[mysql\]/\[mysql\]\ndefault-character-set=utf8mb4/' /etc/mysql/my.cnf +sudo sed -i'' 's/\[mysqld\]/\[mysqld\]\ninnodb_file_per_table=on\ninnodb_file_format=barracuda\ninnodb_large_prefix=on\ncharacter-set-client-handshake=false\ncharacter-set-server=utf8mb4\ncollation-server=utf8mb4_unicode_ci/' /etc/mysql/my.cnf +sudo /etc/init.d/mysql restart +# Travis should fail as soon as possible +mysql -u root -e "SET GLOBAL sql_mode = 'NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES';" +# Avoid "mysql has gone away" errors +mysql -u root -e "SET GLOBAL wait_timeout = 36000;" +mysql -u root -e "SET GLOBAL max_allowed_packet = 134209536;" +# Create database with the correct charset and collation +mysql -u root -e "CREATE DATABASE zds_test CHARACTER SET = utf8mb4 COLLATE utf8mb4_unicode_ci;" diff --git a/scripts/define_function.sh b/scripts/define_function.sh new file mode 100644 index 0000000000..d2d4d199ff --- /dev/null +++ b/scripts/define_function.sh @@ -0,0 +1,55 @@ +## start fold for travis +ZDS_SHOW_TRAVIS_FOLD=0 +if $(_in "--travis-output" $@); then + ZDS_SHOW_TRAVIS_FOLD=1 +fi + + +zds_fold_current_cat="default" +function zds_fold_category { + zds_fold_category=$1 +} + + +zds_fold_current="" +function zds_fold_start { + if [[ $ZDS_SHOW_TRAVIS_FOLD == 1 ]]; then + if [[ $zds_fold_current == $1 ]]; then # for virtualenv fold + return + fi + + zds_fold_current="$1" + echo "travis_fold:start:${zds_fold_current_cat}_${zds_fold_current}" + echo -en "\033[0K" + fi + + print_info "$2" --bold +} + + +function zds_fold_end { + if [[ $ZDS_SHOW_TRAVIS_FOLD == 1 ]] && [[ $zds_fold_current =~ "" ]]; then + echo "travis_fold:end:${zds_fold_current_cat}_${zds_fold_current}" + echo -en "\033[0K" + zds_fold_current="" + fi +} +## end + + +function print_info { + if [[ "$2" == "--bold" ]]; then + echo -en "\033[36;1m" + else + echo -en "\033[0;36m" + fi + echo "$1" + echo -en "\033[00m" +} + + +function print_error { + echo -en "\033[31;1m" + echo "$1" + echo -en "\033[00m" +} diff --git a/scripts/define_script.sh b/scripts/define_script.sh new file mode 100644 index 0000000000..32fce27118 --- /dev/null +++ b/scripts/define_script.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +exVal=0 + +# start elastic +if [[ "$1" == "start_elasticsearch" ]] && [[ "$ZDS_TEST_JOB" == *"zds.searchv2"* ]]; then + zds_fold_start "elasticsearch" "* Start elasticsearch as service" + sudo service elasticsearch start; exVal=$? + zds_fold_end +fi + +# start latex +if [[ "$1" == "start_latex" ]] && [[ "$ZDS_TEST_JOB" == *"zds.tutorialv2"* ]]; then + zds_fold_start "latex" "* Start texhash -> latex" + texhash; exVal=$? + zds_fold_end +fi + +# lint backend +if [[ "$1" == "lint_backend" ]] && [[ "$ZDS_TEST_JOB" == *"zds.gallery"* ]]; then + zds_fold_start "lint_backend" "* Run lint for backend" + ./scripts/no_import_zds_settings.sh + flake8; exVal=$? + flake8 --config=zds/settings/.flake8 zds/settings; exVal=$? + $exVal + zds_fold_end +fi + +# test backend +if [[ "$1" == "test_backend" ]] && [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then + zds_fold_start "test_backend" "* Run test for backend" + python manage.py makemigrations --dry-run --check; exVal=$? + zds_fold_end +fi + +# coverage backend +if [[ "$1" == "coverage_backend" ]] && [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then + zds_fold_start "coverage_backend" "* Run coverage for backend" + coverage run --source='.' manage.py \ + test -v=2\ + --keepdb \ + --settings zds.settings.ci_test \ + --exclude-tag=front \ + ${ZDS_TEST_JOB/front/}; exVal=$? + zds_fold_end +fi + +# print zmarkdown log +if [[ "$1" == "print_zmarkdown_log" ]] && [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then + zds_fold_start "zmarkdown_log" "* Print zmarkdown log" + pm2 logs --nostream --raw --lines 1000; exVal=$? + zds_fold_end +fi + +# selenium test +if [[ "$1" == "selenium_test" ]] && [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then + zds_fold_start "selenium_test" "* Run selenium test for frontend" + xvfb-run --server-args="-screen 0 1280x720x8" python manage.py \ + test -v=2\ + --settings zds.settings.ci_test \ + --tag=front \ + --keepdb; exVal=$? + zds_fold_end +fi + +# build documentation +if [[ "$1" == "build_documentation" ]] && [[ "$ZDS_TEST_JOB" == *"doc"* ]]; then + zds_fold_start "doc" "* Run SphinxBuild to build documentation" + print_info "* Build documentation" + if [[ "$ZDS_TEST_JOB" == *"doc"* ]]; then + make generate-doc; exVal=$? + fi + zds_fold_end +fi + +if [[ $exVal != 0 ]]; then + print_error "!! Some error on the last task ($1)." +fi diff --git a/scripts/install_zds.sh b/scripts/install_zds.sh index 9deb4c5c90..427515d577 100755 --- a/scripts/install_zds.sh +++ b/scripts/install_zds.sh @@ -18,57 +18,6 @@ function _nvm { } -## start fold for travis -ZDS_SHOW_TRAVIS_FOLD=0 -if $(_in "--travis-output" $@); then - ZDS_SHOW_TRAVIS_FOLD=1 -fi - - -zds_fold_current="" -function zds_fold_start { - if [[ $ZDS_SHOW_TRAVIS_FOLD == 1 ]]; then - if [[ $zds_fold_current == $1 ]]; then # for virtualenv fold - return - fi - - zds_fold_current="$1" - echo "travis_fold:start:install_${zds_fold_current}" - echo -en "\033[0K" - fi - - print_info "$2" --bold -} - - -function zds_fold_end { - if [[ $ZDS_SHOW_TRAVIS_FOLD == 1 ]] && [[ $zds_fold_current =~ "" ]]; then - echo "travis_fold:end:install_${zds_fold_current}" - echo -en "\033[0K" - zds_fold_current="" - fi -} -## end - - -function print_info { - if [[ "$2" == "--bold" ]]; then - echo -en "\033[36;1m" - else - echo -en "\033[0;36m" - fi - echo "$1" - echo -en "\033[00m" -} - - -function print_error { - echo -en "\033[31;1m" - echo "$1" - echo -en "\033[00m" -} - - ## start quiet mode function progressfilt { local flag=false c count cr=$'\r' nl=$'\n' @@ -100,6 +49,7 @@ function wget_nv { # variables LOCAL_DIR="$(cd "$(dirname "$0")" && pwd)" source $LOCAL_DIR/define_variable.sh +source $LOCAL_DIR/define_function.sh # zds-site root folder ZDSSITE_DIR=$(pwd) diff --git a/scripts/travis_header.sh b/scripts/travis_header.sh index ea21856d01..080d86b9fc 100755 --- a/scripts/travis_header.sh +++ b/scripts/travis_header.sh @@ -9,24 +9,6 @@ function zds_register_for_install { fi } -function print_info { - echo -n -e "\033[0;36m"; - echo "$1"; - echo -n -e "\033[00m"; -} - -function install_geckodriver { - print_info "* Install webdriver for selenium." - - if [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then - wget "https://github.com/mozilla/geckodriver/releases/download/v0.23.0/geckodriver-v0.23.0-linux64.tar.gz" \ - && mkdir "geckodriver" \ - && tar -xzf "geckodriver-v0.23.0-linux64.tar.gz" -C "geckodriver" \ - && export PATH="$PATH:$PWD/geckodriver" \ - && export DISPLAY=":99.0" - fi -} - function zds_register_module_for_installation { zds_register_for_install "+base +prod" @@ -50,3 +32,58 @@ function zds_register_module_for_installation { print_info "* Argument for installation : $zds_install_argument" } + +function install_geckodriver { + if [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then + wget "https://github.com/mozilla/geckodriver/releases/download/v0.23.0/geckodriver-v0.23.0-linux64.tar.gz" \ + && mkdir "geckodriver" \ + && tar -xzf "geckodriver-v0.23.0-linux64.tar.gz" -C "geckodriver" \ + && export PATH="$PATH:$PWD/geckodriver" \ + && export DISPLAY=":99.0" + fi +} + +source ./scripts/define_variable.sh +source ./scripts/define_function.sh + +zds_fold_category "before_install" + +zds_fold_start "packages" "* update apt-get (for newest dependencies version)" + sudo apt-get update -qq +zds_fold_end + +zds_fold_start "coveralls" "* update apt-get" + pip install -q coveralls +zds_fold_end + +zds_fold_start "ci_turbo" "* Skip task depending on directory changes (task will run only if needed)" + source ./scripts/ci_turbo.sh # This script exports environment variables, it must be sourced +zds_fold_end + +if [[ "$ZDS_TEST_JOB" == *"zds."* ]] || [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then + #display print_info + forwho="" + if [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then + forwho="'zds.*' tasks (-> needed for tests)" + + if [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then + forwho="$forwho and selenium" + fi + else + forwho="selenium" + fi + + zds_fold_start "mysql" "* Install mysql for $forwho." + ./scripts/ci_mysql_setup.sh + zds_fold_end +fi + +if [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then + zds_fold_start "webdriver" "* Install webdriver for selenium" + install_geckodriver + zds_fold_end +fi + +zds_fold_start "register_module" "* Register module for installation" + zds_register_module_for_installation +zds_fold_end diff --git a/scripts/travis_script.sh b/scripts/travis_script.sh index 9081d0bd27..93bfb3aaea 100755 --- a/scripts/travis_script.sh +++ b/scripts/travis_script.sh @@ -1,71 +1,27 @@ #!/bin/bash -function print_info { - echo -n -e "\033[0;36m"; - echo "$1"; - echo -n -e "\033[00m"; -} +zds_fold_category "script" -# start elastic -if [[ "$1" == "start_elasticsearch" ]] && [[ "$ZDS_TEST_JOB" == *"zds.searchv2"* ]]; then - print_info "* Start elasticsearch as service" - sudo service elasticsearch start -fi +print_info "source ./\$ZDS_VENV/bin/activate" -# start latex -if [[ "$1" == "start_elasticsearch" ]] && [[ "$ZDS_TEST_JOB" == *"zds.tutorialv2"* ]]; then - print_info "* Start texhash" - texhash -fi +source ./$ZDS_VENV/bin/activate -# lint backend -if [[ "$1" == "lint_backend" ]] && [[ "$ZDS_TEST_JOB" == *"zds.gallery"* ]]; then - print_info "* Run lint for backend" - ./scripts/no_import_zds_settings.sh \ - && flake8 \ - && flake8 --config=zds/settings/.flake8 zds/settings -fi +./scripts/travis_script.sh "start_elasticsearch" -# test backend -if [[ "$1" == "test_backend" ]] && [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then - print_info "* Run test for backend" - python manage.py makemigrations --dry-run --check -fi +./scripts/travis_script.sh "start_latex" -# coverage backend -if [[ "$1" == "coverage_backend" ]] && [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then - source $HACK_SphinxBuild/bin/activate - coverage run --source='.' manage.py \ - test -v=2\ - --keepdb \ - --settings zds.settings.ci_test \ - --exclude-tag=front \ - ${ZDS_TEST_JOB/front/} - source ./$ZDS_VENV/bin/activate -fi +./scripts/travis_script.sh "lint_backend" -# print zmarkdown log -if [[ "$1" == "print_zmarkdown_log" ]] && [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then - print_info "* Print zmarkdown log" - pm2 logs --nostream --raw --lines 1000 -fi +./scripts/travis_script.sh "test_backend" -# selenium test -if [[ "$1" == "selenium_test" ]] && [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then - print_info "* Run selenium test for frontend" - xvfb-run --server-args="-screen 0 1280x720x8" python manage.py \ - test -v=2\ - --settings zds.settings.ci_test \ - --tag=front \ - --keepdb -fi +./scripts/travis_script.sh "print_zmarkdown_log" -# build documentation -if [[ "$1" == "build_documentation" ]] && [[ "$ZDS_TEST_JOB" == *"doc"* ]]; then - print_info "* Build documentation" - if [[ "$ZDS_TEST_JOB" == *"doc"* ]]; then - source $HACK_SphinxBuild/bin/activate - make generate-doc - source ./$ZDS_VENV/bin/activate - fi -fi +./scripts/travis_script.sh "selenium_test" + +print_info "source ./\$HACK_VIRTUALENV/bin/activate" + +source $HACK_VIRTUALENV/bin/activate # Fix task with "command not found" + +./scripts/travis_script.sh "coverage_backend" + +./scripts/travis_script.sh "build_documentation" From 90460fe76985014bf8e2d90c1554107d4c8d6065 Mon Sep 17 00:00:00 2001 From: A-312 Date: Mon, 1 Apr 2019 13:20:42 +0200 Subject: [PATCH 42/69] fix erreur de syntax --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 48dfe54204..9d0c68ba27 100644 --- a/.travis.yml +++ b/.travis.yml @@ -69,7 +69,7 @@ cache: before_install: - - $HACK_VIRTUALENV=$VIRTUAL_ENV + - HACK_VIRTUALENV=$VIRTUAL_ENV - source ./scripts/travis_header.sh # Need to be sourced From 935e6591d15742a12112b32fc353dd263ca1044c Mon Sep 17 00:00:00 2001 From: A-312 Date: Mon, 1 Apr 2019 13:25:06 +0200 Subject: [PATCH 43/69] unix2dos define_function.sh --- scripts/define_script.sh | 154 +++++++++++++++++++-------------------- 1 file changed, 77 insertions(+), 77 deletions(-) diff --git a/scripts/define_script.sh b/scripts/define_script.sh index 32fce27118..2c2fecc4c8 100644 --- a/scripts/define_script.sh +++ b/scripts/define_script.sh @@ -1,77 +1,77 @@ -#!/bin/bash - -exVal=0 - -# start elastic -if [[ "$1" == "start_elasticsearch" ]] && [[ "$ZDS_TEST_JOB" == *"zds.searchv2"* ]]; then - zds_fold_start "elasticsearch" "* Start elasticsearch as service" - sudo service elasticsearch start; exVal=$? - zds_fold_end -fi - -# start latex -if [[ "$1" == "start_latex" ]] && [[ "$ZDS_TEST_JOB" == *"zds.tutorialv2"* ]]; then - zds_fold_start "latex" "* Start texhash -> latex" - texhash; exVal=$? - zds_fold_end -fi - -# lint backend -if [[ "$1" == "lint_backend" ]] && [[ "$ZDS_TEST_JOB" == *"zds.gallery"* ]]; then - zds_fold_start "lint_backend" "* Run lint for backend" - ./scripts/no_import_zds_settings.sh - flake8; exVal=$? - flake8 --config=zds/settings/.flake8 zds/settings; exVal=$? + $exVal - zds_fold_end -fi - -# test backend -if [[ "$1" == "test_backend" ]] && [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then - zds_fold_start "test_backend" "* Run test for backend" - python manage.py makemigrations --dry-run --check; exVal=$? - zds_fold_end -fi - -# coverage backend -if [[ "$1" == "coverage_backend" ]] && [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then - zds_fold_start "coverage_backend" "* Run coverage for backend" - coverage run --source='.' manage.py \ - test -v=2\ - --keepdb \ - --settings zds.settings.ci_test \ - --exclude-tag=front \ - ${ZDS_TEST_JOB/front/}; exVal=$? - zds_fold_end -fi - -# print zmarkdown log -if [[ "$1" == "print_zmarkdown_log" ]] && [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then - zds_fold_start "zmarkdown_log" "* Print zmarkdown log" - pm2 logs --nostream --raw --lines 1000; exVal=$? - zds_fold_end -fi - -# selenium test -if [[ "$1" == "selenium_test" ]] && [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then - zds_fold_start "selenium_test" "* Run selenium test for frontend" - xvfb-run --server-args="-screen 0 1280x720x8" python manage.py \ - test -v=2\ - --settings zds.settings.ci_test \ - --tag=front \ - --keepdb; exVal=$? - zds_fold_end -fi - -# build documentation -if [[ "$1" == "build_documentation" ]] && [[ "$ZDS_TEST_JOB" == *"doc"* ]]; then - zds_fold_start "doc" "* Run SphinxBuild to build documentation" - print_info "* Build documentation" - if [[ "$ZDS_TEST_JOB" == *"doc"* ]]; then - make generate-doc; exVal=$? - fi - zds_fold_end -fi - -if [[ $exVal != 0 ]]; then - print_error "!! Some error on the last task ($1)." -fi +#!/bin/bash + +exVal=0 + +# start elastic +if [[ "$1" == "start_elasticsearch" ]] && [[ "$ZDS_TEST_JOB" == *"zds.searchv2"* ]]; then + zds_fold_start "elasticsearch" "* Start elasticsearch as service" + sudo service elasticsearch start; exVal=$? + zds_fold_end +fi + +# start latex +if [[ "$1" == "start_latex" ]] && [[ "$ZDS_TEST_JOB" == *"zds.tutorialv2"* ]]; then + zds_fold_start "latex" "* Start texhash -> latex" + texhash; exVal=$? + zds_fold_end +fi + +# lint backend +if [[ "$1" == "lint_backend" ]] && [[ "$ZDS_TEST_JOB" == *"zds.gallery"* ]]; then + zds_fold_start "lint_backend" "* Run lint for backend" + ./scripts/no_import_zds_settings.sh + flake8; exVal=$? + flake8 --config=zds/settings/.flake8 zds/settings; exVal=$? + $exVal + zds_fold_end +fi + +# test backend +if [[ "$1" == "test_backend" ]] && [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then + zds_fold_start "test_backend" "* Run test for backend" + python manage.py makemigrations --dry-run --check; exVal=$? + zds_fold_end +fi + +# coverage backend +if [[ "$1" == "coverage_backend" ]] && [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then + zds_fold_start "coverage_backend" "* Run coverage for backend" + coverage run --source='.' manage.py \ + test -v=2\ + --keepdb \ + --settings zds.settings.ci_test \ + --exclude-tag=front \ + ${ZDS_TEST_JOB/front/}; exVal=$? + zds_fold_end +fi + +# print zmarkdown log +if [[ "$1" == "print_zmarkdown_log" ]] && [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then + zds_fold_start "zmarkdown_log" "* Print zmarkdown log" + pm2 logs --nostream --raw --lines 1000; exVal=$? + zds_fold_end +fi + +# selenium test +if [[ "$1" == "selenium_test" ]] && [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then + zds_fold_start "selenium_test" "* Run selenium test for frontend" + xvfb-run --server-args="-screen 0 1280x720x8" python manage.py \ + test -v=2\ + --settings zds.settings.ci_test \ + --tag=front \ + --keepdb; exVal=$? + zds_fold_end +fi + +# build documentation +if [[ "$1" == "build_documentation" ]] && [[ "$ZDS_TEST_JOB" == *"doc"* ]]; then + zds_fold_start "doc" "* Run SphinxBuild to build documentation" + print_info "* Build documentation" + if [[ "$ZDS_TEST_JOB" == *"doc"* ]]; then + make generate-doc; exVal=$? + fi + zds_fold_end +fi + +if [[ $exVal != 0 ]]; then + print_error "!! Some error on the last task ($1)." +fi From 03646901f82c625c88c86db2c15e49cf9fb47130 Mon Sep 17 00:00:00 2001 From: A-312 Date: Mon, 1 Apr 2019 13:35:37 +0200 Subject: [PATCH 44/69] Fix les permissions --- scripts/define_function.sh | 0 scripts/define_script.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/define_function.sh mode change 100644 => 100755 scripts/define_script.sh diff --git a/scripts/define_function.sh b/scripts/define_function.sh old mode 100644 new mode 100755 diff --git a/scripts/define_script.sh b/scripts/define_script.sh old mode 100644 new mode 100755 From cd54f748d160d2642627b48f1f6b8019b2f84cd9 Mon Sep 17 00:00:00 2001 From: A-312 Date: Mon, 1 Apr 2019 13:39:22 +0200 Subject: [PATCH 45/69] Fix l'erreur de syntax --- scripts/define_function.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/define_function.sh b/scripts/define_function.sh index d2d4d199ff..804d39cdb9 100755 --- a/scripts/define_function.sh +++ b/scripts/define_function.sh @@ -7,7 +7,7 @@ fi zds_fold_current_cat="default" function zds_fold_category { - zds_fold_category=$1 + zds_fold_category="$1" } From 37e10fc2acef48c75ef149ee954974f53531e9c5 Mon Sep 17 00:00:00 2001 From: A-312 Date: Mon, 1 Apr 2019 13:44:05 +0200 Subject: [PATCH 46/69] =?UTF-8?q?Fix=20line=20ending=202=C3=A8me=20essai?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/define_function.sh | 110 ++++++++++++++++++------------------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/scripts/define_function.sh b/scripts/define_function.sh index 804d39cdb9..5714cffeca 100755 --- a/scripts/define_function.sh +++ b/scripts/define_function.sh @@ -1,55 +1,55 @@ -## start fold for travis -ZDS_SHOW_TRAVIS_FOLD=0 -if $(_in "--travis-output" $@); then - ZDS_SHOW_TRAVIS_FOLD=1 -fi - - -zds_fold_current_cat="default" -function zds_fold_category { - zds_fold_category="$1" -} - - -zds_fold_current="" -function zds_fold_start { - if [[ $ZDS_SHOW_TRAVIS_FOLD == 1 ]]; then - if [[ $zds_fold_current == $1 ]]; then # for virtualenv fold - return - fi - - zds_fold_current="$1" - echo "travis_fold:start:${zds_fold_current_cat}_${zds_fold_current}" - echo -en "\033[0K" - fi - - print_info "$2" --bold -} - - -function zds_fold_end { - if [[ $ZDS_SHOW_TRAVIS_FOLD == 1 ]] && [[ $zds_fold_current =~ "" ]]; then - echo "travis_fold:end:${zds_fold_current_cat}_${zds_fold_current}" - echo -en "\033[0K" - zds_fold_current="" - fi -} -## end - - -function print_info { - if [[ "$2" == "--bold" ]]; then - echo -en "\033[36;1m" - else - echo -en "\033[0;36m" - fi - echo "$1" - echo -en "\033[00m" -} - - -function print_error { - echo -en "\033[31;1m" - echo "$1" - echo -en "\033[00m" -} +## start fold for travis +ZDS_SHOW_TRAVIS_FOLD=0 +if $(_in "--travis-output" $@); then + ZDS_SHOW_TRAVIS_FOLD=1 +fi + + +zds_fold_current_cat="default" +function zds_fold_category { + zds_fold_category="$1" +} + + +zds_fold_current="" +function zds_fold_start { + if [[ $ZDS_SHOW_TRAVIS_FOLD == 1 ]]; then + if [[ $zds_fold_current == $1 ]]; then # for virtualenv fold + return + fi + + zds_fold_current="$1" + echo "travis_fold:start:${zds_fold_current_cat}_${zds_fold_current}" + echo -en "\033[0K" + fi + + print_info "$2" --bold +} + + +function zds_fold_end { + if [[ $ZDS_SHOW_TRAVIS_FOLD == 1 ]] && [[ $zds_fold_current =~ "" ]]; then + echo "travis_fold:end:${zds_fold_current_cat}_${zds_fold_current}" + echo -en "\033[0K" + zds_fold_current="" + fi +} +## end + + +function print_info { + if [[ "$2" == "--bold" ]]; then + echo -en "\033[36;1m" + else + echo -en "\033[0;36m" + fi + echo "$1" + echo -en "\033[00m" +} + + +function print_error { + echo -en "\033[31;1m" + echo "$1" + echo -en "\033[00m" +} From 17adac00828e4446bbf03d85ead1e69faa266755 Mon Sep 17 00:00:00 2001 From: A-312 Date: Mon, 1 Apr 2019 13:53:58 +0200 Subject: [PATCH 47/69] add travis_output --- scripts/define_function.sh | 9 +++++++++ scripts/install_zds.sh | 17 ++++++++--------- scripts/travis_header.sh | 2 +- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/scripts/define_function.sh b/scripts/define_function.sh index 5714cffeca..4b80bd75a9 100755 --- a/scripts/define_function.sh +++ b/scripts/define_function.sh @@ -1,3 +1,12 @@ +function _in { + # credits: https://stackoverflow.com/a/8574392 + local e match="$1" + shift + for e; do [[ "$e" == "$match" ]] && return 0; done + return 1 +} + + ## start fold for travis ZDS_SHOW_TRAVIS_FOLD=0 if $(_in "--travis-output" $@); then diff --git a/scripts/install_zds.sh b/scripts/install_zds.sh index 427515d577..8d35379a8f 100755 --- a/scripts/install_zds.sh +++ b/scripts/install_zds.sh @@ -3,15 +3,6 @@ # Install script for the zds-site repository -function _in { - # credits: https://stackoverflow.com/a/8574392 - local e match="$1" - shift - for e; do [[ "$e" == "$match" ]] && return 0; done - return 1 -} - - function _nvm { export NVM_DIR="$HOME/.nvm" [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm @@ -51,6 +42,14 @@ LOCAL_DIR="$(cd "$(dirname "$0")" && pwd)" source $LOCAL_DIR/define_variable.sh source $LOCAL_DIR/define_function.sh + +# enable travis fold +ZDS_SHOW_TRAVIS_FOLD=0 +if $(_in "--travis-output" $@); then + ZDS_SHOW_TRAVIS_FOLD=1 +fi + + # zds-site root folder ZDSSITE_DIR=$(pwd) diff --git a/scripts/travis_header.sh b/scripts/travis_header.sh index 080d86b9fc..0adb8c5e41 100755 --- a/scripts/travis_header.sh +++ b/scripts/travis_header.sh @@ -44,7 +44,7 @@ function install_geckodriver { } source ./scripts/define_variable.sh -source ./scripts/define_function.sh +source ./scripts/define_function.sh --travis-output zds_fold_category "before_install" From fd5b46fa0b762812a08c77df0382f000b54487b9 Mon Sep 17 00:00:00 2001 From: A-312 Date: Mon, 1 Apr 2019 14:03:43 +0200 Subject: [PATCH 48/69] Modifie la nomenclature des fichiers --- .travis.yml | 2 +- scripts/define_function.sh | 3 ++ scripts/define_script.sh | 77 ---------------------------------- scripts/travis_header.sh | 11 +++++ scripts/travis_run.sh | 27 ++++++++++++ scripts/travis_script.sh | 85 ++++++++++++++++++++++++++++++++------ 6 files changed, 114 insertions(+), 91 deletions(-) delete mode 100755 scripts/define_script.sh create mode 100755 scripts/travis_run.sh diff --git a/.travis.yml b/.travis.yml index 9d0c68ba27..d1b65721d3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -79,7 +79,7 @@ install: script: - - ./scripts/travis_script.sh + - ./scripts/travis_run.sh after_success: diff --git a/scripts/define_function.sh b/scripts/define_function.sh index 4b80bd75a9..1253751515 100755 --- a/scripts/define_function.sh +++ b/scripts/define_function.sh @@ -1,3 +1,6 @@ +#!/bin/bash + + function _in { # credits: https://stackoverflow.com/a/8574392 local e match="$1" diff --git a/scripts/define_script.sh b/scripts/define_script.sh deleted file mode 100755 index 2c2fecc4c8..0000000000 --- a/scripts/define_script.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/bash - -exVal=0 - -# start elastic -if [[ "$1" == "start_elasticsearch" ]] && [[ "$ZDS_TEST_JOB" == *"zds.searchv2"* ]]; then - zds_fold_start "elasticsearch" "* Start elasticsearch as service" - sudo service elasticsearch start; exVal=$? - zds_fold_end -fi - -# start latex -if [[ "$1" == "start_latex" ]] && [[ "$ZDS_TEST_JOB" == *"zds.tutorialv2"* ]]; then - zds_fold_start "latex" "* Start texhash -> latex" - texhash; exVal=$? - zds_fold_end -fi - -# lint backend -if [[ "$1" == "lint_backend" ]] && [[ "$ZDS_TEST_JOB" == *"zds.gallery"* ]]; then - zds_fold_start "lint_backend" "* Run lint for backend" - ./scripts/no_import_zds_settings.sh - flake8; exVal=$? - flake8 --config=zds/settings/.flake8 zds/settings; exVal=$? + $exVal - zds_fold_end -fi - -# test backend -if [[ "$1" == "test_backend" ]] && [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then - zds_fold_start "test_backend" "* Run test for backend" - python manage.py makemigrations --dry-run --check; exVal=$? - zds_fold_end -fi - -# coverage backend -if [[ "$1" == "coverage_backend" ]] && [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then - zds_fold_start "coverage_backend" "* Run coverage for backend" - coverage run --source='.' manage.py \ - test -v=2\ - --keepdb \ - --settings zds.settings.ci_test \ - --exclude-tag=front \ - ${ZDS_TEST_JOB/front/}; exVal=$? - zds_fold_end -fi - -# print zmarkdown log -if [[ "$1" == "print_zmarkdown_log" ]] && [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then - zds_fold_start "zmarkdown_log" "* Print zmarkdown log" - pm2 logs --nostream --raw --lines 1000; exVal=$? - zds_fold_end -fi - -# selenium test -if [[ "$1" == "selenium_test" ]] && [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then - zds_fold_start "selenium_test" "* Run selenium test for frontend" - xvfb-run --server-args="-screen 0 1280x720x8" python manage.py \ - test -v=2\ - --settings zds.settings.ci_test \ - --tag=front \ - --keepdb; exVal=$? - zds_fold_end -fi - -# build documentation -if [[ "$1" == "build_documentation" ]] && [[ "$ZDS_TEST_JOB" == *"doc"* ]]; then - zds_fold_start "doc" "* Run SphinxBuild to build documentation" - print_info "* Build documentation" - if [[ "$ZDS_TEST_JOB" == *"doc"* ]]; then - make generate-doc; exVal=$? - fi - zds_fold_end -fi - -if [[ $exVal != 0 ]]; then - print_error "!! Some error on the last task ($1)." -fi diff --git a/scripts/travis_header.sh b/scripts/travis_header.sh index 0adb8c5e41..02f07be361 100755 --- a/scripts/travis_header.sh +++ b/scripts/travis_header.sh @@ -1,5 +1,6 @@ #!/bin/bash + zds_install_argument="" function zds_register_for_install { if [[ "$zds_install_argument" == "" ]]; then @@ -9,6 +10,7 @@ function zds_register_for_install { fi } + function zds_register_module_for_installation { zds_register_for_install "+base +prod" @@ -33,6 +35,7 @@ function zds_register_module_for_installation { print_info "* Argument for installation : $zds_install_argument" } + function install_geckodriver { if [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then wget "https://github.com/mozilla/geckodriver/releases/download/v0.23.0/geckodriver-v0.23.0-linux64.tar.gz" \ @@ -43,23 +46,29 @@ function install_geckodriver { fi } + source ./scripts/define_variable.sh source ./scripts/define_function.sh --travis-output + zds_fold_category "before_install" + zds_fold_start "packages" "* update apt-get (for newest dependencies version)" sudo apt-get update -qq zds_fold_end + zds_fold_start "coveralls" "* update apt-get" pip install -q coveralls zds_fold_end + zds_fold_start "ci_turbo" "* Skip task depending on directory changes (task will run only if needed)" source ./scripts/ci_turbo.sh # This script exports environment variables, it must be sourced zds_fold_end + if [[ "$ZDS_TEST_JOB" == *"zds."* ]] || [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then #display print_info forwho="" @@ -78,12 +87,14 @@ if [[ "$ZDS_TEST_JOB" == *"zds."* ]] || [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; t zds_fold_end fi + if [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then zds_fold_start "webdriver" "* Install webdriver for selenium" install_geckodriver zds_fold_end fi + zds_fold_start "register_module" "* Register module for installation" zds_register_module_for_installation zds_fold_end diff --git a/scripts/travis_run.sh b/scripts/travis_run.sh new file mode 100755 index 0000000000..93bfb3aaea --- /dev/null +++ b/scripts/travis_run.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +zds_fold_category "script" + +print_info "source ./\$ZDS_VENV/bin/activate" + +source ./$ZDS_VENV/bin/activate + +./scripts/travis_script.sh "start_elasticsearch" + +./scripts/travis_script.sh "start_latex" + +./scripts/travis_script.sh "lint_backend" + +./scripts/travis_script.sh "test_backend" + +./scripts/travis_script.sh "print_zmarkdown_log" + +./scripts/travis_script.sh "selenium_test" + +print_info "source ./\$HACK_VIRTUALENV/bin/activate" + +source $HACK_VIRTUALENV/bin/activate # Fix task with "command not found" + +./scripts/travis_script.sh "coverage_backend" + +./scripts/travis_script.sh "build_documentation" diff --git a/scripts/travis_script.sh b/scripts/travis_script.sh index 93bfb3aaea..c625b6907c 100755 --- a/scripts/travis_script.sh +++ b/scripts/travis_script.sh @@ -1,27 +1,86 @@ #!/bin/bash -zds_fold_category "script" -print_info "source ./\$ZDS_VENV/bin/activate" +exVal=0 -source ./$ZDS_VENV/bin/activate -./scripts/travis_script.sh "start_elasticsearch" +# start elastic +if [[ "$1" == "start_elasticsearch" ]] && [[ "$ZDS_TEST_JOB" == *"zds.searchv2"* ]]; then + zds_fold_start "elasticsearch" "* Start elasticsearch as service" + sudo service elasticsearch start; exVal=$? + zds_fold_end +fi -./scripts/travis_script.sh "start_latex" -./scripts/travis_script.sh "lint_backend" +# start latex +if [[ "$1" == "start_latex" ]] && [[ "$ZDS_TEST_JOB" == *"zds.tutorialv2"* ]]; then + zds_fold_start "latex" "* Start texhash -> latex" + texhash; exVal=$? + zds_fold_end +fi -./scripts/travis_script.sh "test_backend" -./scripts/travis_script.sh "print_zmarkdown_log" +# lint backend +if [[ "$1" == "lint_backend" ]] && [[ "$ZDS_TEST_JOB" == *"zds.gallery"* ]]; then + zds_fold_start "lint_backend" "* Run lint for backend" + ./scripts/no_import_zds_settings.sh + flake8; exVal=$? + flake8 --config=zds/settings/.flake8 zds/settings; exVal=$? + $exVal + zds_fold_end +fi -./scripts/travis_script.sh "selenium_test" +# test backend +if [[ "$1" == "test_backend" ]] && [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then + zds_fold_start "test_backend" "* Run test for backend" + python manage.py makemigrations --dry-run --check; exVal=$? + zds_fold_end +fi -print_info "source ./\$HACK_VIRTUALENV/bin/activate" -source $HACK_VIRTUALENV/bin/activate # Fix task with "command not found" +# coverage backend +if [[ "$1" == "coverage_backend" ]] && [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then + zds_fold_start "coverage_backend" "* Run coverage for backend" + coverage run --source='.' manage.py \ + test -v=2\ + --keepdb \ + --settings zds.settings.ci_test \ + --exclude-tag=front \ + ${ZDS_TEST_JOB/front/}; exVal=$? + zds_fold_end +fi -./scripts/travis_script.sh "coverage_backend" -./scripts/travis_script.sh "build_documentation" +# print zmarkdown log +if [[ "$1" == "print_zmarkdown_log" ]] && [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then + zds_fold_start "zmarkdown_log" "* Print zmarkdown log" + pm2 logs --nostream --raw --lines 1000; exVal=$? + zds_fold_end +fi + + +# selenium test +if [[ "$1" == "selenium_test" ]] && [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then + zds_fold_start "selenium_test" "* Run selenium test for frontend" + xvfb-run --server-args="-screen 0 1280x720x8" python manage.py \ + test -v=2\ + --settings zds.settings.ci_test \ + --tag=front \ + --keepdb; exVal=$? + zds_fold_end +fi + + +# build documentation +if [[ "$1" == "build_documentation" ]] && [[ "$ZDS_TEST_JOB" == *"doc"* ]]; then + zds_fold_start "doc" "* Run SphinxBuild to build documentation" + print_info "* Build documentation" + if [[ "$ZDS_TEST_JOB" == *"doc"* ]]; then + make generate-doc; exVal=$? + fi + zds_fold_end +fi + + +if [[ $exVal != 0 ]]; then + print_error "!! Some error on the last task ($1)." +fi From fcef658398d6aa83a85ec6e09273f9431098ff27 Mon Sep 17 00:00:00 2001 From: A-312 Date: Mon, 1 Apr 2019 14:28:29 +0200 Subject: [PATCH 49/69] =?UTF-8?q?Rajoute=20les=20includes=20l=C3=A0,=20o?= =?UTF-8?q?=C3=B9=20c'est=20n=C3=A9cessaire?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/ci_turbo.sh | 1 - scripts/travis_run.sh | 14 ++++++++++---- scripts/travis_script.sh | 3 +++ 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/scripts/ci_turbo.sh b/scripts/ci_turbo.sh index adcd364bf7..bc81a3e0a4 100755 --- a/scripts/ci_turbo.sh +++ b/scripts/ci_turbo.sh @@ -16,7 +16,6 @@ fi changed_files=$(git --no-pager diff --name-only $TRAVIS_COMMIT $(git merge-base $TRAVIS_COMMIT origin/$default_branch)) echo "changed files:" echo $changed_files -echo if ! echo "$changed_files" | egrep -v "^assets" then diff --git a/scripts/travis_run.sh b/scripts/travis_run.sh index 93bfb3aaea..395fb2a29c 100755 --- a/scripts/travis_run.sh +++ b/scripts/travis_run.sh @@ -1,9 +1,14 @@ #!/bin/bash -zds_fold_category "script" -print_info "source ./\$ZDS_VENV/bin/activate" +function print_info { + echo -en "\033[0;36m" + echo "$1" + echo -en "\033[00m" +} + +print_info "source ./\$ZDS_VENV/bin/activate" source ./$ZDS_VENV/bin/activate ./scripts/travis_script.sh "start_elasticsearch" @@ -18,9 +23,10 @@ source ./$ZDS_VENV/bin/activate ./scripts/travis_script.sh "selenium_test" -print_info "source ./\$HACK_VIRTUALENV/bin/activate" +# Use hack for virtualenv (fix some task with "command not found") -source $HACK_VIRTUALENV/bin/activate # Fix task with "command not found" +print_info "source ./\$HACK_VIRTUALENV/bin/activate" +source $HACK_VIRTUALENV/bin/activate ./scripts/travis_script.sh "coverage_backend" diff --git a/scripts/travis_script.sh b/scripts/travis_script.sh index c625b6907c..d01924a3be 100755 --- a/scripts/travis_script.sh +++ b/scripts/travis_script.sh @@ -1,5 +1,8 @@ #!/bin/bash +source ./scripts/define_function.sh --travis-output + +zds_fold_category "script" exVal=0 From 689fa966a3c4436b17c66ab4adbf2d4f5dd5ef62 Mon Sep 17 00:00:00 2001 From: A-312 Date: Mon, 1 Apr 2019 14:55:33 +0200 Subject: [PATCH 50/69] export les variables --- .travis.yml | 2 +- scripts/install_zds.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index d1b65721d3..7536e70c1d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -69,7 +69,7 @@ cache: before_install: - - HACK_VIRTUALENV=$VIRTUAL_ENV + - export HACK_VIRTUALENV=$VIRTUAL_ENV - source ./scripts/travis_header.sh # Need to be sourced diff --git a/scripts/install_zds.sh b/scripts/install_zds.sh index 8d35379a8f..040aa48672 100755 --- a/scripts/install_zds.sh +++ b/scripts/install_zds.sh @@ -225,7 +225,7 @@ else zds_fold_end fi -ZDS_ENV=$(realpath $ZDS_VENV) +export ZDS_ENV=$(realpath $ZDS_VENV) # nvm node & yarn if ! $(_in "-node" $@) && ( $(_in "+node" $@) || $(_in "+base" $@) || $(_in "+full" $@) ); then From 58c81f038d1d8f21e2a60a61d6bfd686b32a2855 Mon Sep 17 00:00:00 2001 From: A-312 Date: Mon, 1 Apr 2019 14:57:15 +0200 Subject: [PATCH 51/69] corrige une erreur de c/c dans echo (incidence visuelle) --- scripts/travis_run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/travis_run.sh b/scripts/travis_run.sh index 395fb2a29c..841445407e 100755 --- a/scripts/travis_run.sh +++ b/scripts/travis_run.sh @@ -25,7 +25,7 @@ source ./$ZDS_VENV/bin/activate # Use hack for virtualenv (fix some task with "command not found") -print_info "source ./\$HACK_VIRTUALENV/bin/activate" +print_info "source \$HACK_VIRTUALENV/bin/activate" source $HACK_VIRTUALENV/bin/activate ./scripts/travis_script.sh "coverage_backend" From 70a20dd27797bd52cb64b14b7e7224d660413a9f Mon Sep 17 00:00:00 2001 From: A-312 Date: Mon, 1 Apr 2019 15:02:28 +0200 Subject: [PATCH 52/69] =?UTF-8?q?J'ai=20oubli=C3=A9=20de=20d=C3=A9finir=20?= =?UTF-8?q?la=20cat=C3=A9gorie=20de=20zds=5Ffold=20pour=20install=5Fzds.sh?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/install_zds.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/install_zds.sh b/scripts/install_zds.sh index 040aa48672..e7caa7ee1b 100755 --- a/scripts/install_zds.sh +++ b/scripts/install_zds.sh @@ -49,6 +49,8 @@ if $(_in "--travis-output" $@); then ZDS_SHOW_TRAVIS_FOLD=1 fi +zds_fold_category "install" + # zds-site root folder ZDSSITE_DIR=$(pwd) From eda8b4691e620eb72a638909ce62f004efb02204 Mon Sep 17 00:00:00 2001 From: A-312 Date: Mon, 1 Apr 2019 15:20:53 +0200 Subject: [PATCH 53/69] (test scope) --- .travis.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7536e70c1d..84b489208e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,14 +33,6 @@ env: jobs: include: - - name: tuto - env: ZDS_TEST_JOB="zds.tutorialv2" - - name: "test1 : member utils forum" - env: ZDS_TEST_JOB="zds.member zds.utils zds.forum" - - name: "test2 : front mp gallery pages featured notification searchv2" - env: ZDS_TEST_JOB="front zds.mp zds.gallery zds.pages zds.featured zds.notification zds.searchv2" - - name: selenium - env: ZDS_TEST_JOB="selenium" - name: doc env: ZDS_TEST_JOB="doc" @@ -73,9 +65,11 @@ before_install: - source ./scripts/travis_header.sh # Need to be sourced + - echo $ZDS_VENV + install: - - ./scripts/install_zds.sh --answer-yes --detect-os-version --travis-output $zds_install_argument + - #./scripts/install_zds.sh --answer-yes --detect-os-version --travis-output $zds_install_argument script: From 56dc6ad3f43076c515c97427bbbd4257f08a59c9 Mon Sep 17 00:00:00 2001 From: A-312 Date: Mon, 1 Apr 2019 15:22:24 +0200 Subject: [PATCH 54/69] (undo test) --- .travis.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 84b489208e..7536e70c1d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,6 +33,14 @@ env: jobs: include: + - name: tuto + env: ZDS_TEST_JOB="zds.tutorialv2" + - name: "test1 : member utils forum" + env: ZDS_TEST_JOB="zds.member zds.utils zds.forum" + - name: "test2 : front mp gallery pages featured notification searchv2" + env: ZDS_TEST_JOB="front zds.mp zds.gallery zds.pages zds.featured zds.notification zds.searchv2" + - name: selenium + env: ZDS_TEST_JOB="selenium" - name: doc env: ZDS_TEST_JOB="doc" @@ -65,11 +73,9 @@ before_install: - source ./scripts/travis_header.sh # Need to be sourced - - echo $ZDS_VENV - install: - - #./scripts/install_zds.sh --answer-yes --detect-os-version --travis-output $zds_install_argument + - ./scripts/install_zds.sh --answer-yes --detect-os-version --travis-output $zds_install_argument script: From f11cd4feea2d0c0fa07512b3759929903247fb18 Mon Sep 17 00:00:00 2001 From: A-312 Date: Mon, 1 Apr 2019 15:24:52 +0200 Subject: [PATCH 55/69] export ZDS_ENV --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 7536e70c1d..9345d3de26 100644 --- a/.travis.yml +++ b/.travis.yml @@ -79,6 +79,8 @@ install: script: + - export ZDS_ENV=$ZDS_ENV + - ./scripts/travis_run.sh From 5cfd8baec4286153ee3cc1590d8fc10a86c03c5c Mon Sep 17 00:00:00 2001 From: A-312 Date: Mon, 1 Apr 2019 15:25:20 +0200 Subject: [PATCH 56/69] exit 1 si erreur --- scripts/travis_script.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/travis_script.sh b/scripts/travis_script.sh index d01924a3be..6002921648 100755 --- a/scripts/travis_script.sh +++ b/scripts/travis_script.sh @@ -86,4 +86,5 @@ fi if [[ $exVal != 0 ]]; then print_error "!! Some error on the last task ($1)." + exit 1 fi From 3a1259afb99ba3a0144a64bf4f621e19e25176d6 Mon Sep 17 00:00:00 2001 From: A-312 Date: Mon, 1 Apr 2019 15:38:58 +0200 Subject: [PATCH 57/69] zds_fold_category n'utilisait pas la bonne variable --- scripts/define_function.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/define_function.sh b/scripts/define_function.sh index 1253751515..3a8a895d99 100755 --- a/scripts/define_function.sh +++ b/scripts/define_function.sh @@ -19,7 +19,7 @@ fi zds_fold_current_cat="default" function zds_fold_category { - zds_fold_category="$1" + zds_fold_current_cat="$1" } From d769cdeec034ffe008c3a81e409770bd707bcc1b Mon Sep 17 00:00:00 2001 From: A-312 Date: Mon, 1 Apr 2019 15:39:26 +0200 Subject: [PATCH 58/69] =?UTF-8?q?ZDS=5FENV,=202=C3=A8me=20essai?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9345d3de26..66f291cf51 100644 --- a/.travis.yml +++ b/.travis.yml @@ -79,9 +79,7 @@ install: script: - - export ZDS_ENV=$ZDS_ENV - - - ./scripts/travis_run.sh + - source ./scripts/travis_run.sh after_success: From e494a79eea70262989073a3f7da80f0227105145 Mon Sep 17 00:00:00 2001 From: A-312 Date: Mon, 1 Apr 2019 16:22:55 +0200 Subject: [PATCH 59/69] =?UTF-8?q?D=C3=A9place=20l'error=20handler=20dans?= =?UTF-8?q?=20le=20script=20parent?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/travis_run.sh | 45 +++++++++++++++++++++++++++++----------- scripts/travis_script.sh | 26 +++++++++-------------- 2 files changed, 43 insertions(+), 28 deletions(-) diff --git a/scripts/travis_run.sh b/scripts/travis_run.sh index 841445407e..70725f0d18 100755 --- a/scripts/travis_run.sh +++ b/scripts/travis_run.sh @@ -8,26 +8,47 @@ function print_info { } -print_info "source ./\$ZDS_VENV/bin/activate" -source ./$ZDS_VENV/bin/activate +function error_handler { + if [[ $exVal != 0 ]]; then + print_error $1 + exit 1 + fi +} + + +function run_script { + ./scripts/travis_script.sh $1; exVal=$? + error_handler "!! Some error on the last task ($1)." +} + -./scripts/travis_script.sh "start_elasticsearch" +function activate_env { + print_info "source $1/bin/activate" + source $1/bin/activate; exVal=$? + error_handler "!! Error: environnement not load.\n - Value = $1" +} -./scripts/travis_script.sh "start_latex" -./scripts/travis_script.sh "lint_backend" +activate_env "./$ZDS_VENV" -./scripts/travis_script.sh "test_backend" + run_script "start_elasticsearch" -./scripts/travis_script.sh "print_zmarkdown_log" + run_script "start_latex" + + run_script "lint_backend" + + run_script "test_backend" + + run_script "print_zmarkdown_log" + + run_script "selenium_test" -./scripts/travis_script.sh "selenium_test" # Use hack for virtualenv (fix some task with "command not found") -print_info "source \$HACK_VIRTUALENV/bin/activate" -source $HACK_VIRTUALENV/bin/activate +activate_env "$HACK_VIRTUALENV" + + run_script "coverage_backend" -./scripts/travis_script.sh "coverage_backend" + run_script "build_documentation" -./scripts/travis_script.sh "build_documentation" diff --git a/scripts/travis_script.sh b/scripts/travis_script.sh index 6002921648..1888795c1a 100755 --- a/scripts/travis_script.sh +++ b/scripts/travis_script.sh @@ -10,7 +10,7 @@ exVal=0 # start elastic if [[ "$1" == "start_elasticsearch" ]] && [[ "$ZDS_TEST_JOB" == *"zds.searchv2"* ]]; then zds_fold_start "elasticsearch" "* Start elasticsearch as service" - sudo service elasticsearch start; exVal=$? + sudo service elasticsearch start zds_fold_end fi @@ -18,7 +18,7 @@ fi # start latex if [[ "$1" == "start_latex" ]] && [[ "$ZDS_TEST_JOB" == *"zds.tutorialv2"* ]]; then zds_fold_start "latex" "* Start texhash -> latex" - texhash; exVal=$? + texhash zds_fold_end fi @@ -26,16 +26,16 @@ fi # lint backend if [[ "$1" == "lint_backend" ]] && [[ "$ZDS_TEST_JOB" == *"zds.gallery"* ]]; then zds_fold_start "lint_backend" "* Run lint for backend" - ./scripts/no_import_zds_settings.sh - flake8; exVal=$? - flake8 --config=zds/settings/.flake8 zds/settings; exVal=$? + $exVal + ./scripts/no_import_zds_settings.sh \ + && flake8 \ + && flake8 --config=zds/settings/.flake8 zds/settings zds_fold_end fi # test backend if [[ "$1" == "test_backend" ]] && [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then zds_fold_start "test_backend" "* Run test for backend" - python manage.py makemigrations --dry-run --check; exVal=$? + python manage.py makemigrations --dry-run --check zds_fold_end fi @@ -48,7 +48,7 @@ if [[ "$1" == "coverage_backend" ]] && [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then --keepdb \ --settings zds.settings.ci_test \ --exclude-tag=front \ - ${ZDS_TEST_JOB/front/}; exVal=$? + ${ZDS_TEST_JOB/front/} zds_fold_end fi @@ -56,7 +56,7 @@ fi # print zmarkdown log if [[ "$1" == "print_zmarkdown_log" ]] && [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then zds_fold_start "zmarkdown_log" "* Print zmarkdown log" - pm2 logs --nostream --raw --lines 1000; exVal=$? + pm2 logs --nostream --raw --lines 1000 zds_fold_end fi @@ -68,7 +68,7 @@ if [[ "$1" == "selenium_test" ]] && [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then test -v=2\ --settings zds.settings.ci_test \ --tag=front \ - --keepdb; exVal=$? + --keepdb zds_fold_end fi @@ -78,13 +78,7 @@ if [[ "$1" == "build_documentation" ]] && [[ "$ZDS_TEST_JOB" == *"doc"* ]]; then zds_fold_start "doc" "* Run SphinxBuild to build documentation" print_info "* Build documentation" if [[ "$ZDS_TEST_JOB" == *"doc"* ]]; then - make generate-doc; exVal=$? + make generate-doc fi zds_fold_end fi - - -if [[ $exVal != 0 ]]; then - print_error "!! Some error on the last task ($1)." - exit 1 -fi From b95fdab577d0b8d90d5f90d6b705df0e8859c94c Mon Sep 17 00:00:00 2001 From: A-312 Date: Mon, 1 Apr 2019 16:25:35 +0200 Subject: [PATCH 60/69] =?UTF-8?q?Apparement=20j'avais=20oubli=C3=A9=20de?= =?UTF-8?q?=20mettre=20le=20lint=20du=20frontend=20:')?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Si l'avion casse la checklist... --- scripts/travis_run.sh | 2 ++ scripts/travis_script.sh | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/scripts/travis_run.sh b/scripts/travis_run.sh index 70725f0d18..5386e2b4b8 100755 --- a/scripts/travis_run.sh +++ b/scripts/travis_run.sh @@ -39,6 +39,8 @@ activate_env "./$ZDS_VENV" run_script "test_backend" + run_script "lint_frontend" + run_script "print_zmarkdown_log" run_script "selenium_test" diff --git a/scripts/travis_script.sh b/scripts/travis_script.sh index 1888795c1a..3507f30dc1 100755 --- a/scripts/travis_script.sh +++ b/scripts/travis_script.sh @@ -40,6 +40,14 @@ if [[ "$1" == "test_backend" ]] && [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then fi +# lint frontend +if [[ "$1" == "lint_frontend" ]] && [[ "$ZDS_TEST_JOB" == *"front"* ]]; then + zds_fold_start "lint_frontend" "* Run lint for frontend" + npm run lint + zds_fold_end +fi + + # coverage backend if [[ "$1" == "coverage_backend" ]] && [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then zds_fold_start "coverage_backend" "* Run coverage for backend" From 80ac3cd82d2cdf7f87b76d47715361a288f1b715 Mon Sep 17 00:00:00 2001 From: A-312 Date: Mon, 1 Apr 2019 16:51:59 +0200 Subject: [PATCH 61/69] =?UTF-8?q?D=C3=A9marre=20zmd=20pour=20coverage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/travis_script.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/scripts/travis_script.sh b/scripts/travis_script.sh index 3507f30dc1..64311ba18b 100755 --- a/scripts/travis_script.sh +++ b/scripts/travis_script.sh @@ -51,12 +51,29 @@ fi # coverage backend if [[ "$1" == "coverage_backend" ]] && [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then zds_fold_start "coverage_backend" "* Run coverage for backend" + + npm run server --prefix zmd/node_modules/zmarkdown -- --silent; exVal=$? + + if [[ $exVal != 0 ]]; then + zds_fold_end + print_error "!! Cannot start zmd to coverage" + exit 1 + fi + coverage run --source='.' manage.py \ test -v=2\ --keepdb \ --settings zds.settings.ci_test \ --exclude-tag=front \ ${ZDS_TEST_JOB/front/} + + + make zmd-stop; exVal=$? + + if [[ $exVal != 0 ]]; then + print_error "Warning: Cannot stop zmd" + fi + zds_fold_end fi From efdde7af377ca1f5734e931b6091469a1f23df30 Mon Sep 17 00:00:00 2001 From: A-312 Date: Mon, 1 Apr 2019 17:15:29 +0200 Subject: [PATCH 62/69] =?UTF-8?q?Ajout=20du=20chemin=20relatif,=20plut?= =?UTF-8?q?=C3=B4t=20que=20la=20commande=20global?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/travis_script.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/travis_script.sh b/scripts/travis_script.sh index 64311ba18b..c8059a68a8 100755 --- a/scripts/travis_script.sh +++ b/scripts/travis_script.sh @@ -67,8 +67,8 @@ if [[ "$1" == "coverage_backend" ]] && [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then --exclude-tag=front \ ${ZDS_TEST_JOB/front/} - - make zmd-stop; exVal=$? + #pm2 kill : + ./zmd/node_modules/pm2/bin/pm2.js kill; exVal=$? if [[ $exVal != 0 ]]; then print_error "Warning: Cannot stop zmd" From 891089d94d8b502323c523669c2ca3dd74251871 Mon Sep 17 00:00:00 2001 From: A-312 Date: Fri, 19 Apr 2019 21:18:49 +0200 Subject: [PATCH 63/69] =?UTF-8?q?Undo=20passage=20apt-get=20=C3=A0=20pip.?= =?UTF-8?q?=20On=20remet=20comme=20c'=C3=A9tait=20avant?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requirements.txt | 3 --- scripts/dependencies/arch.txt | 4 ++++ scripts/dependencies/debian.txt | 4 ++++ scripts/dependencies/fedora.txt | 5 ++++- scripts/dependencies/ubuntu.txt | 4 ++++ 5 files changed, 16 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index d215e8e4d7..b028a1522e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,8 +3,6 @@ social-auth-app-django==3.1.0 elasticsearch==5.5.2 elasticsearch-dsl==5.4.0 sqlparse==0.2.4 -setuptools==40.6.3 -wheel==0.33.1 # Explicit dependencies (references in code) Django==2.1.7 @@ -13,7 +11,6 @@ django-model-utils==3.1.2 django-munin==0.2.1 python-memcached==1.59 lxml==4.2.5 -libxml2-python3==2.9.5 factory-boy==2.8.1 pygeoip==0.3.2 pillow==5.0.0 diff --git a/scripts/dependencies/arch.txt b/scripts/dependencies/arch.txt index d0b6a2d1a8..172f700bb7 100644 --- a/scripts/dependencies/arch.txt +++ b/scripts/dependencies/arch.txt @@ -7,7 +7,11 @@ curl unzip realpath python +python-setuptools +python-sqlparse python3-pip +libxml2 +python-lxml libxslt zlib libffi diff --git a/scripts/dependencies/debian.txt b/scripts/dependencies/debian.txt index 7c70779403..f7392477f4 100644 --- a/scripts/dependencies/debian.txt +++ b/scripts/dependencies/debian.txt @@ -9,6 +9,10 @@ realpath python3-dev python3-pip python3-venv +python3-setuptools +python3-sqlparse +python3-lxml +libxml2-dev libxslt-dev libz-dev libjpeg62-turbo diff --git a/scripts/dependencies/fedora.txt b/scripts/dependencies/fedora.txt index 0dfacc413a..1545d6fde7 100644 --- a/scripts/dependencies/fedora.txt +++ b/scripts/dependencies/fedora.txt @@ -7,9 +7,12 @@ curl unzip realpath python3-devel +python3-setuptools +python3-lxml +libxml2-devel libxslt-devel zlib-devel -libjpeg-turbo-devel +python3-sqlparse libjpeg-turbo-devel freetype freetype-devel diff --git a/scripts/dependencies/ubuntu.txt b/scripts/dependencies/ubuntu.txt index 2c216b5494..3532194d7a 100644 --- a/scripts/dependencies/ubuntu.txt +++ b/scripts/dependencies/ubuntu.txt @@ -9,6 +9,10 @@ realpath python3-dev python3-pip python3-venv +python3-setuptools +python3-sqlparse +python3-lxml +libxml2-dev libxslt1-dev zlib1g-dev libjpeg8 From 6694eafe77852eb3f10f5ab1630668674ff2be3e Mon Sep 17 00:00:00 2001 From: A-312 Date: Fri, 19 Apr 2019 22:49:50 +0200 Subject: [PATCH 64/69] =?UTF-8?q?maj=20versions=20test=C3=A9es?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/dependencies/debian.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/dependencies/debian.txt b/scripts/dependencies/debian.txt index f7392477f4..108c25f6b3 100644 --- a/scripts/dependencies/debian.txt +++ b/scripts/dependencies/debian.txt @@ -1,5 +1,5 @@ #title=Debian -#desc=Utilisation de apt-get / Testé sur : Debian Jessie +#desc=Utilisation de apt-get / Testé sur : Debian Jessie (8), Debian Stretch (9.8). #installcmd=apt-get -y install git wget From 8c7c458d94c532c8a162a9fa392c921dbc03ecc1 Mon Sep 17 00:00:00 2001 From: A-312 Date: Sat, 20 Apr 2019 09:31:57 +0200 Subject: [PATCH 65/69] On installe pas tex --- scripts/travis_header.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/travis_header.sh b/scripts/travis_header.sh index 02f07be361..d8a2f86877 100755 --- a/scripts/travis_header.sh +++ b/scripts/travis_header.sh @@ -23,7 +23,7 @@ function zds_register_module_for_installation { # install latex if [[ "$ZDS_TEST_JOB" == *"zds.tutorialv2"* ]]; then print_info "* Register latex for zds.tutorialv2." - zds_register_for_install "+tex-local +latex-template" + zds_register_for_install "+latex-template" fi # install backend dependencies From 819184339b866853282ac90121b45b1128a48da2 Mon Sep 17 00:00:00 2001 From: A-312 Date: Sat, 20 Apr 2019 09:51:53 +0200 Subject: [PATCH 66/69] =?UTF-8?q?Acc=C3=A9l=C3=A8re=20l'installation=20de?= =?UTF-8?q?=20latex?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/travis_header.sh | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/scripts/travis_header.sh b/scripts/travis_header.sh index d8a2f86877..8eece76919 100755 --- a/scripts/travis_header.sh +++ b/scripts/travis_header.sh @@ -20,12 +20,6 @@ function zds_register_module_for_installation { zds_register_for_install "+jdk-local +elastic-local" fi - # install latex - if [[ "$ZDS_TEST_JOB" == *"zds.tutorialv2"* ]]; then - print_info "* Register latex for zds.tutorialv2." - zds_register_for_install "+latex-template" - fi - # install backend dependencies if ! ( [[ "$ZDS_TEST_JOB" == *"zds."* ]] || [[ "$ZDS_TEST_JOB" == *"selenium"* ]] || [[ "$ZDS_TEST_JOB" == *"doc"* ]] ); then print_info "* Don't register back because zds.* task, selenium and doc are not installed." @@ -94,6 +88,19 @@ if [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then zds_fold_end fi +if [[ "$ZDS_TEST_JOB" == *"zds.tutorialv2"* ]]; then + # install latex + zds_fold_start "register_module" "* Install latex (texlive + latex-template)" + # this script is faster than zds_install.sh +tex-local +latex-template + git clone $ZDS_LATEX_REPO + TEMPLATEDIR=$HOME/.texlive/texmf-local/tex/latex/ + ./latex-template/scripts/install_font.sh \ + && ./latex-template/scripts/install_texlive.sh \ + && export PATH=$HOME/.texlive/bin/x86_64-linux:$PATH \ + && rm -rf $TEMPLATEDIR/latex-template \ + && mkdir -p $TEMPLATEDIR && cp -r ./latex-template $TEMPLATEDIR && texhash + zds_fold_end +fi zds_fold_start "register_module" "* Register module for installation" zds_register_module_for_installation From 28fcd348d61df1a270d4010bc69ffb1dfb40edd4 Mon Sep 17 00:00:00 2001 From: A-312 Date: Sat, 20 Apr 2019 10:15:43 +0200 Subject: [PATCH 67/69] no fixture --- scripts/travis_header.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/travis_header.sh b/scripts/travis_header.sh index 8eece76919..d244da8d61 100755 --- a/scripts/travis_header.sh +++ b/scripts/travis_header.sh @@ -26,6 +26,11 @@ function zds_register_module_for_installation { zds_register_for_install "-back" fi + # fastify : no fixture + if [[ "$ZDS_TEST_JOB" == *"zds.tutorialv2"* ]]; then + zds_register_for_install "-data" + fi + print_info "* Argument for installation : $zds_install_argument" } From f8a5bd0b7741ad0d9dee94288cbbadc1fbff8b82 Mon Sep 17 00:00:00 2001 From: A-312 Date: Sat, 20 Apr 2019 10:36:44 +0200 Subject: [PATCH 68/69] fix `texhash: command not found` --- scripts/travis_run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/travis_run.sh b/scripts/travis_run.sh index 5386e2b4b8..5420eed385 100755 --- a/scripts/travis_run.sh +++ b/scripts/travis_run.sh @@ -33,8 +33,6 @@ activate_env "./$ZDS_VENV" run_script "start_elasticsearch" - run_script "start_latex" - run_script "lint_backend" run_script "test_backend" @@ -50,6 +48,8 @@ activate_env "./$ZDS_VENV" activate_env "$HACK_VIRTUALENV" + run_script "start_latex" + run_script "coverage_backend" run_script "build_documentation" From 0e60225cf32f915351f332dbfe2ba4fdd524f79a Mon Sep 17 00:00:00 2001 From: A-312 Date: Sat, 20 Apr 2019 10:59:42 +0200 Subject: [PATCH 69/69] =?UTF-8?q?texhash=20est=20d=C3=A9j=C3=A0=20ex=C3=A9?= =?UTF-8?q?cut=C3=A9=20directement=20apr=C3=A8s=20l'installation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/travis_header.sh | 2 +- scripts/travis_run.sh | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/travis_header.sh b/scripts/travis_header.sh index d244da8d61..14ac8b7946 100755 --- a/scripts/travis_header.sh +++ b/scripts/travis_header.sh @@ -95,7 +95,7 @@ fi if [[ "$ZDS_TEST_JOB" == *"zds.tutorialv2"* ]]; then # install latex - zds_fold_start "register_module" "* Install latex (texlive + latex-template)" + zds_fold_start "register_module" "* Install latex & Run texhash (install: texlive + latex-template)" # this script is faster than zds_install.sh +tex-local +latex-template git clone $ZDS_LATEX_REPO TEMPLATEDIR=$HOME/.texlive/texmf-local/tex/latex/ diff --git a/scripts/travis_run.sh b/scripts/travis_run.sh index 5420eed385..d5e327d50c 100755 --- a/scripts/travis_run.sh +++ b/scripts/travis_run.sh @@ -48,8 +48,6 @@ activate_env "./$ZDS_VENV" activate_env "$HACK_VIRTUALENV" - run_script "start_latex" - run_script "coverage_backend" run_script "build_documentation"