Feat/kl/new features (#37)

* ci: get frontend packages and deploy on stages * fix: swagger answers * fix: db sessions * fix: envs * chore: rename migrations * fix: ci deploy * fix: redis connection pool * ci: fix github actions
ExpressApp · Jul 29, 2024 · f16da52 · f16da52
1 parent b7d6ac2
commit f16da52
Show file tree

Hide file tree

Showing 24 changed files with 864 additions and 693 deletions.
diff --git a/.github/scripts/test_image b/.github/scripts/test_image
@@ -4,7 +4,7 @@ set -ex
 
 PROJECT_NAME=smartapp-template
 
-python -m copier --defaults . smartapp-template
+python -m copier copy --defaults --trust . smartapp-template
 docker build -t $PROJECT_NAME $PROJECT_NAME
 
 # there should be added `--fail` option to curl command when healthcheck endpoint is added

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -44,11 +44,11 @@ jobs:
 
       - name: Install copier
         run: |
-          pip install copier==7.1.0
+          pip install copier==9.3.1
           pip install copier-templates-extensions
           pip install pyyaml-include==1.4.1
           # hardcode pydantic version to avoid dependency conflict with copier
-          pip install pydantic==1.10.2
+          # pip install pydantic==1.10.2
 
       - name: Test image
         env:
@@ -62,7 +62,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [ "3.8", "3.9", "3.10" ]
+        python-version: [ "3.9", "3.10", "3.11" ]
 
     services:
       postgres:
@@ -93,12 +93,12 @@ jobs:
 
       - name: Install copier
         run: |
-          pip install copier==7.1.0
+          pip install copier==9.3.1
           pip install copier-templates-extensions
           pip install pyyaml-include==1.4.1
           # hardcode pydantic version to avoid dependency conflict with copier
-          pip install pydantic==1.10.2
-          python -m copier --defaults . async-box-bot
+          # pip install pydantic==1.10.2
+          python -m copier copy --defaults --trust . async-box-bot
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v2
@@ -140,20 +140,20 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [ "3.8", "3.9", "3.10" ]
+        python-version: [ "3.9", "3.10", "3.11" ]
 
     steps:
       - name: Checkout
         uses: actions/checkout@v2
 
       - name: Install copier
         run: |
-          pip install copier==7.1.0
+          pip install copier==9.3.1
           pip install copier-templates-extensions
           pip install pyyaml-include==1.4.1
           # hardcode pydantic version to avoid dependency conflict with copier
-          pip install pydantic==1.10.2
-          python -m copier --defaults . async-box-bot
+          # pip install pydantic==1.10.2
+          python -m copier copy --defaults --trust . async-box-bot
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v2

diff --git a/.gitlab-ci.yml.jinja b/.gitlab-ci.yml.jinja
@@ -8,15 +8,16 @@
     -d
     --name ${CONTAINER_NAME}
     --restart always
-    --label traefik.http.routers.${BOT_PROJECT_NAME}.rule="Host(\`${BOT_URL}\`)"
+    --label traefik.http.routers.${CONTAINER_NAME}.rule="Host(\`${BOT_URL}\`)"
     --label traefik.enable=true
-    --label traefik.http.services.${BOT_PROJECT_NAME}.loadbalancer.server.port="8000"
+    --label traefik.http.services.${CONTAINER_NAME}.loadbalancer.server.port="8000"
     --log-opt max-size=10m
     --log-opt max-file=5
     -e POSTGRES_DSN="${POSTGRES_DSN}"
     -e REDIS_DSN="${REDIS_DSN}"
     -e BOT_CREDENTIALS="${BOT_CREDENTIALS}"
     -e DEBUG="${DEBUG:-false}"
+    -e CONTAINER_PREFIX="$CONTAINER_PREFIX"
     $CONTAINER_RELEASE_IMAGE
   {% if add_worker -%}
   # Add envs for worker here
@@ -30,8 +31,9 @@
     -e REDIS_DSN="${REDIS_DSN}"
     -e BOT_CREDENTIALS="${BOT_CREDENTIALS}"
     -e DEBUG="${DEBUG:-false}"
+    -e CONTAINER_PREFIX="$CONTAINER_PREFIX"
     ${CONTAINER_RELEASE_IMAGE}
-    bash -c 'PYTHONPATH="$PYTHONPATH:$PWD" saq app.worker.worker.settings'
+    sh -c 'PYTHONPATH="$PYTHONPATH:$PWD" saq app.worker.worker.settings'
   {%- endif %}
 
 .create_db: &create_db
@@ -61,6 +63,93 @@
   - POSTGRES_HOST=${PROD_POSTGRES_HOST}
   - POSTGRES_DSN=postgres://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB
 
+.download_smartapp_files: &download_smartapp_files
+    - if [ -z ${SMARTAPP_TAG+x} ];
+      then
+      echo "SMARTAPP_TAG is unset, trying to get latest from api" &&
+      SMARTAPP_TAG=`python3 get_latest_frontend_tag.py ${SMARTAPP_GITLAB_ID} ${SMARTAPP_API_READ_TOKEN} ${BACKEND_VERSION}`;
+      fi
+    - echo "Deploing with smartapp tag ${SMARTAPP_TAG}"
+    - 'wget --header="PRIVATE-TOKEN: $SMARTAPP_API_READ_TOKEN" https://gitlab.ccsteam.ru/api/v4/projects/${SMARTAPP_GITLAB_ID}/packages/generic/${SMARTAPP_PACKAGE_NAME}/${SMARTAPP_TAG}/build.tgz'
+    - tar zxvf build.tgz && mkdir -p app/smartapp_files/static/ && mv build/* app/smartapp_files/static/
+    - ls app/smartapp_files/
+
+
+.botstest_before_script: &botstest_before_script
+  - echo "${BOT_PROJECT_NAME}"
+  - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+  - if [ -z ${BOT_PROJECT_NAME:-} ]; then BOT_PROJECT_NAME=${CI_PROJECT_PATH_SLUG#"$CI_PROJECT_NAMESPACE-"}; fi
+  - CONTAINER_NAME="${BOT_PROJECT_NAME}${CONTAINER_PREFIX}"
+  - CONTAINER_RELEASE_IMAGE="$CI_REGISTRY_IMAGE:${CI_COMMIT_TAG:-$CI_COMMIT_REF_SLUG}"
+  - BOT_CREDENTIALS=${DEV_BOT_CREDENTIALS}
+  - BOT_URL="${CONTAINER_NAME}.${DEV_SERVER_HOST}"
+  - REDIS_DSN=redis://${DOCKER_NETWORK_IP}/1
+  - *postgres_envs
+
+.deploy.botstest:
+  image: registry.ccsteam.ru/cicd-images/docker:latest
+  tags:
+    - bots-test
+  only:
+    - branches
+    - tags
+  when: manual
+  variables:
+    # https://docs.gitlab.com/ee/ci/runners/configure_runners.html#git-strategy
+    GIT_STRATEGY: none
+  before_script: *botstest_before_script
+  script:
+    - echo "Use URL 'https://${BOT_URL}/' in your cts admin site"
+    - echo "Using credentials ${BOT_CREDENTIALS}"
+    - echo "Deploying Docker container ${CONTAINER_NAME}"
+    - *create_db
+    - *run_bot
+
+.deploy.botstest.stop:
+  when: manual
+  environment:
+    action: stop
+  extends: .deploy.botstest
+  script:
+    {% if add_worker -%}
+    - docker rm -f ${CONTAINER_NAME} ${CONTAINER_NAME}-worker || true
+    {%- endif %}
+    - docker rm -f ${CONTAINER_NAME} || true
+    - psq- psql -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '${POSTGRES_DB}';" postgres || true
+    - psql -c "drop database \"${POSTGRES_DB}\"" postgres || true
+    - psql -c "drop user \"${POSTGRES_USER}\"" postgres || true
+
+.publish_eXpress_public_registry_before_script: &publish_eXpress_public_registry_before_script
+  - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+  - docker login -u $PUBLIC_REGISTRY_USER -p $PUBLIC_REGISTRY_PASSWORD $PUBLIC_REGISTRY
+  - docker pull $CONTAINER_RELEASE_IMAGE
+  - docker tag $CONTAINER_RELEASE_IMAGE $CONTAINER_PUBLIC_RELEASE_IMAGE
+
+.publish_eXpress_public_registry_script: &publish_eXpress_public_registry_script
+  - docker push $CONTAINER_PUBLIC_RELEASE_IMAGE
+
+.publish_eXpress_public_registry_after_script: &publish_eXpress_public_registry_after_script
+  - docker rmi $CONTAINER_RELEASE_IMAGE || true
+  - docker rmi $CONTAINER_PUBLIC_RELEASE_IMAGE || true
+
+.eXpress_public_registry:
+  stage: publish
+  image: registry.ccsteam.ru/cicd-images/docker:latest
+  tags:
+    - docker
+  only:
+    - tags
+  when: manual
+  environment:
+    name: publish
+  variables:
+    # https://docs.gitlab.com/ee/ci/runners/configure_runners.html#git-strategy
+    GIT_STRATEGY: none
+    CONTAINER_RELEASE_IMAGE: "$CI_REGISTRY_IMAGE:$CI_COMMIT_TAG"
+    PROJECT_PUBLIC_REGISTRY_PATH: "$PUBLIC_REGISTRY/$PUBLIC_REGISTRY_CATALOG/$CI_PROJECT_NAME"
+    CONTAINER_PUBLIC_RELEASE_IMAGE: "$PROJECT_PUBLIC_REGISTRY_PATH:$CI_COMMIT_TAG"
+    CONTAINER_PUBLIC_LATEST_RELEASE_IMAGE: "$PROJECT_PUBLIC_REGISTRY_PATH:latest"
+
 # Jobs
 variables:
   GIT_DEPTH: 1  # Fetch only latest commit
@@ -70,13 +159,15 @@ stages:
   - check
   - build
   - security
+  - deploy-test
   - deploy
+  - publish
 
 default:
   interruptible: true
 
 lint:
-  image: python:3.10
+  image: python:3.11
   stage: check
   tags:
     - docker
@@ -87,13 +178,18 @@ lint:
     - poetry run ./scripts/lint
 
 test:
-  image: python:3.10
+  image: python:3.11
   stage: check
   tags:
     - docker
   services:
-    - postgres:15.3-alpine
-    - redis:7.0-alpine
+    - name: postgres:15.3-alpine
+      alias: postgres
+      variables:
+        POSTGRES_USER: postgres
+        POSTGRES_PASSWORD: postgres
+    - name: redis:7.0-alpine
+      alias: redis
   cache: *cache_dependencies
   variables:
     BOT_CREDENTIALS: cts.example.com@secret@123e4567-e89b-12d3-a456-426655440000
@@ -103,7 +199,7 @@ test:
     - *install_dependencies
   script:
     - poetry run pytest --cov-config=setup.cfg
-  coverage: '/Total coverage: \d\d\d.\d\d%/'
+  coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)/'
 
 security:
   stage: security
@@ -113,7 +209,7 @@ security:
       - project: devsecops/pipelines
         file: integration_templates/python.yml
 
-  
+
 build:
   image: docker:latest
   stage: build
@@ -125,67 +221,49 @@ build:
     - CONTAINER_RELEASE_IMAGE="$CI_REGISTRY_IMAGE:${CI_COMMIT_TAG:-$CI_COMMIT_REF_SLUG}"
   script:
     - docker pull $CONTAINER_RELEASE_IMAGE || true
+    - apk add python3
+    - apk add jq
+    - apk add grep
+    - python3 --version
+    - BACKEND_VERSION=$(grep -m 1 version pyproject.toml | tr -s ' ' | tr -d '"' | tr -d "'" | cut -d' ' -f3)
+    - *download_smartapp_files
     - docker build
       --cache-from $CONTAINER_RELEASE_IMAGE
       --build-arg GIT_HOST=$GIT_HOST
       --build-arg CI_JOB_TOKEN=$CI_JOB_TOKEN
       --build-arg CI_COMMIT_SHA=$CI_COMMIT_SHA
+      --label BackendVersion="${BACKEND_VERSION}"
+      --label SmartappVersion="${SMARTAPP_TAG}"
       --force-rm
       -t $CONTAINER_RELEASE_IMAGE .
     - docker push $CONTAINER_RELEASE_IMAGE
+    - docker image inspect $CONTAINER_RELEASE_IMAGE | jq '.[0] .Config .Labels'
     - docker rmi $CONTAINER_RELEASE_IMAGE
 
 deploy.botstest:
-  image: docker:latest
-  stage: deploy
-  tags:
-    - bots-test
-  only:
-    - branches
-  when: manual
+  stage: deploy-test
+  extends:
+    - .deploy.botstest
   environment:
-    name: test
+    name: S1
     on_stop: deploy.botstest.stop
   variables:
-    # https://docs.gitlab.com/ee/ci/runners/configure_runners.html#git-strategy
-    GIT_STRATEGY: none
-  before_script:
-    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-    - if [ -z ${BOT_PROJECT_NAME:-} ]; then BOT_PROJECT_NAME=${CI_PROJECT_PATH_SLUG#"$CI_PROJECT_NAMESPACE-"}; fi
-    - CONTAINER_NAME=$BOT_PROJECT_NAME
-    - CONTAINER_RELEASE_IMAGE="$CI_REGISTRY_IMAGE:${CI_COMMIT_TAG:-$CI_COMMIT_REF_SLUG}"
-    - BOT_URL="${BOT_PROJECT_NAME}.${DEV_SERVER_HOST}"
-    - BOT_CREDENTIALS=$DEV_BOT_CREDENTIALS
-    - *postgres_envs
-    - REDIS_DSN=redis://${DOCKER_NETWORK_IP}/1
-  script:
-    - echo "Use URL 'https://${BOT_URL}/' in your cts admin site"
-    - echo "Using credentials ${BOT_CREDENTIALS}"
-    - echo "Deploing Docker container ${CONTAINER_NAME}"
-    - *create_db
-    - *run_bot
+    CONTAINER_PREFIX: ""
 
 deploy.botstest.stop:
-  when: manual
+  stage: deploy-test
+  extends:
+    - .deploy.botstest.stop
   environment:
-    name: test
-    action: stop
-  extends: deploy.botstest
-  script:
-    - docker rm -f ${CONTAINER_NAME} || true
-    {% if add_worker -%}
-    - docker rm -f ${CONTAINER_NAME} ${CONTAINER_NAME}-worker || true
-    {%- endif %}
-    - psql -c "select pg_terminate_backend(pid) from pg_stat_activity \
-      where datname = '${POSTGRES_DB}';" postgres || true
-    - psql -c "drop database \"${POSTGRES_DB}\"" postgres || true
-    - psql -c "drop user \"${POSTGRES_USER}\"" postgres || true
+    name: S1
+  variables:
+    CONTAINER_PREFIX: ""
 
 deploy.botsprod:
   stage: deploy
   image: docker:latest
   tags:
-    - bots-prod
+    - botsprod-yac
   only:
     # Note the bots-prod worker requires branch to be protected
     - master
@@ -211,3 +289,21 @@ deploy.botsprod:
     - *run_bot
   needs:
     - job: security
+
+eXpress_public_registry:
+  extends: .eXpress_public_registry
+  before_script: *publish_eXpress_public_registry_before_script
+  script: *publish_eXpress_public_registry_script
+  after_script: *publish_eXpress_public_registry_after_script
+
+eXpress_public_registry.as_latest:
+  extends: .eXpress_public_registry
+  before_script:
+    - *publish_eXpress_public_registry_before_script
+    - docker tag $CONTAINER_RELEASE_IMAGE $CONTAINER_PUBLIC_LATEST_RELEASE_IMAGE
+  script:
+    - *publish_eXpress_public_registry_script
+    - docker push $CONTAINER_PUBLIC_LATEST_RELEASE_IMAGE
+  after_script:
+    - *publish_eXpress_public_registry_after_script
+    - docker rmi $CONTAINER_PUBLIC_LATEST_RELEASE_IMAGE || true
diff --git a/README.md b/README.md
@@ -19,9 +19,10 @@
 
 ## 1. Развертывание из шаблона и структура проекта
 
-Для развертывания проекта необходимо установить [copier](https://github.com/copier-org/copier) и выполнить команду:
+Для развертывания проекта необходимо установить [copier](https://github.com/copier-org/copier) 
+и [copier-templates-extensions](https://github.com/copier-org/copier-templates-extensions) и выполнить команду:
 ```bash
-$ copier smartapp-template smartapp-example
+$ copier copy --trust smartapp-template smartapp-example
 ```  
 
 Структура шаблонного SmartApp состоит из нескольких следующих пакетов и модулей: