From 5e3440da912541557bea1446651f409d7c49ca58 Mon Sep 17 00:00:00 2001 From: Blake Imsland Date: Wed, 19 Apr 2017 10:23:14 -0700 Subject: [PATCH] Update Circle CI for our workflow - Use new master / rc release release strategy (#440) - Migrate Circle CI 2.0 (#488, #502, #923) - Install redash-stmo. In the long run we'll be able to install additional dependencies by having an own Dockerfile to build images based on the Redash image but that installs additional Python dependencies. But until we have a fork with lots of changes ourselves we need to do it this way. Redash-stmo contains the ability to hook up our own Dockerflow library. Disable handlers test that tests the login page and the existence of the remote auth link there. We override this functionality for user experience reasons via redash-stmo and redirect from the login page to the remote auth URL. Update metrics test since our auth changes add a database call which also gets timed, and therefore assert this was called only once fails. Refs #13 Refs #37 --- .circleci/config.yml | 109 +++++++++++++++++++--------------- .circleci/docker_build | 16 ++--- .circleci/update_version | 4 +- bin/dockerflow-version | 13 ++++ requirements_bundles.txt | 2 + tests/metrics/test_request.py | 2 +- tests/test_handlers.py | 38 ++++++------ 7 files changed, 104 insertions(+), 80 deletions(-) create mode 100755 bin/dockerflow-version diff --git a/.circleci/config.yml b/.circleci/config.yml index a18c644cb2..dd220650e2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,17 +1,5 @@ version: 2.0 -build-docker-image-job: &build-docker-image-job - docker: - - image: circleci/node:12 - steps: - - setup_remote_docker - - checkout - - run: sudo apt update - - run: sudo apt install python3-pip - - run: sudo pip3 install -r requirements_bundles.txt - - run: .circleci/update_version - - run: npm run bundle - - run: .circleci/docker_build jobs: backend-lint: docker: @@ -83,68 +71,95 @@ jobs: name: Run Visualizations Tests command: (cd viz-lib && npm test) - run: npm run lint - frontend-e2e-tests: - environment: - COMPOSE_FILE: .circleci/docker-compose.cypress.yml - COMPOSE_PROJECT_NAME: cypress - PERCY_TOKEN_ENCODED: ZGRiY2ZmZDQ0OTdjMzM5ZWE0ZGQzNTZiOWNkMDRjOTk4Zjg0ZjMxMWRmMDZiM2RjOTYxNDZhOGExMjI4ZDE3MA== - CYPRESS_PROJECT_ID_ENCODED: OTI0Y2th - CYPRESS_RECORD_KEY_ENCODED: YzA1OTIxMTUtYTA1Yy00NzQ2LWEyMDMtZmZjMDgwZGI2ODgx + build-docker-image: docker: - image: circleci/node:12 steps: - setup_remote_docker - checkout - - run: - name: Install npm dependencies - command: | - npm ci - - run: - name: Setup Redash server - command: | - npm run cypress build - npm run cypress start -- --skip-db-seed - docker-compose run cypress npm run cypress db-seed - - run: - name: Execute Cypress tests - command: npm run cypress run-ci - build-docker-image: *build-docker-image-job - build-preview-docker-image: *build-docker-image-job + - run: echo "export MOZILLA_VERSION=master" >> $BASH_ENV + - run: sudo apt update + - run: sudo apt install python3-pip + - run: sudo pip3 install -r requirements_bundles.txt + - run: .circleci/update_version + - run: npm run bundle + - run: .circleci/docker_build + build-docker-image-rc: + docker: + - image: circleci/node:12 + steps: + - setup_remote_docker + - checkout + - run: echo "export MOZILLA_VERSION=rc" >> $BASH_ENV + - run: sudo apt update + - run: sudo apt install python3-pip + - run: sudo pip3 install -r requirements_bundles.txt + - run: .circleci/update_version + - run: npm run bundle + - run: .circleci/docker_build + build-docker-image-tag: + docker: + - image: circleci/node:12 + steps: + - setup_remote_docker + - checkout + - run: echo "export MOZILLA_VERSION=$CIRCLE_TAG" >> $BASH_ENV + - run: sudo apt update + - run: sudo apt install python3-pip + - run: sudo pip3 install -r requirements_bundles.txt + - run: .circleci/update_version + - run: npm run bundle + - run: .circleci/docker_build + # Create alias from tag to "latest": + - run: docker tag $DOCKERHUB_REPO:$CIRCLE_TAG $DOCKERHUB_REPO:latest + - run: docker push $DOCKERHUB_REPO:latest workflows: version: 2 build: jobs: - - backend-lint + - backend-lint: + filters: + tags: + only: /^m[0-9]+(\.[0-9]+)?$/ - backend-unit-tests: + filters: + tags: + only: /^m[0-9]+(\.[0-9]+)?$/ requires: - backend-lint - - frontend-lint + - frontend-lint: + filters: + tags: + only: /^m[0-9]+(\.[0-9]+)?$/ - frontend-unit-tests: + filters: + tags: + only: /^m[0-9]+(\.[0-9]+)?$/ requires: - backend-lint - frontend-lint - - frontend-e2e-tests: - requires: - - frontend-lint - - build-preview-docker-image: + - build-docker-image: requires: - backend-unit-tests - frontend-unit-tests - - frontend-e2e-tests filters: branches: only: - master - - hold: - type: approval + - build-docker-image-rc: requires: - backend-unit-tests - frontend-unit-tests - - frontend-e2e-tests filters: branches: only: - - /release\/.*/ - - build-docker-image: + - release + - build-docker-image-tag: requires: - - hold + - backend-unit-tests + - frontend-unit-tests + filters: + branches: + ignore: /.*/ + tags: + only: /^m[0-9]+(\.[0-9]+)?$/ diff --git a/.circleci/docker_build b/.circleci/docker_build index e105c584e0..b4da012ae8 100755 --- a/.circleci/docker_build +++ b/.circleci/docker_build @@ -1,17 +1,9 @@ #!/bin/bash -VERSION=$(jq -r .version package.json) -VERSION_TAG=$VERSION.b$CIRCLE_BUILD_NUM +VERSION_TAG="$MOZILLA_VERSION" docker login -u $DOCKER_USER -p $DOCKER_PASS -if [ $CIRCLE_BRANCH = master ] || [ $CIRCLE_BRANCH = preview-image ] -then - docker build --build-arg skip_dev_deps=true -t redash/redash:preview -t redash/preview:$VERSION_TAG . - docker push redash/redash:preview - docker push redash/preview:$VERSION_TAG -else - docker build --build-arg skip_dev_deps=true -t redash/redash:$VERSION_TAG . - docker push redash/redash:$VERSION_TAG -fi +docker build -t $DOCKERHUB_REPO:$VERSION_TAG . +docker push $DOCKERHUB_REPO:$VERSION_TAG -echo "Built: $VERSION_TAG" \ No newline at end of file +echo "Built: $VERSION_TAG" diff --git a/.circleci/update_version b/.circleci/update_version index d397fb23df..338517b939 100755 --- a/.circleci/update_version +++ b/.circleci/update_version @@ -1,6 +1,8 @@ #!/bin/bash +bin/dockerflow-version "$MOZILLA_VERSION" + VERSION=$(jq -r .version package.json) -FULL_VERSION=$VERSION+b$CIRCLE_BUILD_NUM +FULL_VERSION=$VERSION+b$CIRCLE_BUILD_NUM-$MOZILLA_VERSION sed -ri "s/^__version__ = '([A-Za-z0-9.-]*)'/__version__ = '$FULL_VERSION'/" redash/__init__.py sed -i "s/dev/$CIRCLE_SHA1/" client/app/version.json diff --git a/bin/dockerflow-version b/bin/dockerflow-version new file mode 100755 index 0000000000..027d61971f --- /dev/null +++ b/bin/dockerflow-version @@ -0,0 +1,13 @@ +#!/bin/bash + +set -eo pipefail + +VERSION="$1" + +printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \ + "$CIRCLE_SHA1" \ + "$VERSION" \ + "$CIRCLE_PROJECT_USERNAME" \ + "$CIRCLE_PROJECT_REPONAME" \ + "$CIRCLE_BUILD_URL" \ +> version.json diff --git a/requirements_bundles.txt b/requirements_bundles.txt index 3f57a20dd2..1a2cbbec20 100644 --- a/requirements_bundles.txt +++ b/requirements_bundles.txt @@ -2,6 +2,8 @@ # loading mechanism need on Python 2 and can be removed # when moved to Python 3. # It's automatically installed when running npm run bundle +# This is the Mozilla Redash extension +redash-stmo>=2019.9.0 # These can be removed when upgrading to Python 3.x importlib-metadata>=1.6 # remove when on 3.8 diff --git a/tests/metrics/test_request.py b/tests/metrics/test_request.py index c04f69a3f5..58c7f05a42 100644 --- a/tests/metrics/test_request.py +++ b/tests/metrics/test_request.py @@ -6,4 +6,4 @@ class TestRequestMetrics(BaseTestCase): def test_flask_request_records_statsd_metrics(self, timing): self.client.get("/ping") - timing.assert_called_once_with("requests.redash_ping.get", ANY) + timing.assert_called_with("requests.redash_ping.get", ANY) diff --git a/tests/test_handlers.py b/tests/test_handlers.py index 162c9f5e87..2d0549b809 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -101,25 +101,25 @@ def test_get_login_form(self): rv = self.client.get("/default/login") self.assertEqual(rv.status_code, 200) - def test_get_login_form_remote_auth(self): - """Make sure the remote auth link can be rendered correctly on the - login page when the remote user login feature is enabled""" - old_remote_user_enabled = settings.REMOTE_USER_LOGIN_ENABLED - old_ldap_login_enabled = settings.LDAP_LOGIN_ENABLED - try: - settings.REMOTE_USER_LOGIN_ENABLED = True - settings.LDAP_LOGIN_ENABLED = True - rv = self.client.get("/default/login") - self.assertEqual(rv.status_code, 200) - self.assertIn( - "/{}/remote_user/login".format(self.factory.org.slug), rv.data.decode() - ) - self.assertIn( - "/{}/ldap/login".format(self.factory.org.slug), rv.data.decode() - ) - finally: - settings.REMOTE_USER_LOGIN_ENABLED = old_remote_user_enabled - settings.LDAP_LOGIN_ENABLED = old_ldap_login_enabled + # def test_get_login_form_remote_auth(self): + # """Make sure the remote auth link can be rendered correctly on the + # login page when the remote user login feature is enabled""" + # old_remote_user_enabled = settings.REMOTE_USER_LOGIN_ENABLED + # old_ldap_login_enabled = settings.LDAP_LOGIN_ENABLED + # try: + # settings.REMOTE_USER_LOGIN_ENABLED = True + # settings.LDAP_LOGIN_ENABLED = True + # rv = self.client.get("/default/login") + # self.assertEqual(rv.status_code, 200) + # self.assertIn( + # "/{}/remote_user/login".format(self.factory.org.slug), rv.data.decode() + # ) + # self.assertIn( + # "/{}/ldap/login".format(self.factory.org.slug), rv.data.decode() + # ) + # finally: + # settings.REMOTE_USER_LOGIN_ENABLED = old_remote_user_enabled + # settings.LDAP_LOGIN_ENABLED = old_ldap_login_enabled def test_submit_non_existing_user(self): with patch("redash.handlers.authentication.login_user") as login_user_mock: