diff --git a/.github/actions/gnocchi-container/Dockerfile b/.github/actions/gnocchi-container/Dockerfile new file mode 100644 index 000000000..021750808 --- /dev/null +++ b/.github/actions/gnocchi-container/Dockerfile @@ -0,0 +1,57 @@ +FROM ubuntu:20.04 +ENV GNOCCHI_WORKSPACE /github/workspace +ENV DEBIAN_FRONTEND noninteractive + +#NOTE(tobias-urdin): need gnupg for apt-key +RUN apt-get update -y && apt-get install -qy gnupg +RUN echo 'deb http://ppa.launchpad.net/deadsnakes/ppa/ubuntu focal main' >> /etc/apt/sources.list +RUN apt-key adv --recv-keys --keyserver keyserver.ubuntu.com F23C5A6CF475977595C89F51BA6932366A755776 +RUN apt-get update -y && apt-get install -qy \ + locales \ + git \ + wget \ + curl \ + nodejs \ + npm \ + python3 \ + python3-dev \ + python3-pip \ + python3.6 \ + python3.6-dev \ +# Needed for uwsgi core routing support + build-essential \ + libffi-dev \ + libpq-dev \ + postgresql \ + memcached \ + mysql-client \ + mysql-server \ +# For Ceph + librados-dev \ + liberasurecode-dev \ + python3-rados \ + ceph \ +# For prometheus + libsnappy-dev \ + libprotobuf-dev \ +# For redis + redis-server \ + && rm -rf /var/lib/apt/lists/* + +#NOTE(sileht): really no utf-8 in 2017 !? +ENV LANG en_US.UTF-8 +RUN update-locale +RUN locale-gen $LANG + +#NOTE(sileht): Upgrade python dev tools +RUN python3 -m pip install -U pip tox virtualenv +RUN python3.6 -m pip install -U pip tox virtualenv + +RUN npm install s3rver@3.7.0 --global + +RUN groupadd --gid 1001 tester +RUN useradd --uid 1001 --gid 1001 --home-dir $GNOCCHI_WORKSPACE --shell /bin/bash tester +USER tester + +COPY entrypoint.sh /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] diff --git a/.github/actions/gnocchi-container/action.yml b/.github/actions/gnocchi-container/action.yml new file mode 100644 index 000000000..5eb2cedc3 --- /dev/null +++ b/.github/actions/gnocchi-container/action.yml @@ -0,0 +1,12 @@ +name: 'Gnocchi Container' +description: 'Run command inside Gnocchi Container' +author: 'Tobias Urdin' +inputs: + command: + description: 'Command to run' + required: true +runs: + using: 'docker' + image: 'Dockerfile' + args: + - ${{ inputs.command }} diff --git a/.github/actions/gnocchi-container/entrypoint.sh b/.github/actions/gnocchi-container/entrypoint.sh new file mode 100755 index 000000000..a4a32ab8e --- /dev/null +++ b/.github/actions/gnocchi-container/entrypoint.sh @@ -0,0 +1,2 @@ +#!/bin/sh +$@ diff --git a/.github/workflows/gnocchi.yml b/.github/workflows/gnocchi.yml new file mode 100644 index 000000000..e776edbec --- /dev/null +++ b/.github/workflows/gnocchi.yml @@ -0,0 +1,69 @@ +name: Gnocchi + +on: + push: + branches: [master] + pull_request: + branches: [master] + +# NOTE(tobias-urdin): If you change any jobs make sure to modify +# the Mergify.io config in .mergify.yml to include the jobs! + +jobs: + doc: + runs-on: ubuntu-latest + strategy: + matrix: + env: + - docs + - docs-gnocchi-web + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - uses: ./.github/actions/gnocchi-container + with: + command: tox -e ${{ matrix.env }} + + check: + runs-on: ubuntu-latest + strategy: + matrix: + env: + - pep8 + steps: + - uses: actions/checkout@v2 + - uses: ./.github/actions/gnocchi-container + with: + command: tox -e ${{ matrix.env }} + + test: + runs-on: ubuntu-latest + strategy: + matrix: + python: + - py36 + - py38 + env: + - mysql-ceph-upgrade-from-4.3 + - postgresql-file-upgrade-from-4.3 + - mysql-file + - mysql-swift + - mysql-s3 + - mysql-ceph + - postgresql-file + - postgresql-swift + - postgresql-s3 + - postgresql-ceph + exclude: + - env: mysql-ceph-upgrade-from-4.3 + python: py36 + - env: mysql-ceph + python: py36 + - env: postgresql-ceph + python: py36 + steps: + - uses: actions/checkout@v2 + - uses: ./.github/actions/gnocchi-container + with: + command: tox -e ${{ matrix.python }}-${{ matrix.env }} diff --git a/.mergify.yml b/.mergify.yml index d7861e749..09121c8a7 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -8,7 +8,30 @@ pull_request_rules: conditions: - label!=work-in-progress - '#approved-reviews-by>=1' - - status-success=Travis CI - Pull Request + - status-success=doc (docs) + - status-success=doc (docs-gnocchi-web) + + - status-success=check (pep8) + + - status-success=test (py36-postgresql-file-upgrade-from-4.3) + - status-success=test (py36-mysql-file) + - status-success=test (py36-mysql-swift) + - status-success=test (py36-mysql-s3) + - status-success=test (py36-postgresql-file) + - status-success=test (py36-postgresql-swift) + - status-success=test (py36-postgresql-s3) + + - status-success=test (py38-mysql-ceph-upgrade-from-4.3) + - status-success=test (py38-postgresql-file-upgrade-from-4.3) + - status-success=test (py38-mysql-file) + - status-success=test (py38-mysql-swift) + - status-success=test (py38-mysql-s3) + - status-success=test (py38-mysql-ceph) + - status-success=test (py38-postgresql-file) + - status-success=test (py38-postgresql-swift) + - status-success=test (py38-postgresql-s3) + - status-success=test (py38-postgresql-ceph) + - name: automatic merge backports from Mergify actions: merge: @@ -20,6 +43,7 @@ pull_request_rules: - label!=work-in-progress - author=mergify[bot] - status-success=Travis CI - Pull Request + - name: dismiss reviews on additional commit actions: dismiss_reviews: {} @@ -29,42 +53,14 @@ pull_request_rules: - actions: backport: branches: - - stable/3.0 - conditions: - - label=backport-to-3.0 - name: backport stable/3.0 - - actions: - backport: - branches: - - stable/3.1 - conditions: - - label=backport-to-3.1 - name: backport stable/3.1 - - actions: - backport: - branches: - - stable/4.0 - conditions: - - label=backport-to-4.0 - name: backport stable/4.0 - - actions: - backport: - branches: - - stable/4.1 - conditions: - - label=backport-to-4.1 - name: backport stable/4.1 - - actions: - backport: - branches: - - stable/4.2 + - stable/4.3 conditions: - - label=backport-to-4.2 - name: backport stable/4.2 + - label=backport-to-4.3 + name: backport stable/4.3 - actions: backport: branches: - - stable/4.3 + - stable/4.4 conditions: - - label=backport-to-4.3 - name: backport stable/4.3 + - label=backport-to-4.4 + name: backport stable/4.4 diff --git a/.travis.yml b/.travis.yml index a3eb395b3..90ae6ae90 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,81 +1,16 @@ +os: linux dist: bionic language: generic -sudo: required - -services: - - docker - -cache: - directories: - - ~/.cache/pip -env: - - TARGET: pep8 - - TARGET: docs - - TARGET: docs-gnocchi-web - - - TARGET: py36-mysql-ceph-upgrade-from-4.3 - - TARGET: py36-postgresql-file-upgrade-from-4.3 - - - TARGET: py36-mysql-file - - TARGET: py36-mysql-swift - - TARGET: py36-mysql-s3 - - TARGET: py36-postgresql-file - - TARGET: py36-postgresql-swift - - TARGET: py36-postgresql-s3 - -before_script: - # NOTE(sileht): We need to fetch all tags/branches for documentation. - # For the multiversioning, we change all remotes refs to point to - # the pull request checkout. So the "master" branch will be the PR sha and not - # real "master" branch. This ensures the doc build use the PR code for initial - # doc setup. - - if \[ "$TRAVIS_PULL_REQUEST" != "false" -o -n "$TRAVIS_TAG" \]; then - set -x; - case $TARGET in - docs*) - git config --get-all remote.origin.fetch; - git config --unset-all remote.origin.fetch; - git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/*; - git config --get-all remote.origin.fetch; - git fetch --unshallow --tags; - ;; - esac ; - case $TARGET in - docs-gnocchi-web) - git branch -a | sed -n "/\/HEAD /d; /\/master$/d; s,remotes/origin/,,p;" | xargs -i git branch {} origin/{} ; - git branch -D master; - git checkout -b master; - git remote set-url origin file:///home/tester/src; - git ls-remote --heads --tags | grep heads; - ;; - esac ; - set +x; - fi -install: - - if \[ "$TRAVIS_PULL_REQUEST" != "false" -o -n "$TRAVIS_TAG" \]; then - docker pull gnocchixyz/ci-tools:latest; - fi -script: - - if \[ "$TRAVIS_PULL_REQUEST" != "false" -o -n "$TRAVIS_TAG" \]; then - docker run -v ~/.cache/pip:/home/tester/.cache/pip -v $(pwd):/home/tester/src gnocchixyz/ci-tools:latest tox -e ${TARGET} ; - fi - -notifications: - email: false - irc: - on_success: change - on_failure: always - skip_join: true - channels: - - "irc.oftc.net#gnocchi" before_deploy: - - pip install --user --upgrade pip - - pip install --user --upgrade six + - sudo apt update -y + - sudo apt install -y python3 python3-pip python3-virtualenv + - python3 -m virtualenv env + - source env/bin/activate deploy: provider: pypi - user: jd + username: jd password: secure: c+Ccx3SHCWepiy0PUxDJ7XO9r3aNYnHjkzxF5c/kjV8QaCJayAJEgXJnBKhvjroqwgn7JPUgpD6QdSWdB4FqjbZYQ3I3oHOO1YL0vYYa8wHG5HuMsMp4J8qvzgs3QNQDECPI1mXsPevn3VMfGszUN+6BQrHB3FbZsTtOmE+Kmgok5NCT+obsfEhVea/UOD0XFUkVW9VJhPjQ2ytvYvFIc46/73GQf2Er/5DCa/4GGDEBSD++bDJgp3kQj438xslCAFeZWDwGsa+cTc43PI0Y0+E144ySVY7QyVbZ1B66a1BGWVrXJuM+gW/eIBCMN1FJXmD7CDdPa22azKI8dfMF7qaH3Oiv3cVovPWpubOvhTUHUFwG8+W7Fx+zUKktCWiLer/fZvEd3W8tcgby2kNOdcUfKfDB2ImZJ+P694/OJ4jJ8T5TQerruNoP2OstzcBMon77Ry0XawXR15SZd4JhbqhSi+h7XV6EYmct1UN4zoysA7fx/cWHcBxdnm2G6R0gzmOiiGUd74ptU8lZ3IlEP6EZckK/OZOdy1I8EQeUe7aiTooXZDAn07iPkDZliYRr2e36ij/xjtWCe1AjCksn/xdKfHOKJv5UVob495DU2GuNObe01ewXzexcnldjfp9Sb8SVEFuhHx6IvH5OC+vAq+BVYu2jwvMcVfXi3VSOkB4= skip_existing: true diff --git a/README.rst b/README.rst index 9012f7167..e73a3295b 100644 --- a/README.rst +++ b/README.rst @@ -2,10 +2,6 @@ Gnocchi - Metric as a Service =============================== -.. image:: https://travis-ci.org/gnocchixyz/gnocchi.png?branch=master - :target: https://travis-ci.org/gnocchixyz/gnocchi - :alt: Build Status - .. image:: https://badge.fury.io/py/gnocchi.svg :target: https://badge.fury.io/py/gnocchi diff --git a/doc/source/contributing.rst b/doc/source/contributing.rst index 4b5e0ac64..727c60aa4 100644 --- a/doc/source/contributing.rst +++ b/doc/source/contributing.rst @@ -33,14 +33,14 @@ When opening a pull-request, make sure that: `git rebase --interactive` and/or `git commit --amend`. * We recommend using `git pull-request`_ to send your pull-requests. -All sent pull-requests are checked using `Travis-CI`_, which is in charge of +All sent pull-requests are checked using GitHub Actions, which is in charge of running the tests suites. There are different scenarios being run: `PEP 8`_ compliance tests, upgrade tests, unit and functional tests. All pull-requests must be reviewed by `members of the Gnocchi project`_. -When a pull-request is approved by at least two of the members and when -Travis-CI confirms that all the tests run fine, the patch will be merged. +When a pull-request is approved by a team member and the GitHub Actions +confirms that all the tests run fine, the patch will be merged. The Gnocchi project leverages `Mergify`_ in order to schedule the merge of the different pull-requests. Mergify is in charge of making sure that the @@ -52,7 +52,6 @@ make sure that no pull-request can break another one. .. _`git pull-request`: https://github.com/jd/git-pull-request .. _`PEP 8`: https://www.python.org/dev/peps/pep-0008/ -.. _`Travis-CI`: http://travis-ci.org .. _`members of the Gnocchi project`: https://github.com/orgs/gnocchixyz/people .. _`Mergify`: https://mergify.io .. _`Gnocchi's Mergify dashboard`: https://gh.mergify.io/gnocchixyz @@ -66,26 +65,25 @@ a virtual environment for each test environment, so make sure you are using an up to date version of `virtualenv `_. Different test environments and configurations can be found by running the -``tox -l`` command. For example, to run tests with Python 3.6, PostgreSQL as +``tox -l`` command. For example, to run tests with Python 3.8, PostgreSQL as indexer, and file as storage backend: :: - tox -e py36-postgresql-file + tox -e py38-postgresql-file To run tests with MySQL as indexer, and Ceph as storage backend: :: - tox -e py36-mysql-ceph + tox -e py38-mysql-ceph -In order to run the tests like they do in travis CI, you could create -a user named tester, use UID 2000 and GID 2000 and run a command like -this +In order to run the tests like they do in the CI, you could create +a user with UID 1001 and GID 1001 and run a command like this :: - docker run -v ~/.cache/pip:/home/tester/.cache/pip -v $(pwd):/home/tester/src gnocchixyz/ci-tools:latest tox + docker run -v $(pwd):/github/workspace gnocchixyz/ci-tools:latest tox Make sure the machine executing the tests has as least 4 GB of RAM. diff --git a/gnocchi/tests/base.py b/gnocchi/tests/base.py index 3cb9a46c5..a7087d1e7 100644 --- a/gnocchi/tests/base.py +++ b/gnocchi/tests/base.py @@ -323,9 +323,9 @@ def setUp(self): self.conf.set_override('s3_endpoint_url', os.getenv("GNOCCHI_STORAGE_HTTP_URL"), group="storage") - self.conf.set_override('s3_access_key_id', "gnocchi", + self.conf.set_override('s3_access_key_id', "S3RVER", group="storage") - self.conf.set_override('s3_secret_access_key', "anythingworks", + self.conf.set_override('s3_secret_access_key', "S3RVER", group="storage") storage_driver = os.getenv("GNOCCHI_TEST_STORAGE_DRIVER", "file") diff --git a/gnocchi/tests/functional/fixtures.py b/gnocchi/tests/functional/fixtures.py index dd098a0cb..a1db758fd 100644 --- a/gnocchi/tests/functional/fixtures.py +++ b/gnocchi/tests/functional/fixtures.py @@ -150,8 +150,8 @@ def start_fixture(self): conf.set_override('s3_endpoint_url', os.getenv("GNOCCHI_STORAGE_HTTP_URL"), group="storage") - conf.set_override('s3_access_key_id', "gnocchi", group="storage") - conf.set_override('s3_secret_access_key', "anythingworks", + conf.set_override('s3_access_key_id', "S3RVER", group="storage") + conf.set_override('s3_secret_access_key', "S3RVER", group="storage") conf.set_override("s3_bucket_prefix", str(uuid.uuid4())[:26], "storage") diff --git a/run-func-tests.sh b/run-func-tests.sh index afd60b0da..22ec69223 100755 --- a/run-func-tests.sh +++ b/run-func-tests.sh @@ -41,6 +41,7 @@ for storage in ${GNOCCHI_TEST_STORAGE_DRIVERS}; do export PATH=$PWD/npm-s3rver/bin:$PATH fi eval $(pifpaf -e STORAGE run s3rver) + STORAGE_URL=s3://S3RVER:S3RVER@localhost:4568 ;; file) STORAGE_URL=file:// diff --git a/run-tests.sh b/run-tests.sh index 320198ae6..325b4b8d4 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -28,7 +28,7 @@ do s3) if ! which s3rver >/dev/null 2>&1 then - mkdir npm-s3rver + mkdir -p npm-s3rver export NPM_CONFIG_PREFIX=npm-s3rver npm install s3rver --global export PATH=$PWD/npm-s3rver/bin:$PATH @@ -53,5 +53,5 @@ do PIDS="" # TODO(sileht): the output can be a mess with this # Create a less verbose testrun output (with dot like nose ?) - # merge all subunit output and print it in after_script in travis + # merge all subunit output and print it after. done diff --git a/tox.ini b/tox.ini index 7265ba537..7fb077dbf 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] minversion = 2.4 -envlist = py36-{postgresql,mysql}{,-file,-swift,-ceph,-s3},pep8 +envlist = {py36,py38}-{postgresql,mysql}{,-file,-swift,-ceph,-s3},pep8 skipsdist = True [testenv] @@ -45,13 +45,16 @@ deps = -e .[test,redis,prometheus,amqp1,{env:GNOCCHI_STORAGE_DEPS:},{env:GNOCCHI_INDEXER_DEPS:}] {env:GNOCCHI_TEST_TARBALLS:} + # TODO(tobias-urdin): Remove this pin and use pifpaf directly instead when this is + # merged and released https://github.com/jd/pifpaf/pull/150 + git+https://github.com/tobias-urdin/pifpaf@51f74a3d8743a7ac33259413df7efc30df993460 cliff!=2.9.0 gnocchiclient>=2.8.0,!=7.0.7 commands = {toxinidir}/run-tests.sh {posargs} {toxinidir}/run-func-tests.sh {posargs} -[testenv:py36-postgresql-file-upgrade-from-4.3] +[testenv:{py36,py38}-postgresql-file-upgrade-from-4.3] # We should always recreate since the script upgrade # Gnocchi we can't reuse the virtualenv recreate = True @@ -59,12 +62,14 @@ setenv = GNOCCHI_VERSION_FROM=stable/4.3 GNOCCHI_VARIANT=test,postgresql,file deps = - pifpaf>=0.13 + # TODO(tobias-urdin): Remove this pin and use pifpaf directly instead when this is + # merged and released https://github.com/jd/pifpaf/pull/150 + git+https://github.com/tobias-urdin/pifpaf@51f74a3d8743a7ac33259413df7efc30df993460 gnocchiclient>=2.8.0,!=7.0.7 xattr!=0.9.4 commands = {toxinidir}/run-upgrade-tests.sh postgresql-file -[testenv:py36-mysql-ceph-upgrade-from-4.3] +[testenv:{py36,py38}-mysql-ceph-upgrade-from-4.3] # We should always recreate since the script upgrade # Gnocchi we can't reuse the virtualenv recreate = True @@ -72,19 +77,21 @@ setenv = GNOCCHI_VERSION_FROM=stable/4.3 GNOCCHI_VARIANT=test,mysql,ceph,ceph_recommended_lib deps = + # TODO(tobias-urdin): Remove this pin and use pifpaf directly instead when this is + # merged and released https://github.com/jd/pifpaf/pull/150 + git+https://github.com/tobias-urdin/pifpaf@51f74a3d8743a7ac33259413df7efc30df993460 gnocchiclient>=2.8.0,!=7.0.7 - pifpaf[ceph]>=0.13 xattr!=0.9.4 commands = {toxinidir}/run-upgrade-tests.sh mysql-ceph [testenv:pep8] basepython = python3 -deps = hacking>=0.12 +deps = hacking>=3.2.0,<4.0.0 commands = flake8 {posargs} allowlist_externals = /usr/bin/flake8 -[testenv:py36-cover] +[testenv:{py36,py38}-cover] commands = pifpaf -g GNOCCHI_INDEXER_URL run postgresql -- python setup.py testr --coverage --testr-args="{posargs}" [flake8] @@ -110,14 +117,16 @@ commands = doc8 --ignore-path doc/source/rest.rst,doc/source/comparison-table.rs [testenv:docs-gnocchi-web] basepython = python3 -whitelist_externals = bash rm +whitelist_externals = + /bin/bash + /bin/rm setenv = GNOCCHI_STORAGE_DEPS=file GNOCCHI_TEST_DEBUG=1 install_command = pip install -U {opts} {packages} deps = {[testenv:docs]deps} setuptools commands = - rm -rf doc/build/html + /bin/rm -rf doc/build/html pifpaf -g GNOCCHI_INDEXER_URL run postgresql -- python setup.py build_sphinx [doc8]