Skip to content

TT-13421 create indexes on sharded sql pumps #2635

TT-13421 create indexes on sharded sql pumps

TT-13421 create indexes on sharded sql pumps #2635

Workflow file for this run

# yamllint disable rule:line-length rule:truthy
name: Release
# Generated by: gromit policy
# Distribution channels covered by this workflow
# - Ubuntu and Debian
# - RHEL and AL
# - docker hub
# - devenv ECR
# - Cloudsmith
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
on:
# Trigger release every monday at midnight for master CI images
schedule:
- cron: "0 0 * * 1"
pull_request:
push:
branches:
- master
- release-**
tags:
- 'v*'
env:
GOPRIVATE: github.com/TykTechnologies
VARIATION: inverted
DOCKER_BUILD_SUMMARY: false
DOCKER_BUILD_RECORD_UPLOAD: false
# startsWith covers pull_request_target too
BASE_REF: ${{startsWith(github.event_name, 'pull_request') && github.base_ref || github.ref_name}}
jobs:
goreleaser:
if: github.event.pull_request.draft == false
name: '${{ matrix.golang_cross }}'
runs-on: ubuntu-latest-m
permissions:
id-token: write # AWS OIDC JWT
contents: read # actions/checkout
strategy:
fail-fast: false
matrix:
golang_cross:
- 1.22-bookworm
include:
- golang_cross: 1.22-bookworm
goreleaser: 'ci/goreleaser/goreleaser.yml'
cgo: 0
rpmvers: 'el/7 el/8 el/9 amazon/2 amazon/2023'
debvers: 'ubuntu/xenial ubuntu/bionic ubuntu/focal ubuntu/jammy debian/jessie debian/buster debian/bullseye debian/bookworm'
outputs:
tags: ${{ steps.ci_metadata.outputs.tags }}
commit_author: ${{ steps.set_outputs.outputs.commit_author}}
steps:
- name: Checkout of tyk-pump
uses: actions/checkout@v4
with:
fetch-depth: 1
- name: Set some outputs for later
id: set_outputs
shell: bash
env:
HEAD_REF: ${{github.head_ref}}
run: |
echo "commit_author=$(git show -s --format='%ae' HEAD)" >> $GITHUB_OUTPUT
echo "branch=${HEAD_REF##*/}" >> $GITHUB_OUTPUT
- uses: docker/setup-qemu-action@v3
- uses: docker/setup-buildx-action@v3
- name: Login to DockerHub
if: startsWith(github.ref, 'refs/tags')
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to Cloudsmith
if: startsWith(github.ref, 'refs/tags')
uses: docker/login-action@v3
with:
registry: docker.tyk.io
username: ${{ secrets.CLOUDSMITH_USERNAME }}
password: ${{ secrets.CLOUDSMITH_API_KEY }}
- uses: actions/cache@v4
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
- name: Build
env:
NFPM_PASSPHRASE: ${{ secrets.SIGNING_KEY_PASSPHRASE }}
PKG_SIGNING_KEY: ${{ secrets.SIGNING_KEY }}
PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}
run: |
echo '#!/bin/sh
ci/bin/unlock-agent.sh
git config --global url."https://${{ secrets.ORG_GH_TOKEN }}@github.com".insteadOf "https://github.com"
git config --global --add safe.directory /go/src/github.com/TykTechnologies/tyk-pump
goreleaser release --clean -f ${{ matrix.goreleaser }} ${{ !startsWith(github.ref, 'refs/tags/') && ' --snapshot --skip=sign' || '' }}' | tee /tmp/build.sh
chmod +x /tmp/build.sh
docker run --rm --privileged -e GITHUB_TOKEN=${{ github.token }} \
-e GOPRIVATE=github.com/TykTechnologies \
-e DEBVERS='${{ matrix.debvers }}' \
-e RPMVERS='${{ matrix.rpmvers }}' \
-e CGO_ENABLED=${{ matrix.cgo }} \
-e NFPM_PASSPHRASE="$NFPM_PASSPHRASE" \
-e GPG_FINGERPRINT=12B5D62C28F57592D1575BD51ED14C59E37DAC20 \
-e PKG_SIGNING_KEY="$PKG_SIGNING_KEY" \
-e PACKAGECLOUD_TOKEN=$PACKAGECLOUD_TOKEN \
-v ${{github.workspace}}:/go/src/github.com/TykTechnologies/tyk-pump \
-v /var/run/docker.sock:/var/run/docker.sock \
-v ~/.docker/config.json:/root/.docker/config.json \
-e GOCACHE=/cache/go-build \
-e GOMODCACHE=/go/pkg/mod \
-v ~/go/pkg/mod:/go/pkg/mod \
-v ~/.cache/go-build:/cache/go-build \
-v /tmp/build.sh:/tmp/build.sh \
-w /go/src/github.com/TykTechnologies/tyk-pump \
tykio/golang-cross:${{ matrix.golang_cross }} /tmp/build.sh
- uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: arn:aws:iam::754489498669:role/ecr_rw_tyk
role-session-name: cipush
aws-region: eu-central-1
# Don't mask to pass it across job boundaries
mask-aws-account-id: false
- uses: aws-actions/amazon-ecr-login@v2
id: ecr
if: ${{ matrix.golang_cross == '1.22-bookworm' }}
with:
mask-password: 'true'
- name: Docker metadata for CI
id: ci_metadata
if: ${{ matrix.golang_cross == '1.22-bookworm' }}
uses: docker/metadata-action@v5
with:
images: ${{ steps.ecr.outputs.registry }}/tyk-pump
flavor: |
latest=false
tags: |
type=ref,event=branch
type=ref,event=pr
type=sha,format=long
type=semver,pattern={{major}},prefix=v
type=semver,pattern={{major}}.{{minor}},prefix=v
type=semver,pattern={{version}},prefix=v
- name: push image to CI
if: ${{ matrix.golang_cross == '1.22-bookworm' }}
uses: docker/build-push-action@v6
with:
context: "dist"
platforms: linux/amd64,linux/arm64
file: ci/Dockerfile.distroless
provenance: mode=max
sbom: true
push: true
cache-from: type=gha
cache-to: type=gha,mode=max
tags: ${{ steps.ci_metadata.outputs.tags }}
labels: ${{ steps.tag_metadata.outputs.labels }}
- name: Docker metadata for tag push
id: tag_metadata
uses: docker/metadata-action@v5
with:
images: |
tykio/tyk-pump-docker-pub
docker.tyk.io/tyk-pump/tyk-pump
flavor: |
latest=false
prefix=v
tags: |
type=semver,pattern={{major}}.{{minor}}
type=semver,pattern={{version}}
labels: "org.opencontainers.image.title=tyk-pump (distroless) \norg.opencontainers.image.description=Tyk Analytics Pump to move analytics data from Redis to any supported back end (multiple back ends can be written to at once).\norg.opencontainers.image.vendor=tyk.io\norg.opencontainers.image.version=${{ github.ref_name }}\n"
- name: push image to prod
if: ${{ matrix.golang_cross == '1.22-bookworm' }}
uses: docker/build-push-action@v6
with:
context: "dist"
platforms: linux/amd64,linux/arm64
file: ci/Dockerfile.distroless
provenance: mode=max
sbom: true
cache-from: type=gha
cache-to: type=gha,mode=max
push: ${{ startsWith(github.ref, 'refs/tags') }}
tags: ${{ steps.tag_metadata.outputs.tags }}
labels: ${{ steps.tag_metadata.outputs.labels }}
- name: save deb
uses: actions/upload-artifact@v4
if: ${{ matrix.golang_cross == '1.22-bookworm' }}
with:
name: deb
retention-days: 1
path: |
dist/*.deb
!dist/*PAYG*.deb
!dist/*fips*.deb
- name: save rpm
uses: actions/upload-artifact@v4
if: ${{ matrix.golang_cross == '1.22-bookworm' }}
with:
name: rpm
retention-days: 1
path: |
dist/*.rpm
!dist/*PAYG*.rpm
!dist/*fips*.rpm
test-controller-api:
if: github.event.pull_request.draft == false
needs:
- goreleaser
runs-on: ubuntu-latest
outputs:
envfiles: ${{ steps.params.outputs.envfiles }}
pump: ${{ steps.params.outputs.pump }}
sink: ${{ steps.params.outputs.sink }}
steps:
- name: set params
id: params
shell: bash
run: |
set -eo pipefail
curl -s --retry 5 --retry-delay 10 --fail-with-body "http://tui.internal.dev.tyk.technology/v2/$VARIATION/tyk-pump/$BASE_REF/${{ github.event_name}}/api.gho" | tee -a "$GITHUB_OUTPUT"
if ! [[ $VARIATION =~ prod ]] ;then
echo "::warning file=.github/workflows/release.yml,line=24,col=1,endColumn=8::Using non-prod variation"
echo "### :warning: You are using VARIATION=${VARIATION} in test-controller-api" >> $GITHUB_STEP_SUMMARY
fi
api-tests:
needs:
- test-controller-api
- goreleaser
runs-on: ubuntu-latest-m-2
env:
XUNIT_REPORT_PATH: ${{ github.workspace}}/test-results.xml
permissions:
id-token: write # This is required for requesting the Github JWT
contents: read # This is required for actions/checkout
strategy:
fail-fast: false
matrix:
envfiles: ${{ fromJson(needs.test-controller-api.outputs.envfiles) }}
sink: ${{ fromJson(needs.test-controller-api.outputs.sink) }}
steps:
- uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: arn:aws:iam::754489498669:role/ecr_rw_tyk
role-session-name: cipush
aws-region: eu-central-1
- id: ecr
uses: aws-actions/amazon-ecr-login@v2
with:
mask-password: 'true'
- name: Setup tmate session only in debug mode
uses: mxschmitt/action-tmate@v3
if: runner.debug == '1'
with:
detached: true
limit-access-to-actor: true
# Only ${{ github.actor }} has access
# See https://github.com/mxschmitt/action-tmate#use-registered-public-ssh-keys
- name: fetch env from tyk-pro
env:
GH_TOKEN: ${{ github.token }}
run: |
gh release download --repo github.com/tyklabs/tyk-pro --archive tar.gz -O env.tgz
mkdir auto && tar --strip-components=1 -C auto -xzvf env.tgz
- name: env up
shell: bash
working-directory: auto
id: env_up
env:
pull_policy: 'if_not_present'
GH_TOKEN: ${{ secrets.ORG_GH_TOKEN }}
TYK_DB_LICENSEKEY: ${{ secrets.DASH_LICENSE }}
TYK_MDCB_LICENSE: ${{ secrets.MDCB_LICENSE }}
run: |
match_tag=${{steps.ecr.outputs.registry}}/tyk-pump:$BASE_REF
tags=(${{ needs.goreleaser.outputs.tags }})
set -eaxo pipefail
docker run -q --rm -v ~/.docker/config.json:/root/.docker/config.json tykio/gromit policy match ${tags[0]} ${match_tag} 2>versions.env
echo '# alfa and beta have to come after the override
tyk_alfa_image=$tyk_image
tyk_beta_image=$tyk_image
ECR=${{steps.ecr.outputs.registry}}
tyk_sink_image=${{matrix.sink}}
confs_dir=./pro-ha
env_file=local.env' >> versions.env
cat ./confs/${{ matrix.envfiles.config }}.env local-${{ matrix.envfiles.db }}.env > local.env
echo "::group::versions"
cat versions.env local.env
echo "::endgroup::"
# bring up env, the project name is important
docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.envfiles.db }}.yml -f ${{ matrix.envfiles.cache }}.yml --env-file versions.env --profile master-datacenter up --quiet-pull -d
./dash-bootstrap.sh http://localhost:3000
docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.envfiles.db }}.yml -f ${{ matrix.envfiles.cache }}.yml --env-file versions.env --profile slave-datacenter up --quiet-pull -d
echo "$(cat pytest.env | grep USER_API_SECRET)" >> $GITHUB_OUTPUT
echo "ts=$(date +%s%N)" >> $GITHUB_OUTPUT
- uses: actions/checkout@v4
with:
repository: TykTechnologies/tyk-analytics
path: tyk-analytics
token: ${{ secrets.ORG_GH_TOKEN }}
fetch-depth: 0
sparse-checkout: tests/api
- name: Choosing test code branch
working-directory: tyk-analytics/tests/api
run: |
if [[ ${{ github.event_name }} == "pull_request" ]]; then
PR_BRANCH=${{ github.event.pull_request.head.ref }}
TARGET_BRANCH=${{ github.event.pull_request.base.ref }}
echo "Looking for PR_BRANCH:$PR_BRANCH or TARGET_BRANCH:$TARGET_BRANCH..."
if git rev-parse --verify "origin/$PR_BRANCH" >/dev/null 2>&1; then
echo "PR branch $PR_BRANCH exists. Checking out..."
git checkout "$PR_BRANCH"
elif git rev-parse --verify "origin/$TARGET_BRANCH" >/dev/null 2>&1; then
echo "Target branch $TARGET_BRANCH exists. Checking out..."
git checkout "$TARGET_BRANCH"
fi
fi
if [[ ${{ github.event_name }} == "push" ]]; then
PUSH_BRANCH=${{ github.ref_name }}
echo "Looking for PUSH_BRANCH:$PUSH_BRANCH..."
if git rev-parse --verify "origin/$PUSH_BRANCH" >/dev/null 2>&1; then
echo "Push branch $PUSH_BRANCH exists. Checking out..."
git checkout "$PUSH_BRANCH"
fi
fi
echo "Current commit: $(git rev-parse HEAD)"
- uses: actions/setup-python@v5
with:
cache: 'pip'
python-version: '3.10'
- name: Run API tests
id: test_execution
working-directory: tyk-analytics/tests/api
run: |
pytest="pytest --ci --random-order --force-flaky --no-success-flaky-report --maxfail=3 --junitxml=${XUNIT_REPORT_PATH} --cache-clear --ignore=./tests/mdcb -v --log-cli-level=ERROR"
pip install -r requirements.txt
cat >pytest.env <<-EOF
TYK_TEST_BASE_URL=http://localhost:3000/
TYK_TEST_GW_URL=https://localhost:8080/
TYK_TEST_GW_1_ALFA_URL=https://localhost:8181/
TYK_TEST_GW_1_BETA_URL=https://localhost:8182/
TYK_TEST_GW_2_ALFA_URL=https://localhost:8281/
TYK_TEST_GW_2_BETA_URL=https://localhost:8282/
TYK_TEST_MONGODB=localhost:27017
TYK_TEST_REDIS=localhost
TYK_TEST_DB_ADMIN=12345
TYK_TEST_GW_SECRET=352d20ee67be67f6340b4c0605b044b7
TYK_TEST_DB_NAME=tyk_analytics
TYK_TEST_FEDERATION_HOST=federation
TYK_TEST_GRAPHQL_FAKER_HOST=graphql-faker
GATEWAY_CONTAINER_NAME=tyk
USER_API_SECRET=${{ steps.env_up.outputs.USER_API_SECRET }}
EOF
env $(cat pytest.env | xargs) $pytest -m "${{ matrix.envfiles.apimarkers }}"
- name: Generate metadata and upload test reports
id: metadata_report
if: always() && (steps.test_execution.conclusion != 'skipped')
env:
REPORT_NAME: ${{ github.repository }}_${{ github.run_id }}_${{ github.run_attempt }}-${{steps.env_up.outputs.ts}}
METADATA_REPORT_PATH: metadata.toml
run: |
# Generate metadata report
set -eo pipefail
echo "[metadata]
repo = ${{ github.repository }}
branch = ${{ github.ref }}
commit = ${{ github.sha }}
test_suite_version = $BASE_REF
test_suite_name = ${{ github.job }}
test_suite_run = ${{ github.run_id }}-${{ github.run_attempt }}
db = ${{ matrix.envfiles.db }}
conf = ${{ matrix.envfiles.config }}
cache = ${{ matrix.envfiles.cache }}
pump_compatibility = ${{ matrix.pump }}
sink_compatibility = ${{ matrix.sink }}
" | tee ${METADATA_REPORT_PATH}
aws s3 cp ${XUNIT_REPORT_PATH} s3://assets.dev.tyk.technology/testreports/${REPORT_NAME#*/}.xml
aws s3 cp ${METADATA_REPORT_PATH} s3://assets.dev.tyk.technology/testreports/${REPORT_NAME#*/}.metadata.toml
- name: Docker logs for all components
if: failure() && (steps.test_execution.outcome != 'success' || steps.env_up.outcome != 'success')
working-directory: auto
env:
pull_policy: 'if_not_present'
GH_TOKEN: ${{ secrets.ORG_GH_TOKEN }}
TYK_DB_LICENSEKEY: ${{ secrets.DASH_LICENSE }}
TYK_MDCB_LICENSE: ${{ secrets.MDCB_LICENSE }}
ECR: ${{ steps.ecr.outputs.registry }}
run: |
docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.envfiles.db }}.yml -f ${{ matrix.envfiles.cache }}.yml --env-file versions.env --profile all logs | sort > ${{ github.workspace }}/docker-compose.log
echo "::group::DockerLogs"
cat ${{ github.workspace }}/docker-compose.log
echo "::endgroup::"
- name: Upload compose logs
uses: actions/upload-artifact@v4
if: failure() && (steps.test_execution.outcome != 'success' || steps.env_up.outcome != 'success')
with:
name: docker-compose-logs-${{ github.job }}-${{ matrix.envfiles.db }}-${{ matrix.envfiles.conf }}-${{ github.run_id }}
path: ${{ github.workspace }}/docker-compose.log
retention-days: 3
overwrite: true
test-controller-distros:
if: github.event.pull_request.draft == false
needs:
- goreleaser
runs-on: ubuntu-latest
outputs:
deb: ${{ steps.params.outputs.deb }}
rpm: ${{ steps.params.outputs.rpm }}
steps:
- name: set params
id: params
shell: bash
env:
# startsWith covers pull_request_target too
BASE_REF: ${{startsWith(github.event_name, 'pull_request') && github.base_ref || github.ref_name}}
run: |
set -eo pipefail
curl -s --retry 5 --retry-delay 10 --fail-with-body "http://tui.internal.dev.tyk.technology/v2/$VARIATION/tyk-pump/$BASE_REF/${{ github.event_name}}/api/Distros.gho" | tee -a "$GITHUB_OUTPUT"
if ! [[ $VARIATION =~ prod ]];then
echo "::warning file=.github/workflows/release.yml,line=24,col=1,endColumn=8::Using test variation"
fi
upgrade-deb:
services:
httpbin.org:
image: kennethreitz/httpbin
runs-on: ubuntu-latest
needs:
- test-controller-distros
strategy:
fail-fast: true
matrix:
arch:
- amd64
- arm64
distro: ${{ fromJson(needs.test-controller-distros.outputs.deb) }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 1
sparse-checkout: ci
- uses: actions/download-artifact@v4
with:
name: deb
- uses: docker/setup-qemu-action@v3
- uses: docker/setup-buildx-action@v3
- name: generate dockerfile
run: |
echo 'FROM ${{ matrix.distro }}
ARG TARGETARCH
COPY tyk-pump*_${TARGETARCH}.deb /tyk-pump.deb
RUN apt-get update && apt-get install -y curl
RUN curl -fsSL https://packagecloud.io/install/repositories/tyk/tyk-pump/script.deb.sh | bash && apt-get install -y tyk-pump=1.6.0
RUN dpkg -i /tyk-pump.deb
' | tee Dockerfile
- name: install on ${{ matrix.distro }}
uses: docker/build-push-action@v6
with:
context: "."
platforms: linux/${{ matrix.arch }}
build-args: |
RHELARCH=${{ startsWith(matrix.arch, 'arm64') && 'aarch64' || 'x86_64' }}
cache-from: type=gha
cache-to: type=gha,mode=max
file: Dockerfile
push: false
upgrade-rpm:
services:
httpbin.org:
image: kennethreitz/httpbin
runs-on: ubuntu-latest
needs:
- test-controller-distros
strategy:
fail-fast: true
matrix:
arch:
- amd64
- arm64
distro: ${{ fromJson(needs.test-controller-distros.outputs.rpm) }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 1
sparse-checkout: ci
- uses: actions/download-artifact@v4
with:
name: rpm
- uses: docker/setup-qemu-action@v3
- uses: docker/setup-buildx-action@v3
- name: generate dockerfile
run: |
echo 'FROM ${{ matrix.distro }}
ARG RHELARCH
COPY tyk-pump*.${RHELARCH}.rpm /tyk-pump.rpm
RUN command -v curl || yum install -y curl
RUN command -v useradd || yum install -y shadow-utils
RUN curl -fsSL https://packagecloud.io/install/repositories/tyk/tyk-pump/script.rpm.sh | bash && yum install -y tyk-pump-1.6.0-1
RUN curl https://keyserver.tyk.io/tyk.io.rpm.signing.key.2020 -o tyk-pump.key && rpm --import tyk-pump.key
RUN rpm --checksig /tyk-pump.rpm
RUN rpm -Uvh --force /tyk-pump.rpm
' | tee Dockerfile
- name: install on ${{ matrix.distro }}
uses: docker/build-push-action@v6
with:
context: "."
platforms: linux/${{ matrix.arch }}
build-args: |
RHELARCH=${{ startsWith(matrix.arch, 'arm64') && 'aarch64' || 'x86_64' }}
cache-from: type=gha
cache-to: type=gha,mode=max
file: Dockerfile
push: false
sbom:
needs: goreleaser
uses: TykTechnologies/github-actions/.github/workflows/sbom.yaml@main
secrets:
DEPDASH_URL: ${{ secrets.DEPDASH_URL }}
DEPDASH_KEY: ${{ secrets.DEPDASH_KEY }}
ORG_GH_TOKEN: ${{ secrets.ORG_GH_TOKEN }}