-
Notifications
You must be signed in to change notification settings - Fork 320
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'main' into helm-chart-fixes
- Loading branch information
Showing
516 changed files
with
3,488 additions
and
638 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,113 @@ | ||
| #!/bin/bash | ||
| # | ||
| # Copyright 2018-2023 contributors to the Marquez project | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
| # | ||
| # A script used in CI to load test HTTP API server by: | ||
| # (1) Starting HTTP API server | ||
| # (2) Generating random dataset, job, and run metadata | ||
| # (3) Running load test using k6 | ||
| # (4) Writing load test results to 'k6/results' for analysis | ||
| # | ||
| # Usage: $ ./api-load-test.sh | ||
|
|
||
| set -e | ||
|
|
||
| # Build version of Marquez | ||
| readonly MARQUEZ_VERSION="0.30.0-SNAPSHOT" | ||
| # Fully qualified path to marquez.jar | ||
| readonly MARQUEZ_JAR="api/build/libs/marquez-api-${MARQUEZ_VERSION}.jar" | ||
|
|
||
| readonly MARQUEZ_HOST="localhost" | ||
| readonly MARQUEZ_ADMIN_PORT=8081 | ||
| readonly MARQUEZ_URL="http://${MARQUEZ_HOST}:${MARQUEZ_ADMIN_PORT}" | ||
| readonly MARQUEZ_DB="marquez-db" | ||
|
|
||
| readonly METADATA_FILE="api/load-testing/metadata.json" | ||
| readonly METADATA_STATS_QUERY=$(cat <<-END | ||
| SELECT run_uuid,COUNT(*) | ||
| FROM lineage_events | ||
| GROUP BY run_uuid; | ||
| END | ||
| ) | ||
|
|
||
| # marquez.yml | ||
| cat > marquez.yml <<EOF | ||
| server: | ||
| applicationConnectors: | ||
| - type: http | ||
| port: 8080 | ||
| httpCompliance: RFC7230_LEGACY | ||
| adminConnectors: | ||
| - type: http | ||
| port: 8081 | ||
| db: | ||
| driverClass: org.postgresql.Driver | ||
| url: jdbc:postgresql://localhost:5432/marquez | ||
| user: marquez | ||
| password: marquez | ||
| migrateOnStartup: true | ||
| EOF | ||
|
|
||
| log() { | ||
| echo -e "\033[1m>>\033[0m ${1}" | ||
| } | ||
|
|
||
| cpu_and_mem_info() { | ||
| log "CPU info:" | ||
| cat /proc/cpuinfo | ||
| log "MEM info:" | ||
| cat /proc/meminfo | ||
| } | ||
|
|
||
| metadata_stats() { | ||
| # Query db for metadata stats | ||
| log "load test metadata stats:" | ||
| docker exec "${MARQUEZ_DB}" \ | ||
| psql -U marquez -c "${METADATA_STATS_QUERY}" | ||
| } | ||
|
|
||
| # Change working directory to project root | ||
| project_root=$(git rev-parse --show-toplevel) | ||
| cd "${project_root}" | ||
|
|
||
| # (1) Start db | ||
| log "start db:" | ||
| docker-compose -f docker-compose.db.yml up --detach | ||
|
|
||
| # (2) Build HTTP API server | ||
| log "build http API server..." | ||
| ./gradlew --no-daemon :api:build -x test > /dev/null 2>&1 | ||
|
|
||
| # (3) Start HTTP API server | ||
| log "start http API server..." | ||
| mkdir marquez && \ | ||
| java -jar "${MARQUEZ_JAR}" server marquez.yml > marquez/http.log 2>&1 & | ||
|
|
||
| # (4) Wait for HTTP API server | ||
| log "waiting for http API server (${MARQUEZ_URL})..." | ||
| until curl --output /dev/null --silent --head --fail "${MARQUEZ_URL}/ping"; do | ||
| sleep 5 | ||
| done | ||
| # When available, print status | ||
| log "http API server is ready!" | ||
|
|
||
| # (5) Use metadata command to generate random dataset, job, and run metadata | ||
| log "generate load test metadata (${METADATA_FILE}):" | ||
| java -jar "${MARQUEZ_JAR}" metadata --runs 10 --bytes-per-event 16384 --output "${METADATA_FILE}" | ||
|
|
||
| # Display CPU/MEM | ||
| cpu_and_mem_info | ||
|
|
||
| # (6) Run load test | ||
| log "start load test:" | ||
| mkdir -p k6/results && \ | ||
| k6 run --vus 25 --duration 30s api/load-testing/http.js \ | ||
| --out json=k6/results/full.json --summary-export=k6/results/summary.json | ||
|
|
||
| # Display metadata stats | ||
| metadata_stats | ||
|
|
||
| echo "DONE!" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -15,7 +15,6 @@ only-on-release: &only-on-release | |
| ignore: /.*/ | ||
|
|
||
| orbs: | ||
| # https://circleci.com/orbs/registry/orb/codecov/codecov | ||
| codecov: codecov/[email protected] | ||
|
|
||
| jobs: | ||
|
|
@@ -148,6 +147,30 @@ jobs: | |
| - run: npm install --prefix=${HOME}/.local --global redoc-cli | ||
| - run: redoc-cli bundle spec/openapi.yml | ||
|
|
||
| load-test-api: | ||
| working_directory: ~/marquez | ||
| machine: | ||
| image: ubuntu-2004:current | ||
| steps: | ||
| - checkout | ||
| - run: ./.circleci/get-docker-compose.sh | ||
| - run: ./.circleci/get-jdk17.sh | ||
| - run: ./.circleci/get-k6.sh | ||
| - run: ./.circleci/api-load-test.sh | ||
| - store_artifacts: | ||
| path: marquez | ||
| - store_artifacts: | ||
| path: k6 | ||
|
|
||
| migrate-db: | ||
| working_directory: ~/marquez | ||
| machine: | ||
| image: ubuntu-2004:current | ||
| steps: | ||
| - checkout | ||
| - run: ./.circleci/get-docker-compose.sh | ||
| - run: ./.circleci/db-migration.sh | ||
|
|
||
| release-java: | ||
| working_directory: ~/marquez | ||
| machine: | ||
|
|
@@ -195,6 +218,12 @@ workflows: | |
| - unit-test-web | ||
| - unit-test-client-python | ||
| - lint-spec-api | ||
| - load-test-api: | ||
| requires: | ||
| - build-api | ||
| - migrate-db: | ||
| requires: | ||
| - build-api | ||
| release: | ||
| jobs: | ||
| - build-client-python: | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,89 @@ | ||
| #!/bin/bash | ||
| # | ||
| # Copyright 2018-2023 contributors to the Marquez project | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
| # | ||
| # A script used in CI to test database migrations by: | ||
| # (1) Applying db migrations on latest Marquez release | ||
| # (2) Taking a backup of db from Step 1 | ||
| # (3) Applying db migrations on latest Marquez build using backup | ||
| # | ||
| # Usage: $ ./db-migration.sh | ||
|
|
||
| # Version of PostgreSQL | ||
| readonly POSTGRES_VERSION="12.1" | ||
| # Version of Marquez | ||
| readonly MARQUEZ_VERSION="0.29.0" | ||
| # Build version of Marquez | ||
| readonly MARQUEZ_BUILD_VERSION="$(git log --pretty=format:'%h' -n 1)" # SHA1 | ||
|
|
||
| readonly DB_MIGRATION_VOLUME="marquez_db-backup" | ||
| readonly DB_MIGRATION_BACKUP="db-migration-backup" | ||
| readonly DB_MIGRATION_QUERY=$(cat <<-END | ||
| SELECT version,installed_on,checksum | ||
| FROM flyway_schema_history | ||
| WHERE version IS NOT NULL | ||
| ORDER BY installed_on DESC LIMIT 1; | ||
| END | ||
| ) | ||
|
|
||
| log() { | ||
| echo -e "\033[1m>>\033[0m ${1}" | ||
| } | ||
|
|
||
| error() { | ||
| echo -e "\033[0;31merror: ${1}\033[0m" | ||
| } | ||
|
|
||
| exit_with_cause() { | ||
| log "please view container logs for more details on cause:" | ||
| docker-compose logs | ||
| exit 1 | ||
| } | ||
|
|
||
| query_db_migration() { | ||
| # Start db using backup | ||
| [[ $(docker ps -f "name=${DB_MIGRATION_BACKUP}" --format '{{.Names}}') == "${DB_MIGRATION_BACKUP}" ]] || \ | ||
| docker run -d --name "${DB_MIGRATION_BACKUP}" \ | ||
| -v "${DB_MIGRATION_VOLUME}:/var/lib/postgresql/data" \ | ||
| "postgres:${POSTGRES_VERSION}" | ||
| # Query applied db migrations | ||
| log "latest migration applied to db:" | ||
| docker exec "${DB_MIGRATION_BACKUP}" \ | ||
| psql -U marquez -c "${DB_MIGRATION_QUERY}" | ||
| } | ||
|
|
||
| # Change working directory to project root | ||
| project_root=$(git rev-parse --show-toplevel) | ||
| cd "${project_root}/" | ||
|
|
||
| # (1) Apply db migrations on latest Marquez release | ||
| log "start db with latest migrations (marquez=${MARQUEZ_VERSION}):" | ||
| if ! ./docker/up.sh \ | ||
| --args "--exit-code-from seed_marquez" \ | ||
| --tag "${MARQUEZ_VERSION}" \ | ||
| --no-web \ | ||
| --seed > /dev/null; then | ||
| error "failed to start db using backup!" | ||
| exit_with_cause | ||
| fi | ||
|
|
||
| # Query, then display schema migration applied | ||
| query_db_migration | ||
|
|
||
| # (2) Apply db migrations on latest Marquez build using backup | ||
| log "start db using backup (marquez=${MARQUEZ_BUILD_VERSION}):" | ||
| if ! ./docker/up.sh \ | ||
| --args "--exit-code-from seed_marquez" \ | ||
| --no-web \ | ||
| --no-volumes \ | ||
| --build \ | ||
| --seed > /dev/null; then | ||
| error "failed to start db using backup!" | ||
| exit_with_cause | ||
| fi | ||
|
|
||
| # Query, then display additional schema migration applied on backup (if any) | ||
| query_db_migration | ||
|
|
||
| log "DONE!" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,24 +1,18 @@ | ||
| #!/bin/bash | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
| # Copyright 2018-2022 contributors to the Marquez project | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
| # | ||
| # Usage: $ ./get-docker-compose.sh | ||
|
|
||
| set -e | ||
|
|
||
| curl -L https://github.com/docker/compose/releases/download/1.25.3/docker-compose-`uname -s`-`uname -m` > ~/docker-compose | ||
| chmod +x ~/docker-compose | ||
| sudo mv ~/docker-compose /usr/local/bin/docker-compose | ||
| docker-compose --version | ||
| # Download docker compose | ||
| curl -L https://github.com/docker/compose/releases/download/1.29.2/docker-compose-`uname -s`-`uname -m` > ~/docker-compose | ||
|
|
||
| # Change permissions, relocate docker compose, then verify | ||
| chmod +x ~/docker-compose && \ | ||
| sudo mv ~/docker-compose /usr/local/bin/docker-compose && \ | ||
| docker-compose --version | ||
|
|
||
| echo "DONE!" | ||
| echo "DONE!" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| #!/bin/bash | ||
| # | ||
| # Copyright 2018-2023 contributors to the Marquez project | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
| # | ||
| # Usage: $ ./get-k6.sh | ||
|
|
||
| set -e | ||
|
|
||
| # Delete existing key (if present) | ||
| sudo apt-key del k6 | ||
|
|
||
| # Add k6 key and update the repository | ||
| sudo gpg -k && \ | ||
| sudo gpg --no-default-keyring --keyring /usr/share/keyrings/k6-archive-keyring.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys C5AD17C747E3415A3642D57D77C6C491D6AC1D69 | ||
| echo "deb [signed-by=/usr/share/keyrings/k6-archive-keyring.gpg] https://dl.k6.io/deb stable main" | sudo tee /etc/apt/sources.list.d/k6.list | ||
|
|
||
| # Install k6, then verify | ||
| sudo snap install k6 && k6 version | ||
|
|
||
| echo "DONE!" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.