Merge branch 'main' into ol-facets/PR1-feed-new-tables

.circleci/api-load-test.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+#
+# Copyright 2018-2023 contributors to the Marquez project
+# SPDX-License-Identifier: Apache-2.0
+#
+# A script used in CI to load test HTTP API server by:
+#   (1) Starting HTTP API server
+#   (2) Generating random dataset, job, and run metadata
+#   (3) Running load test using k6
+#   (4) Writing load test results to 'k6/results' for analysis
+#
+# Usage: $ ./api-load-test.sh
+
+set -e
+
+# Build version of Marquez
+readonly MARQUEZ_VERSION="0.30.0-SNAPSHOT"
+# Fully qualified path to marquez.jar
+readonly MARQUEZ_JAR="api/build/libs/marquez-api-${MARQUEZ_VERSION}.jar"
+
+readonly MARQUEZ_HOST="localhost"
+readonly MARQUEZ_ADMIN_PORT=5001 # Use default 'dev' admin port
+readonly MARQUEZ_URL="http://${MARQUEZ_HOST}:${MARQUEZ_ADMIN_PORT}"
+readonly MARQUEZ_DB="marquez-db"
+
+readonly METADATA_FILE="api/load-testing/metadata.json"
+readonly METADATA_STATS_QUERY=$(cat <<-END
+  SELECT run_uuid,COUNT(*)
+    FROM lineage_events
+   GROUP BY run_uuid;
+END
+)
+
+log() {
+  echo -e "\033[1m>>\033[0m ${1}"
+}
+
+cpu_and_mem_info() {
+  log "CPU info:"
+  cat /proc/cpuinfo
+  log "MEM info:"
+  cat /proc/meminfo
+}
+
+ol_events_stats() {
+    # Query db for OL events stats
+  log "load test metadata stats:"
+  docker exec "${MARQUEZ_DB}" \
+    psql -U marquez -c "${METADATA_STATS_QUERY}"
+}
+
+# Change working directory to project root
+project_root=$(git rev-parse --show-toplevel)
+cd "${project_root}"
+
+# (1) Start db
+log "start db:"
+docker-compose -f docker-compose.db.yml up --detach
+
+# (2) Build HTTP API server
+log "build http API server..."
+./gradlew --no-daemon :api:build -x test > /dev/null 2>&1
+
+# (3) Start HTTP API server
+log "start http API server..."
+mkdir marquez && \
+  java -jar "${MARQUEZ_JAR}" server marquez.dev.yml > marquez/http.log 2>&1 &
+
+# (4) Wait for HTTP API server
+log "waiting for http API server (${MARQUEZ_URL})..."
+until curl --output /dev/null --silent --head --fail "${MARQUEZ_URL}/ping"; do
+    sleep 5
+done
+# When available, print status
+log "http API server is ready!"
+
+# (5) Use metadata command to generate random dataset, job, and run metadata
+log "generate load test metadata (${METADATA_FILE}):"
+java -jar "${MARQUEZ_JAR}" metadata --runs 10 --bytes-per-event 16384 --output "${METADATA_FILE}"
+
+# Display CPU/MEM
+cpu_and_mem_info
+
+# (6) Run load test
+log "start load test:"
+mkdir -p k6/results && \
+  k6 run --vus 25 --duration 30s api/load-testing/http.js \
+    --out json=k6/results/full.json --summary-export=k6/results/summary.json
+
+# Display OL event stats
+ol_events_stats
+
+echo "DONE!"

.circleci/config.yml

@@ -15,7 +15,6 @@ only-on-release: &only-on-release
       ignore: /.*/
 
 orbs:
-  # https://circleci.com/orbs/registry/orb/codecov/codecov
   codecov: codecov/[email protected]
 
 jobs:
@@ -148,6 +147,21 @@ jobs:
       - run: npm install --prefix=${HOME}/.local --global redoc-cli
       - run: redoc-cli bundle spec/openapi.yml
 
+  load-test-api:
+    working_directory: ~/marquez
+    machine:
+      image: ubuntu-2004:current
+    steps:
+      - checkout
+      - run: ./.circleci/get-docker-compose.sh
+      - run: ./.circleci/get-jdk17.sh
+      - run: ./.circleci/get-k6.sh
+      - run: ./.circleci/api-load-test.sh
+      - store_artifacts:
+          path: marquez
+      - store_artifacts:
+          path: k6
+
   migrate-db:
     working_directory: ~/marquez
     machine:
@@ -204,6 +218,9 @@ workflows:
       - unit-test-web
       - unit-test-client-python
       - lint-spec-api
+      - load-test-api:
+          requires:
+            - build-api
       - migrate-db:
           requires:
             - build-api

.circleci/db-migration.sh

@@ -1,11 +1,11 @@
 #!/bin/bash
 #
-# Copyright 2018-2022 contributors to the Marquez project
+# Copyright 2018-2023 contributors to the Marquez project
 # SPDX-License-Identifier: Apache-2.0
 #
 # A script used in CI to test database migrations by:
 #   (1) Applying db migrations on latest Marquez release
-#   (2) Take a backup of db from Step 1
+#   (2) Taking a backup of db from Step 1
 #   (3) Applying db migrations on latest Marquez build using backup
 #
 # Usage: $ ./db-migration.sh

.circleci/get-jdk17.sh

@@ -1,19 +1,12 @@
 #!/bin/bash
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Copyright 2018-2023 contributors to the Marquez project
+# SPDX-License-Identifier: Apache-2.0
 #
 # Usage: $ ./get-jdk17.sh
 
+set -e
+
 wget -qO - https://adoptium.jfrog.io/adoptium/api/gpg/key/public | sudo apt-key add -
 sudo add-apt-repository --yes https://adoptium.jfrog.io/adoptium/deb
 sudo apt-get update --allow-releaseinfo-change && sudo apt-get install --yes temurin-17-jdk

.circleci/get-k6.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Copyright 2018-2023 contributors to the Marquez project
+# SPDX-License-Identifier: Apache-2.0
+#
+# Usage: $ ./get-k6.sh
+
+set -e
+
+# Delete existing key (if present)
+sudo apt-key del k6
+
+# Add k6 key and update the repository
+sudo gpg -k && \
+  sudo gpg --no-default-keyring --keyring /usr/share/keyrings/k6-archive-keyring.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys C5AD17C747E3415A3642D57D77C6C491D6AC1D69
+echo "deb [signed-by=/usr/share/keyrings/k6-archive-keyring.gpg] https://dl.k6.io/deb stable main" | sudo tee /etc/apt/sources.list.d/k6.list
+
+# Install k6, then verify
+sudo snap install k6 && k6 version
+
+echo "DONE!"

api/load-testing/http.js

@@ -0,0 +1,27 @@
+import { SharedArray } from 'k6/data';
+import http from 'k6/http';
+import { check, sleep } from 'k6';
+import { Rate } from 'k6/metrics';
+
+export const errorRate = new Rate('errors');
+
+const metadata = new SharedArray('metadata', function () {
+  return JSON.parse(open('./metadata.json'));
+});
+
+export default function () {
+  const url = 'http://localhost:5000/api/v1/lineage';
+  const params = {
+    headers: {
+      'Content-Type': 'application/json',
+    },
+  };
+
+  var ol_event = metadata[__VU-1]
+
+  check(http.post(url, JSON.stringify(ol_event), params), {
+    'status is 201': (r) => r.status == 201,
+  }) || errorRate.add(1);
+
+  sleep(1);
+}

docs/load-testing.md → api/load-testing/load-testing.md

@@ -90,5 +90,5 @@ export default function () {
    > **Note:** To learn how to run tests locally with `k6`, see [_Running k6_](https://k6.io/docs/getting-started/running-k6).
 
 ----
-SPDX-License-Identifier: Apache-2.0 
-Copyright 2018-2023 contributors to the Marquez project.
+SPDX-License-Identifier: Apache-2.0
+Copyright 2018-2023 contributors to the Marquez project.

build.gradle

@@ -54,7 +54,7 @@ subprojects {
     }
 
     ext {
-        assertjVersion = '3.23.1'
+        assertjVersion = '3.24.1'
         dropwizardVersion = '2.1.4'
         jacocoVersion = '0.8.8'
         junit5Version = '5.9.1'

docker-compose.db.yml

@@ -9,7 +9,7 @@ services:
       - POSTGRES_USER=postgres
       - POSTGRES_PASSWORD=password
       - MARQUEZ_DB=marquez
-      - MARQUEZ_USER=buendia
-      - MARQUEZ_PASSWORD=macondo
+      - MARQUEZ_USER=marquez
+      - MARQUEZ_PASSWORD=marquez
     volumes:
       - ./docker/init-db.sh:/docker-entrypoint-initdb.d/init-db.sh

docker/up.sh

@@ -131,9 +131,9 @@ fi
 
 # Enable web UI
 if [[ "${NO_WEB}" = "false" ]]; then
-  # Enable building web UI from source; otherwise use 'latest' build
-  [[ "${BUILD}" = "true" ]] && compose_files+=" -f docker-compose.web-dev.yml" \
-    || compose_files+=" -f docker-compose.web.yml"
+  compose_files+=" -f docker-compose.web.yml"
+  # Enable building web UI from source
+  [[ "${BUILD}" = "true" ]] && compose_files+=" -f docker-compose.web-dev.yml"
 fi
 
 # Create docker volumes for Marquez

marquez.dev.yml

@@ -9,7 +9,7 @@ server:
 
 db:
   driverClass: org.postgresql.Driver
-  url: jdbc:postgresql://postgres:5432/marquez
+  url: jdbc:postgresql://${POSTGRES_HOST:-localhost}:5432/marquez
   user: marquez
   password: marquez
 

web/src/__tests__/components/Dialog.test.tsx

@@ -11,7 +11,8 @@ describe('Dialog Component', () => {
   const mockProps = {
     dialogIsOpen: true,
     dialogToggle: dialogToggle,
-    ignoreWarning: ignoreWarning
+    ignoreWarning: ignoreWarning,
+    editWarningField: 'Description of dialog...'
   }
 
   const ignoreWarning = () => {
@@ -31,5 +32,4 @@ describe('Dialog Component', () => {
   it('renders a snapshot that matches previous', () => {
     expect(wrapper).toMatchSnapshot()
   })
-
 })

web/src/__tests__/components/__snapshots__/Dialog.test.tsx.snap

@@ -1,29 +1,41 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
 exports[`Dialog Component renders a snapshot that matches previous 1`] = `
-<div>
-  <WithStyles(ForwardRef(Dialog))
-    open={true}
-  >
-    <WithStyles(ForwardRef(DialogTitle)) />
-    <WithStyles(ForwardRef(DialogContent))>
-      <WithStyles(ForwardRef(DialogContentText)) />
-    </WithStyles(ForwardRef(DialogContent))>
-    <WithStyles(ForwardRef(DialogActions))>
-      <WithStyles(ForwardRef(Button))
-        className="dialogButton"
-        color="secondary"
-      >
-        Continue
-      </WithStyles(ForwardRef(Button))>
-      <WithStyles(ForwardRef(Button))
-        className="dialogButton"
-        color="primary"
-        onClick={[Function]}
-      >
-        Cancel
-      </WithStyles(ForwardRef(Button))>
-    </WithStyles(ForwardRef(DialogActions))>
-  </WithStyles(ForwardRef(Dialog))>
-</div>
+<WithStyles(ForwardRef(Dialog))
+  open={true}
+>
+  <WithStyles(ForwardRef(DialogTitle)) />
+  <WithStyles(ForwardRef(DialogContent))>
+    <WithStyles(ForwardRef(DialogContentText))>
+      Description of dialog...
+    </WithStyles(ForwardRef(DialogContentText))>
+  </WithStyles(ForwardRef(DialogContent))>
+  <WithStyles(ForwardRef(DialogContent))>
+    <WithStyles(ForwardRef(DialogContentText))>
+      Description of dialog...
+    </WithStyles(ForwardRef(DialogContentText))>
+  </WithStyles(ForwardRef(DialogContent))>
+  <WithStyles(ForwardRef(DialogActions))>
+    <WithStyles(ForwardRef(Button))
+      className="dialogButton"
+      color="primary"
+      onClick={[Function]}
+      style={
+        Object {
+          "backgroundColor": "#ee7b7b",
+          "color": "#fff",
+        }
+      }
+    >
+      Cancel
+    </WithStyles(ForwardRef(Button))>
+    <WithStyles(ForwardRef(Button))
+      className="dialogButton"
+      color="primary"
+      variant="outlined"
+    >
+      Continue
+    </WithStyles(ForwardRef(Button))>
+  </WithStyles(ForwardRef(DialogActions))>
+</WithStyles(ForwardRef(Dialog))>
 `;

web/src/__tests__/reducers/datasets.test.ts

@@ -14,7 +14,12 @@ describe('datasets reducer', () => {
         datasets: datasets
       }
     }
-    expect(datasetsReducer(initialState, action)).toStrictEqual({init: true, isLoading: false, result: datasets})
+    expect(datasetsReducer(initialState, action)).toStrictEqual({
+      init: true, 
+      isLoading: false, 
+      result: datasets, 
+      deletedDatasetName: ''
+    })
   })
 
 })

web/src/__tests__/reducers/jobs.test.ts

@@ -16,7 +16,7 @@ describe('jobs reducer', () => {
         jobs: jobs
       }
     }
-    expect(jobsReducer(initialState, action)).toStrictEqual({ isLoading: false, result: jobs, init: true })
+    expect(jobsReducer(initialState, action)).toStrictEqual({ isLoading: false, result: jobs, init: true, deletedJobName: '' })
   })
 })