add supavisor bench

examples/supavisor/simple-select/k6/Makefile

@@ -0,0 +1,19 @@
+.PHONY: db_test
+
+MAKEFLAGS += -j2
+
+export
+
+conns ?= 20
+shift ?= 1000
+rampingduration ?= 10
+consecutiveduration ?= 20
+rampscount ?= 5
+requests ?= 1
+rand = $(shell bash -c 'echo $$RANDOM')
+testrun ?= "random-run-$(rand)"
+
+load:
+	@RAMPING_DURATION=$(rampingduration) CONSECUTIVE_DURATION=$(consecutiveduration) RAMPS_COUNT=$(rampscount) \
+		REQUESTS=$(requests) CONNS=$(conns) SHIFT=$(shift) TEST_RUN=$(testrun) ./k6 run load.js \
+		--tag testrun=$(testrun) --tag system='storage_api' -o 'prometheus=namespace=k6'

examples/supavisor/simple-select/k6/common.js

@@ -0,0 +1,61 @@
+/**
+ * Return a random integer between the minimum (inclusive)
+ * and maximum (exclusive) values
+ * @param {number} min - The minimum value to return.
+ * @param {number} max - The maximum value you want to return.
+ * @return {number} The random number between the min and max.
+ */
+export function getRandomInt(min, max) {
+  min = Math.ceil(min)
+  max = Math.floor(max)
+  // The maximum is exclusive and the minimum is inclusive
+  return Math.floor(Math.random() * (max - min) + min)
+}
+
+/**
+ * Generate default k6 ramping-vus scenario.
+ * @param {number} baseDuration - Total duration of the scenario.
+ * @param {number} conns - max number of vus during the scenario execution.
+ *
+ * It starts with 0 VUs, ramps up to half the number of connections in 1/12 of total duration then
+ * it remains on this number for 1/4 of total duration time.
+ * Then ramps down to a quarter of the number of connections in 1/12 of total duration.
+ * Then ramps up to the full number of connections in 1/6 of total duration and
+ * it remains on this number for 1/3 of total duration time.
+ * Then ramps down to a quarter of the number of connections in 1/12 of total duration,
+ * then ramps down to 0 VUs in 10s.
+ */
+export function scenario(rampingDuration, consecutiveDuration, ramps, conns) {
+  const stages = [
+    {
+      duration: `${parseInt(rampingDuration) * 5}s`,
+      target: parseInt(conns) / 2,
+    },
+  ]
+  for (let i = 1; i <= ramps; i++) {
+    stages.push({
+      duration: `${parseInt(rampingDuration)}s`,
+      target:
+        (i * parseInt(conns)) / (parseInt(ramps) * 2) + parseInt(conns) / 2,
+    })
+    stages.push({
+      duration: `${parseInt(consecutiveDuration)}s`,
+      target:
+        (i * parseInt(conns)) / (parseInt(ramps) * 2) + parseInt(conns) / 2,
+    })
+  }
+  stages.push({
+    duration: `${parseInt(consecutiveDuration) * 5}s`,
+    target: parseInt(conns),
+  })
+
+  return {
+    executor: 'ramping-vus',
+    startVUs: 0,
+    stages: stages,
+    gracefulRampDown: '60s',
+  }
+}
+
+/* Exporting an array of default summaryTrendStats to be used in summary result. */
+export const trends = ['avg', 'med', 'p(99)', 'p(95)', 'p(0.1)', 'count']

examples/supavisor/simple-select/k6/load.js

@@ -0,0 +1,111 @@
+import { check, sleep, group } from 'k6'
+import { vu, scenario } from 'k6/execution'
+import { Rate, Trend, Counter } from 'k6/metrics'
+import sql from 'k6/x/sql'
+import { randomIntBetween } from 'https://jslib.k6.io/k6-utils/1.3.0/index.js'
+
+import { scenario as sc, trends } from './common.js'
+export { handleSummary } from './summary.js'
+
+const pgConnectionStringsRaw = __ENV.BASE_URI
+  ? __ENV.BASE_URI
+  : `['postgres://postgres_user:postgres_pass@$postgres_host:6543/postgres?sslmode=disable']`
+
+const conns = __ENV.CONNS ? parseInt(__ENV.CONNS) : 10
+let requests = __ENV.REQUESTS ? parseFloat(__ENV.REQUESTS) : 1
+const rampingDuration = __ENV.RAMPING_DURATION
+  ? parseInt(__ENV.RAMPING_DURATION)
+  : 20
+const consecutiveDuration = __ENV.CONSECUTIVE_DURATION
+  ? parseInt(__ENV.CONSECUTIVE_DURATION)
+  : 40
+const ramps = __ENV.RAMPS_COUNT ? parseInt(__ENV.RAMPS_COUNT) : 10
+const testRun = __ENV.TEST_RUN ? __ENV.TEST_RUN : 'default'
+
+const myFailRate = new Rate('failed_requests')
+const counterQueries = new Counter('queries')
+const counterFailed = new Counter('failed')
+const queryTrend = new Trend('query_trend', true)
+
+const to = {
+  failed_requests: ['rate<0.1'],
+  query_trend: ['p(95)<1000'],
+}
+
+export const options = {
+  vus: 1,
+  thresholds: to,
+  summaryTrendStats: trends,
+  scenarios: {
+    supavisor_select: sc(rampingDuration, consecutiveDuration, ramps, conns),
+  },
+}
+
+// const pgConnectionStrings = JSON.parse(pgConnectionStringsRaw)
+const pgConnectionStrings = JSON.parse(
+  '["postgresql://postgres._tenant_:_password_@_address_:7654/postgres?sslmode=disable","postgresql://postgres._tenant_:_password_@_address2_:7654/postgres?sslmode=disable"]'
+)
+
+let timeslot = 1000
+if (requests < 1 && requests >= 0.1) {
+  timeslot = 1000 * 10
+  requests = 10 * requests
+} else if (requests < 0.1 && requests >= 0.01) {
+  timeslot = 1000 * 100
+  requests = 100 * requests
+} else {
+  requests = __ENV.REQUESTS ? parseInt(__ENV.REQUESTS) : 1
+}
+
+export default () => {
+  const pgConnectionString =
+    pgConnectionStrings[randomIntBetween(0, pgConnectionStrings.length - 1)]
+  try {
+    if (scenario.progress >= 0.98) {
+      sleep(10)
+      return
+    }
+
+    const db = sql.open('postgres', pgConnectionString)
+    while (scenario.progress < 0.95) {
+      const start = new Date()
+      for (let i = 1; i <= requests; i++) {
+        const exStart = new Date()
+        try {
+          db.exec(
+            "select * from (values (1, 'one'), (2, 'two'), (3, 'three')) as t (num,letter);"
+          )
+          myFailRate.add(false)
+        } catch (e) {
+          console.log(e)
+          myFailRate.add(true)
+          counterFailed.add(1)
+        }
+        const exFinish = new Date()
+        counterQueries.add(1)
+        queryTrend.add(exFinish - exStart)
+
+        const finish = new Date()
+        if (finish - start > timeslot) {
+          break
+        }
+        sleep(
+          (timeslot - (finish - start)) /
+            1000 /
+            (requests + randomIntBetween(0, requests))
+        )
+      }
+      const finish = new Date()
+      if (finish - start < timeslot) {
+        sleep((timeslot - (finish - start)) / 1000)
+      }
+    }
+  } finally {
+    db.close()
+  }
+}
+
+export function teardown(data) {
+  // db.exec("delete from public.positions where title='Load Tester'")
+  // db.close()
+}

examples/supavisor/simple-select/k6/summary.js

@@ -0,0 +1,79 @@
+import http from 'k6/http'
+import { textSummary } from 'https://jslib.k6.io/k6-summary/0.0.1/index.js'
+
+/* Setting up the environment variables for the test run. */
+const testrun = __ENV.TEST_RUN
+const origin = __ENV.TEST_ORIGIN
+const benchmark = __ENV.BENCHMARK_ID
+const run = __ENV.RUN_ID
+const token = __ENV.SUPABENCH_TOKEN
+const supabench_uri = __ENV.SUPABENCH_URI
+  ? __ENV.SUPABENCH_URI
+  : 'http://localhost:8090'
+
+/**
+ * Handle summary implementation that additionally sends the data to the reports server.
+ */
+export function handleSummary(data) {
+  console.log('Preparing the end-of-test summary...')
+  const started = Date.now()
+
+  // Send the results to remote server
+  if (!run) {
+    const report = {
+      output: textSummary(data, { indent: ' ', enableColors: false }),
+      raw: data,
+      benchmark_id: benchmark,
+      name: testrun ? testrun : null,
+      status: 'success',
+      origin: origin,
+      started_at: `${started - 60 * 1000}`,
+      ended_at: `${
+        started + parseInt(data.state.testRunDurationMs) + 60 * 1000
+      }`,
+    }
+
+    const resp = http.post(
+      `${supabench_uri}/api/collections/runs/records`,
+      JSON.stringify(report),
+      {
+        headers: {
+          'Content-Type': 'application/json',
+          Authorization: `Admin ${token}`,
+        },
+      }
+    )
+    if (resp.status != 200) {
+      console.error('Could not send summary, got status ' + resp.status)
+    }
+  } else {
+    const report = {
+      output: textSummary(data, { indent: ' ', enableColors: false }),
+      raw: data,
+      status: 'success',
+      started_at: `${started - 120 * 1000}`,
+      ended_at: `${
+        started + parseInt(data.state.testRunDurationMs) + 15 * 1000
+      }`,
+    }
+
+    const resp = http.patch(
+      `${supabench_uri}/api/collections/runs/records/${run}`,
+      JSON.stringify(report),
+      {
+        headers: {
+          'Content-Type': 'application/json',
+          Authorization: `Admin ${token}`,
+        },
+      }
+    )
+    if (resp.status != 200) {
+      console.error('Could not send summary, got status ' + resp.status)
+    }
+  }
+
+  return {
+    stdout: textSummary(data, { indent: ' ', enableColors: true }), // Show the text summary to stdout...
+    'summary.json': JSON.stringify(data), // and a JSON with all the details...
+  }
+}

examples/supavisor/simple-select/main.tf

@@ -0,0 +1,53 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "4.0.0"
+    }
+  }
+}
+
+provider "aws" {
+  region = "eu-central-1"
+}
+
+# Create an infrastructure with System Under Test (SUT).
+# module "setup_infra" {
+#   source = "./modules/setup"
+
+#   app_name = var.app_name
+#   fly_access_token = var.fly_access_token
+# }
+
+module "script" {
+  source = "./modules/script"
+
+  ami_id               = var.ami_id
+  instance_type        = var.instance_type
+  instances_count      = var.instances_count
+  security_group_id    = var.security_group_id
+  subnet_id            = var.subnet_id
+  sut_name             = var.sut_name
+  key_name             = var.key_name
+  private_key_location = var.private_key_location
+
+  testrun_name    = var.testrun_name
+  testrun_id      = var.testrun_id
+  test_origin     = var.test_origin
+  benchmark_id    = var.benchmark_id
+  supabench_token = var.supabench_token
+  supabench_uri   = var.supabench_uri
+
+  anon_token          = var.anon_token
+  service_token       = var.service_token
+  base_uri            = var.base_uri
+  conns               = var.conns
+  requests            = var.requests
+  rampscount          = var.rampscount
+  rampingduration     = var.rampingduration
+  consecutiveduration = var.consecutiveduration
+
+  # depends_on = [
+  #   module.setup_infra.ready,
+  # ]
+}

examples/supavisor/simple-select/modules/script/entrypoint.sh.tpl

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# update golang and make sure go is in path
+wget https://golang.org/dl/go1.19.linux-amd64.tar.gz
+sudo rm -rf /usr/local/go && sudo tar -C /usr/local -xzf go1.19.linux-amd64.tar.gz
+export PATH=$PATH:/usr/local/go/bin
+
+# build k6 with xk6 plugins, you may add some extra plugins here if needed
+export K6_VERSION='v0.37.0'
+~/go/bin/xk6 build --output /tmp/k6/k6 \
+  --with github.com/jdheyburn/[email protected] \
+  --with github.com/grafana/xk6-sql@659485a
+
+# run telegraf to collect metrics from k6 and host and push them to prometheus
+telegraf --config telegraf.conf &>/dev/null &
+
+# go to k6 dir and run k6
+cd /tmp/k6 || exit 1
+
+# leave these as is. Supabench will pass it and it is needed to upload the report.
+export RUN_ID="${testrun_id}"
+export BENCHMARK_ID="${benchmark_id}"
+export TEST_RUN="${testrun_name}"
+export TEST_ORIGIN="${test_origin}"
+export SUPABENCH_TOKEN="${supabench_token}"
+export SUPABENCH_URI="${supabench_uri}"
+
+# this is the place to add your variables, required by benchmark.
+export ANON_TOKEN="${anon_token}"
+export SERVICE_TOKEN="${service_token}"
+export BASE_URI="${base_uri}"
+
+# make command from the k6 folder to run k6 benchmark, you can add some extra vars here if needed
+# Leave testrun_name as it is passed to k6 command to add global tag to all metrics for grafana!
+make load \
+  rampingduration="${rampingduration}" consecutiveduration="${consecutiveduration}" rampscount="${rampscount}" \
+  requests="${requests}" conns="${conns}" shift="${shift}" testrun="${testrun_name}"