diff --git a/.gitignore b/.gitignore index 58501031..c66edc4e 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,8 @@ *.so *.dylib bin/* +dashboard/containers/** +dashboard/bin/* **/main **/*.out diff --git a/README.md b/README.md index 867422e6..6c5fff43 100644 --- a/README.md +++ b/README.md @@ -15,31 +15,29 @@ The main executable is a [custom controller] that manages resources of kind [LoadTest]. This controller must be deployed to the cluster before load tests can be run on it. The controller is implemented with [kubebuilder]. -There is also a set of tools used to prepare prebuilt images and run batches of -tests. These tools are used to generate the dashboard linked from the [gRPC -performance benchmarking] page. For more information, see -[tools](tools/README.md). +There is also a set of [tools](tools/README.md) used to generate load test +configurations, prepare prebuilt images and run batches of tests. These tools +are used to run batches of tests for continuous integration. [Examples](config/samples/README.md) of load test configurations in the supported languages are also provided. [custom controller]: cmd/controller/main.go -[grpc performance benchmarking]: https://grpc.io/docs/guides/benchmarking/ [kubebuilder]: https://kubebuilder.io [loadtest]: config/crd/bases/e2etest.grpc.io_loadtests.yaml -## Contributing +## Dashboard -Welcome! Please read [how to contribute](CONTRIBUTING.md) before proceeding. +The data generated in continuous integration are saved to [BigQuery], and +displayed on a public dashboard linked from the [gRPC performance benchmarking] +page. -This project includes third party dependencies as git submodules. Be sure to -initialize and update them when setting up a development environment: +For more information, and to build your own dashboard, see +[dashboard](dashboard/README.md). -```shell -# Init/update during the clone -git clone --recursive https://github.com/grpc/test-infra.git # HTTPS -git clone --recursive git@github.com:grpc/test-infra.git # SSH +[bigquery]: https://cloud.google.com/bigquery +[grpc performance benchmarking]: https://grpc.io/docs/guides/benchmarking/ -# (or) Init/update after the clone -git submodule update --init -``` +## Contributing + +Welcome! Please read [how to contribute](CONTRIBUTING.md) before proceeding. diff --git a/dashboard/Makefile b/dashboard/Makefile index 77c1379f..19c84cfd 100644 --- a/dashboard/Makefile +++ b/dashboard/Makefile @@ -1,25 +1,28 @@ GOCMD ?= go -GOARGS= -trimpath -REPLICATOR_OUTPUT_DIR ?= bin +GOARGS = -trimpath CONFIG_TEMPLATE_DIR ?= config +DASHBOARDS_CONFIG_DIR ?= $(CONFIG_TEMPLATE_DIR)/grafana/dashboards/default +REPLICATOR_CONFIG_TEMPLATE ?= $(CONFIG_TEMPLATE_DIR)/postgres_replicator/default/config.yaml +REPLICATOR_OUTPUT_DIR ?= bin GRAFANA_CONTAINER_OUTPUT_DIR ?= containers/grafana REPLICATOR_CONTAINER_OUTPUT_DIR ?= containers/replicator configure-grafana: scripts/check_env.sh GCP_GRAFANA_SERVICE PG_USER PG_PASS PG_DATABASE GRAFANA_ADMIN_PASS CLOUD_SQL_INSTANCE mkdir -p $(GRAFANA_CONTAINER_OUTPUT_DIR) - cp -r $(CONFIG_TEMPLATE_DIR)/grafana/dashboards/default $(GRAFANA_CONTAINER_OUTPUT_DIR)/dashboards + cp -r $(DASHBOARDS_CONFIG_DIR) $(GRAFANA_CONTAINER_OUTPUT_DIR)/dashboards cp -r grafana/* $(GRAFANA_CONTAINER_OUTPUT_DIR) cp $(CONFIG_TEMPLATE_DIR)/grafana/Dockerfile $(GRAFANA_CONTAINER_OUTPUT_DIR)/Dockerfile envsubst < $(CONFIG_TEMPLATE_DIR)/grafana/app.yaml > $(GRAFANA_CONTAINER_OUTPUT_DIR)/app.yaml configure-replicator: scripts/check_env.sh GCP_PROJECT_ID GCP_DATA_TRANSFER_SERVICE BQ_PROJECT_ID PG_USER PG_PASS PG_DATABASE CLOUD_SQL_INSTANCE - mkdir -p containers/replicator + mkdir -p $(REPLICATOR_CONTAINER_OUTPUT_DIR) + envsubst < $(REPLICATOR_CONFIG_TEMPLATE) > $(REPLICATOR_CONTAINER_OUTPUT_DIR)/config.yaml cp $(CONFIG_TEMPLATE_DIR)/postgres_replicator/Dockerfile $(REPLICATOR_CONTAINER_OUTPUT_DIR)/Dockerfile envsubst < $(CONFIG_TEMPLATE_DIR)/postgres_replicator/app.yaml > $(REPLICATOR_CONTAINER_OUTPUT_DIR)/app.yaml - envsubst < $(CONFIG_TEMPLATE_DIR)/postgres_replicator/default/config.yaml > $(REPLICATOR_CONTAINER_OUTPUT_DIR)/config.yaml +# Replicator target is for testing only. Production binary is built by Docker. replicator: fmt vet $(GOCMD) build $(GOARGS) -o $(REPLICATOR_OUTPUT_DIR)/replicator cmd/postgres_replicator/main.go diff --git a/dashboard/README.md b/dashboard/README.md new file mode 100644 index 00000000..c5be0056 --- /dev/null +++ b/dashboard/README.md @@ -0,0 +1,76 @@ +# Dashboard + +This folder contains the components necessary to build and deploy a dashboard to +visualize gRPC OSS benchmarking results. + +gRPC OSS benchmarks save results to [BigQuery]. The dashboard consists of two +components: + +1. A [Postgres replicator], to transfer the results to a Postgres database. +1. A Grafana dashboard, to displays the results from the Postgres database. + +These components can be built and deployed manually using the +[Makefile](Makefile) (see [manual build](#manual-build)). + +Notice that the dashboard build is independent from the top-level build. + +## Configuration + +The configuration of the Postgres replicator is defined in a YAML file. The +default configuration is defined here, in template form: + +- [config/postgres_replicator/default/config.yaml][postgres replicator config] + +For more information, see [Postgres replicator]. + +The configuration of the Grafana dashboard is defined in a set of dashboards +specified in JSON files. The default configuration is defined here: + +- [config/grafana/dashboards/default/][grafana dashboard config] + +The continuous integration dashboard linked from the [gRPC performance +benchmarking] page uses the default configuration. The variables +`REPLICATOR_CONFIG_TEMPLATE` and `DASHBOARDS_CONFIG_DIR` can be set to build +dashboards with different configurations. + +[bigquery]: https://cloud.google.com/bigquery +[grafana dashboard config]: config/grafana/dashboards/default/ +[grpc performance benchmarking]: https://grpc.io/docs/guides/benchmarking/ +[postgres replicator]: cmd/postgres_replicator/README.md +[postgres replicator config]: config/postgres_replicator/default/config.yaml + +## Manual build + +Several environment variables must be set before building and deploying. The +table below shows the names and values of the variables in our main dashboard: + +| Variable | Value | +| --------------------------- | --------------------------------------------- | +| `BQ_PROJECT_ID` | `grpc-testing` | +| `CLOUD_SQL_INSTANCE` | `grpc-testing:us-central1:grafana-datasource` | +| `GCP_DATA_TRANSFER_SERVICE` | `postgres-replicator` | +| `GCP_GRAFANA_SERVICE` | `grafana` | +| `GCP_PROJECT_ID` | `grpc-testing` | +| `GRAFANA_ADMIN_PASS` | \*\*\* | +| `PG_DATABASE` | `datasource` | +| `PG_PASS` | \*\*\* | +| `PG_USER` | `grafana-user` | + +Docker files that can be used to build and deploy the Postgres replicator and +Grafana dashboard are then created with the following commands: + +```shell +make configure-replicator +make configure-dashboard +``` + +## Cloud build + +The continuous integration dashboard is built and deployed with [Cloud Build], +using the configuration specified in [cloudbuild.yaml](cloudbuild.yaml). + +The use of Cloud Build allows the dashboard to be redeployed automatically on +configuration changes. In addition, it allows passwords such as `PG_PASS` and +`GRAFANA_ADMIN_PASS` to be stored as secrets in the cloud project. + +[cloud build]: https://cloud.google.com/build diff --git a/dashboard/cmd/postgres_replicator/README.md b/dashboard/cmd/postgres_replicator/README.md index 71350bc9..c2c6880d 100644 --- a/dashboard/cmd/postgres_replicator/README.md +++ b/dashboard/cmd/postgres_replicator/README.md @@ -4,7 +4,7 @@ This tool replicates streaming data from BigQuery into PostgreSQL. ## Configuration -It is configured with a YAML file. Here is an example configuration file: +The tool is configured with a YAML file. Here is an example configuration file: ```yaml # Source database settings @@ -52,7 +52,7 @@ This port number can be overridden via the `PORT` environment variable. ## Running -From the test-infra project root, run `make replicator`, then +From the dashboard project root, run `make replicator`, then `bin/replicator -c `. When the replicator receives a `GET` request for `/run`, it will transfer new @@ -64,8 +64,8 @@ ignore additional requests to `/run` (but still return `200`). 1. The data in the database must be sequentially ordered by time. Specifically, there must be a column of BigQuery datatype TIMESTAMP. This timestamp should strictly increase for new data. -2. Configuration only allows for one GCP project at a time. -3. All table names must be unique, even across multiple BigQuery datasets +1. Configuration only allows for one GCP project at a time. +1. All table names must be unique, even across multiple BigQuery datasets ## How it works @@ -87,13 +87,14 @@ BigQuery table. For non-nested columns, the replicator currently supports -- FLOAT -- STRING -- TIMESTAMP +- `FLOAT` +- `STRING` +- `TIMESTAMP` BigQuery table fields that are of the `RECORD`/`REPEATED`/`STRUCT`/`ARRAY` type are converted to JSON and stored in Postgres as the JSON datatype. To retrieve -values from the Postgres table, see the [JSON Functions and Operators] page. +values from the Postgres table, see the documentation on [JSON Functions and +Operators]. For example, if your BigQuery table has the FLOAT field `stats.client1.latency`, this could then be queried and typecast with the following in Postgres: diff --git a/dashboard/config/postgres_replicator/Dockerfile b/dashboard/config/postgres_replicator/Dockerfile index 6563cfeb..d973559e 100644 --- a/dashboard/config/postgres_replicator/Dockerfile +++ b/dashboard/config/postgres_replicator/Dockerfile @@ -21,8 +21,8 @@ ARG REPOSITORY=grpc/test-infra ARG GITREF=master RUN git clone https://github.com/$REPOSITORY.git src \ - && cd src/dashboard && git checkout $GITREF \ - && make replicator REPLICATOR_OUTPUT_DIR=/ + && cd src/dashboard && git checkout $GITREF \ + && make replicator REPLICATOR_OUTPUT_DIR=/ # Copy replicator binary and run it FROM golang:1.16 diff --git a/dashboard/scripts/check_env.sh b/dashboard/scripts/check_env.sh index f53117a2..34d6a41f 100755 --- a/dashboard/scripts/check_env.sh +++ b/dashboard/scripts/check_env.sh @@ -2,15 +2,16 @@ # check_env checks that all required environment variables are set check_env () { - env_vars=("$@") - missing_var=0 + local -a env_vars=( "$@" ) + local missing_var=0 + local var for var in "${env_vars[@]}"; do if [ -z "${!var}" ]; then echo "${var} not set" missing_var=1 fi done - return $missing_var + return ${missing_var} } check_env "$@"