From 64a9f152c223eb1f8fd8c2e4fa28d26ea62829ce Mon Sep 17 00:00:00 2001 From: Sanjay Vasandani Date: Wed, 26 Apr 2023 12:38:51 -0700 Subject: [PATCH] Update GKE correctness test guide to use GcsCorrectnessTest. This also updates all GKE guides to use Kustomize. --- build/k8s/defs.bzl | 7 +- build/repositories.bzl | 6 +- build/variables.bzl | 14 +- docs/gke/cluster-config.md | 2 +- docs/gke/correctness-test.md | 567 ++++-------------- docs/gke/duchy-deployment.md | 188 ++---- docs/gke/kingdom-deployment.md | 261 +++----- docs/gke/reporting-server-deployment.md | 168 ++---- src/main/docker/BUILD.bazel | 70 ++- src/main/docker/images.bzl | 12 +- src/main/docker/macros.bzl | 40 ++ src/main/k8s/BUILD.bazel | 24 - src/main/k8s/base.cue | 26 +- src/main/k8s/dev/BUILD.bazel | 228 +++++-- src/main/k8s/dev/config.cue | 9 +- .../k8s/dev/config_files_kustomization.yaml | 18 + src/main/k8s/dev/duchy_gke.cue | 2 +- .../dev/duchy_secret_kustomization.tmpl.yaml | 24 + src/main/k8s/dev/edp_simulator_gke.cue | 4 +- src/main/k8s/dev/frontend_simulator_gke.cue | 40 -- .../k8s/dev/kingdom_secret_kustomization.yaml | 24 + .../dev/panel_match_resource_setup_gke.cue | 31 - .../reporting_config_files_kustomization.yaml | 19 + src/main/k8s/dev/reporting_gke.cue | 3 +- src/main/k8s/dev/resource_setup_gke.cue | 26 - src/main/k8s/frontend_simulator.cue | 48 -- src/main/k8s/local/BUILD.bazel | 23 - src/main/k8s/local/README.md | 3 +- src/main/k8s/local/frontend_simulator.cue | 35 -- src/main/k8s/panel_match_resource_setup.cue | 54 -- src/main/k8s/resource_setup.cue | 90 --- .../k8s/{local => testing}/resource_setup.sh | 2 +- src/main/k8s/testing/secretfiles/BUILD.bazel | 6 + .../deploy/common/daemon/herald/BUILD.bazel | 4 +- .../daemon/mill/liquidlegionsv2/BUILD.bazel | 4 +- .../duchy/deploy/common/job/BUILD.bazel | 4 +- .../duchy/deploy/common/server/BUILD.bazel | 16 +- .../daemon/mill/liquidlegionsv2/BUILD.bazel | 4 +- .../duchy/deploy/gcloud/server/BUILD.bazel | 10 +- .../deploy/gcloud/spanner/tools/BUILD.bazel | 23 +- .../kingdom/deploy/common/server/BUILD.bazel | 6 +- .../kingdom/deploy/gcloud/server/BUILD.bazel | 4 +- .../deploy/gcloud/spanner/tools/BUILD.bazel | 20 +- .../loadtest/dataprovider/BUILD.bazel | 8 +- .../measurement/loadtest/frontend/BUILD.bazel | 76 --- ...ForwardedStorageFrontendSimulatorRunner.kt | 36 -- .../frontend/FrontendSimulatorFlags.kt | 112 ---- .../frontend/FrontendSimulatorRunner.kt | 107 ---- .../frontend/GcsFrontendSimulatorRunner.kt | 37 -- .../panelmatchresourcesetup/BUILD.bazel | 4 +- .../loadtest/resourcesetup/BUILD.bazel | 4 +- .../deploy/common/server/BUILD.bazel | 4 +- .../deploy/gcloud/postgres/server/BUILD.bazel | 4 +- .../deploy/gcloud/postgres/tools/BUILD.bazel | 23 +- .../deploy/postgres/server/BUILD.bazel | 4 +- .../deploy/postgres/tools/BUILD.bazel | 23 +- 56 files changed, 845 insertions(+), 1766 deletions(-) create mode 100644 src/main/docker/macros.bzl create mode 100644 src/main/k8s/dev/config_files_kustomization.yaml create mode 100644 src/main/k8s/dev/duchy_secret_kustomization.tmpl.yaml delete mode 100644 src/main/k8s/dev/frontend_simulator_gke.cue create mode 100644 src/main/k8s/dev/kingdom_secret_kustomization.yaml delete mode 100644 src/main/k8s/dev/panel_match_resource_setup_gke.cue create mode 100644 src/main/k8s/dev/reporting_config_files_kustomization.yaml delete mode 100644 src/main/k8s/dev/resource_setup_gke.cue delete mode 100644 src/main/k8s/frontend_simulator.cue delete mode 100644 src/main/k8s/local/frontend_simulator.cue delete mode 100644 src/main/k8s/panel_match_resource_setup.cue delete mode 100644 src/main/k8s/resource_setup.cue rename src/main/k8s/{local => testing}/resource_setup.sh (98%) delete mode 100644 src/main/kotlin/org/wfanet/measurement/loadtest/frontend/ForwardedStorageFrontendSimulatorRunner.kt delete mode 100644 src/main/kotlin/org/wfanet/measurement/loadtest/frontend/FrontendSimulatorFlags.kt delete mode 100644 src/main/kotlin/org/wfanet/measurement/loadtest/frontend/FrontendSimulatorRunner.kt delete mode 100644 src/main/kotlin/org/wfanet/measurement/loadtest/frontend/GcsFrontendSimulatorRunner.kt diff --git a/build/k8s/defs.bzl b/build/k8s/defs.bzl index 0ef0ce43f63..623419ed91f 100644 --- a/build/k8s/defs.bzl +++ b/build/k8s/defs.bzl @@ -244,7 +244,10 @@ def _kustomization_file_impl(ctx): dir_info = src[KustomizationDirInfo] resources.append(_relative_path(dir_info.path, dir_path)) else: - resources.append(src.files.to_list()[0].basename) + src_name = src.files.to_list()[0].basename + if not src_name.endswith(".yaml"): + fail("%s does not end in .yaml" % src_name) + resources.append(src_name) content_lines = ["resources:"] + [ "- " + resource @@ -263,7 +266,7 @@ _kustomization_file = rule( attrs = { "srcs": attr.label_list( doc = "Items to list in resources", - allow_files = [".yaml"], + allow_files = True, providers = [ [DefaultInfo], [KustomizationDirInfo], diff --git a/build/repositories.bzl b/build/repositories.bzl index cb7c1febb4b..8d70792a8ea 100644 --- a/build/repositories.bzl +++ b/build/repositories.bzl @@ -18,6 +18,8 @@ Adds external repos necessary for wfa_measurement_system. load("//build/wfa:repositories.bzl", "wfa_repo_archive") +MEASUREMENT_SYSTEM_REPO = "https://github.com/world-federation-of-advertisers/cross-media-measurement" + def wfa_measurement_system_repositories(): """Imports all direct dependencies for wfa_measurement_system.""" @@ -73,6 +75,6 @@ def wfa_measurement_system_repositories(): wfa_repo_archive( name = "wfa_rules_cue", repo = "rules_cue", - sha256 = "652379dec5174ed7fa8fe4223d0adf9a1d610ff0aa02e1bd1e74f79834b526a6", - version = "0.2.0", + sha256 = "0261b7797fa9083183536667958b1094fc732725fc48fca5cb68e6f731cdce2f", + version = "0.3.0", ) diff --git a/build/variables.bzl b/build/variables.bzl index cd73db966c6..f716bde0d78 100644 --- a/build/variables.bzl +++ b/build/variables.bzl @@ -56,23 +56,25 @@ TEST_K8S_SETTINGS = struct( GCLOUD_SETTINGS = struct( project = "$(google_cloud_project)", spanner_instance = "$(spanner_instance)", + postgres_instance = "$(postgres_instance)", + postgres_region = "$(postgres_region)", ) # Settings for Kingdom Kubernetes deployments. KINGDOM_K8S_SETTINGS = struct( - secret_name = "$(k8s_kingdom_secret_name)", + public_api_target = "$(kingdom_public_api_target)", + system_api_target = "$(kingdom_system_api_target)", ) # Settings for Duchy Kubernetes deployments. DUCHY_K8S_SETTINGS = struct( - secret_name = "$(k8s_duchy_secret_name)", certificate_id = "$(duchy_cert_id)", storage_bucket = "$(duchy_storage_bucket)", + public_api_target = "$(duchy_public_api_target)", ) # Settings for simulator Kubernetes deployments. SIMULATOR_K8S_SETTINGS = struct( - secret_name = "$(k8s_simulator_secret_name)", storage_bucket = "$(simulator_storage_bucket)", mc_name = "$(mc_name)", mc_api_key = "$(mc_api_key)", @@ -84,12 +86,6 @@ SIMULATOR_K8S_SETTINGS = struct( edp6_name = "$(edp6_name)", ) -# Settings for Reporting server Kubernetes deployments. -REPORTING_K8S_SETTINGS = struct( - secret_name = "$(k8s_reporting_secret_name)", - mc_config_secret_name = "$(k8s_reporting_mc_config_secret_name)", -) - # Settings for Grafana Kubernetes deployments. GRAFANA_K8S_SETTINGS = struct( secret_name = "$(k8s_grafana_secret_name)", diff --git a/docs/gke/cluster-config.md b/docs/gke/cluster-config.md index 999a70e3c4c..92831fcbcad 100644 --- a/docs/gke/cluster-config.md +++ b/docs/gke/cluster-config.md @@ -100,7 +100,7 @@ the `gcloud` and `bq` CLI tools. ```shell gcloud projects add-iam-policy-binding halo-cmm-dev \ --role=roles/bigquery.jobUser \ - --member='serviceAccount:simulator@halo-cmm-dev.iam.gserviceaccount.com` + --member='serviceAccount:simulator@halo-cmm-dev.iam.gserviceaccount.com' ``` ```shell diff --git a/docs/gke/correctness-test.md b/docs/gke/correctness-test.md index f032c785261..a734df95314 100644 --- a/docs/gke/correctness-test.md +++ b/docs/gke/correctness-test.md @@ -1,175 +1,147 @@ -# Multi-cluster Correctness Test on GKE +# Correctness Test on GKE -This documentation provides step-by-step instructions on how to complete a GKE -multi-cluster correctness test for the Cross-Media Measurement system. - -For the purposes of the correctness test, we use -[testing files](../../src/main/k8s/testing) that are checked-in to the -repository. These are *not* suitable for a production environment. +How to run the Kubernetes correctness test against a CMMS using simulators on +GKE. ## Overview -In the correctness test, we will create - -- 4 Spanner databases - - One for each Duchy as well as the Kingdom. - - These can all be in the same Spanner instance. -- 1 BigQuery table (`demo.labelled_events`) - - This will contain pre-generated Synthetic test data that will be used by - all EDP simulators. -- 5 Kubernetes clusters - - Kingdom - - Duchy `aggregator` - - Duchy `worker1` - - Duchy `worker2` - - Simulators - - 6 EDP simulators - - 1 MC frontend simulator - -***Since we are doing multi-cluster deployment, you will need your own domain to -manage the DNS records of the Kingdom and Duchies' public and system APIs. Make -sure you have a domain you can configure.*** +In order to run the correctness test, it is assumed that the CMMS instance has a +Kingdom as well as Duchies named `worker1`, `worker2`, and `aggregator`. -## Step 1. Deploy the Kingdom +See the [Kingdom deployment guide](kingdom-deployment.md) and +[Duchy deployment guide](duchy-deployment.md). -Follow the steps in [Kingdom Deployment](kingdom-deployment.md), with the -following modifications: +Note: The test currently also assumes that the CMMS instance is using the +[testing secret files](kingdom-deployment.md#secret-files-for-testing). +Therefore, the correctness test cannot be run on a production CMMS instance. -* Use the - [testing secret files](kingdom-deployment.md#secret-files-for-testing) for - your K8s secret. +## Run ResourceSetup -## Step 2. Set up Kingdom API resources +The `ResourceSetup` tool will create API resources for testing. If you have not +yet run the `ResourceSetup` tool against this CMMS instance, you will need to do +so. Note that this can only be done once per instance, and requires access to +the Kingdom cluster. -We have a `resource-setup` job that can be run in the Kingdom cluster to create -the Kingdom API resources that we'll use for the test. +First, build the tool: -This will create: +```shell +bazel build //src/main/kotlin/org/wfanet/measurement/loadtest/resourcesetup:ResourceSetup +``` -* 1 `Account` -* 1 `MeasurementConsumer` -* 1 API key -* 6 `DataProvider`s, named `edp1` through `edp6`. -* 3 Duchy `Certificate`s, one for each of the three Duchies. +We'll then need to be able to access the internal API from the host machine. +This can be done by forwarding the service port: -### Build and push container image +```shell +kubectl port-forward --address=localhost services/gcp-kingdom-data-server 9443:8443 +``` -You can build and push the container for this job by running (substituting the -values for your container registry): +Then run the tool, outputting to some directory (e.g. `/tmp/resource-setup`): ```shell -bazel run //src/main/docker:push_resource_setup_runner_image \ - -c opt --define container_registry=gcr.io \ - --define image_repo_prefix=halo-cmm-demo --define image_tag=build-0001 +src/main/k8s/testing/resource_setup.sh \ + --kingdom-public-api-target=v2alpha.kingdom.dev.halo-cmm.org:8443 \ + --kingdom-internal-api-target=localhost:9443 \ + --bazel-config-name=halo-dev \ + --output-dir=/tmp/resource-setup ``` -### Create and apply K8s manifest +Tip: The job will output a `resource-setup.bazelrc` file with `--define` options +that you can include in your `.bazelrc` file. You can then specify +`--config=halo-dev` to Bazel commands instead of those individual options. + +### Update the Kingdom + +After running the `ResourceSetup` tool, you will need to update the Kingdom +using its output. Copy the entries from the +`authority_key_identifier_to_principal_map.textproto` file output by the +`ResourceSetup` tool into your Kingdom Kustomization directory. You can then +apply the Kustomization to update the running Kingdom. -You can generate the K8s manifest for this job using the -`//src/main/k8s/dev:resource_setup_gke` Bazel target. Run the following -(substituting your Kingdom secret name): +Assuming your KUBECONFIG is pointing at the Kingdom cluster, run the following +from the Kustomization directory: ```shell -bazel build //src/main/k8s/dev:resource_setup_gke \ - --define=k8s_kingdom_secret_name=certs-and-configs-gb46dm7468 \ - --define container_registry=gcr.io \ - --define image_repo_prefix=halo-cmm-demo --define image_tag=build-0001 +kubectl apply -k src/main/k8s/dev/kingdom ``` -By default, this will generate the manifest at -`bazel-bin/src/main/k8s/dev/resource_setup_gke.yaml`. Copy this file to another -location and modify it to point to where you pushed the image. You can then -apply it using `kubectl`. +## Deploy EDP simulators -### Update ConfigMap +The correctness test assumes that you have six Event Data Provider (EDP) +simulators running, each acting as a different fake `DataProvider`. All of these +must write their sketches to a single Google Cloud Storage bucket. -Look at the log for the resource setup job: +### Initial Setup -```shell -kubectl logs -f jobs/resource-setup-job -``` +1. Create a Cloud Storage bucket -You should get something like + This can be done from the + [Console](https://console.cloud.google.com/storage/browser). Note that + bucket names are public, globally unique, and cannot be changed once + created. See + [Bucket naming guidelines](https://cloud.google.com/storage/docs/naming-buckets). -``` -Jan 26, 2022 9:50:43 PM org.wfanet.measurement.loadtest.resourcesetup.ResourceSetup process -INFO: Starting with RunID: 2022-01-2621-50-31-988 ... -Jan 26, 2022 9:50:51 PM org.wfanet.measurement.loadtest.resourcesetup.ResourceSetup process -INFO: Successfully created data provider: dataProviders/HRL1wWehTSM -Jan 26, 2022 9:50:51 PM org.wfanet.measurement.loadtest.resourcesetup.ResourceSetup process -INFO: Successfully created data provider: dataProviders/djQdz2ehSSE -Jan 26, 2022 9:50:51 PM org.wfanet.measurement.loadtest.resourcesetup.ResourceSetup process -INFO: Successfully created data provider: dataProviders/SQ99TmehSA8 -Jan 26, 2022 9:50:51 PM org.wfanet.measurement.loadtest.resourcesetup.ResourceSetup process -INFO: Successfully created data provider: dataProviders/TBZkB5heuL0 -Jan 26, 2022 9:50:51 PM org.wfanet.measurement.loadtest.resourcesetup.ResourceSetup process -INFO: Successfully created data provider: dataProviders/HOCBxZheuS8 -Jan 26, 2022 9:50:51 PM org.wfanet.measurement.loadtest.resourcesetup.ResourceSetup process -INFO: Successfully created data provider: dataProviders/VGExFmehRhY -Jan 26, 2022 9:50:58 PM org.wfanet.measurement.loadtest.resourcesetup.ResourceSetup process -INFO: Successfully created measurement consumer: measurementConsumers/TGWOaWehLQ8 -Jan 26, 2022 9:50:58 PM org.wfanet.measurement.loadtest.resourcesetup.ResourceSetup process -INFO: API key for measurement consumer measurementConsumers/TGWOaWehLQ8: ZEhkVZhe1Q0 -Jan 26, 2022 9:50:59 PM org.wfanet.measurement.loadtest.resourcesetup.ResourceSetup process -INFO: Successfully created certificate duchies/aggregator/certificates/DTDmi5he1do -Jan 26, 2022 9:50:59 PM org.wfanet.measurement.loadtest.resourcesetup.ResourceSetup process -INFO: Successfully created certificate duchies/worker1/certificates/Vr9cWmehKZM -Jan 26, 2022 9:50:59 PM org.wfanet.measurement.loadtest.resourcesetup.ResourceSetup process -INFO: Successfully created certificate duchies/worker2/certificates/QBC5Lphe1p0 -``` + As the data in this bucket need not be exposed to the public internet, + select "Enforce public access prevention on this bucket". -Note: We will use the values from the log in future commands. You may wish to -save it. +1. Create a K8s cluster -Tip: The job will output a `resource-setup.bazelrc` file with `--define` options -that you can include in your `.bazelrc` file. You can then specify -`--config=halo-kind` to Bazel commands instead of those individual options. + The simulators can run in their own cluster. You can use the Google Cloud + SDK to create a new one, substituting your own + [Use least privilege service account](https://cloud.google.com/kubernetes-engine/docs/how-to/hardening-your-cluster#use_least_privilege_sa) + address: -Now that we know the `DataProvider` resource names, we can update the AKID -mapping in the `config-files` ConfigMap. + ```shell + gcloud container clusters create simulators \ + --service-account="gke-cluster@halo-cmm-demo.iam.gserviceaccount.com" \ + --num-nodes=4 --enable-autoscaling --min-nodes=4 --max-nodes=8 \ + --machine-type=e2-small + ``` -The resource setup job will output an -`authority_key_identifier_to_principal_map.textproto` file with entries for each -of the test EDPs, using the AKIDs from the test certificates in -[secretfiles](../../src/main/k8s/testing/secretfiles). You can copy this file -and use it to replace the ConfigMap: + Point your KUBECONFIG to this cluster: -```shell -kubectl create configmap config-files --output=yaml --dry-run=client \ - --from-file=authority_key_identifier_to_principal_map.textproto \ - | kubectl replace -f - -``` + ```shell + gcloud container clusters get-credentials simulators + ``` -For more information on the file format, see -[Creating Resources](../operations/creating-resources.md). +1. Create a `simulator` K8s service account -You can then restart the Kingdom deployments that depend on `config-files`. At -the moment, this is just the public API server. + The underlying IAM service account must be able to access the Cloud Storage + bucket, create BigQuery jobs, and access the `labelled_events` BigQuery + table. See the [configuration guide](cluster-config.md#workload-identity) + for details. + +### Build and push simulator image + +If you aren't using pre-built release images, you can build the image yourself +from source and push them to a container registry. For example, if you're using +the [Google Container Registry](https://cloud.google.com/container-registry), +you would specify `gcr.io` as your container registry and your Cloud project +name as your image repository prefix. + +Assuming a project named `halo-cmm-demo` and an image tag `build-0001`, run the +following to build and push the image: ```shell -kubectl rollout restart deployments v2alpha-public-api-server-deployment +bazel run -c opt //src/main/docker:push_gcs_edp_simulator_runner_image \ + --define container_registry=gcr.io \ + --define image_repo_prefix=halo-cmm-demo --define image_tag=build-0001 ``` -## Step 3. Prepare EDP test data +### Prepare EDP test data The EDP simulators read their labelled events from a dataset in Google Cloud BigQuery. We can upload pre-generated synthetic test data from the [synthetic-labelled-events.csv](../../src/main/k8s/testing/data/synthetic-labelled-events.csv) file. -***(Note that in M1B, the EDP simulators do a fixed query since the EventFilter -implementation is not done yet. In other words, all measurements will have the -same result but with random noise. In M2, we will be able to specify query -parameters in the Frontend simulator and create different measurements.)*** - 1. Visit the GCloud console [Bigquery](https://console.cloud.google.com/bigquery) page. Enable the BiqQuery API in the project if you haven't done it yet. 2. Create a dataset - - Click the three-dot button to the right of the `halo-cmm-demo` project + - Click the three-dot button to the right of the project - Click `Create dataset` - Set the "Dataset ID" to `demo` - - Set the "data location" to `us-central1` + - Set the "data location" to (e.g. `us-central1`) - Click `CREATE DATASET` 3. Create a table - Click the three-dot button to the right of the `demo` data set. @@ -188,303 +160,61 @@ parameters in the Frontend simulator and create different measurements.)*** ![image-step-4-1](step-4-1.png)![image-step-4-1](step-4-2.png) -Now this synthetic test data is ready to use in the correctness test. - -## Step 4. Deploy the Duchies - -Follow the steps in the [Duchy deployment guide](duchy-deployment.md). You'll -need to repeat the process for each of the three Duchies for the correctness -test: `aggregator`, `worker1`, and `worker2`. +You will need to ensure that the simulator service account has access to this +table. See +[Granting BigQuery table access](cluster-config.md#granting-bigquery-table-access). -There are separate BUILD targets to generate a manifest for each of the three -Duchies: - -* `//src/main/k8s/dev:aggregator_duchy_gke` -* `//src/main/k8s/dev:worker1_duchy_gke` -* `//src/main/k8s/dev:worker2_duchy_gke` - -You'll only need to build and push the images once,and you can share the IAM -service accounts across the Duchies. You will need a separate cluster for each -Duchy. - -Tip: Don't forget to use the `gcloud` CLI to switch which cluster `kubectl` is -connected to. e.g. - -```shell -gcloud container clusters get-credentials halo-cmm-aggregator-demo-cluster -``` +Now this synthetic test data is ready to use in the correctness test. -## Step 5. Update Duchy DNS records +### Generate K8s Kustomization -You can obtain the public IPs of a Duchy's public API and system API servers -using the following command +Run the following, substituting your own values: ```shell -kubectl get services -``` - -You should see something like the following: - -``` -NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE -aggregator-async-computation-control-server ClusterIP 10.16.12.88 8443/TCP 97m -aggregator-computation-control-server LoadBalancer 10.16.4.6 34.133.4.73 8443:30898/TCP 97m -aggregator-requisition-fulfillment-server LoadBalancer 10.16.1.214 35.224.63.238 8443:31523/TCP 97m -aggregator-spanner-computations-server ClusterIP 10.16.2.183 8443/TCP 97m -kubernetes ClusterIP 10.16.0.1 443/TCP 32h +bazel build //src/main/k8s/dev:edp_simulators.tar \ + --define=google_cloud_project=halo-cmm-demo \ + --define=simulator_storage_bucket=halo-cmm-demo-bucket \ + --define=kingdom_public_api_target=v2alpha.kingdom.dev.halo-cmm.org:8443 \ + --define=duchy_public_api_target=public.worker1.dev.halo-cmm.org:8443 \ + --define=mc_name=measurementConsumers/TGWOaWehLQ8 \ + --define=edp1_name=dataProviders/HRL1wWehTSM \ + --define=edp2_name=dataProviders/djQdz2ehSSE \ + --define=edp3_name=dataProviders/SQ99TmehSA8 \ + --define=edp4_name=dataProviders/TBZkB5heuL0 \ + --define=edp5_name=dataProviders/HOCBxZheuS8 \ + --define=edp6_name=dataProviders/VGExFmehRhY \ + --define container_registry=gcr.io \ + --define image_repo_prefix=halo-cmm-demo --define image_tag=build-0001 ``` -The above example is from an aggregator cluster. The public API is -`aggregator-requisition-fulfillment-server`, and we point a subdomain (e.g. -`public.aggregator.dev.halo-cmm.org`) to `35.224.63.238` The system API is -`aggregator-computation-control-server`, and we point a subdomain (e.g. -`system.aggregator.dev.halo-cmm.org`) to `34.133.4.73`. +Extract the generated archive to some directory. -You'll need to do this for each Duchy. +### Apply K8s Kustomization -### Verifying a Duchy is running - -Take `worker2` as an example. If you run +From the Kustomization directory, run: ```shell -kubectl get pods +kubectl apply -k src/main/k8s/dev/edp_simulators ``` -in that cluster, you should see something like the following - -``` -NAME READY STATUS RESTARTS AGE -worker2-async-computation-control-server-deployment-5f6b7dc6gt4 1/1 Running 0 107s -worker2-computation-control-server-deployment-6574d489db-4nlvd 1/1 Running 0 107s -worker2-herald-daemon-deployment-f4464fdbb-lwrmm 1/1 Running 0 107s -worker2-liquid-legions-v2-mill-daemon-deployment-55cdf8f78d7gxf 1/1 Running 0 107s -worker2-push-spanner-schema-job-cd2xk 0/1 Completed 0 107s -worker2-requisition-fulfillment-server-deployment-64b4d6bbfnz94 1/1 Running 0 107s -worker2-spanner-computations-server-deployment-f57b576bc-wvvhk 1/1 Running 1 107s -``` +## Run the correctness test -View the logs of the Mill +Run the following, substituting your own values: ```shell -kubectl logs deployment/worker2-liquid-legions-v2-mill-daemon-deployment +bazel test //src/test/kotlin/org/wfanet/measurement/integration/k8s:GcsCorrectnessTest + --test_output=streamed \ + --define=kingdom_public_api_target=v2alpha.kingdom.dev.halo-cmm.org:8443 \ + --define=google_cloud_project=halo-cmm-demo \ + --define=simulator_storage_bucket=cmm-demo-simulators \ + --define=mc_name=measurementConsumers/Rcn7fKd25C8 \ + --define=mc_api_key=W9q4zad246g ``` -It should show that it is running `pollAndProcessNextComputation`, but none is -available yet. (You may see connection errors earlier in the logs. This is fine, -as it could mean dependent servers weren't yet ready.) +The test generally takes around 6 minutes to complete, since that is how long +the MPC protocol takes to finish. Eventually, you should see logs like this ``` -... -Jan 26, 2022 11:35:49 PM org.wfanet.measurement.duchy.daemon.mill.MillBase pollAndProcessNextComputation -INFO: @Mill worker2-liquid-legions-v2-mill-daemon-deployment-55cdf8f78d7gxf: -No computation available, waiting for the next poll... -Jan 26, 2022 11:35:50 PM org.wfanet.measurement.duchy.daemon.mill.MillBase pollAndProcessNextComputation -INFO: @Mill worker2-liquid-legions-v2-mill-daemon-deployment-55cdf8f78d7gxf: -Polling available computations... -Jan 26, 2022 11:35:50 PM org.wfanet.measurement.duchy.daemon.mill.MillBase pollAndProcessNextComputation -INFO: @Mill worker2-liquid-legions-v2-mill-daemon-deployment-55cdf8f78d7gxf: -No computation available, waiting for the next poll... ... -``` - -## Step 6. Deploy the EDP Simulators - -In this step, we deploy 6 EDP simulators in the same GCP clusters. Each of them -acts as one of the 6 different EDPs. - -1. Create Cloud Storage bucket - - While it may be possible to share one of a Duchy bucket, it's probably safer - to create a separate storage bucket for the simulators. Follow the same - process as creating a storage bucket for a Duchy. - -1. Create K8s cluster - - You may use the same cluster service account as the Kingdom/Duchies. - - ```shell - gcloud container clusters create halo-cmm-simulator-demo-cluster \ - --service-account="gke-cluster@halo-cmm-demo.iam.gserviceaccount.com" \ - --num-nodes=4 --enable-autoscaling --min-nodes=4 --max-nodes=8 \ - --machine-type=e2-small - ``` - -1. Configure `kubectl` to connect to the cluster - - ```shell - gcloud container clusters get-credentials halo-cmm-simulator-demo-cluster - ``` - -1. Create a `simulator` K8s service account - - Follow the same steps as in Kingdom/Duchy deployment. The underlying IAM - service account must be able to access the Cloud Storage bucket, create - BigQuery jobs, and access the `labelled_events` BigQuery table. See the - [configuration guide](cluster-config.md#workload-identity) for details. - -1. Create the k8s secret which contains the certificates and config files used - by the EDP simulators. - - ```shell - bazel run //src/main/k8s/testing/secretfiles:apply_kustomization - ``` - - Since we are using the same files to create the secret, the name of the k8s - secret should be the same as the one in the kingdom cluster. And in our - case, it is certs-and-configs-gb46dm7468 - -1. Push container image - - ```shell - bazel run -c opt //src/main/docker:push_gcs_edp_simulator_runner_image \ - --define container_registry=gcr.io \ - --define image_repo_prefix=halo-cmm-demo --define image_tag=build-0001 - ``` - -1. Generate K8s manifest - - The CUE files for the EDP simulators are - [`config.cue`](../../src/main/k8s/dev/config.cue) and - [`edp_simulator_gke.cue`](../../src/main/k8s/dev/edp_simulator_gke.cue). - - Update the definitions in these files for your configuration. For example: - - ```cue - #GloudProject: "halo-cmm-demo" - #ContainerRegistry: "gcr.io" - ``` - - ```cue - #KingdomPublicApiTarget: "your kingdom public API domain/subdomain:8443" - #DuchyPublicApiTarget: "your kingdom system API domain/subdomain:8443" - ``` - - Generate the manifest using the `//src/main/k8s/dev:edp_simulator_gke` - target: - - ```shell - bazel build //src/main/k8s/dev:edp_simulator_gke \ - --define=k8s_simulator_secret_name=certs-and-configs-gb46dm7468 \ - --define=simulator_storage_bucket=halo-cmm-demo-bucket \ - --define=mc_name=measurementConsumers/TGWOaWehLQ8 \ - --define=edp1_name=dataProviders/HRL1wWehTSM \ - --define=edp2_name=dataProviders/djQdz2ehSSE \ - --define=edp3_name=dataProviders/SQ99TmehSA8 \ - --define=edp4_name=dataProviders/TBZkB5heuL0 \ - --define=edp5_name=dataProviders/HOCBxZheuS8 \ - --define=edp6_name=dataProviders/VGExFmehRhY \ - --define container_registry=gcr.io \ - --define image_repo_prefix=halo-cmm-demo --define image_tag=build-0001 - ``` - -1. Apply the K8s manifest - - ```shell - k8s apply -f bazel-bin/src/main/k8s/dev/edp_simulator_gke.yaml - ``` - -1. Verify everything is fine. - - ```shell - kubectl get pods - ``` - - Example output showing all pods are running and ready: - - ``` - NAME READY STATUS RESTARTS AGE - edp1-simulator-deployment-d8bddf566-tm2sh 1/1 Running 0 78s - edp2-simulator-deployment-775bb96f55-2jzbm 1/1 Running 0 78s - edp3-simulator-deployment-5d48c86954-xlrfm 1/1 Running 0 78s - edp4-simulator-deployment-f44c67c66-6c8nn 1/1 Running 0 78s - edp5-simulator-deployment-86888c855b-l77fg 1/1 Running 0 77s - edp6-simulator-deployment-598d8f7d49-2qzh9 1/1 Running 0 77s - ``` - - Check the logs of any one of them: - - ```shell - kubectl logs -f deployments/edp1-simulator-deployment - ``` - - You should see something like this: - - ``` - Jan 26, 2022 10:55:16 PM org.wfanet.measurement.loadtest.dataprovider.EdpSimulator createEventGroup - INFO: Successfully created eventGroup dataProviders/HRL1wWehTSM/eventGroups/NeQ2xZiZsN0... - Jan 26, 2022 10:55:16 PM org.wfanet.measurement.loadtest.dataprovider.EdpSimulator executeRequisitionFulfillingWorkflow - INFO: Executing requisitionFulfillingWorkflow... - Jan 26, 2022 10:55:18 PM org.wfanet.measurement.loadtest.dataprovider.EdpSimulator executeRequisitionFulfillingWorkflow - INFO: No unfulfilled requisition. Polling again later... - Jan 26, 2022 10:55:18 PM org.wfanet.measurement.loadtest.dataprovider.EdpSimulator executeRequisitionFulfillingWorkflow - INFO: Executing requisitionFulfillingWorkflow... - Jan 26, 2022 10:55:18 PM org.wfanet.measurement.loadtest.dataprovider.EdpSimulator executeRequisitionFulfillingWorkflow - INFO: No unfulfilled requisition. Polling again later... ... - ``` - -## Step 7. Deploy the Frontend simulator test runner - -Now the kingdom + 3 duchies + 6 EDP simulators are all deployed and running. We -can deploy the Frontend simulator to act as the measurement consumer and create -a measurement. The Frontend simulator acts as the test runner for the -correctness test. It works by: - -- periodically (time interval of 30s) polling from the kingdom to get the - result of the measurement. -- then read the raw data provided by the EDP simulators and compute the - expected measurement result. -- compare the measured result from the kingdom to the expected result and make - sure they pass. - -1. Connect to the simulator cluster in `kubectl` by running: - - ```shell - gcloud container clusters get-credentials halo-cmm-simulator-demo-cluster - ``` - -2. Create the K8s secret - - ```shell - bazel run //src/main/k8s/testing/secretfiles:apply_kustomization - ``` - -3. Push container image - - ```shell - bazel run -c opt //src/main/docker:gcs_frontend_simulator_runner_image \ - --define container_registry=gcr.io \ - --define image_repo_prefix=halo-cmm-demo - ``` - -4. Generate the K8s manifest - - Similar to what we did for the EDP simulator, you may need to adjust some of - the definitions in - [`frontend_simulator_gke.cue`](../../src/main/k8s/dev/frontend_simulator_gke.cue). - - Generate the manifest using the `//src/main/k8s/dev:frontend_simulator_gke` - target: - - ```shell - bazel build //src/main/k8s/dev:frontend_simulator_gke \ - --define=k8s_simulator_secret_name=certs-and-configs-gb46dm7468 \ - --define=simulator_storage_bucket=halo-cmm-demo-bucket \ - --define=mc_name=measurementConsumers/TGWOaWehLQ8 \ - --define=mc_api_key=ZEhkVZhe1Q0 \ - --define container_registry=gcr.io \ - --define image_repo_prefix=halo-cmm-demo --define image_tag=build-0001 - ``` - -5. Apply the K8s manifest - - ```shell - k8s apply -f bazel-bin/src/main/k8s/dev/frontend_simulator_gke.yaml - ``` - -The frontend simulator job takes about 6 minutes to complete, since that is how -long the MPC protocol takes to finish. Eventually, you should see logs like this - -```shell -$ kubectl logs -f job.batch/frontend-simulator-job Jan 27, 2022 12:47:01 AM org.wfanet.measurement.loadtest.frontend.FrontendSimulator process INFO: Created measurement measurementConsumers/TGWOaWehLQ8/measurements/Y6gTFpj__3g. Jan 27, 2022 12:47:02 AM org.wfanet.measurement.loadtest.frontend.FrontendSimulator process @@ -536,8 +266,8 @@ databases. If something is wrong, we will see logs to debug. 1. Visit the GCloud console [spanner](https://console.cloud.google.com/spanner/instances) page. -2. Select demo-instance -3. Select kingdom +2. Select your instance +3. Select the `kingdom` database. 4. Click Query on the left ### Query the measurement status @@ -611,41 +341,6 @@ Example result ![query-3](query-3.png) -## Cleanup - -It costs money to keep all the kingdom, duchies and simulators running, even if -there is no measurement being computed. With the setting of the clusters in this -docs, it costs several tens of dollars per 24 hours. Most cost comes from these -Kubernetes clusters. - -If you are just curious about the project and want to try running the -correctnessTest once and don't plan to run it again in the near future. - -- Just delete the GCP project. Everything will be gone. - -If you plan to run the correctnessTest once in a while, but not frequently, for -example once a week. - -- Delete all GKE clusters in the GCP console -- No need to touch the GCloud storage bucket and BigQuery tables - -If you plan to run the correctnessTest lots of time, but want an empty database -before a certain run. - -- Delete all tables in the Spanner instance. -- or simply delete the Spanner instance and create it again. - -Note: In Step 2, the resources created are persisted in the Spanner database. -You can use the same resourceNames to complete as many correctnessTest as you -like. - -You only need to run step 2 if you have reset the kingdom Spanner database. In -other words, delete/recreate the clusters or redeploy the kingdom or any other -components, doesn't have any impact on the resources, thus the parameters in -those commands don't change. As a result, you should keep a copy of those result -names in the log of the ResourceSetupJob, if you want to reuse the same -resources (EDP, MC, etc.) for different measurements. - ## Troubleshooting If anything is wrong, first check diff --git a/docs/gke/duchy-deployment.md b/docs/gke/duchy-deployment.md index f7492b706c3..842a500e6c0 100644 --- a/docs/gke/duchy-deployment.md +++ b/docs/gke/duchy-deployment.md @@ -79,13 +79,15 @@ Duchy operator. ## Step 2. Create the database -The Duchy expects its own database within your Spanner instance. You can create -one with the `gcloud` CLI. For example, a database named -`worker1_duchy_computations` in the `dev-instance` instance. +The Duchy expects its own database within your Spanner instance. The `dev` +configuration assumes that this is named `_duchy_computations`. + +You can create a database using the `gcloud` CLI. For example with a Duchy named +`worker1` in the `halo-cmms` instance: ```shell gcloud spanner databases create worker1_duchy_computations \ - --instance=dev-instance + --instance=halo-cmms ``` ## Step 3. Create the Cloud Storage Bucket @@ -100,35 +102,24 @@ As the data in this bucket need not be exposed to the public internet, select ## Step 4. Build and push the container images -The `dev` configuration uses the -[Container Registry](https://cloud.google.com/container-registry) to store our -docker images. Enable the Google Container Registry API in the console if you -haven't done it. If you use other repositories, adjust the commands accordingly. +If you aren't using pre-built release images, you can build the images yourself +from source and push them to a container registry. For example, if you're using +the [Google Container Registry](https://cloud.google.com/container-registry), +you would specify `gcr.io` as your container registry and your Cloud project +name as your image repository prefix. Assuming a project named `halo-worker1-demo` and an image tag `build-0001`, run -the following to build the images: +the following to build and push the images: ```shell -bazel query 'filter("push_duchy", kind("container_push", //src/main/docker:all))' | - xargs bazel build -c opt --define container_registry=gcr.io \ - --define image_repo_prefix=halo-worker1-demo --define image_tag=build-0001 -``` - -and then push them: - -```shell -bazel query 'filter("push_duchy", kind("container_push", //src/main/docker:all))' | - xargs -n 1 bazel run -c opt --define container_registry=gcr.io \ +bazel run -c opt //src/main/docker:push_all_duchy_gke_images \ + --define container_registry=gcr.io \ --define image_repo_prefix=halo-worker1-demo --define image_tag=build-0001 ``` -You should see output like "Successfully pushed Docker image to -gcr.io/halo-worker1-demo/duchy/spanner-update-schema:build-0001" - Tip: If you're using [Hybrid Development](../building.md#hybrid-development) for containerized builds, replace `bazel build` with `tools/bazel-container build` -and `bazel run` with `tools/bazel-container-run`. You'll also want to pass the -`-o` option to `xargs`. +and `bazel run` with `tools/bazel-container-run`. ## Step 5. Create the Cluster @@ -178,10 +169,35 @@ service accounts. The `dev` configuration assumes that they are named See [Metrics Deployment](metrics-deployment.md). -## Step 6. Create Kubernetes secrets +## Step 6. Generate the K8s Kustomization + +Populating a cluster is generally done by applying a K8s Kustomization. You can +use the `dev` configuration as a base to get started. The Kustomization is +generated using Bazel rules from files written in [CUE](https://cuelang.org/). + +To generate the `dev` Kustomization, run the following (substituting your own +values): + +```shell +bazel build //src/main/k8s/dev:worker1_duchy.tar \ + --define kingdom_public_api_target=v2alpha.kingdom.dev.halo-cmm.org:8443 \ + --define google_cloud_project=halo-kingdom-demo \ + --define spanner_instance=halo-cmms \ + --define duchy_cert_id=SVVse4xWHL0 \ + --define duchy_storage_bucket=worker1-duchy \ + --define container_registry=gcr.io \ + --define image_repo_prefix=halo-worker1-demo --define image_tag=build-0001 +``` + +Extract the generated archive to some directory. -***(Note: this step does not use any halo code, and you don't need to do it -within the cross-media-measurement repo.)*** +You can customize this generated object configuration with your own settings +such as the number of replicas per deployment, the memory and CPU requirements +of each container, and the JVM options of each container. + +## Step 7. Customize the K8s secret + +We use a K8s secret to hold sensitive information, such as private keys. The Duchy binaries are configured to read certificates and config files from a mounted Kubernetes secret volume. @@ -234,128 +250,44 @@ files are required in a Duchy: - Set the role (aggregator or non_aggregator) in the config appropriately - [Example](../../src/main/k8s/testing/secretfiles/aggregator_protocols_setup_config.textproto) -Put all above files in the same folder (anywhere in your local machine), and -create a file named `kustomization.yaml` with the following content, -substituting the appropriate version of protocols setup config: - -```yaml -secretGenerator: -- name: certs-and-configs - files: - - all_root_certs.pem - - worker1_tls.pem - - worker1_tls.key - - worker1_cs_cert.der - - worker1_cs_private.der - - duchy_cert_config.textproto - - xxx_protocols_setup_config.textproto -``` - -and run - -```shell -kubectl apply -k -``` - -Now the secret is created in the `halo-cmm-worker1-demo-cluster`. You should be -able to see the secret by running - -```shell -kubectl get secrets -``` - -We assume the name is `certs-and-configs-abcdedf` and will use it in the -following documents. +Place these files into the `src/main/k8s/dev/worker1_duchy_secret/` path within +the Kustomization directory. ### Secret files for testing There are some [secret files](../../src/main/k8s/testing/secretfiles) within the -repository. These can be used to generate a secret for testing, but **must not** -be used for production environments as doing so would be highly insecure. +repository. These can be used for testing, but **must not** be used for +production environments as doing so would be highly insecure. + +Generate the archive: ```shell -bazel run //src/main/k8s/testing/secretfiles:apply_kustomization +bazel build //src/main/k8s/testing/secretfiles:archive ``` -## Step 7. Create the configmap +Extract the generated archive to the `src/main/k8s/dev/worker1_duchy_secret/` +path within the Kustomization directory. + +## Step 8. Customize the K8s configMap Configuration that may frequently change is stored in a K8s configMap. The `dev` configuration uses one named `config-files` containing the file -`authority_key_identifier_to_principal_map.textproto`. This file is initially -empty. +`authority_key_identifier_to_principal_map.textproto`. -```shell -kubectl create configmap config-files \ - --from-file=authority_key_identifier_to_principal_map.textproto=/dev/null -``` +Place this file in the `src/main/k8s/dev/config_files/` path within the +Kustomization directory. See [Creating Resources](../operations/creating-resources.md) for information on this file format. -## Step 8. Create the K8s manifest - -Deploying the Duchy to the cluster is generally done by applying a K8s manifest. -You can use the `dev` configuration as a base to get started. The `dev` manifest -is a YAML file that is generated from files written in -[CUE](https://cuelang.org/) using Bazel rules. - -The main file for the `dev` Duchy is -[`duchy_gke.cue`](../../src/main/k8s/dev/duchy_gke.cue). Some configuration is -in [`config.cue`](../../src/main/k8s/dev/config.cue) You can modify these file -to specify your own values for your Google Cloud project and Spanner instance. -**Do not** push your modifications to the repository. - -For example, - -``` -# KingdomSystemApiTarget: "your kingdom's system API domain or subdomain:8443" -# GloudProject: "halo-worker1-demo" -# SpannerInstance: "halo-worker1-instance" -# CloudStorageBucket: "halo-worker1-bucket" -``` - -``` -_computation_control_targets: { - "aggregator": "your aggregator's system API domain:8443" - "worker1": "your worker1's system API domain:8443" - "worker2": "your worker2's system API domain:8443" -} -``` - -You can also modify things such as the number of replicas per deployment, the -memory and CPU requirements of each container, and the JVM options of each -container. - -To generate the YAML manifest from the CUE files, run the following -(substituting your own values for the `--define` options): - -```shell -bazel build //src/main/k8s/dev:worker1_duchy_gke \ - --define k8s_duchy_secret_name=certs-and-configs-abcdedg \ - --define duchy_cert_id=SVVse4xWHL0 \ - --define duchy_storage_bucket=worker1-duchy \ - --define container_registry=gcr.io \ - --define image_repo_prefix=halo-worker1-demo --define image_tag=build-0001 -``` - -You can also do your customization to the generated YAML file rather than to the -CUE file. +## Step 9. Apply the K8s Kustomization -Note: The `dev` configuration does not specify a tag or digest for the container -images. You likely want to change this for a production environment. - -## Step 9. Apply the K8s manifest - -If you're using a manifest generated by the -`//src/main/k8s/dev:worker1_duchy_gke` Bazel target, the command to apply that -manifest is +Use `kubectl` to apply the Kustomization. From the Kustomization directory run: ```shell -kubectl apply -f bazel-bin/src/main/k8s/dev/worker1_duchy_gke.yaml +kubectl apply -k src/main/k8s/dev/worker1_duchy ``` -Substitute that path if you're using a different K8s manifest. - Now all Duchy components should be successfully deployed to your GKE cluster. You can verify by running diff --git a/docs/gke/kingdom-deployment.md b/docs/gke/kingdom-deployment.md index d04fe2c2c63..5cf456c4e8b 100644 --- a/docs/gke/kingdom-deployment.md +++ b/docs/gke/kingdom-deployment.md @@ -71,57 +71,42 @@ following: 2. Enable the `Cloud Spanner API` if you have not done so yet. 3. Click Create Instance - Notes: - - * Our `dev` configuration uses `dev-instance` as the instance name. - * 100 processing units is the current minimum value. This should be enough - to test things out, but you will likely want to adjust this depending on - expected load. +Note: 100 processing units is the current minimum value. This should be enough +to test things out, but you will likely want to adjust this depending on +expected load. ## Step 1. Create the database -The Kingdom expects its own database within your Spanner instance. You can -create one with the `gcloud` CLI. For example, a database named `kingdom` in the -`dev-instance` instance. +The Kingdom expects its own database within your Spanner instance. The `dev` +configuration assumes that this is named `kingdom`. + +You can create a database using the `gcloud` CLI. For example in the `halo-cmms` +instance: ```shell -gcloud spanner databases create kingdom --instance=dev-instance +gcloud spanner databases create kingdom --instance=halo-cmms ``` ## Step 2. Build and push the container images -The `dev` configuration uses the -[Container Registry](https://cloud.google.com/container-registry) to store our -docker images. Enable the Google Container Registry API in the console if you -haven't done it. If you use other repositories, adjust the commands accordingly. +If you aren't using pre-built release images, you can build the images yourself +from source and push them to a container registry. For example, if you're using +the [Google Container Registry](https://cloud.google.com/container-registry), +you would specify `gcr.io` as your container registry and your Cloud project +name as your image repository prefix. Assuming a project named `halo-kingdom-demo` and an image tag `build-0001`, run -the following to build the images: - -```shell -bazel query 'filter("push_kingdom", kind("container_push", //src/main/docker:all))' | - xargs bazel build -c opt --define container_registry=gcr.io \ - --define image_repo_prefix=halo-kingdom-demo --define image_tag=build-0001 -``` - -and then push them: +the following to build and push the images: ```shell -bazel query 'filter("push_kingdom", kind("container_push", //src/main/docker:all))' | - xargs -n 1 bazel run -c opt --define container_registry=gcr.io \ +bazel run -c opt //src/main/docker:push_all_kingdom_gke_images \ + --define container_registry=gcr.io \ --define image_repo_prefix=halo-kingdom-demo --define image_tag=build-0001 ``` -You should see output like "Successfully pushed Docker image to -gcr.io/halo-kingdom-demo/kingdom/data-server:build-0001" - Tip: If you're using [Hybrid Development](../building.md#hybrid-development) for containerized builds, replace `bazel build` with `tools/bazel-container build` -and `bazel run` with `tools/bazel-container-run`. You'll also want to pass the -`-o` option to `xargs`. - -Note: You may want to add a specific tag for the images in your container -registry. +and `bazel run` with `tools/bazel-container-run`. ## Step 3. Create resources for the cluster @@ -214,10 +199,30 @@ kubectl annotate serviceaccount internal-server \ iam.gke.io/gcp-service-account=kingdom-internal@halo-kingdom-demo.iam.gserviceaccount.com ``` -## Step 6. Create K8s secret +## Step 6. Generate the K8s Kustomization + +Populating a cluster is generally done by applying a K8s Kustomization. You can +use the `dev` configuration as a base to get started. The Kustomization is +generated using Bazel rules from files written in [CUE](https://cuelang.org/). + +To generate the `dev` Kustomization, run the following (substituting your own +values): + +```shell +bazel build //src/main/k8s/dev:kingdom.tar \ + --define google_cloud_project=halo-kingdom-demo \ + --define spanner_instance=halo-cmms \ + --define container_registry=gcr.io \ + --define image_repo_prefix=halo-kingdom-demo --define image_tag=build-0001 +``` + +Extract the generated archive to some directory. + +You can customize this generated object configuration with your own settings +such as the number of replicas per deployment, the memory and CPU requirements +of each container, and the JVM options of each container. -***(Note: this step does not use any Halo code, and you don't need to do it -within the cross-media-measurement repo.)*** +## Step 7. Customize the K8s secret We use a K8s secret to hold sensitive information, such as private keys. @@ -244,6 +249,10 @@ First, prepare all the files we want to include in the Kubernetes secret. The Note: This assumes that all your root certificate PEM files end in newline. +1. `kingdom_root.pem` + + The root certificate of the Kingdom's CA. + 1. `kingdom_tls.pem` The Kingdom's TLS certificate. @@ -258,6 +267,12 @@ First, prepare all the files we want to include in the Kubernetes secret. The - [Example](../../src/main/k8s/testing/secretfiles/duchy_cert_config.textproto) +1. `duchy_id_config.textproto` + + Configuration mapping external (public) Duchy IDs to internal Duchy IDs. + + - [Example](../../src/main/k8s/testing/secretfiles/duchy_id_config.textproto) + 1. `llv2_protocol_config_config.textproto` Configuration for the Liquid Legions v2 protocol. @@ -267,113 +282,44 @@ First, prepare all the files we want to include in the Kubernetes secret. The ***The private keys are confidential to the Kingdom, and are generated by the Kingdom's certificate authority (CA).*** -To generate the secret, put all above files in the same folder (on your local -machine), and create a file with name `kustomization.yaml` with the following -content: - -``` -secretGenerator: -- name: certs-and-configs - files: - - all_root_certs.pem - - kingdom_tls.key - - kingdom_tls.pem - - duchy_cert_config.textproto - - llv2_protocol_config_config.textproto -``` - -and run - -```shell -kubectl apply -k -``` - -Now the secret is created in the cluster. You should be able to see the secret -by running - -```shell -kubectl get secrets -``` - -We assume the name is `certs-and-configs-abcdedf` and will use it in the -following documents. +Place these files into the `src/main/k8s/dev/kingdom_secret/` path within the +Kustomization directory. ### Secret files for testing There are some [secret files](../../src/main/k8s/testing/secretfiles) within the -repository. These can be used to generate a secret for testing, but **must not** -be used for production environments as doing so would be highly insecure. +repository. These can be used for testing, but **must not** be used for +production environments as doing so would be highly insecure. + +Generate the archive: ```shell -bazel run //src/main/k8s/testing/secretfiles:apply_kustomization +bazel build //src/main/k8s/testing/secretfiles:archive ``` -## Step 7. Create the K8s configMap +Extract the generated archive to the `src/main/k8s/dev/kingdom_secret/` path +within the Kustomization directory. + +## Step 8. Customize the K8s configMap Configuration that may frequently change is stored in a K8s configMap. The `dev` configuration uses one named `config-files` containing the file -`authority_key_identifier_to_principal_map.textproto`. This file is initially -empty. +`authority_key_identifier_to_principal_map.textproto`. -```shell -kubectl create configmap config-files \ - --from-file=authority_key_identifier_to_principal_map.textproto=/dev/null -``` +Place this file in the `src/main/k8s/dev/config_files/` path within the +Kustomization directory. See [Creating Resources](../operations/creating-resources.md) for information on this file format. -## Step 8. Create the K8s manifest - -Deploying the Kingdom to the cluster is generally done by applying a K8s -manifest. You can use the `dev` configuration as a base to get started. The -`dev` manifest is a YAML file that is generated from files written in -[CUE](https://cuelang.org/) using Bazel rules. - -The main file for the `dev` Kingdom is -[`kingdom_gke.cue`](../../src/main/k8s/dev/kingdom_gke.cue). Some configuration -is in [`config.cue`](../../src/main/k8s/dev/config.cue) You can modify these -file to specify your own values for your Spanner instance. **Do not** push your -modifications to the repository. - -For example, - -``` -# GloudProject: "halo-kingdom-demo" -# SpannerInstance: "halo-kingdom-demo-instance" -``` - -You can also modify things such as the number of replicas per deployment, the -memory and CPU requirements of each container, and the JVM options of each -container. - -To generate the YAML manifest from the CUE files, run the following -(substituting your own secret name and image tag): - -```shell -bazel build //src/main/k8s/dev:kingdom_gke \ - --define=k8s_kingdom_secret_name=certs-and-configs-abcdedg \ - --define container_registry=gcr.io \ - --define image_repo_prefix=halo-kingdom-demo --define image_tag=build-0001 -``` - -You can also do your customization to the generated YAML file rather than to the -CUE file. - -Note: The `dev` configuration does not specify a tag or digest for the container -images. You likely want to change this for a production environment. - -## Step 9. Apply the K8s manifest +## Step 9. Apply the K8s Kustomization -If you're using a manifest generated by the `//src/main/k8s/dev:kingdom_gke` -Bazel target, the command to apply that manifest is +Use `kubectl` to apply the Kustomization. From the Kustomization directory run: ```shell -kubectl apply -f bazel-bin/src/main/k8s/dev/kingdom_gke.yaml +kubectl apply -k src/main/k8s/dev/kingdom ``` -Substitute that path if you're using a different K8s manifest. - Now all Kingdom components should be successfully deployed to your GKE cluster. You can verify by running @@ -429,65 +375,6 @@ For example, in the halo dev instance, we have subdomains: The domains/subdomains are what the EDPs/MPs/MCs/Duchies use to communicate with the kingdom. -## Additional setting you may want to make - -After finishing the above steps, we have - -- 1 system API, 1 public API and 1 internal API running. -- Only gRPC requests are allowed and connections are via mTLS. -- All communications between pods within the cluster are also encrypted via - mTLS. -- Network policy is set such that - - only the system API and public API are accessible via the external IP - - only the Internal API is allowed to send requests outside (We plan to - restrict the target to only Cloud Spanner, not down yet). - -In this section, we list some additional settings/configurations you may want to -consider. They are mostly for enhancing security. - -### 1. Application-layer secrets - -encryption Those certifications and configurations we stored in Kubernetes -secret are encrypted on the storage layer, but not on the application layer. In -other works, whoever has access to the cluster resource can just call - -```shell -kubectl get secrets secret_name -o json -``` - -to see the content of the files in the secret. - -This may not be an issue if there are only a small number of people that have -access to the cluster resources. These people should already have access to -those secret files if they need to be able to create them. - -However, if we want, we can enable Application-layer secrets encryption in the -cluster. - -- Go to Console -> Kubernetes Engine -> - [Clusters](https://console.cloud.google.com/kubernetes/list) -- Open the cluster you want to config Under Security, -- edit the "Application-layer secrets encryption" - -Note that you need to enable -[Cloud KMS](https://console.cloud.google.com/security/kms) in your GCP project -and create a private key for encrypting the secret. You also need to grant the -service account "cloudkms.cryptoKeyEncrypterDecrypter" role in the Console -> -[IAM & Admin](https://console.cloud.google.com/iam-admin) page. Check the -"include Google-provided role grants" to see the service account you are looking -for. - -(Note: Whether this part works or not is not confirmed yet.) - -### 2. Role Based Access Control - -You can use both IAM and Kubernetes -[RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) to control -access to your GKE cluster. GCloud provides the "Google Groups for RBAC" -feature. Follow this -[instruction](https://cloud.google.com/kubernetes-engine/docs/how-to/role-based-access-control) -if you want to set it up. - ## Q/A ### Q1. How to generate certificates/key pairs? @@ -509,10 +396,9 @@ Certificate requirements: Encryption keys can be generated using the [Tinkey tool](https://github.com/google/tink/blob/master/docs/TINKEY.md). -### Q2. What if the secret files need to be updated? +### Q2. What if the secret or configuration files need to be updated? -You'll need to recreate the K8s secret and update your cluster resources -accordingly. One way to do this is to update the K8s manifest and re-apply it. +Modify the Kustomization directory and re-apply it. ### Q3. How to test if the kingdom is working properly? @@ -520,7 +406,6 @@ Follow the ["How to complete multi-cluster correctnessTest on GKE"](correctness-test.md) doc and complete a correctness test using the Kingdom you have deployed. -If you don't want to deploy duchies and simulators, you can just deploy the -resourceSetupJob in the same kingdom cluster to see if you can create the -resources successfully. If yes, you can consider the Kingdom is working -properly. +If you don't want to deploy Duchies and simulators, you can just run +ResourceSetup to see if you can create the resources successfully. If yes, you +can consider the Kingdom is working properly. diff --git a/docs/gke/reporting-server-deployment.md b/docs/gke/reporting-server-deployment.md index 54c1ad62985..04802c1d54a 100644 --- a/docs/gke/reporting-server-deployment.md +++ b/docs/gke/reporting-server-deployment.md @@ -55,35 +55,24 @@ gcloud sql databases create reporting --instance=dev-postgres ## Build and push the container images -The `dev` configuration uses the -[Container Registry](https://cloud.google.com/container-registry) to store our -docker images. Enable the Google Container Registry API in the console if you -haven't done it. If you use other repositories, adjust the commands accordingly. +If you aren't using pre-built release images, you can build the images yourself +from source and push them to a container registry. For example, if you're using +the [Google Container Registry](https://cloud.google.com/container-registry), +you would specify `gcr.io` as your container registry and your Cloud project +name as your image repository prefix. Assuming a project named `halo-cmm-dev` and an image tag `build-0001`, run the following to build and push the images: ```shell -bazel query 'filter("reporting", kind("container_push", //src/main/docker:all))' | - xargs bazel build -c opt --define container_registry=gcr.io \ - --define image_repo_prefix=halo-cmm-dev --define image_tag=build-0001 -``` - -and then push them: - -```shell -bazel query 'filter("reporting", kind("container_push", //src/main/docker:all))' | - xargs -n 1 bazel run -c opt --define container_registry=gcr.io \ +bazel run -c opt //src/main/docker:push_all_reporting_gke_images \ + --define container_registry=gcr.io \ --define image_repo_prefix=halo-cmm-dev --define image_tag=build-0001 ``` Tip: If you're using [Hybrid Development](../building.md#hybrid-development) for containerized builds, replace `bazel build` with `tools/bazel-container build` -and `bazel run` with `tools/bazel-container-run`. You'll also want to pass the -`-o` option to `xargs`. - -Note: You may want to add a specific tag for the images in your container -registry. +and `bazel run` with `tools/bazel-container-run`. ## Create resources for the cluster @@ -182,7 +171,32 @@ kubectl annotate serviceaccount internal-reporting-server \ iam.gke.io/gcp-service-account=reporting-internal@halo-cmm-dev.iam.gserviceaccount.com ``` -## Create the K8s Secrets +## Generate the K8s Kustomization + +Populating a cluster is generally done by applying a K8s Kustomization. You can +use the `dev` configuration as a base to get started. The Kustomization is +generated using Bazel rules from files written in [CUE](https://cuelang.org/). + +To generate the `dev` Kustomization, run the following (substituting your own +values): + +```shell +bazel build //src/main/k8s/dev:reporting.tar \ + --define google_cloud_project=halo-cmm-dev \ + --define postgres_instance=dev-postgres \ + --define postgres_region=us-central1 \ + --define kingdom_public_api_target=v2alpha.kingdom.dev.halo-cmm.org:8443 \ + --define container_registry=gcr.io \ + --define image_repo_prefix=halo-kingdom-demo --define image_tag=build-0001 +``` + +Extract the generated archive to some directory. + +You can customize this generated object configuration with your own settings +such as the number of replicas per deployment, the memory and CPU requirements +of each container, and the JVM options of each container. + +## Customize the K8s secrets We use K8s secrets to hold sensitive information, such as private keys. @@ -231,10 +245,15 @@ There are some [testing keys](../../src/main/k8s/testing/secretfiles) within the repository. These can be used to create the above secret for testing, but **must not** be used for production environments as doing so would be highly insecure. +Generate the archive: + ```shell -bazel run //src/main/k8s/testing/secretfiles:apply_kustomization +bazel build //src/main/k8s/testing/secretfiles:archive ``` +Extract the generated archive to the `src/main/k8s/dev/reporting_secrets/` path +within the Kustomization directory. + ### Measurement Consumer config Contents: @@ -246,8 +265,11 @@ Contents: ### Generator -To generate secrets, put the files in a directory and create a -`kustomization.yaml` file within it to specify the contents of each secret. +Place the above files into the `src/main/k8s/dev/reporting_secrets/` path within +the Kustomization directory. + +Create a `kustomization.yaml` file in that path with the following content, +substituting the names of your own keys: ```yaml secretGenerator: @@ -264,21 +286,7 @@ secretGenerator: - measurement_consumer_config.textproto ``` -Apply the above to create the secrets: - -```shell -kubectl apply -k -``` - -The generated secret names will be suffixed with a hash. We'll assume `abcdef` -is the hash for convenience. If you lose track of the secret names, you can find -them again using - -```shell -kubectl get secrets -``` - -## Create the K8s ConfigMap +## Customize the K8s ConfigMap Configuration that may frequently change is stored in a K8s configMap. The `dev` configuration uses one named `config-files`, which contains configuration files @@ -290,69 +298,17 @@ in * `encryption_key_pair_config.textproto` - [`EncryptionKeyPairConfig`](../../src/main/proto/wfa/measurement/config/reporting/encryption_key_pair_config.proto) -You can pass all of these files to the `kubectl create configmap` command: - -```shell -kubectl create configmap config-files \ - --from-file=authority_key_identifier_to_principal_map.textproto \ - --from-file=encryption_key_pair_config.textproto -``` - -## Create the K8s manifest - -Deploying to the cluster is generally done by applying a K8s manifest. You can -use the `dev` configuration as a base to get started. The `dev` manifest is a -YAML file that is generated from files written in [CUE](https://cuelang.org/) -using Bazel rules. +Place these files into the `src/main/k8s/dev/reporting_config_files/` path +within the Kustomization directory. -The main file for the `dev` Reporting server is -[`reporting_gke.cue`](../../src/main/k8s/dev/reporting_gke.cue). Some -configuration is in [`config.cue`](../../src/main/k8s/dev/config.cue) You can -modify these file to specify your own values for your own DB instance. **Do -not** push your modifications to the repository. - -For example, - -```cue -#GCloudProject: "foo-measurement" -#PostgresConfig: { - project: #GCloudProject - instance: "psql" - region: "europe-west2" -} -``` +## Apply the K8s Kustomization -You can also modify things such as the memory and CPU request/limit of each pod, -as well as the number of replicas per deployment. - -To generate the YAML manifest from the CUE files, run the following -(substituting your own secret name): +Within the Kustomization directory, run ```shell -bazel build //src/main/k8s/dev:reporting_gke \ - --define=k8s_reporting_secret_name=signing-abcdef - --define=k8s_reporting_mc_config_secret_name=mc-config-abcdef \ - --define container_registry=gcr.io \ - --define image_repo_prefix=halo-cmm-dev --define image_tag=build-0001 +kubectl apply -k src/main/k8s/dev/reporting ``` -You can also do your customization to the generated YAML file rather than to the -CUE file. - -Note: The `dev` configuration does not specify a tag or digest for the container -images. You likely want to change this for a production environment. - -## Apply the K8s manifest - -If you're using a manifest generated by the above Bazel target, the command to -apply that manifest is - -```shell -kubectl apply -f bazel-bin/src/main/k8s/dev/reporting_gke.yaml -``` - -Substitute that path if you're using a different K8s manifest. - Now all components should be successfully deployed to your GKE cluster. You can verify by running @@ -369,24 +325,24 @@ kubectl get services You should see something like the following: ``` -NAME READY UP-TO-DATE AVAILABLE AGE -postgres-reporting-data-server-deployment 1/1 1 1 16h -v1alpha-public-api-server-deployment 1/1 1 1 16h +NAME READY UP-TO-DATE AVAILABLE AGE +postgres-reporting-data-server-deployment 1/1 1 1 254d +reporting-public-api-v1alpha-server-deployment 1/1 1 1 9m2s ``` ``` -NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE -kubernetes ClusterIP 10.16.32.1 443/TCP 6d21h -postgres-reporting-data-server ClusterIP 10.16.39.47 8443/TCP 20h -v1alpha-public-api-server LoadBalancer 10.16.46.241 34.135.79.68 8443:30290/TCP 20h +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +kubernetes ClusterIP 10.16.32.1 443/TCP 260d +postgres-reporting-data-server ClusterIP 10.16.39.47 8443/TCP 254d +reporting-public-api-v1alpha-server LoadBalancer 10.16.32.255 34.135.79.68 8443:30104/TCP 8m45s ``` ## Reserve an external IP -The `v1alpha-public-api-server` has an external load balancer IP so that it can -be accessed from outside the cluster. By default, the assigned IP address is -ephemeral. We can reserve a static IP to make it easier to access. See -[Reserving External IPs](cluster-config.md#reserving-external-ips). +The `reporting-public-api-v1alpha-server` service has an external load balancer +IP so that it can be accessed from outside the cluster. By default, the assigned +IP address is ephemeral. We can reserve a static IP to make it easier to access. +See [Reserving External IPs](cluster-config.md#reserving-external-ips). ## Appendix diff --git a/src/main/docker/BUILD.bazel b/src/main/docker/BUILD.bazel index 2c17388a4e4..e0a6690e436 100644 --- a/src/main/docker/BUILD.bazel +++ b/src/main/docker/BUILD.bazel @@ -5,7 +5,7 @@ load( ) load("@io_bazel_rules_docker//contrib:push-all.bzl", "docker_push") load("//build:variables.bzl", "IMAGE_REPOSITORY_SETTINGS") -load(":images.bzl", "ALL_IMAGES", "ALL_LOCAL_IMAGES") +load(":images.bzl", "ALL_GKE_IMAGES", "ALL_IMAGES", "ALL_LOCAL_IMAGES", "ALL_REPORTING_GKE_IMAGES") package( default_testonly = True, #TODO: delete when InMemoryKeyStore and FakeHybridCipher are not used. @@ -40,8 +40,76 @@ container_bundle( visibility = ["//visibility:private"], ) +container_bundle( + name = "all_gke_images", + images = {"{registry}/{name}:{tag}".format( + name = image_spec.repository, + registry = IMAGE_REPOSITORY_SETTINGS.container_registry, + tag = IMAGE_REPOSITORY_SETTINGS.image_tag, + ): image_spec.image for image_spec in ALL_GKE_IMAGES}, + tags = ["manual"], + visibility = ["//visibility:private"], +) + +container_bundle( + name = "kingdom_gke_images", + images = {"{registry}/{name}:{tag}".format( + name = image_spec.repository, + registry = IMAGE_REPOSITORY_SETTINGS.container_registry, + tag = IMAGE_REPOSITORY_SETTINGS.image_tag, + ): image_spec.image for image_spec in ALL_IMAGES if image_spec.name.startswith("kingdom_")}, + tags = ["manual"], + visibility = ["//visibility:private"], +) + +container_bundle( + name = "duchy_gke_images", + images = {"{registry}/{name}:{tag}".format( + name = image_spec.repository, + registry = IMAGE_REPOSITORY_SETTINGS.container_registry, + tag = IMAGE_REPOSITORY_SETTINGS.image_tag, + ): image_spec.image for image_spec in ALL_IMAGES if image_spec.name.startswith("duchy_")}, + tags = ["manual"], + visibility = ["//visibility:private"], +) + +container_bundle( + name = "reporting_gke_images", + images = {"{registry}/{name}:{tag}".format( + name = image_spec.repository, + registry = IMAGE_REPOSITORY_SETTINGS.container_registry, + tag = IMAGE_REPOSITORY_SETTINGS.image_tag, + ): image_spec.image for image_spec in ALL_REPORTING_GKE_IMAGES}, + tags = ["manual"], + visibility = ["//visibility:private"], +) + docker_push( name = "push_all_local_images", bundle = ":all_local_images", tags = ["manual"], ) + +docker_push( + name = "push_all_gke_images", + bundle = ":all_gke_images", + tags = ["manual"], +) + +docker_push( + name = "push_all_kingdom_gke_images", + bundle = ":kingdom_gke_images", + tags = ["manual"], +) + +docker_push( + name = "push_all_duchy_gke_images", + bundle = ":duchy_gke_images", + tags = ["manual"], +) + +docker_push( + name = "push_all_reporting_gke_images", + bundle = ":reporting_gke_images", + tags = ["manual"], +) diff --git a/src/main/docker/images.bzl b/src/main/docker/images.bzl index 0ea1c147af8..d98fd4b4c05 100644 --- a/src/main/docker/images.bzl +++ b/src/main/docker/images.bzl @@ -96,11 +96,6 @@ GKE_IMAGES = [ image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/daemon/mill/liquidlegionsv2:gcs_liquid_legions_v2_mill_daemon_image", repository = _PREFIX + "/duchy/liquid-legions-v2-mill", ), - struct( - name = "gcs_frontend_simulator_runner_image", - image = "//src/main/kotlin/org/wfanet/measurement/loadtest/frontend:gcs_frontend_simulator_runner_image", - repository = _PREFIX + "/loadtest/frontend-simulator", - ), struct( name = "gcs_edp_simulator_runner_image", image = "//src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider:gcs_edp_simulator_runner_image", @@ -130,11 +125,6 @@ LOCAL_IMAGES = [ image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/server:forwarded_storage_requisition_fulfillment_server_image", repository = _PREFIX + "/duchy/local-requisition-fulfillment", ), - struct( - name = "forwarded_storage_frontend_simulator_runner_image", - image = "//src/main/kotlin/org/wfanet/measurement/loadtest/frontend:forwarded_storage_frontend_simulator_runner_image", - repository = _PREFIX + "/simulator/local-mc", - ), struct( name = "forwarded_storage_edp_simulator_runner_image", image = "//src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider:forwarded_storage_edp_simulator_runner_image", @@ -186,3 +176,5 @@ ALL_GKE_IMAGES = COMMON_IMAGES + GKE_IMAGES + REPORTING_COMMON_IMAGES + REPORTIN ALL_LOCAL_IMAGES = COMMON_IMAGES + LOCAL_IMAGES + REPORTING_COMMON_IMAGES + REPORTING_LOCAL_IMAGES ALL_IMAGES = COMMON_IMAGES + LOCAL_IMAGES + GKE_IMAGES + REPORTING_COMMON_IMAGES + REPORTING_LOCAL_IMAGES + REPORTING_GKE_IMAGES + +ALL_REPORTING_GKE_IMAGES = REPORTING_COMMON_IMAGES + REPORTING_GKE_IMAGES diff --git a/src/main/docker/macros.bzl b/src/main/docker/macros.bzl new file mode 100644 index 00000000000..022c91b1c10 --- /dev/null +++ b/src/main/docker/macros.bzl @@ -0,0 +1,40 @@ +# Copyright 2023 The Cross-Media Measurement Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@io_bazel_rules_docker//java:image.bzl", "DEFAULT_JAVA_BASE", "jar_app_layer") +load("//build:repositories.bzl", "MEASUREMENT_SYSTEM_REPO") + +def java_image( + name, + binary, + main_class = None, + args = None, + base = None, + visibility = None, + **kwargs): + """Java container image. + + This is a replacement for the java_image rule from rules_docker which sets + common attrs. + """ + jar_app_layer( + name = name, + base = base or DEFAULT_JAVA_BASE, + binary = binary, + labels = {"org.opencontainers.image.source": MEASUREMENT_SYSTEM_REPO}, + main_class = main_class, + visibility = visibility, + args = args, + **kwargs + ) diff --git a/src/main/k8s/BUILD.bazel b/src/main/k8s/BUILD.bazel index f35951a9967..5bb719192eb 100644 --- a/src/main/k8s/BUILD.bazel +++ b/src/main/k8s/BUILD.bazel @@ -29,21 +29,6 @@ cue_library( srcs = ["spanner.cue"], ) -cue_library( - name = "resource_setup", - srcs = ["resource_setup.cue"], - deps = [ - ":base", - ":config", - ], -) - -cue_library( - name = "panel_match_resource_setup", - srcs = ["panel_match_resource_setup.cue"], - deps = [":base"], -) - cue_library( name = "kingdom", srcs = ["kingdom.cue"], @@ -63,15 +48,6 @@ cue_library( ], ) -cue_library( - name = "frontend_simulator", - srcs = ["frontend_simulator.cue"], - deps = [ - ":base", - ":config", - ], -) - cue_library( name = "edp_simulator", srcs = ["edp_simulator.cue"], diff --git a/src/main/k8s/base.cue b/src/main/k8s/base.cue index b460f2b2062..81ffdcfbec6 100644 --- a/src/main/k8s/base.cue +++ b/src/main/k8s/base.cue @@ -144,12 +144,28 @@ objects: [ for objectSet in objectSets for object in objectSet {object}] #CommonTarget: { host: string port: uint32 | string - target: "\(host):\(port)" + target: string } -#ServiceTarget: { +#HostPortTarget: this={ + #CommonTarget + + target: "\(this.host):\(this.port)" +} + +#StringTarget: this={ #CommonTarget + let parts = strings.Split(this.target, ":") + host: parts[0] + if len(parts) > 1 { + port: parts[1] + } +} + +#ServiceTarget: { + #HostPortTarget + serviceName: string let ServiceNameVar = strings.Replace(strings.ToUpper(serviceName), "-", "_", -1) @@ -157,7 +173,7 @@ objects: [ for objectSet in objectSets for object in objectSet {object}] port: "$(" + ServiceNameVar + "_SERVICE_PORT)" } -#Target: #CommonTarget | *#ServiceTarget | { +#Target: #HostPortTarget | #StringTarget | #ServiceTarget | { #ServiceTarget name: string @@ -165,7 +181,9 @@ objects: [ for objectSet in objectSets for object in objectSet {object}] } #GrpcTarget: GrpcTarget={ - *#CommonTarget | #ServiceTarget + #ServiceTarget | #StringTarget | #HostPortTarget + + port: _ | *443 certificateHost?: string diff --git a/src/main/k8s/dev/BUILD.bazel b/src/main/k8s/dev/BUILD.bazel index d0d1063dbaa..374a63ade68 100644 --- a/src/main/k8s/dev/BUILD.bazel +++ b/src/main/k8s/dev/BUILD.bazel @@ -2,13 +2,21 @@ load("@wfa_rules_cue//cue:defs.bzl", "cue_library") load( "//build:variables.bzl", "DUCHY_K8S_SETTINGS", + "GCLOUD_SETTINGS", "GRAFANA_K8S_SETTINGS", "IMAGE_REPOSITORY_SETTINGS", "KINGDOM_K8S_SETTINGS", - "REPORTING_K8S_SETTINGS", "SIMULATOR_K8S_SETTINGS", ) +load("@wfa_common_jvm//build:defs.bzl", "expand_template") load("//src/main/k8s:macros.bzl", "cue_dump") +load("//build/k8s:defs.bzl", "kustomization_dir") + +SECRET_NAME = "certs-and-configs" + +MC_CONFIG_SECRET_NAME = "mc-config" + +SIGNING_SECRET_NAME = "signing" cue_library( name = "base_gke", @@ -29,10 +37,12 @@ cue_dump( name = "kingdom_gke", srcs = ["kingdom_gke.cue"], cue_tags = { - "secret_name": KINGDOM_K8S_SETTINGS.secret_name, + "secret_name": SECRET_NAME, "container_registry": IMAGE_REPOSITORY_SETTINGS.container_registry, "image_repo_prefix": IMAGE_REPOSITORY_SETTINGS.repository_prefix, "image_tag": IMAGE_REPOSITORY_SETTINGS.image_tag, + "google_cloud_project": GCLOUD_SETTINGS.project, + "spanner_instance": GCLOUD_SETTINGS.spanner_instance, }, tags = ["manual"], deps = [ @@ -42,37 +52,29 @@ cue_dump( ], ) -cue_dump( - name = "resource_setup_gke", - srcs = ["resource_setup_gke.cue"], - cue_tags = { - "secret_name": KINGDOM_K8S_SETTINGS.secret_name, - "container_registry": IMAGE_REPOSITORY_SETTINGS.container_registry, - "image_repo_prefix": IMAGE_REPOSITORY_SETTINGS.repository_prefix, - "image_tag": IMAGE_REPOSITORY_SETTINGS.image_tag, - }, - tags = ["manual"], - deps = [ - ":base_gke", - ":config", - "//src/main/k8s:resource_setup", - ], +kustomization_dir( + name = "kingdom_secret", + srcs = ["kingdom_secret_kustomization.yaml"], + renames = {"kingdom_secret_kustomization.yaml": "kustomization.yaml"}, ) -cue_dump( - name = "panel_match_resource_setup_gke", - srcs = ["panel_match_resource_setup_gke.cue"], - cue_tags = { - "secret_name": KINGDOM_K8S_SETTINGS.secret_name, - "container_registry": IMAGE_REPOSITORY_SETTINGS.container_registry, - "image_repo_prefix": IMAGE_REPOSITORY_SETTINGS.repository_prefix, - "image_tag": IMAGE_REPOSITORY_SETTINGS.image_tag, - }, +kustomization_dir( + name = "config_files", + srcs = ["config_files_kustomization.yaml"], + renames = {"config_files_kustomization.yaml": "kustomization.yaml"}, +) + +kustomization_dir( + name = "kingdom", + srcs = [ + "resource_requirements.yaml", + ":kingdom_gke", + ], + generate_kustomization = True, tags = ["manual"], deps = [ - ":base_gke", - ":config", - "//src/main/k8s:panel_match_resource_setup", + "config_files", + ":kingdom_secret", ], ) @@ -82,12 +84,15 @@ cue_dump( cue_tags = { "duchy_name": "aggregator", "duchy_protocols_setup_config": "aggregator_protocols_setup_config.textproto", - "secret_name": DUCHY_K8S_SETTINGS.secret_name, + "secret_name": SECRET_NAME, "certificate_id": DUCHY_K8S_SETTINGS.certificate_id, "cloud_storage_bucket": DUCHY_K8S_SETTINGS.storage_bucket, "container_registry": IMAGE_REPOSITORY_SETTINGS.container_registry, "image_repo_prefix": IMAGE_REPOSITORY_SETTINGS.repository_prefix, "image_tag": IMAGE_REPOSITORY_SETTINGS.image_tag, + "google_cloud_project": GCLOUD_SETTINGS.project, + "spanner_instance": GCLOUD_SETTINGS.spanner_instance, + "kingdom_system_api_target": KINGDOM_K8S_SETTINGS.system_api_target, }, tags = ["manual"], deps = [ @@ -104,12 +109,15 @@ cue_dump( cue_tags = { "duchy_name": "worker1", "duchy_protocols_setup_config": "non_aggregator_protocols_setup_config.textproto", - "secret_name": DUCHY_K8S_SETTINGS.secret_name, + "secret_name": SECRET_NAME, "certificate_id": DUCHY_K8S_SETTINGS.certificate_id, "cloud_storage_bucket": DUCHY_K8S_SETTINGS.storage_bucket, "container_registry": IMAGE_REPOSITORY_SETTINGS.container_registry, "image_repo_prefix": IMAGE_REPOSITORY_SETTINGS.repository_prefix, "image_tag": IMAGE_REPOSITORY_SETTINGS.image_tag, + "google_cloud_project": GCLOUD_SETTINGS.project, + "spanner_instance": GCLOUD_SETTINGS.spanner_instance, + "kingdom_system_api_target": KINGDOM_K8S_SETTINGS.system_api_target, }, tags = ["manual"], deps = [ @@ -126,12 +134,15 @@ cue_dump( cue_tags = { "duchy_name": "worker2", "duchy_protocols_setup_config": "non_aggregator_protocols_setup_config.textproto", - "secret_name": DUCHY_K8S_SETTINGS.secret_name, + "secret_name": SECRET_NAME, "certificate_id": DUCHY_K8S_SETTINGS.certificate_id, "cloud_storage_bucket": DUCHY_K8S_SETTINGS.storage_bucket, "container_registry": IMAGE_REPOSITORY_SETTINGS.container_registry, "image_repo_prefix": IMAGE_REPOSITORY_SETTINGS.repository_prefix, "image_tag": IMAGE_REPOSITORY_SETTINGS.image_tag, + "google_cloud_project": GCLOUD_SETTINGS.project, + "spanner_instance": GCLOUD_SETTINGS.spanner_instance, + "kingdom_system_api_target": KINGDOM_K8S_SETTINGS.system_api_target, }, tags = ["manual"], deps = [ @@ -142,10 +153,101 @@ cue_dump( ], ) +expand_template( + name = "gen_aggregator_secret_kustomization", + out = "aggregator_secret_kustomization.yaml", + substitutions = { + "{duchy_id}": "aggregator", + "{duchy_role}": "aggregator", + }, + template = "duchy_secret_kustomization.tmpl.yaml", +) + +expand_template( + name = "gen_worker1_secret_kustomization", + out = "worker1_secret_kustomization.yaml", + substitutions = { + "{duchy_id}": "worker1", + "{duchy_role}": "non_aggregator", + }, + template = "duchy_secret_kustomization.tmpl.yaml", +) + +expand_template( + name = "gen_worker2_secret_kustomization", + out = "worker2_secret_kustomization.yaml", + substitutions = { + "{duchy_id}": "worker2", + "{duchy_role}": "non_aggregator", + }, + template = "duchy_secret_kustomization.tmpl.yaml", +) + +kustomization_dir( + name = "aggregator_duchy_secret", + srcs = [":aggregator_secret_kustomization.yaml"], + renames = {"aggregator_secret_kustomization.yaml": "kustomization.yaml"}, +) + +kustomization_dir( + name = "worker1_duchy_secret", + srcs = [":worker1_secret_kustomization.yaml"], + renames = {"worker1_secret_kustomization.yaml": "kustomization.yaml"}, +) + +kustomization_dir( + name = "worker2_duchy_secret", + srcs = [":worker2_secret_kustomization.yaml"], + renames = {"worker2_secret_kustomization.yaml": "kustomization.yaml"}, +) + +kustomization_dir( + name = "aggregator_duchy", + srcs = [ + "resource_requirements.yaml", + ":aggregator_duchy_gke", + ], + generate_kustomization = True, + tags = ["manual"], + deps = [ + ":aggregator_duchy_secret", + ":config_files", + ], +) + +kustomization_dir( + name = "worker1_duchy", + srcs = [ + "resource_requirements.yaml", + ":worker1_duchy_gke", + ], + generate_kustomization = True, + tags = ["manual"], + deps = [ + ":config_files", + ":worker1_duchy_secret", + ], +) + +kustomization_dir( + name = "worker2_duchy", + srcs = [ + "resource_requirements.yaml", + ":worker2_duchy_gke", + ], + generate_kustomization = True, + tags = ["manual"], + deps = [ + ":config_files", + ":worker2_duchy_secret", + ], +) + cue_dump( name = "edp_simulator_gke", srcs = ["edp_simulator_gke.cue"], cue_tags = { + "secret_name": SECRET_NAME, "mc_name": SIMULATOR_K8S_SETTINGS.mc_name, "edp1_name": SIMULATOR_K8S_SETTINGS.edp1_name, "edp2_name": SIMULATOR_K8S_SETTINGS.edp2_name, @@ -153,11 +255,13 @@ cue_dump( "edp4_name": SIMULATOR_K8S_SETTINGS.edp4_name, "edp5_name": SIMULATOR_K8S_SETTINGS.edp5_name, "edp6_name": SIMULATOR_K8S_SETTINGS.edp6_name, - "secret_name": SIMULATOR_K8S_SETTINGS.secret_name, + "google_cloud_project": GCLOUD_SETTINGS.project, "cloud_storage_bucket": SIMULATOR_K8S_SETTINGS.storage_bucket, "container_registry": IMAGE_REPOSITORY_SETTINGS.container_registry, "image_repo_prefix": IMAGE_REPOSITORY_SETTINGS.repository_prefix, "image_tag": IMAGE_REPOSITORY_SETTINGS.image_tag, + "kingdom_public_api_target": KINGDOM_K8S_SETTINGS.public_api_target, + "duchy_public_api_target": DUCHY_K8S_SETTINGS.public_api_target, }, tags = ["manual"], deps = [ @@ -167,24 +271,17 @@ cue_dump( ], ) -cue_dump( - name = "frontend_simulator_gke", - srcs = ["frontend_simulator_gke.cue"], - cue_tags = { - "mc_name": SIMULATOR_K8S_SETTINGS.mc_name, - "mc_api_key": SIMULATOR_K8S_SETTINGS.mc_api_key, - "secret_name": SIMULATOR_K8S_SETTINGS.secret_name, - "cloud_storage_bucket": SIMULATOR_K8S_SETTINGS.storage_bucket, - "container_registry": IMAGE_REPOSITORY_SETTINGS.container_registry, - "image_repo_prefix": IMAGE_REPOSITORY_SETTINGS.repository_prefix, - "image_tag": IMAGE_REPOSITORY_SETTINGS.image_tag, - }, +kustomization_dir( + name = "edp_simulators", + testonly = True, + srcs = [ + "resource_requirements.yaml", + ":edp_simulator_gke", + ], + generate_kustomization = True, tags = ["manual"], deps = [ - ":base_gke", - ":config", - "//src/main/k8s:base", - "//src/main/k8s:frontend_simulator", + "//src/main/k8s/testing/secretfiles:kustomization", ], ) @@ -192,11 +289,15 @@ cue_dump( name = "reporting_gke", srcs = ["reporting_gke.cue"], cue_tags = { - "secret_name": REPORTING_K8S_SETTINGS.secret_name, - "mc_config_secret_name": REPORTING_K8S_SETTINGS.mc_config_secret_name, + "secret_name": SIGNING_SECRET_NAME, + "mc_config_secret_name": MC_CONFIG_SECRET_NAME, "container_registry": IMAGE_REPOSITORY_SETTINGS.container_registry, "image_repo_prefix": IMAGE_REPOSITORY_SETTINGS.repository_prefix, "image_tag": IMAGE_REPOSITORY_SETTINGS.image_tag, + "google_cloud_project": GCLOUD_SETTINGS.project, + "postgres_instance": GCLOUD_SETTINGS.postgres_instance, + "postgres_region": GCLOUD_SETTINGS.postgres_region, + "kingdom_public_api_target": KINGDOM_K8S_SETTINGS.public_api_target, }, tags = ["manual"], deps = [ @@ -206,6 +307,30 @@ cue_dump( ], ) +kustomization_dir( + name = "reporting_config_files", + srcs = ["reporting_config_files_kustomization.yaml"], + renames = {"reporting_config_files_kustomization.yaml": "kustomization.yaml"}, +) + +kustomization_dir( + name = "reporting_secrets", +) + +kustomization_dir( + name = "reporting", + srcs = [ + "resource_requirements.yaml", + ":reporting_gke", + ], + generate_kustomization = True, + tags = ["manual"], + deps = [ + ":reporting_config_files", + ":reporting_secrets", + ], +) + cue_dump( name = "open_telemetry_gke", srcs = ["open_telemetry_gke.cue"], @@ -218,6 +343,7 @@ cue_dump( cue_dump( name = "prometheus_gke", srcs = ["prometheus_gke.cue"], + cue_tags = {"google_cloud_project": GCLOUD_SETTINGS.project}, tags = ["manual"], deps = [ ":base_gke", diff --git a/src/main/k8s/dev/config.cue b/src/main/k8s/dev/config.cue index 18a2adbd342..5dcfc2c2895 100644 --- a/src/main/k8s/dev/config.cue +++ b/src/main/k8s/dev/config.cue @@ -14,8 +14,7 @@ package k8s -#GCloudProject: "halo-cmm-dev" -#SpannerInstance: "dev-instance" +#GCloudProject: string @tag("google_cloud_project") #GCloudConfig: { project: #GCloudProject @@ -23,7 +22,7 @@ package k8s #SpannerConfig: { project: #GCloudProject - instance: #SpannerInstance + instance: string @tag("spanner_instance") readyTimeout: "30s" } @@ -60,8 +59,8 @@ package k8s #PostgresConfig: { project: #GCloudProject - instance: "dev-postgres" - region: "us-central1" + instance: string @tag("postgres_instance") + region: string @tag("postgres_region") } #PrometheusFrontendPort: 9090 diff --git a/src/main/k8s/dev/config_files_kustomization.yaml b/src/main/k8s/dev/config_files_kustomization.yaml new file mode 100644 index 00000000000..2ffb149570a --- /dev/null +++ b/src/main/k8s/dev/config_files_kustomization.yaml @@ -0,0 +1,18 @@ +# Copyright 2022 The Cross-Media Measurement Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +configMapGenerator: +- name: config-files + files: + - authority_key_identifier_to_principal_map.textproto diff --git a/src/main/k8s/dev/duchy_gke.cue b/src/main/k8s/dev/duchy_gke.cue index 7c81d9f8012..dffc2d6a563 100644 --- a/src/main/k8s/dev/duchy_gke.cue +++ b/src/main/k8s/dev/duchy_gke.cue @@ -22,7 +22,7 @@ _certificateId: string @tag("certificate_id") _duchy_cert_name: "duchies/\(_duchy_name)/certificates/\(_certificateId)" -#KingdomSystemApiTarget: "system.kingdom.dev.halo-cmm.org:8443" +#KingdomSystemApiTarget: string @tag("kingdom_system_api_target") #InternalServerServiceAccount: "internal-server" #StorageServiceAccount: "storage" #InternalServerResourceRequirements: #ResourceRequirements & { diff --git a/src/main/k8s/dev/duchy_secret_kustomization.tmpl.yaml b/src/main/k8s/dev/duchy_secret_kustomization.tmpl.yaml new file mode 100644 index 00000000000..22faf8c7d0d --- /dev/null +++ b/src/main/k8s/dev/duchy_secret_kustomization.tmpl.yaml @@ -0,0 +1,24 @@ +# Copyright 2023 The Cross-Media Measurement Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +secretGenerator: +- name: certs-and-configs + files: + - all_root_certs.pem + - {duchy_id}_tls.pem + - {duchy_id}_tls.key + - {duchy_id}_cs_cert.der + - {duchy_id}_cs_private.der + - duchy_cert_config.textproto + - {duchy_role}_protocols_setup_config.textproto diff --git a/src/main/k8s/dev/edp_simulator_gke.cue b/src/main/k8s/dev/edp_simulator_gke.cue index b419664bdfc..cc80246f5c6 100644 --- a/src/main/k8s/dev/edp_simulator_gke.cue +++ b/src/main/k8s/dev/edp_simulator_gke.cue @@ -25,8 +25,8 @@ _edpResourceNames: [_edp1_name, _edp2_name, _edp3_name, _edp4_name, _edp5_name, _secret_name: string @tag("secret_name") _cloudStorageBucket: string @tag("cloud_storage_bucket") -#KingdomPublicApiTarget: "public.kingdom.dev.halo-cmm.org:8443" -#DuchyPublicApiTarget: "public.worker1.dev.halo-cmm.org:8443" +#KingdomPublicApiTarget: string @tag("kingdom_public_api_target") +#DuchyPublicApiTarget: string @tag("duchy_public_api_target") #BigQueryDataSet: "demo" #BigQueryTable: "labelled_events" #ServiceAccount: "simulator" diff --git a/src/main/k8s/dev/frontend_simulator_gke.cue b/src/main/k8s/dev/frontend_simulator_gke.cue deleted file mode 100644 index 45382134599..00000000000 --- a/src/main/k8s/dev/frontend_simulator_gke.cue +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2021 The Cross-Media Measurement Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package k8s - -_mc_name: string @tag("mc_name") -_mc_api_key: string @tag("mc_api_key") -_secret_name: string @tag("secret_name") -_cloudStorageBucket: string @tag("cloud_storage_bucket") - -#KingdomPublicApiTarget: "public.kingdom.dev.halo-cmm.org:8443" -#ServiceAccount: "simulator" - -objectSets: [frontend_simulator] - -_cloudStorageConfig: #CloudStorageConfig & { - bucket: _cloudStorageBucket -} -frontend_simulator: #FrontendSimulator & { - _mc_resource_name: _mc_name - _mc_api_authentication_key: _mc_api_key - _mc_secret_name: _secret_name - _kingdom_public_api_target: #KingdomPublicApiTarget - _simulator_image: _imageConfig.image - _blob_storage_flags: _cloudStorageConfig.flags - job: spec: template: spec: #ServiceAccountPodSpec & { - serviceAccountName: #ServiceAccount - } -} diff --git a/src/main/k8s/dev/kingdom_secret_kustomization.yaml b/src/main/k8s/dev/kingdom_secret_kustomization.yaml new file mode 100644 index 00000000000..94c2ba00beb --- /dev/null +++ b/src/main/k8s/dev/kingdom_secret_kustomization.yaml @@ -0,0 +1,24 @@ +# Copyright 2023 The Cross-Media Measurement Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +secretGenerator: +- name: certs-and-configs + files: + - all_root_certs.pem + - kingdom_root.pem + - kingdom_tls.key + - kingdom_tls.pem + - duchy_cert_config.textproto + - duchy_id_config.textproto + - llv2_protocol_config_config.textproto diff --git a/src/main/k8s/dev/panel_match_resource_setup_gke.cue b/src/main/k8s/dev/panel_match_resource_setup_gke.cue deleted file mode 100644 index ec104fad507..00000000000 --- a/src/main/k8s/dev/panel_match_resource_setup_gke.cue +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2022 The Cross-Media Measurement Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package k8s - -_secret_name: string @tag("secret_name") - -objectSets: [ - job, -] - -_imageConfig: #ImageConfig & { - repoSuffix: "loadtest/panel-match-resource-setup" -} - -job: #PanelMatchResourceSetup & { - _edp_display_name: "edp1" - _job_image: _imageConfig.image - _resource_setup_secret_name: _secret_name -} diff --git a/src/main/k8s/dev/reporting_config_files_kustomization.yaml b/src/main/k8s/dev/reporting_config_files_kustomization.yaml new file mode 100644 index 00000000000..317d17a9e95 --- /dev/null +++ b/src/main/k8s/dev/reporting_config_files_kustomization.yaml @@ -0,0 +1,19 @@ +# Copyright 2022 The Cross-Media Measurement Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +configMapGenerator: +- name: config-files + files: + - authority_key_identifier_to_principal_map.textproto + - encryption_key_pair_config.textproto diff --git a/src/main/k8s/dev/reporting_gke.cue b/src/main/k8s/dev/reporting_gke.cue index b7da0a5e716..f32d1a4ac96 100644 --- a/src/main/k8s/dev/reporting_gke.cue +++ b/src/main/k8s/dev/reporting_gke.cue @@ -18,8 +18,7 @@ _reportingSecretName: string @tag("secret_name") _reportingMcConfigSecretName: string @tag("mc_config_secret_name") #KingdomApiTarget: #GrpcTarget & { - host: "public.kingdom.dev.halo-cmm.org" - port: 8443 + target: string @tag("kingdom_public_api_target") } // Name of K8s service account for the internal API server. diff --git a/src/main/k8s/dev/resource_setup_gke.cue b/src/main/k8s/dev/resource_setup_gke.cue deleted file mode 100644 index da44322c83d..00000000000 --- a/src/main/k8s/dev/resource_setup_gke.cue +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2021 The Cross-Media Measurement Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package k8s - -_secret_name: string @tag("secret_name") - -objectSets: [resourceSetup] - -resourceSetup: #ResourceSetup & { - _edp_display_names: ["edp1", "edp2", "edp3", "edp4", "edp5", "edp6"] - _duchy_ids: ["aggregator", "worker1", "worker2"] - _resource_setup_secret_name: _secret_name - _dependencies: ["gcp-kingdom-data-server"] -} diff --git a/src/main/k8s/frontend_simulator.cue b/src/main/k8s/frontend_simulator.cue deleted file mode 100644 index 2dc85b05d48..00000000000 --- a/src/main/k8s/frontend_simulator.cue +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2021 The Cross-Media Measurement Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package k8s - -#FrontendSimulator: { - _mc_resource_name: string - _mc_secret_name: string - _mc_api_authentication_key: string - _kingdom_public_api_target: string - _blob_storage_flags: [...string] - _imageConfig: #ImageConfig & { - repoSuffix: string | *"loadtest/frontend-simulator" - } - - job: #Job & { - _name: "frontend-simulator" - _secretName: _mc_secret_name - _container: { - image: _imageConfig.image - args: [ - "--tls-cert-file=/var/run/secrets/files/mc_tls.pem", - "--tls-key-file=/var/run/secrets/files/mc_tls.key", - "--cert-collection-file=/var/run/secrets/files/all_root_certs.pem", - "--kingdom-public-api-target=\(_kingdom_public_api_target)", - "--kingdom-public-api-cert-host=localhost", - "--mc-resource-name=\(_mc_resource_name)", - "--api-authentication-key=\(_mc_api_authentication_key)", - "--mc-consent-signaling-cert-der-file=/var/run/secrets/files/mc_cs_cert.der", - "--mc-consent-signaling-key-der-file=/var/run/secrets/files/mc_cs_private.der", - "--mc-encryption-private-keyset=/var/run/secrets/files/mc_enc_private.tink", - "--output-differential-privacy-epsilon=0.1", - "--output-differential-privacy-delta=0.000001", - ] + _blob_storage_flags - } - } -} diff --git a/src/main/k8s/local/BUILD.bazel b/src/main/k8s/local/BUILD.bazel index 2ae3c2c2bbc..618d8f86630 100644 --- a/src/main/k8s/local/BUILD.bazel +++ b/src/main/k8s/local/BUILD.bazel @@ -147,29 +147,6 @@ cue_dump( ], ) -cue_library( - name = "frontend_simulator_cue", - srcs = ["frontend_simulator.cue"], - deps = [ - ":config_cue", - "//src/main/k8s:frontend_simulator", - ], -) - -cue_dump( - name = "mc_frontend_simulator", - cue_tags = { - "secret_name": SECRET_NAME, - "container_registry": IMAGE_REPOSITORY_SETTINGS.container_registry, - "image_repo_prefix": IMAGE_REPOSITORY_SETTINGS.repository_prefix, - "image_tag": IMAGE_REPOSITORY_SETTINGS.image_tag, - "mc_name": TEST_K8S_SETTINGS.mc_name, - "mc_api_key": TEST_K8S_SETTINGS.mc_api_key, - }, - tags = ["manual"], - deps = [":frontend_simulator_cue"], -) - cue_dump( name = "postgres_database", srcs = ["postgres_database.cue"], diff --git a/src/main/k8s/local/README.md b/src/main/k8s/local/README.md index c0c53a4e7ff..b2a1d9297c4 100644 --- a/src/main/k8s/local/README.md +++ b/src/main/k8s/local/README.md @@ -109,9 +109,10 @@ kubectl port-forward --address=localhost services/gcp-kingdom-data-server 9443:8 Then run the tool, outputting to some directory (e.g. `/tmp/resource-setup`): ```shell -src/main/k8s/local/resource_setup.sh \ +src/main/k8s/testing/resource_setup.sh \ --kingdom-public-api-target=localhost:8443 \ --kingdom-internal-api-target=localhost:9443 \ + --bazel-config-name=halo-local \ --output-dir=/tmp/resource-setup ``` diff --git a/src/main/k8s/local/frontend_simulator.cue b/src/main/k8s/local/frontend_simulator.cue deleted file mode 100644 index 4fb41d193cb..00000000000 --- a/src/main/k8s/local/frontend_simulator.cue +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2021 The Cross-Media Measurement Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package k8s - -_mc_name: string @tag("mc_name") -_mc_api_key: string @tag("mc_api_key") -_secret_name: string @tag("secret_name") - -#KingdomPublicApiTarget: (#Target & {name: "v2alpha-public-api-server"}).target - -objectSets: [frontendSimulator] - -frontendSimulator: #FrontendSimulator & { - _imageConfig: repoSuffix: "simulator/local-mc" - _mc_resource_name: _mc_name - _mc_secret_name: _secret_name - _mc_api_authentication_key: _mc_api_key - _kingdom_public_api_target: #KingdomPublicApiTarget - _blob_storage_flags: [ - "--forwarded-storage-service-target=" + (#Target & {name: "fake-storage-server"}).target, - "--forwarded-storage-cert-host=localhost", - ] -} diff --git a/src/main/k8s/panel_match_resource_setup.cue b/src/main/k8s/panel_match_resource_setup.cue deleted file mode 100644 index a0d9efef283..00000000000 --- a/src/main/k8s/panel_match_resource_setup.cue +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2022 The Cross-Media Measurement Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package k8s - -#PanelMatchResourceSetup: { - _edp_display_name: string - _resource_setup_secret_name: string - _job_image: string - _job_image_pull_policy: string | *"Always" - _tls_cert_key_files_flags: [ - "--tls-cert-file=/var/run/secrets/files/mc_tls.pem", - "--tls-key-file=/var/run/secrets/files/mc_tls.key", - "--cert-collection-file=/var/run/secrets/files/all_root_certs.pem", - ] - _kingdom_internal_api_flags: [ - "--kingdom-internal-api-target=" + (#Target & {name: "gcp-kingdom-data-server"}).target, - "--kingdom-internal-api-cert-host=localhost", - ] - _edp_cert_key_files_flags: [ - "--edp-display-name=\(_edp_display_name)", - "--edp-cert-der-file=/var/run/secrets/files/edp1_cs_cert.der", - "--edp-key-der-file=/var/run/secrets/files/edp1_cs_private.der", - "--edp-encryption-public-keyset=/var/run/secrets/files/edp1_enc_public.tink", - ] - _exchange_workflow_flag: [ - "--exchange-workflow=/var/run/secrets/files/exchange_workflow.textproto", - ] - - resource_setup_job: #Job & { - _name: "resource-setup" - _secretName: _resource_setup_secret_name - _container: { - image: _job_image - imagePullPolicy: _job_image_pull_policy - args: - _tls_cert_key_files_flags + - _kingdom_internal_api_flags + - _edp_cert_key_files_flags + - _exchange_workflow_flag - } - } -} diff --git a/src/main/k8s/resource_setup.cue b/src/main/k8s/resource_setup.cue deleted file mode 100644 index 9ed1c2890f1..00000000000 --- a/src/main/k8s/resource_setup.cue +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2021 The Cross-Media Measurement Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package k8s - -#ResourceSetup: ResourceSetup={ - _edp_display_names: [...string] - _duchy_ids: [...string] - _resource_setup_secret_name: string - _imageConfig: #ImageConfig & { - repoSuffix: "loadtest/resource-setup" - } - _dependencies: [...string] - _bazelConfigName?: string - _edp_cert_key_files_flags: - [ - for d in _edp_display_names { - "--edp-consent-signaling-cert-der-files=\(d)=/var/run/secrets/files/\(d)_cs_cert.der" - }, - ] + [ - for d in _edp_display_names { - "--edp-consent-signaling-key-der-files=\(d)=/var/run/secrets/files/\(d)_cs_private.der" - }, - ] + [ - for d in _edp_display_names { - "--edp-encryption-public-keysets=\(d)=/var/run/secrets/files/\(d)_enc_public.tink" - }, - ] - _mc_cert_key_files_flags: [ - "--mc-consent-signaling-cert-der-file=/var/run/secrets/files/mc_cs_cert.der", - "--mc-consent-signaling-key-der-file=/var/run/secrets/files/mc_cs_private.der", - "--mc-encryption-public-keyset=/var/run/secrets/files/mc_enc_public.tink", - ] - _tls_cert_key_files_flags: [ - "--tls-cert-file=/var/run/secrets/files/kingdom_tls.pem", - "--tls-key-file=/var/run/secrets/files/kingdom_tls.key", - "--cert-collection-file=/var/run/secrets/files/kingdom_root.pem", - ] - _duchy_cs_cert_files_flags: [ - for d in _duchy_ids { - "--duchy-consent-signaling-cert-der-files=\(d)=/var/run/secrets/files/\(d)_cs_cert.der" - }, - ] - _kingdom_public_api_flags: [ - "--kingdom-public-api-target=" + (#Target & {name: "v2alpha-public-api-server"}).target, - "--kingdom-public-api-cert-host=localhost", - ] - _kingdom_internal_api_flags: [ - "--kingdom-internal-api-target=" + (#Target & {name: "gcp-kingdom-data-server"}).target, - "--kingdom-internal-api-cert-host=localhost", - ] - - resource_setup_job: #Job & { - _name: "resource-setup" - _secretName: _resource_setup_secret_name - _container: { - image: _imageConfig.image - args: - _edp_cert_key_files_flags + - _mc_cert_key_files_flags + - _tls_cert_key_files_flags + - _duchy_cs_cert_files_flags + - _kingdom_public_api_flags + - _kingdom_internal_api_flags + [ - if _bazelConfigName != _|_ { - "--bazel-config-name=\(_bazelConfigName)" - }, - ] - } - - spec: { - backoffLimit: 0 // Don't retry. - template: spec: { - _dependencies: ResourceSetup._dependencies - restartPolicy: "Never" - } - } - } -} diff --git a/src/main/k8s/local/resource_setup.sh b/src/main/k8s/testing/resource_setup.sh similarity index 98% rename from src/main/k8s/local/resource_setup.sh rename to src/main/k8s/testing/resource_setup.sh index 11beaeda969..d0d8e4ced25 100755 --- a/src/main/k8s/local/resource_setup.sh +++ b/src/main/k8s/testing/resource_setup.sh @@ -44,4 +44,4 @@ exec $BAZEL_BIN/src/main/kotlin/org/wfanet/measurement/loadtest/resourcesetup/Re --edp-encryption-public-keysets=edp4=src/main/k8s/testing/secretfiles/edp4_enc_public.tink \ --edp-encryption-public-keysets=edp5=src/main/k8s/testing/secretfiles/edp5_enc_public.tink \ --edp-encryption-public-keysets=edp6=src/main/k8s/testing/secretfiles/edp6_enc_public.tink \ ---bazel-config-name=halo-local "$@" +"$@" diff --git a/src/main/k8s/testing/secretfiles/BUILD.bazel b/src/main/k8s/testing/secretfiles/BUILD.bazel index 88223f0e60c..ea420109665 100644 --- a/src/main/k8s/testing/secretfiles/BUILD.bazel +++ b/src/main/k8s/testing/secretfiles/BUILD.bazel @@ -1,5 +1,6 @@ load("@bazel_skylib//rules:write_file.bzl", "write_file") load("//build/k8s:defs.bzl", "k8s_apply", "kustomization_dir") +load("@rules_pkg//pkg:pkg.bzl", "pkg_tar") package( default_testonly = True, @@ -181,6 +182,11 @@ filegroup( srcs = SECRET_FILES, ) +pkg_tar( + name = "archive", + srcs = [":secret_files"], +) + write_file( name = "gen_kustomization", out = "kustomization.yaml", diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/herald/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/herald/BUILD.bazel index 579ee8f41a2..0402f690f94 100644 --- a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/herald/BUILD.bazel +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/herald/BUILD.bazel @@ -1,6 +1,6 @@ load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") load("@rules_java//java:defs.bzl", "java_binary") -load("@io_bazel_rules_docker//java:image.bzl", "java_image") +load("//src/main/docker:macros.bzl", "java_image") kt_jvm_library( name = "herald_daemon", @@ -29,7 +29,7 @@ java_binary( java_image( name = "herald_daemon_image", + binary = ":HeraldDaemon", main_class = "org.wfanet.measurement.duchy.deploy.common.daemon.herald.HeraldDaemonKt", visibility = ["//src:docker_image_deployment"], - runtime_deps = [":herald_daemon"], ) diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/liquidlegionsv2/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/liquidlegionsv2/BUILD.bazel index ac926631ecb..9233b17b46f 100644 --- a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/liquidlegionsv2/BUILD.bazel +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/liquidlegionsv2/BUILD.bazel @@ -1,6 +1,6 @@ load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") load("@rules_java//java:defs.bzl", "java_binary") -load("@io_bazel_rules_docker//java:image.bzl", "java_image") +load("//src/main/docker:macros.bzl", "java_image") load("//build/platforms:constraints.bzl", "DISTROLESS_JAVA") package(default_testonly = True) #TODO: delete when InMemoryKeyStore and FakeHybridCipher are not used. @@ -56,8 +56,8 @@ java_binary( java_image( name = "forwarded_storage_liquid_legions_v2_mill_daemon_image", + binary = ":ForwardedStorageLiquidLegionsV2MillDaemon", main_class = "org.wfanet.measurement.duchy.deploy.common.daemon.mill.liquidlegionsv2.ForwardedStorageLiquidLegionsV2MillDaemonKt", target_compatible_with = DISTROLESS_JAVA, visibility = ["//src:docker_image_deployment"], - runtime_deps = [":forwarded_storage_liquid_legions_v2_mill_daemon"], ) diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/job/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/job/BUILD.bazel index 0c29afaebde..eb977815550 100644 --- a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/job/BUILD.bazel +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/job/BUILD.bazel @@ -1,6 +1,6 @@ load("@rules_java//java:defs.bzl", "java_binary") load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") -load("@io_bazel_rules_docker//java:image.bzl", "java_image") +load("//src/main/docker:macros.bzl", "java_image") kt_jvm_library( name = "computations_cleaner_job", @@ -25,7 +25,7 @@ java_binary( java_image( name = "computations_cleaner_image", + binary = ":ComputationsCleanerJob", main_class = "org.wfanet.measurement.duchy.deploy.common.job.ComputationsCleanerJobKt", visibility = ["//src:docker_image_deployment"], - runtime_deps = [":computations_cleaner_job"], ) diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/server/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/server/BUILD.bazel index 1253ef89d06..5c2c72a7626 100644 --- a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/server/BUILD.bazel +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/server/BUILD.bazel @@ -1,6 +1,6 @@ load("@rules_java//java:defs.bzl", "java_binary") load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") -load("@io_bazel_rules_docker//java:image.bzl", "java_image") +load("//src/main/docker:macros.bzl", "java_image") kt_jvm_library( name = "async_computation_control_server", @@ -27,9 +27,9 @@ java_binary( java_image( name = "async_computation_control_server_image", + binary = ":AsyncComputationControlServer", main_class = "org.wfanet.measurement.duchy.deploy.common.server.AsyncComputationControlServerKt", visibility = ["//src:docker_image_deployment"], - runtime_deps = [":async_computation_control_server"], ) kt_jvm_library( @@ -69,11 +69,9 @@ java_binary( java_image( name = "forwarded_storage_computation_control_server_image", + binary = ":ForwardedStorageComputationControlServer", main_class = "org.wfanet.measurement.duchy.deploy.common.server.ForwardedStorageComputationControlServerKt", - visibility = [ - "//src:docker_image_deployment", - ], - runtime_deps = [":forwarded_storage_computation_control_server"], + visibility = ["//src:docker_image_deployment"], ) kt_jvm_library( @@ -129,9 +127,7 @@ java_binary( java_image( name = "forwarded_storage_requisition_fulfillment_server_image", + binary = ":ForwardedStorageRequisitionFulfillmentServer", main_class = "org.wfanet.measurement.duchy.deploy.common.server.ForwardedStorageRequisitionFulfillmentServerKt", - visibility = [ - "//src:docker_image_deployment", - ], - runtime_deps = [":forwarded_storage_requisition_fulfillment_server"], + visibility = ["//src:docker_image_deployment"], ) diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/daemon/mill/liquidlegionsv2/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/daemon/mill/liquidlegionsv2/BUILD.bazel index b7f44b00c80..f1a363beae9 100644 --- a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/daemon/mill/liquidlegionsv2/BUILD.bazel +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/daemon/mill/liquidlegionsv2/BUILD.bazel @@ -1,7 +1,7 @@ load("@wfa_common_jvm//build:defs.bzl", "test_target") load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") load("@rules_java//java:defs.bzl", "java_binary") -load("@io_bazel_rules_docker//java:image.bzl", "java_image") +load("//src/main/docker:macros.bzl", "java_image") load("//build/platforms:constraints.bzl", "DISTROLESS_JAVA") package( @@ -33,8 +33,8 @@ java_binary( java_image( name = "gcs_liquid_legions_v2_mill_daemon_image", + binary = ":GcsLiquidLegionsV2MillDaemon", main_class = "org.wfanet.measurement.duchy.deploy.gcloud.daemon.mill.liquidlegionsv2.GcsLiquidLegionsV2MillDaemonKt", target_compatible_with = DISTROLESS_JAVA, visibility = ["//src:docker_image_deployment"], - runtime_deps = [":gcs_liquid_legions_v2_mill_daemon"], ) diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/server/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/server/BUILD.bazel index 4c57c5c4d8f..529288452c1 100644 --- a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/server/BUILD.bazel +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/server/BUILD.bazel @@ -1,6 +1,6 @@ load("@rules_java//java:defs.bzl", "java_binary") load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") -load("@io_bazel_rules_docker//java:image.bzl", "java_image") +load("//src/main/docker:macros.bzl", "java_image") kt_jvm_library( name = "gcs_computation_control_server", @@ -22,11 +22,11 @@ java_binary( java_image( name = "gcs_computation_control_server_image", + binary = ":GcsComputationControlServer", main_class = "org.wfanet.measurement.duchy.deploy.gcloud.server.GcsComputationControlServerKt", visibility = [ "//src:docker_image_deployment", ], - runtime_deps = [":gcs_computation_control_server"], ) kt_jvm_library( @@ -51,9 +51,9 @@ java_binary( java_image( name = "forwarded_storage_spanner_computations_server_image", + binary = ":ForwardedStorageSpannerComputationsServer", main_class = "org.wfanet.measurement.duchy.deploy.gcloud.server.ForwardedStorageSpannerComputationServerKt", visibility = ["//src:docker_image_deployment"], - runtime_deps = [":forwarded_storage_spanner_computations_server"], ) kt_jvm_library( @@ -78,9 +78,9 @@ java_binary( java_image( name = "gcs_spanner_computations_server_image", + binary = ":GcsSpannerComputationServer", main_class = "org.wfanet.measurement.duchy.deploy.gcloud.server.GcsSpannerComputationsServerKt", visibility = ["//src:docker_image_deployment"], - runtime_deps = [":gcs_spanner_computations_server"], ) kt_jvm_library( @@ -102,9 +102,9 @@ java_binary( java_image( name = "gcs_requisition_fulfillment_server_image", + binary = ":GcsRequisitionFulfillmentServer", main_class = "org.wfanet.measurement.duchy.deploy.gcloud.server.GcsRequisitionFulfillmentServerKt", visibility = [ "//src:docker_image_deployment", ], - runtime_deps = [":gcs_requisition_fulfillment_server"], ) diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/spanner/tools/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/spanner/tools/BUILD.bazel index 586fbbdc5fe..6db3fe388d7 100644 --- a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/spanner/tools/BUILD.bazel +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/spanner/tools/BUILD.bazel @@ -1,17 +1,20 @@ -load("@io_bazel_rules_docker//java:image.bzl", "java_image") +load("@rules_java//java:defs.bzl", "java_binary") +load("//src/main/docker:macros.bzl", "java_image") -java_image( - name = "update_schema_image", - args = [ - "--changelog=duchy/spanner/changelog.yaml", - ], +java_binary( + name = "UpdateSchema", + args = ["--changelog=duchy/spanner/changelog.yaml"], main_class = "org.wfanet.measurement.gcloud.spanner.tools.UpdateSchema", resources = ["//src/main/resources/duchy/spanner"], - visibility = [ - "//src/main/docker:__pkg__", - "//src/main/k8s:__subpackages__", - ], runtime_deps = [ "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/gcloud/spanner/tools:update_schema", ], ) + +java_image( + name = "update_schema_image", + args = ["--changelog=duchy/spanner/changelog.yaml"], + binary = ":UpdateSchema", + main_class = "org.wfanet.measurement.gcloud.spanner.tools.UpdateSchema", + visibility = ["//src:docker_image_deployment"], +) diff --git a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/server/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/server/BUILD.bazel index 11426a33f33..171da709a04 100644 --- a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/server/BUILD.bazel +++ b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/server/BUILD.bazel @@ -1,6 +1,6 @@ load("@rules_java//java:defs.bzl", "java_binary") load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") -load("@io_bazel_rules_docker//java:image.bzl", "java_image") +load("//src/main/docker:macros.bzl", "java_image") kt_jvm_library( name = "kingdom_data_server", @@ -68,9 +68,9 @@ java_binary( java_image( name = "v2alpha_public_api_server_image", + binary = ":V2alphaPublicApiServer", main_class = "org.wfanet.measurement.kingdom.deploy.common.server.V2alphaPublicApiServerKt", visibility = ["//src:docker_image_deployment"], - runtime_deps = [":v2alpha_public_api_server"], ) kt_jvm_library( @@ -98,7 +98,7 @@ java_binary( java_image( name = "system_api_server_image", + binary = ":SystemApiServer", main_class = "org.wfanet.measurement.kingdom.deploy.common.server.SystemApiServerKt", visibility = ["//src:docker_image_deployment"], - runtime_deps = [":system_api_server"], ) diff --git a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/server/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/server/BUILD.bazel index 4318eba15e8..be1f9fed34e 100644 --- a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/server/BUILD.bazel +++ b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/server/BUILD.bazel @@ -1,6 +1,6 @@ -load("@io_bazel_rules_docker//java:image.bzl", "java_image") load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") load("@rules_java//java:defs.bzl", "java_binary") +load("//src/main/docker:macros.bzl", "java_image") kt_jvm_library( name = "spanner_kingdom_data_server", @@ -22,7 +22,7 @@ java_binary( java_image( name = "gcp_kingdom_data_server_image", + binary = ":SpannerKingdomDataServer", main_class = "org.wfanet.measurement.kingdom.deploy.gcloud.server.SpannerKingdomDataServerKt", visibility = ["//src:docker_image_deployment"], - runtime_deps = [":spanner_kingdom_data_server"], ) diff --git a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/tools/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/tools/BUILD.bazel index e26e600106c..07d0d91b6a3 100644 --- a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/tools/BUILD.bazel +++ b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/tools/BUILD.bazel @@ -1,17 +1,27 @@ -load("@io_bazel_rules_docker//java:image.bzl", "java_image") +load("@rules_java//java:defs.bzl", "java_binary") +load("//src/main/docker:macros.bzl", "java_image") + +java_binary( + name = "UpdateSchema", + args = [ + "--changelog=kingdom/spanner/changelog.yaml", + ], + main_class = "org.wfanet.measurement.gcloud.spanner.tools.UpdateSchema", + resources = ["//src/main/resources/kingdom/spanner"], + runtime_deps = [ + "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/gcloud/spanner/tools:update_schema", + ], +) java_image( name = "update_schema_image", args = [ "--changelog=kingdom/spanner/changelog.yaml", ], + binary = ":UpdateSchema", main_class = "org.wfanet.measurement.gcloud.spanner.tools.UpdateSchema", - resources = ["//src/main/resources/kingdom/spanner"], visibility = [ "//src/main/docker:__pkg__", "//src/main/k8s:__subpackages__", ], - runtime_deps = [ - "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/gcloud/spanner/tools:update_schema", - ], ) diff --git a/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/BUILD.bazel index 494ef310c1f..a09afe4aa7b 100644 --- a/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/BUILD.bazel +++ b/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/BUILD.bazel @@ -1,6 +1,6 @@ load("@rules_java//java:defs.bzl", "java_binary") load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") -load("@io_bazel_rules_docker//java:image.bzl", "java_image") +load("//src/main/docker:macros.bzl", "java_image") load("//build/platforms:constraints.bzl", "DISTROLESS_JAVA") package( @@ -98,12 +98,12 @@ java_binary( java_image( name = "forwarded_storage_edp_simulator_runner_image", - #Use java_debug_image_base which has TAR binary to run kubectl cp + # Use the debug image as it includes the `tar` binary needed to use `kubectl cp` to copy in a CSV file. base = "@java_debug_image_base//image", + binary = ":ForwardStorageEdpSimulatorRunner", main_class = "org.wfanet.measurement.loadtest.dataprovider.ForwardStorageEdpSimulatorRunnerKt", target_compatible_with = DISTROLESS_JAVA, visibility = ["//src:docker_image_deployment"], - runtime_deps = [":forwarded_storage_edp_simulator_runner"], ) kt_jvm_library( @@ -125,8 +125,8 @@ java_binary( java_image( name = "gcs_edp_simulator_runner_image", + binary = ":GcsEdpSimulatorRunner", main_class = "org.wfanet.measurement.loadtest.dataprovider.GcsEdpSimulatorRunnerKt", target_compatible_with = DISTROLESS_JAVA, visibility = ["//src:docker_image_deployment"], - runtime_deps = [":gcs_edp_simulator_runner"], ) diff --git a/src/main/kotlin/org/wfanet/measurement/loadtest/frontend/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/loadtest/frontend/BUILD.bazel index 0f21c1e6aa0..b90839ad608 100644 --- a/src/main/kotlin/org/wfanet/measurement/loadtest/frontend/BUILD.bazel +++ b/src/main/kotlin/org/wfanet/measurement/loadtest/frontend/BUILD.bazel @@ -1,7 +1,4 @@ load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") -load("@rules_java//java:defs.bzl", "java_binary") -load("@io_bazel_rules_docker//java:image.bzl", "java_image") -load("//build/platforms:constraints.bzl", "DISTROLESS_JAVA") package( default_testonly = True, @@ -50,76 +47,3 @@ kt_jvm_library( "@wfa_consent_signaling_client//src/main/kotlin/org/wfanet/measurement/consent/client/measurementconsumer", ], ) - -kt_jvm_library( - name = "frontend_simulator_runner", - srcs = [ - "FrontendSimulatorFlags.kt", - "FrontendSimulatorRunner.kt", - ], - deps = [ - ":frontend_simulator", - "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common:flags", - "//src/main/kotlin/org/wfanet/measurement/loadtest:service_flags", - "//src/main/kotlin/org/wfanet/measurement/loadtest/config:event_filters", - "//src/main/proto/wfa/measurement/api/v2alpha:event_groups_service_kt_jvm_grpc_proto", - "//src/main/proto/wfa/measurement/api/v2alpha:measurements_service_kt_jvm_grpc_proto", - "@wfa_common_jvm//imports/java/io/grpc:api", - "@wfa_common_jvm//imports/java/picocli", - "@wfa_common_jvm//imports/kotlin/kotlinx/coroutines:core", - "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/common/crypto/testing", - "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/common/crypto/tink", - ], -) - -kt_jvm_library( - name = "forwarded_storage_frontend_simulator_runner", - srcs = ["ForwardedStorageFrontendSimulatorRunner.kt"], - deps = [ - ":frontend_simulator_runner", - "@wfa_common_jvm//imports/java/picocli", - "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/common", - "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/storage/forwarded", - ], -) - -java_binary( - name = "ForwardedStorageFrontendSimulatorRunner", - main_class = "org.wfanet.measurement.loadtest.frontend.ForwardedStorageFrontendSimulatorRunnerKt", - runtime_deps = [":forwarded_storage_frontend_simulator_runner"], -) - -java_image( - name = "forwarded_storage_frontend_simulator_runner_image", - main_class = "org.wfanet.measurement.loadtest.frontend.ForwardedStorageFrontendSimulatorRunnerKt", - target_compatible_with = DISTROLESS_JAVA, - visibility = [ - "//src:docker_image_deployment", - ], - runtime_deps = [":forwarded_storage_frontend_simulator_runner"], -) - -kt_jvm_library( - name = "gcs_frontend_simulator_runner", - srcs = ["GcsFrontendSimulatorRunner.kt"], - deps = [ - ":frontend_simulator_runner", - "@wfa_common_jvm//imports/java/picocli", - "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/common", - "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/gcloud/gcs", - ], -) - -java_binary( - name = "GcsFrontendSimulatorRunner", - main_class = "org.wfanet.measurement.loadtest.frontend.GcsFrontendSimulatorRunnerKt", - runtime_deps = [":gcs_frontend_simulator_runner"], -) - -java_image( - name = "gcs_frontend_simulator_runner_image", - main_class = "org.wfanet.measurement.loadtest.frontend.GcsFrontendSimulatorRunnerKt", - target_compatible_with = DISTROLESS_JAVA, - visibility = ["//src:docker_image_deployment"], - runtime_deps = [":gcs_frontend_simulator_runner"], -) diff --git a/src/main/kotlin/org/wfanet/measurement/loadtest/frontend/ForwardedStorageFrontendSimulatorRunner.kt b/src/main/kotlin/org/wfanet/measurement/loadtest/frontend/ForwardedStorageFrontendSimulatorRunner.kt deleted file mode 100644 index fdf135f9cf6..00000000000 --- a/src/main/kotlin/org/wfanet/measurement/loadtest/frontend/ForwardedStorageFrontendSimulatorRunner.kt +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2021 The Cross-Media Measurement Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.wfanet.measurement.loadtest.frontend - -import org.wfanet.measurement.common.commandLineMain -import org.wfanet.measurement.storage.forwarded.ForwardedStorageFromFlags -import picocli.CommandLine - -/** Implementation of [FrontendSimulatorRunner] using Fake Storage Service. */ -@CommandLine.Command( - name = "ForwardedStorageFrontendSimulatorRunnerDaemon", - description = ["Daemon for ForwardedStorageFrontendSimulatorRunner."], - mixinStandardHelpOptions = true, - showDefaultValues = true -) -class ForwardedStorageFrontendSimulatorRunner : FrontendSimulatorRunner() { - @CommandLine.Mixin private lateinit var forwardedStorageFlags: ForwardedStorageFromFlags.Flags - - override fun run() { - run(ForwardedStorageFromFlags(forwardedStorageFlags, flags.tlsFlags).storageClient) - } -} - -fun main(args: Array) = commandLineMain(ForwardedStorageFrontendSimulatorRunner(), args) diff --git a/src/main/kotlin/org/wfanet/measurement/loadtest/frontend/FrontendSimulatorFlags.kt b/src/main/kotlin/org/wfanet/measurement/loadtest/frontend/FrontendSimulatorFlags.kt deleted file mode 100644 index 672b8c02423..00000000000 --- a/src/main/kotlin/org/wfanet/measurement/loadtest/frontend/FrontendSimulatorFlags.kt +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright 2021 The Cross-Media Measurement Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.wfanet.measurement.loadtest.frontend - -import java.io.File -import java.time.Duration -import java.time.Instant -import java.time.ZoneOffset -import java.time.format.DateTimeFormatter -import kotlin.properties.Delegates -import org.wfanet.measurement.common.grpc.TlsFlags -import org.wfanet.measurement.loadtest.KingdomPublicApiFlags -import picocli.CommandLine - -class FrontendSimulatorFlags { - - /** The FrontendSimulator pod's own tls certificates. */ - @CommandLine.Mixin - lateinit var tlsFlags: TlsFlags - private set - - @CommandLine.Mixin - lateinit var kingdomPublicApiFlags: KingdomPublicApiFlags - private set - - @CommandLine.Option( - names = ["--mc-resource-name"], - description = ["The resource name of the measurement consumer."], - required = true - ) - lateinit var mcResourceName: String - private set - - @CommandLine.Option( - names = ["--mc-consent-signaling-cert-der-file"], - description = ["The MC's consent signaling cert (DER format) file."], - required = true - ) - lateinit var mcCsCertDerFile: File - private set - - @CommandLine.Option( - names = ["--mc-consent-signaling-key-der-file"], - description = ["The MC's consent signaling private key (DER format) file."], - required = true - ) - lateinit var mcCsPrivateKeyDerFile: File - private set - - @CommandLine.Option( - names = ["--mc-encryption-private-keyset"], - description = ["The MC's encryption private Tink Keyset."], - required = true - ) - lateinit var mcEncryptionPrivateKeyset: File - private set - - @set:CommandLine.Option( - names = ["--output-differential-privacy-epsilon"], - description = ["The common epsilon used for all output's differential privacy noises."], - required = true - ) - var outputDpEpsilon by Delegates.notNull() - private set - - @set:CommandLine.Option( - names = ["--output-differential-privacy-delta"], - description = ["The common delta used for all output's differential privacy noises."], - required = true - ) - var outputDpDelta by Delegates.notNull() - private set - - @CommandLine.Option( - names = ["--run-id"], - description = ["Unique identifier of the run, if not set, timestamp will be used."], - required = false - ) - var runId: String = - DateTimeFormatter.ofPattern("yyyy-MM-ddHH-mm-ss-SSS") - .withZone(ZoneOffset.UTC) - .format(Instant.now()) - private set - - @CommandLine.Option( - names = ["--api-authentication-key"], - description = ["API authentication key for measurement consumer authentication."], - required = true - ) - lateinit var apiAuthenticationKey: String - private set - - @CommandLine.Option( - names = ["--result-polling-delay"], - description = ["Duration to delay when polling for Measurement result"], - defaultValue = "30s", - ) - lateinit var resultPollingDelay: Duration - private set -} diff --git a/src/main/kotlin/org/wfanet/measurement/loadtest/frontend/FrontendSimulatorRunner.kt b/src/main/kotlin/org/wfanet/measurement/loadtest/frontend/FrontendSimulatorRunner.kt deleted file mode 100644 index cba97cefca3..00000000000 --- a/src/main/kotlin/org/wfanet/measurement/loadtest/frontend/FrontendSimulatorRunner.kt +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright 2021 The Cross-Media Measurement Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.wfanet.measurement.loadtest.frontend - -import io.grpc.ManagedChannel -import java.util.logging.Logger -import kotlinx.coroutines.launch -import kotlinx.coroutines.runBlocking -import org.wfanet.measurement.api.v2alpha.CertificatesGrpcKt.CertificatesCoroutineStub -import org.wfanet.measurement.api.v2alpha.DataProvidersGrpcKt.DataProvidersCoroutineStub -import org.wfanet.measurement.api.v2alpha.DifferentialPrivacyParams -import org.wfanet.measurement.api.v2alpha.EventGroupsGrpcKt.EventGroupsCoroutineStub -import org.wfanet.measurement.api.v2alpha.MeasurementConsumersGrpcKt.MeasurementConsumersCoroutineStub -import org.wfanet.measurement.api.v2alpha.MeasurementsGrpcKt.MeasurementsCoroutineStub -import org.wfanet.measurement.api.v2alpha.RequisitionsGrpcKt.RequisitionsCoroutineStub -import org.wfanet.measurement.common.crypto.SigningCerts -import org.wfanet.measurement.common.crypto.testing.loadSigningKey -import org.wfanet.measurement.common.crypto.tink.loadPrivateKey -import org.wfanet.measurement.common.grpc.buildMutualTlsChannel -import org.wfanet.measurement.loadtest.config.EventFilters.EVENT_TEMPLATES_TO_FILTERS_MAP -import org.wfanet.measurement.loadtest.storage.SketchStore -import org.wfanet.measurement.storage.StorageClient -import picocli.CommandLine - -/** The base class of the frontend simulator runner. */ -abstract class FrontendSimulatorRunner : Runnable { - @CommandLine.Mixin - protected lateinit var flags: FrontendSimulatorFlags - private set - - protected fun run(storageClient: StorageClient) { - val clientCerts = - SigningCerts.fromPemFiles( - certificateFile = flags.tlsFlags.certFile, - privateKeyFile = flags.tlsFlags.privateKeyFile, - trustedCertCollectionFile = flags.tlsFlags.certCollectionFile - ) - val v2alphaPublicApiChannel: ManagedChannel = - buildMutualTlsChannel( - flags.kingdomPublicApiFlags.target, - clientCerts, - flags.kingdomPublicApiFlags.certHost - ) - val dataProvidersStub = DataProvidersCoroutineStub(v2alphaPublicApiChannel) - val eventGroupsStub = EventGroupsCoroutineStub(v2alphaPublicApiChannel) - val requisitionsStub = RequisitionsCoroutineStub(v2alphaPublicApiChannel) - val measurementsStub = MeasurementsCoroutineStub(v2alphaPublicApiChannel) - val measurementConsumersStub = MeasurementConsumersCoroutineStub(v2alphaPublicApiChannel) - val certificatesStub = CertificatesCoroutineStub(v2alphaPublicApiChannel) - - val mcName = flags.mcResourceName - - val measurementConsumerData = - MeasurementConsumerData( - mcName, - loadSigningKey(flags.mcCsCertDerFile, flags.mcCsPrivateKeyDerFile), - loadPrivateKey(flags.mcEncryptionPrivateKeyset), - flags.apiAuthenticationKey - ) - val outputDpParams = - DifferentialPrivacyParams.newBuilder() - .apply { - epsilon = flags.outputDpEpsilon - delta = flags.outputDpDelta - } - .build() - val frontendSimulator = - FrontendSimulator( - measurementConsumerData, - outputDpParams, - dataProvidersStub, - eventGroupsStub, - measurementsStub, - requisitionsStub, - measurementConsumersStub, - certificatesStub, - SketchStore(storageClient), - flags.resultPollingDelay, - flags.tlsFlags.signingCerts.trustedCertificates, - EVENT_TEMPLATES_TO_FILTERS_MAP, - ) - - runBlocking { - // Run the tests in parallel. - launch { frontendSimulator.executeReachAndFrequency(flags.runId + "-reach_frequency") } - launch { frontendSimulator.executeImpression(flags.runId + "-impression") } - launch { frontendSimulator.executeDuration(flags.runId + "-duration") } - } - logger.info("Correctness test passed") - } - - companion object { - private val logger: Logger = Logger.getLogger(this::class.java.name) - } -} diff --git a/src/main/kotlin/org/wfanet/measurement/loadtest/frontend/GcsFrontendSimulatorRunner.kt b/src/main/kotlin/org/wfanet/measurement/loadtest/frontend/GcsFrontendSimulatorRunner.kt deleted file mode 100644 index fc1aff498dd..00000000000 --- a/src/main/kotlin/org/wfanet/measurement/loadtest/frontend/GcsFrontendSimulatorRunner.kt +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2021 The Cross-Media Measurement Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.wfanet.measurement.loadtest.frontend - -import org.wfanet.measurement.common.commandLineMain -import org.wfanet.measurement.gcloud.gcs.GcsFromFlags -import org.wfanet.measurement.gcloud.gcs.GcsStorageClient -import picocli.CommandLine - -@CommandLine.Command( - name = "GcsFrontendSimulatorRunner", - description = ["Daemon for GcsFrontendSimulatorRunner."], - mixinStandardHelpOptions = true, - showDefaultValues = true -) -class GcsFrontendSimulatorRunner : FrontendSimulatorRunner() { - @CommandLine.Mixin private lateinit var gcsFlags: GcsFromFlags.Flags - - override fun run() { - val gcs = GcsFromFlags(gcsFlags) - run(GcsStorageClient.fromFlags(gcs)) - } -} - -fun main(args: Array) = commandLineMain(GcsFrontendSimulatorRunner(), args) diff --git a/src/main/kotlin/org/wfanet/measurement/loadtest/panelmatchresourcesetup/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/loadtest/panelmatchresourcesetup/BUILD.bazel index 2ccaba201d2..b50394d45f6 100644 --- a/src/main/kotlin/org/wfanet/measurement/loadtest/panelmatchresourcesetup/BUILD.bazel +++ b/src/main/kotlin/org/wfanet/measurement/loadtest/panelmatchresourcesetup/BUILD.bazel @@ -1,7 +1,7 @@ load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") load("//src/main/proto/wfa/measurement/internal/kingdom:all_protos.bzl", "KINGDOM_INTERNAL_PROTOS") # buildifier: disable=bzl-visibility load("@rules_java//java:defs.bzl", "java_binary") -load("@io_bazel_rules_docker//java:image.bzl", "java_image") +load("//src/main/docker:macros.bzl", "java_image") package( default_testonly = True, @@ -36,7 +36,7 @@ java_binary( java_image( name = "panel_match_resource_setup_runner_image", + binary = ":PanelMatchResourceSetupRunner", main_class = "org.wfanet.measurement.loadtest.panelmatchresourcesetup.PanelMatchResourceSetupRunnerKt", visibility = ["//src:docker_image_deployment"], - runtime_deps = [":panelmatchresourcesetup"], ) diff --git a/src/main/kotlin/org/wfanet/measurement/loadtest/resourcesetup/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/loadtest/resourcesetup/BUILD.bazel index d05640b4256..35a807c143b 100644 --- a/src/main/kotlin/org/wfanet/measurement/loadtest/resourcesetup/BUILD.bazel +++ b/src/main/kotlin/org/wfanet/measurement/loadtest/resourcesetup/BUILD.bazel @@ -1,6 +1,6 @@ load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") load("@rules_java//java:defs.bzl", "java_binary") -load("@io_bazel_rules_docker//java:image.bzl", "java_image") +load("//src/main/docker:macros.bzl", "java_image") package( default_testonly = True, @@ -82,7 +82,7 @@ java_binary( java_image( name = "resource_setup_runner_image", + binary = ":ResourceSetup", main_class = "org.wfanet.measurement.loadtest.resourcesetup.ResourceSetupRunnerKt", visibility = ["//src:docker_image_deployment"], - runtime_deps = [":resource_setup_runner"], ) diff --git a/src/main/kotlin/org/wfanet/measurement/reporting/deploy/common/server/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/reporting/deploy/common/server/BUILD.bazel index f9975ff574a..53afbfcd9ac 100644 --- a/src/main/kotlin/org/wfanet/measurement/reporting/deploy/common/server/BUILD.bazel +++ b/src/main/kotlin/org/wfanet/measurement/reporting/deploy/common/server/BUILD.bazel @@ -1,6 +1,6 @@ load("@rules_java//java:defs.bzl", "java_binary") load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") -load("@io_bazel_rules_docker//java:image.bzl", "java_image") +load("//src/main/docker:macros.bzl", "java_image") kt_jvm_library( name = "reporting_data_server", @@ -65,7 +65,7 @@ java_binary( java_image( name = "v1alpha_public_api_server_image", + binary = ":V1AlphaPublicApiServer", main_class = "org.wfanet.measurement.reporting.deploy.common.server.V1AlphaPublicApiServerKt", visibility = ["//src:docker_image_deployment"], - runtime_deps = [":v1alpha_public_api_server"], ) diff --git a/src/main/kotlin/org/wfanet/measurement/reporting/deploy/gcloud/postgres/server/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/reporting/deploy/gcloud/postgres/server/BUILD.bazel index c6b199ab840..a6bf379dc03 100644 --- a/src/main/kotlin/org/wfanet/measurement/reporting/deploy/gcloud/postgres/server/BUILD.bazel +++ b/src/main/kotlin/org/wfanet/measurement/reporting/deploy/gcloud/postgres/server/BUILD.bazel @@ -1,6 +1,6 @@ -load("@io_bazel_rules_docker//java:image.bzl", "java_image") load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") load("@rules_java//java:defs.bzl", "java_binary") +load("//src/main/docker:macros.bzl", "java_image") kt_jvm_library( name = "gcloud_postgres_reporting_data_server", @@ -26,7 +26,7 @@ java_binary( java_image( name = "gcloud_postgres_reporting_data_server_image", + binary = ":GCloudPostgresReportingDataServer", main_class = "org.wfanet.measurement.reporting.deploy.gcloud.postgres.server.GCloudPostgresReportingDataServerKt", visibility = ["//src:docker_image_deployment"], - runtime_deps = [":gcloud_postgres_reporting_data_server"], ) diff --git a/src/main/kotlin/org/wfanet/measurement/reporting/deploy/gcloud/postgres/tools/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/reporting/deploy/gcloud/postgres/tools/BUILD.bazel index 37f9a7fb10b..a6993a90733 100644 --- a/src/main/kotlin/org/wfanet/measurement/reporting/deploy/gcloud/postgres/tools/BUILD.bazel +++ b/src/main/kotlin/org/wfanet/measurement/reporting/deploy/gcloud/postgres/tools/BUILD.bazel @@ -1,17 +1,20 @@ -load("@io_bazel_rules_docker//java:image.bzl", "java_image") +load("@rules_java//java:defs.bzl", "java_binary") +load("//src/main/docker:macros.bzl", "java_image") -java_image( - name = "update_schema_image", - args = [ - "--changelog=reporting/postgres/changelog.yaml", - ], +java_binary( + name = "UpdateSchema", + args = ["--changelog=reporting/postgres/changelog.yaml"], main_class = "org.wfanet.measurement.gcloud.postgres.tools.UpdateSchema", resources = ["//src/main/resources/reporting/postgres"], - visibility = [ - "//src/main/docker:__pkg__", - "//src/main/k8s:__subpackages__", - ], runtime_deps = [ "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/gcloud/postgres/tools:update_schema", ], ) + +java_image( + name = "update_schema_image", + args = ["--changelog=reporting/postgres/changelog.yaml"], + binary = ":UpdateSchema", + main_class = "org.wfanet.measurement.gcloud.postgres.tools.UpdateSchema", + visibility = ["//src:docker_image_deployment"], +) diff --git a/src/main/kotlin/org/wfanet/measurement/reporting/deploy/postgres/server/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/reporting/deploy/postgres/server/BUILD.bazel index 047f2d29df2..e7d9e7bd776 100644 --- a/src/main/kotlin/org/wfanet/measurement/reporting/deploy/postgres/server/BUILD.bazel +++ b/src/main/kotlin/org/wfanet/measurement/reporting/deploy/postgres/server/BUILD.bazel @@ -1,6 +1,6 @@ -load("@io_bazel_rules_docker//java:image.bzl", "java_image") load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") load("@rules_java//java:defs.bzl", "java_binary") +load("//src/main/docker:macros.bzl", "java_image") kt_jvm_library( name = "postgres_reporting_data_server", @@ -23,7 +23,7 @@ java_binary( java_image( name = "postgres_reporting_data_server_image", + binary = ":PostgresReportingDataServer", main_class = "org.wfanet.measurement.reporting.deploy.postgres.server.PostgresReportingDataServerKt", visibility = ["//src:docker_image_deployment"], - runtime_deps = [":postgres_reporting_data_server"], ) diff --git a/src/main/kotlin/org/wfanet/measurement/reporting/deploy/postgres/tools/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/reporting/deploy/postgres/tools/BUILD.bazel index c7c2db74f44..31c1f110702 100644 --- a/src/main/kotlin/org/wfanet/measurement/reporting/deploy/postgres/tools/BUILD.bazel +++ b/src/main/kotlin/org/wfanet/measurement/reporting/deploy/postgres/tools/BUILD.bazel @@ -1,17 +1,20 @@ -load("@io_bazel_rules_docker//java:image.bzl", "java_image") +load("@rules_java//java:defs.bzl", "java_binary") +load("//src/main/docker:macros.bzl", "java_image") -java_image( - name = "update_schema_image", - args = [ - "--changelog=reporting/postgres/changelog.yaml", - ], +java_binary( + name = "UpdateSchema", + args = ["--changelog=reporting/postgres/changelog.yaml"], main_class = "org.wfanet.measurement.common.db.postgres.tools.UpdateSchema", resources = ["//src/main/resources/reporting/postgres"], - visibility = [ - "//src/main/docker:__pkg__", - "//src/main/k8s:__subpackages__", - ], runtime_deps = [ "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/common/db/postgres/tools:update_schema", ], ) + +java_image( + name = "update_schema_image", + args = ["--changelog=reporting/postgres/changelog.yaml"], + binary = ":UpdateSchema", + main_class = "org.wfanet.measurement.common.db.postgres.tools.UpdateSchema", + visibility = ["//src:docker_image_deployment"], +)