diff --git a/src/main/docker/images.bzl b/src/main/docker/images.bzl index 912c0290fd3..d773379c399 100644 --- a/src/main/docker/images.bzl +++ b/src/main/docker/images.bzl @@ -36,6 +36,11 @@ COMMON_IMAGES = [ image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/spanner/tools:update_schema_image", repository = _PREFIX + "/duchy/spanner-update-schema", ), + struct( + name = "duchy_postgres_update_schema_image", + image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/postgres/tools:update_schema_image", + repository = _PREFIX + "/duchy/postgres-update-schema", + ), struct( name = "duchy_computations_cleaner_image", image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/job:computations_cleaner_image", @@ -111,6 +116,11 @@ GKE_IMAGES = [ image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/server:gcs_spanner_computations_server_image", repository = _PREFIX + "/duchy/spanner-computations", ), + struct( + name = "duchy_gcs_postgres_internal_server_image", + image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/server:gcs_postgres_internal_server_image", + repository = _PREFIX + "/duchy/postgres-internal-server", + ), struct( name = "duchy_requisition_fulfillment_server_image", image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/server:gcs_requisition_fulfillment_server_image", @@ -126,6 +136,11 @@ GKE_IMAGES = [ image = "//src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider:bigquery_edp_simulator_runner_image", repository = _PREFIX + "/simulator/bigquery-edp", ), + struct( + name = "duchy_gcloud_postgres_update_schema_image", + image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/postgres/tools:update_schema_image", + repository = _PREFIX + "/duchy/gcloud-postgres-update-schema", + ), ] # List of image build rules that are only used locally (e.g. in Kind). @@ -145,6 +160,11 @@ LOCAL_IMAGES = [ image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/server:forwarded_storage_spanner_computations_server_image", repository = _PREFIX + "/duchy/local-spanner-computations", ), + struct( + name = "forwarded_storage_postgres_data_server_image", + image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/server:forwarded_storage_postgres_duchy_data_server_image", + repository = _PREFIX + "/duchy/local-postgres-internal-server", + ), struct( name = "forwarded_storage_requisition_fulfillment_server_image", image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/server:forwarded_storage_requisition_fulfillment_server_image", diff --git a/src/main/k8s/BUILD.bazel b/src/main/k8s/BUILD.bazel index 15bd92bceb0..e815499a116 100644 --- a/src/main/k8s/BUILD.bazel +++ b/src/main/k8s/BUILD.bazel @@ -48,6 +48,22 @@ cue_library( ], ) +cue_library( + name = "spanner_duchy", + srcs = ["spanner_duchy.cue"], + deps = [ + ":duchy", + ], +) + +cue_library( + name = "postgres_duchy", + srcs = ["postgres_duchy.cue"], + deps = [ + ":duchy", + ], +) + cue_library( name = "edp_simulator", srcs = ["edp_simulator.cue"], diff --git a/src/main/k8s/duchy.cue b/src/main/k8s/duchy.cue index b4d54baabb6..a6564861127 100644 --- a/src/main/k8s/duchy.cue +++ b/src/main/k8s/duchy.cue @@ -30,9 +30,6 @@ import ("strings") _computationsTimeToLive: string | *"180d" _duchyMillParallelism: uint | *2 _kingdom_system_api_target: string - _spannerConfig: #SpannerConfig & { - database: "\(_duchy.name)_duchy_computations" - } _blob_storage_flags: [...string] _verbose_grpc_logging: "true" | "false" @@ -49,8 +46,6 @@ import ("strings") "herald-daemon": string | *"duchy/herald" "liquid-legions-v2-mill-daemon": string | *"duchy/liquid-legions-v2-mill" "requisition-fulfillment-server": string | *"duchy/requisition-fulfillment" - "spanner-computations-server": string | *"duchy/spanner-computations" - "update-duchy-schema": string | *"duchy/spanner-update-schema" "computations-cleaner": string | *"duchy/computations-cleaner" } _imageConfigs: [string]: #ImageConfig @@ -66,18 +61,19 @@ import ("strings") } _millPollingInterval?: string + _duchyInternalServerContainerArgs: [...string] _akid_to_principal_map_file_flag: "--authority-key-identifier-to-principal-map-file=/etc/\(#AppName)/config-files/authority_key_identifier_to_principal_map.textproto" _async_computations_control_service_target_flag: "--async-computation-control-service-target=" + (#Target & {name: "\(_name)-async-computation-control-server"}).target _async_computations_control_service_cert_host_flag: "--async-computation-control-service-cert-host=localhost" - _computations_service_target_flag: "--computations-service-target=" + (#Target & {name: "\(_name)-spanner-computations-server"}).target - _computations_service_cert_host_flag: "--computations-service-cert-host=localhost" _duchy_name_flag: "--duchy-name=\(_name)" _duchy_info_config_flag: "--duchy-info-config=/var/run/secrets/files/duchy_cert_config.textproto" _duchy_protocols_setup_config_flag: "--protocols-setup-config=/var/run/secrets/files/\(_protocols_setup_config)" _duchy_tls_cert_file_flag: "--tls-cert-file=/var/run/secrets/files/\(_name)_tls.pem" _duchy_tls_key_file_flag: "--tls-key-file=/var/run/secrets/files/\(_name)_tls.key" _duchy_cert_collection_file_flag: "--cert-collection-file=/var/run/secrets/files/all_root_certs.pem" + _duchyInternalApiTargetFlag: "--computations-service-target=" + (#Target & {name: "\(_name)-internal-api-server"}).target + _duchyInternalApiCertHostFlag: "--computations-service-cert-host=localhost" _duchyComputationsTimeToLiveFlag: "--computations-time-to-live=\(_computationsTimeToLive)" _duchyDryRunRetentionPolicyFlag: "--dry-run" _duchyMillParallelismFlag: "--parallelism=\(_duchyMillParallelism)" @@ -99,7 +95,7 @@ import ("strings") services: { "async-computation-control-server": {} "computation-control-server": _type: "LoadBalancer" - "spanner-computations-server": {} + "internal-api-server": {} "requisition-fulfillment-server": _type: "LoadBalancer" } @@ -116,8 +112,8 @@ import ("strings") deployments: { "herald-daemon-deployment": { _container: args: [ - _computations_service_target_flag, - _computations_service_cert_host_flag, + _duchyInternalApiTargetFlag, + _duchyInternalApiCertHostFlag, _duchy_name_flag, _duchy_tls_cert_file_flag, _duchy_tls_key_file_flag, @@ -128,13 +124,13 @@ import ("strings") _debug_verbose_grpc_client_logging_flag, ] + _duchyDeletableStatesFlag spec: template: spec: _dependencies: [ - "\(_name)-spanner-computations-server", + "\(_name)-internal-api-server", ] } "liquid-legions-v2-mill-daemon-deployment": Deployment={ _container: args: [ - _computations_service_target_flag, - _computations_service_cert_host_flag, + _duchyInternalApiTargetFlag, + _duchyInternalApiCertHostFlag, _duchy_name_flag, _duchy_info_config_flag, _duchy_tls_cert_file_flag, @@ -151,13 +147,13 @@ import ("strings") "--otel-service-name=\(Deployment.metadata.name)", ] + _blob_storage_flags + _computation_control_target_flags spec: template: spec: _dependencies: [ - "\(_name)-spanner-computations-server", "\(_name)-computation-control-server", + "\(_name)-internal-api-server", "\(_name)-computation-control-server", ] } "async-computation-control-server-deployment": #ServerDeployment & { _container: args: [ - _computations_service_target_flag, - _computations_service_cert_host_flag, + _duchyInternalApiTargetFlag, + _duchyInternalApiCertHostFlag, _duchy_name_flag, _duchy_info_config_flag, _duchy_tls_cert_file_flag, @@ -182,7 +178,7 @@ import ("strings") "--health-port=8080", ] + _blob_storage_flags } - "spanner-computations-server-deployment": #ServerDeployment & { + "internal-api-server-deployment": #ServerDeployment & { _container: args: [ _debug_verbose_grpc_server_logging_flag, _duchy_name_flag, @@ -195,11 +191,11 @@ import ("strings") "--channel-shutdown-timeout=3s", "--port=8443", "--health-port=8080", - ] + _spannerConfig.flags + _blob_storage_flags + ] + _duchyInternalServerContainerArgs + _blob_storage_flags _updateSchemaContainer: #Container & { image: _images["update-duchy-schema"] imagePullPolicy?: _container.imagePullPolicy - args: _spannerConfig.flags + args: _duchyInternalServerContainerArgs } spec: template: spec: { _initContainers: { @@ -215,8 +211,8 @@ import ("strings") _duchy_tls_cert_file_flag, _duchy_tls_key_file_flag, _duchy_cert_collection_file_flag, - _computations_service_target_flag, - _computations_service_cert_host_flag, + _duchyInternalApiTargetFlag, + _duchyInternalApiCertHostFlag, _kingdom_system_api_target_flag, _kingdom_system_api_cert_host_flag, "--port=8443", @@ -224,7 +220,7 @@ import ("strings") ] + _blob_storage_flags spec: template: spec: { _mounts: "config-files": #ConfigMapMount - _dependencies: ["\(_name)-spanner-computations-server"] + _dependencies: ["\(_name)-internal-api-server"] } } } @@ -242,8 +238,8 @@ import ("strings") cronjobs: { "computations-cleaner": { _container: args: [ - _computations_service_target_flag, - _computations_service_cert_host_flag, + _duchyInternalApiTargetFlag, + _duchyInternalApiCertHostFlag, _duchy_tls_cert_file_flag, _duchy_tls_key_file_flag, _duchy_cert_collection_file_flag, @@ -260,8 +256,8 @@ import ("strings") } // TODO(@wangyaopw): Consider setting GCS and spanner destinations explicityly. networkPolicies: { - "spanner-computations-server": { - _app_label: _object_prefix + "spanner-computations-server-app" + "internal-api-server": { + _app_label: _object_prefix + "internal-api-server-app" _sourceMatchLabels: [ _object_prefix + "herald-daemon-app", _object_prefix + "liquid-legions-v2-mill-daemon-app", @@ -295,7 +291,7 @@ import ("strings") _object_prefix + "computation-control-server-app", ] _destinationMatchLabels: [ - _object_prefix + "spanner-computations-server-app", + _object_prefix + "internal-api-server-app", "opentelemetry-collector-app", ] } @@ -331,7 +327,7 @@ import ("strings") "computations-cleaner": { _app_label: _object_prefix + "computations-cleaner-app" _destinationMatchLabels: [ - _object_prefix + "spanner-computations-server-app", + _object_prefix + "internal-api-server-app", ] } } diff --git a/src/main/k8s/local/BUILD.bazel b/src/main/k8s/local/BUILD.bazel index 7e5460d659a..2870c82a6cc 100644 --- a/src/main/k8s/local/BUILD.bazel +++ b/src/main/k8s/local/BUILD.bazel @@ -77,7 +77,9 @@ cue_library( srcs = ["duchies.cue"], deps = [ ":config_cue", - "//src/main/k8s:duchy", + "//src/main/k8s:postgres", + "//src/main/k8s:postgres_duchy", + "//src/main/k8s:spanner_duchy", ], ) @@ -85,6 +87,7 @@ cue_dump( name = "duchies", cue_tags = { "secret_name": SECRET_NAME, + "db_secret_name": DB_SECRET_NAME, "container_registry": IMAGE_REPOSITORY_SETTINGS.container_registry, "image_repo_prefix": IMAGE_REPOSITORY_SETTINGS.repository_prefix, "image_tag": IMAGE_REPOSITORY_SETTINGS.image_tag, @@ -315,11 +318,13 @@ kustomization_dir( ":edp_simulators", ":emulators", ":kingdom", + ":postgres_database", ], generate_kustomization = True, tags = ["manual"], deps = [ ":config_files", + ":db_creds", "//src/main/k8s/testing/secretfiles:kustomization", ], ) diff --git a/src/main/k8s/local/duchies.cue b/src/main/k8s/local/duchies.cue index 4365d6efff6..e3d312363a8 100644 --- a/src/main/k8s/local/duchies.cue +++ b/src/main/k8s/local/duchies.cue @@ -16,6 +16,7 @@ package k8s _secret_name: string @tag("secret_name") _aggregator_cert_name: string @tag("aggregator_cert_name") +_duchyDbSecretName: string @tag("db_secret_name") _worker1_cert_name: string @tag("worker1_cert_name") _worker2_cert_name: string @tag("worker2_cert_name") @@ -24,6 +25,7 @@ _worker2_cert_name: string @tag("worker2_cert_name") #DuchyConfig: { let duchyName = name name: string + databaseType: string protocolsSetupConfig: string certificateResourceName: string computationControlServiceTarget: (#Target & {name: "\(duchyName)-computation-control-server"}).target @@ -35,14 +37,34 @@ _duchyConfigs: { "aggregator": { protocolsSetupConfig: "aggregator_protocols_setup_config.textproto" certificateResourceName: _aggregator_cert_name + databaseType: "spanner" } "worker1": { protocolsSetupConfig: "non_aggregator_protocols_setup_config.textproto" certificateResourceName: _worker1_cert_name + databaseType: "spanner" } "worker2": { protocolsSetupConfig: "non_aggregator_protocols_setup_config.textproto" certificateResourceName: _worker2_cert_name + databaseType: "postgres" + } +} + +let EnvVars = #EnvVarMap & { + "POSTGRES_USER": { + valueFrom: + secretKeyRef: { + name: _duchyDbSecretName + key: "username" + } + } + "POSTGRES_PASSWORD": { + valueFrom: + secretKeyRef: { + name: _duchyDbSecretName + key: "password" + } } } @@ -54,26 +76,58 @@ _computationControlTargets: { } } -duchies: [ for duchyConfig in _duchyConfigs { - #Duchy & { - _imageSuffixes: { - "computation-control-server": "duchy/local-computation-control" - "liquid-legions-v2-mill-daemon": "duchy/local-liquid-legions-v2-mill" - "requisition-fulfillment-server": "duchy/local-requisition-fulfillment" - "spanner-computations-server": "duchy/local-spanner-computations" +_baseDuchyConfig: { + _imageSuffixes: { + "computation-control-server": "duchy/local-computation-control" + "liquid-legions-v2-mill-daemon": "duchy/local-liquid-legions-v2-mill" + "requisition-fulfillment-server": "duchy/local-requisition-fulfillment" + } + _duchy_secret_name: _secret_name + _computation_control_targets: _computationControlTargets + _kingdom_system_api_target: #KingdomSystemApiTarget + _blob_storage_flags: [ + "--forwarded-storage-service-target=" + (#Target & {name: "fake-storage-server"}).target, + "--forwarded-storage-cert-host=localhost", + ] + _verbose_grpc_logging: "true" +} + +duchies: [ + for duchyConfig in _duchyConfigs { + if (duchyConfig.databaseType == "spanner") { + #SpannerDuchy & _baseDuchyConfig & { + _imageSuffixes: { + "internal-api-server": "duchy/local-spanner-computations" + } + _duchy: { + name: duchyConfig.name + protocols_setup_config: duchyConfig.protocolsSetupConfig + cs_cert_resource_name: duchyConfig.certificateResourceName + } + } } - _duchy: { - name: duchyConfig.name - protocols_setup_config: duchyConfig.protocolsSetupConfig - cs_cert_resource_name: duchyConfig.certificateResourceName + if (duchyConfig.databaseType == "postgres") { + #PostgresDuchy & _baseDuchyConfig & { + _imageSuffixes: { + "internal-api-server": "duchy/local-postgres-internal-server" + } + _duchy: { + name: duchyConfig.name + protocols_setup_config: duchyConfig.protocolsSetupConfig + cs_cert_resource_name: duchyConfig.certificateResourceName + } + _postgresConfig: { + serviceName: "postgres" + password: "$(POSTGRES_PASSWORD)" + user: "$(POSTGRES_USER)" + } + deployments: { + "internal-api-server-deployment": { + _container: _envVars: EnvVars + _updateSchemaContainer: _envVars: EnvVars + } + } + } } - _duchy_secret_name: _secret_name - _computation_control_targets: _computationControlTargets - _kingdom_system_api_target: #KingdomSystemApiTarget - _blob_storage_flags: [ - "--forwarded-storage-service-target=" + (#Target & {name: "fake-storage-server"}).target, - "--forwarded-storage-cert-host=localhost", - ] - _verbose_grpc_logging: "true" - } -}] + }, +] diff --git a/src/main/k8s/local/testing/BUILD.bazel b/src/main/k8s/local/testing/BUILD.bazel index 27f08f6b82e..560f45db9b1 100644 --- a/src/main/k8s/local/testing/BUILD.bazel +++ b/src/main/k8s/local/testing/BUILD.bazel @@ -11,10 +11,13 @@ package( SECRET_NAME = "certs-and-configs" +DB_SECRET_NAME = "db-creds" + cue_dump( name = "duchies", cue_tags = { "secret_name": SECRET_NAME, + "db_secret_name": DB_SECRET_NAME, "container_registry": IMAGE_REPOSITORY_SETTINGS.container_registry, "image_repo_prefix": IMAGE_REPOSITORY_SETTINGS.repository_prefix, "image_tag": IMAGE_REPOSITORY_SETTINGS.image_tag, @@ -56,6 +59,14 @@ kustomization_dir( renames = {"config_files_kustomization.yaml": "kustomization.yaml"}, ) +kustomization_dir( + name = "db_creds", + srcs = ["db_creds_kustomization.yaml"], + renames = { + "db_creds_kustomization.yaml": "kustomization.yaml", + }, +) + kustomization_dir( name = "cmms", testonly = True, @@ -64,11 +75,13 @@ kustomization_dir( ":edp_simulators", "//src/main/k8s/local:emulators", "//src/main/k8s/local:kingdom", + "//src/main/k8s/local:postgres_database", ], generate_kustomization = True, tags = ["manual"], deps = [ ":config_files", + ":db_creds", "//src/main/k8s/testing/secretfiles:kustomization", ], ) diff --git a/src/main/k8s/local/testing/db_creds_kustomization.yaml b/src/main/k8s/local/testing/db_creds_kustomization.yaml new file mode 100644 index 00000000000..02e4abf30f0 --- /dev/null +++ b/src/main/k8s/local/testing/db_creds_kustomization.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 The Cross-Media Measurement Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +secretGenerator: +- name: db-creds + literals: + - username=db-user + - password=hunter2 diff --git a/src/main/k8s/postgres_duchy.cue b/src/main/k8s/postgres_duchy.cue new file mode 100644 index 00000000000..2138d0f453e --- /dev/null +++ b/src/main/k8s/postgres_duchy.cue @@ -0,0 +1,29 @@ +// Copyright 2023 The Cross-Media Measurement Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package k8s + +#PostgresDuchy: { + #Duchy + + _duchy: _duchy + _postgresConfig: #PostgresConfig + + _imageSuffixes: { + "internal-api-server": string | *"duchy/postgres-internal-server" + "update-duchy-schema": string | *"duchy/postgres-update-schema" + } + + _duchyInternalServerContainerArgs: _postgresConfig.flags +} diff --git a/src/main/k8s/spanner_duchy.cue b/src/main/k8s/spanner_duchy.cue new file mode 100644 index 00000000000..d3bdb60b039 --- /dev/null +++ b/src/main/k8s/spanner_duchy.cue @@ -0,0 +1,31 @@ +// Copyright 2023 The Cross-Media Measurement Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package k8s + +#SpannerDuchy: { + #Duchy + + _duchy: _duchy + _spannerConfig: #SpannerConfig & { + database: "\(_duchy.name)_duchy_computations" + } + + _imageSuffixes: { + "internal-api-server": string | *"duchy/spanner-computations" + "update-duchy-schema": string | *"duchy/spanner-update-schema" + } + + _duchyInternalServerContainerArgs: _spannerConfig.flags +} diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/postgres/PostgresComputationsService.kt b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/postgres/PostgresComputationsService.kt index ec9d7d5ecee..043448cce89 100644 --- a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/postgres/PostgresComputationsService.kt +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/postgres/PostgresComputationsService.kt @@ -420,11 +420,11 @@ class PostgresComputationsService( ): CreateComputationLogEntryRequest { return createComputationLogEntryRequest { parent = ComputationParticipantKey(globalId, duchyName).toName() - computationLogEntry { + computationLogEntry = computationLogEntry { // TODO: maybe set participantChildReferenceId logMessage = "Computation $globalId at stage ${computationStage.name}, " + "attempt $attempt" - stageAttempt { + stageAttempt = stageAttempt { stage = computationStage.number stageName = computationStage.name stageStartTime = clock.protoTimestamp() diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/postgres/readers/ComputationStageAttemptReader.kt b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/postgres/readers/ComputationStageAttemptReader.kt index 275d6df7330..d58086b7f29 100644 --- a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/postgres/readers/ComputationStageAttemptReader.kt +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/postgres/readers/ComputationStageAttemptReader.kt @@ -26,7 +26,6 @@ class ComputationStageAttemptReader { data class UnfinishedAttempt( val computationId: Long, - val protocol: Long, val computationStage: Long, val attempt: Long, val details: ComputationStageAttemptDetails @@ -35,7 +34,6 @@ class ComputationStageAttemptReader { row: ResultRow ) : this( computationId = row["ComputationId"], - protocol = row["Protocol"], computationStage = row["ComputationStage"], attempt = row["Attempt"], details = row.getProtoMessage("Details", ComputationStageAttemptDetails.parser()) diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/server/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/server/BUILD.bazel index 472934405ec..a9be4be2636 100644 --- a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/server/BUILD.bazel +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/server/BUILD.bazel @@ -151,3 +151,30 @@ kt_jvm_library( "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/common/identity", ], ) + +kt_jvm_library( + name = "forwarded_storage_postgres_duchy_data_server", + srcs = ["ForwardedStoragePostgresDuchyDataServer.kt"], + deps = [ + "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/server:duchy_data_server", + "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/service:postgres_duchy_data_services", + "@wfa_common_jvm//imports/java/picocli", + "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/common", + "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/common/db/postgres:flags", + "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/common/db/r2dbc/postgres", + "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/storage/forwarded", + ], +) + +java_binary( + name = "ForwardedStoragePostgresDuchyDataServer", + main_class = "org.wfanet.measurement.duchy.deploy.common.server.ForwardedStoragePostgresDuchyDataServerKt", + runtime_deps = [":forwarded_storage_postgres_duchy_data_server"], +) + +java_image( + name = "forwarded_storage_postgres_duchy_data_server_image", + binary = ":ForwardedStoragePostgresDuchyDataServer", + main_class = "org.wfanet.measurement.duchy.deploy.common.server.ForwardedStoragePostgresDuchyDataServerKt", + visibility = ["//src:docker_image_deployment"], +) diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/server/DuchyDataServer.kt b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/server/DuchyDataServer.kt index 59379e111df..0fac3d2b86a 100644 --- a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/server/DuchyDataServer.kt +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/server/DuchyDataServer.kt @@ -56,18 +56,18 @@ abstract class DuchyDataServer : Runnable { lateinit var systemApiFlags: SystemApiFlags private set - val clientCerts = - SigningCerts.fromPemFiles( - certificateFile = serverFlags.tlsFlags.certFile, - privateKeyFile = serverFlags.tlsFlags.privateKeyFile, - trustedCertCollectionFile = serverFlags.tlsFlags.certCollectionFile - ) - val channel: ManagedChannel = - buildMutualTlsChannel(systemApiFlags.target, clientCerts, systemApiFlags.certHost) - .withShutdownTimeout(channelShutdownTimeout) - - val computationLogEntriesClient = + protected val computationLogEntriesClient by lazy { + val clientCerts = + SigningCerts.fromPemFiles( + certificateFile = serverFlags.tlsFlags.certFile, + privateKeyFile = serverFlags.tlsFlags.privateKeyFile, + trustedCertCollectionFile = serverFlags.tlsFlags.certCollectionFile + ) + val channel: ManagedChannel = + buildMutualTlsChannel(systemApiFlags.target, clientCerts, systemApiFlags.certHost) + .withShutdownTimeout(channelShutdownTimeout) ComputationLogEntriesCoroutineStub(channel).withDuchyId(duchyFlags.duchyName) + } protected suspend fun run(services: DuchyDataServices) { val server = CommonServer.fromFlags(serverFlags, this::class.simpleName!!, services.toList()) diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/server/ForwardedStoragePostgresDuchyDataServer.kt b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/server/ForwardedStoragePostgresDuchyDataServer.kt new file mode 100644 index 00000000000..850ff0e4aa7 --- /dev/null +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/server/ForwardedStoragePostgresDuchyDataServer.kt @@ -0,0 +1,58 @@ +// Copyright 2020 The Cross-Media Measurement Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.wfanet.measurement.duchy.deploy.common.server + +import java.time.Clock +import kotlinx.coroutines.runBlocking +import org.wfanet.measurement.common.commandLineMain +import org.wfanet.measurement.common.db.postgres.PostgresFlags +import org.wfanet.measurement.common.db.r2dbc.postgres.PostgresDatabaseClient +import org.wfanet.measurement.common.identity.RandomIdGenerator +import org.wfanet.measurement.duchy.deploy.common.service.PostgresDuchyDataServices +import org.wfanet.measurement.storage.forwarded.ForwardedStorageFromFlags +import picocli.CommandLine + +/** Implementation of [DuchyDataServer] using Postgres and a fake Storage Service. */ +@CommandLine.Command( + name = "ForwardedStoragePostgresDuchyDataServer", + description = ["Server daemon for ${DuchyDataServer.SERVICE_NAME} service."], + mixinStandardHelpOptions = true, + showDefaultValues = true +) +class ForwardedStoragePostgresDuchyDataServer : DuchyDataServer() { + @CommandLine.Mixin private lateinit var postgresFlags: PostgresFlags + @CommandLine.Mixin private lateinit var forwardedStorageFlags: ForwardedStorageFromFlags.Flags + + override fun run() = runBlocking { + val clock = Clock.systemUTC() + val idGenerator = RandomIdGenerator(clock) + val storageClient = + ForwardedStorageFromFlags(forwardedStorageFlags, serverFlags.tlsFlags).storageClient + + val client = PostgresDatabaseClient.fromFlags(postgresFlags) + + run( + PostgresDuchyDataServices.create( + storageClient, + computationLogEntriesClient, + duchyFlags.duchyName, + idGenerator, + client + ) + ) + } +} + +fun main(args: Array) = commandLineMain(ForwardedStoragePostgresDuchyDataServer(), args) diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/service/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/service/BUILD.bazel index 3d20425c430..85f67d4f9f5 100644 --- a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/service/BUILD.bazel +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/service/BUILD.bazel @@ -19,8 +19,8 @@ kt_jvm_library( name = "postgres_duchy_data_services", srcs = ["PostgresDuchyDataServices.kt"], visibility = [ + "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/server:__pkg__", "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/server:__pkg__", - "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/postgres/server:__pkg__", "//src/test/kotlin/org/wfanet/measurement/integration/common/duchy:__pkg__", ], deps = [ diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/postgres/tools/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/postgres/tools/BUILD.bazel new file mode 100644 index 00000000000..5d29bed2132 --- /dev/null +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/postgres/tools/BUILD.bazel @@ -0,0 +1,20 @@ +load("@rules_java//java:defs.bzl", "java_binary") +load("//src/main/docker:macros.bzl", "java_image") + +java_binary( + name = "UpdateSchema", + args = ["--changelog=duchy/postgres/changelog.yaml"], + main_class = "org.wfanet.measurement.gcloud.postgres.tools.UpdateSchema", + resources = ["//src/main/resources/duchy/postgres"], + runtime_deps = [ + "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/gcloud/postgres/tools:update_schema", + ], +) + +java_image( + name = "update_schema_image", + args = ["--changelog=duchy/postgres/changelog.yaml"], + binary = ":UpdateSchema", + main_class = "org.wfanet.measurement.gcloud.postgres.tools.UpdateSchema", + visibility = ["//src:docker_image_deployment"], +) diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/server/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/server/BUILD.bazel index 529288452c1..c50a12a4282 100644 --- a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/server/BUILD.bazel +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/server/BUILD.bazel @@ -5,6 +5,7 @@ load("//src/main/docker:macros.bzl", "java_image") kt_jvm_library( name = "gcs_computation_control_server", srcs = ["GcsComputationControlServer.kt"], + runtime_deps = ["@wfa_common_jvm//imports/java/com/google/cloud/sql/postgres:r2dbc"], deps = [ "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/server:computation_control_server", "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/spanner/computation", @@ -108,3 +109,31 @@ java_image( "//src:docker_image_deployment", ], ) + +kt_jvm_library( + name = "gcs_postgres_duchy_data_server", + srcs = ["GcsPostgresDuchyDataServer.kt"], + deps = [ + "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/server:duchy_data_server", + "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/service:postgres_duchy_data_services", + "@wfa_common_jvm//imports/java/picocli", + "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/common", + "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/common/db/r2dbc/postgres", + "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/gcloud/gcs", + "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/gcloud/postgres:factories", + "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/gcloud/postgres:flags", + ], +) + +java_binary( + name = "GcsPostgresDuchyDataServer", + main_class = "org.wfanet.measurement.duchy.deploy.gcloud.server.GcsPostgresDuchyDataServerKt", + runtime_deps = [":gcs_postgres_duchy_data_server"], +) + +java_image( + name = "gcs_postgres_internal_server_image", + binary = ":GcsPostgresDuchyDataServer", + main_class = "org.wfanet.measurement.duchy.deploy.gcloud.server.GcsPostgresDuchyDataServerKt", + visibility = ["//src:docker_image_deployment"], +) diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/server/GcsPostgresDuchyDataServer.kt b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/server/GcsPostgresDuchyDataServer.kt new file mode 100644 index 00000000000..0d078d62733 --- /dev/null +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/server/GcsPostgresDuchyDataServer.kt @@ -0,0 +1,63 @@ +// Copyright 2023 The Cross-Media Measurement Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.wfanet.measurement.duchy.deploy.gcloud.server + +import java.time.Clock +import kotlinx.coroutines.runBlocking +import org.wfanet.measurement.common.commandLineMain +import org.wfanet.measurement.common.db.r2dbc.postgres.PostgresDatabaseClient +import org.wfanet.measurement.common.identity.RandomIdGenerator +import org.wfanet.measurement.duchy.deploy.common.server.DuchyDataServer +import org.wfanet.measurement.duchy.deploy.common.service.PostgresDuchyDataServices +import org.wfanet.measurement.gcloud.gcs.GcsFromFlags +import org.wfanet.measurement.gcloud.gcs.GcsStorageClient +import org.wfanet.measurement.gcloud.postgres.PostgresConnectionFactories +import org.wfanet.measurement.gcloud.postgres.PostgresFlags as GCloudPostgresFlags +import picocli.CommandLine + +/** + * Implementation of [DuchyDataServer] using Google Cloud Postgres and Google Cloud Storage (GCS). + */ +@CommandLine.Command( + name = "GcsPostgresDuchyDataServer", + description = ["Server daemon for ${DuchyDataServer.SERVICE_NAME} service."], + mixinStandardHelpOptions = true, + showDefaultValues = true +) +class GcsPostgresDuchyDataServer : DuchyDataServer() { + @CommandLine.Mixin private lateinit var gcsFlags: GcsFromFlags.Flags + @CommandLine.Mixin private lateinit var gCloudPostgresFlags: GCloudPostgresFlags + + override fun run() = runBlocking { + val clock = Clock.systemUTC() + val idGenerator = RandomIdGenerator(clock) + val storageClient = GcsStorageClient.fromFlags(GcsFromFlags(gcsFlags)) + + val factory = PostgresConnectionFactories.buildConnectionFactory(gCloudPostgresFlags) + val client = PostgresDatabaseClient.fromConnectionFactory(factory) + + run( + PostgresDuchyDataServices.create( + storageClient, + computationLogEntriesClient, + duchyFlags.duchyName, + idGenerator, + client + ) + ) + } +} + +fun main(args: Array) = commandLineMain(GcsPostgresDuchyDataServer(), args)