Manage external IP addresses in Terraform.

.github/workflows/configure-duchy.yml

@@ -89,14 +89,10 @@ jobs:
     - name: Write ~/.bazelrc
       env:
         IMAGE_TAG: ${{ inputs.image-tag }}
-        POSTGRES_HOST: ${{ vars.AWS_POSTGRES_HOST }}
-        POSTGRES_CRED_SECRET_NAME: ${{ vars.AWS_POSTGRES_CRED_SECRET_NAME }}
         KINGDOM_SYSTEM_API_TARGET: ${{ vars.KINGDOM_SYSTEM_API_TARGET }}
         AGGREGATOR_SYSTEM_API_TARGET: ${{ vars.AGGREGATOR_SYSTEM_API_TARGET }}
         WORKER1_SYSTEM_API_TARGET: ${{ vars.WORKER1_SYSTEM_API_TARGET }}
         WORKER2_SYSTEM_API_TARGET: ${{ vars.WORKER2_SYSTEM_API_TARGET }}
-        COMPUTATION_CONTROL_SERVER_EIPS: ${{ vars.AWS_COMPUTATION_CONTROL_SERVER_EIPS }}
-        S3_BUCKET: ${{ vars.AWS_S3_BUCKET }}
       run: |
         cat << EOF > ~/.bazelrc
         common --config=ci
@@ -107,14 +103,9 @@ jobs:
         build --define aggregator_system_api_target=$AGGREGATOR_SYSTEM_API_TARGET
         build --define worker1_system_api_target=$WORKER1_SYSTEM_API_TARGET
         build --define worker2_system_api_target=$WORKER2_SYSTEM_API_TARGET
-        build --define s3_bucket=$S3_BUCKET
-        build --define s3_region=$AWS_REGION
         build --define duchy_cert_id=$DUCHY_CERT_ID
-        build --define postgres_host=$POSTGRES_HOST
-        build --define postgres_port=5432
-        build --define postgres_region=$AWS_REGION
-        build --define postgres_credential_secret_name=$POSTGRES_CRED_SECRET_NAME
-        build --define computation_control_server_eips=$COMPUTATION_CONTROL_SERVER_EIPS
+        build --define duchy_public_api_address_name=${DUCHY_NAME}-v2alpha
+        build --define duchy_system_api_address_name=${DUCHY_NAME}-system-v1alpha
         EOF
 
     - name: Export BAZEL_BIN

.github/workflows/configure-kingdom.yml

@@ -78,6 +78,8 @@ jobs:
           build --define "image_tag=$IMAGE_TAG"
           build --define "google_cloud_project=$GCLOUD_PROJECT"
           build --define "spanner_instance=$SPANNER_INSTANCE"
+          build --define kingdom_public_api_address_name=kingdom-v2alpha
+          build --define kingdom_system_api_address_name=kingdom-system-v1alpha
           EOF
 
       - name: Export BAZEL_BIN

.github/workflows/configure-reporting-v2.yml

@@ -82,6 +82,7 @@ jobs:
         build --define "postgres_instance=$POSTGRES_INSTANCE"
         build --define "postgres_region=$GCLOUD_REGION"
         build --define "kingdom_public_api_target=$KINGDOM_PUBLIC_API_TARGET"
+        build --define reporting_public_api_address_name=reporting-v2alpha
         EOF
 
     - name: Export BAZEL_BIN

.github/workflows/configure-reporting.yml

@@ -82,6 +82,7 @@ jobs:
         build --define "postgres_instance=$POSTGRES_INSTANCE"
         build --define "postgres_region=$GCLOUD_REGION"
         build --define "kingdom_public_api_target=$KINGDOM_PUBLIC_API_TARGET"
+        build --define reporting_public_api_address_name=reporting-v1alpha
         EOF
 
 

build/variables.bzl

@@ -84,6 +84,8 @@ AWS_SETTINGS = struct(
 KINGDOM_K8S_SETTINGS = struct(
     public_api_target = "$(kingdom_public_api_target)",
     system_api_target = "$(kingdom_system_api_target)",
+    public_api_address_name = "$(kingdom_public_api_address_name)",
+    system_api_address_name = "$(kingdom_system_api_address_name)",
 )
 
 # Settings for Duchy Kubernetes deployments.
@@ -97,6 +99,13 @@ DUCHY_K8S_SETTINGS = struct(
     worker2_system_api_target = "$(worker2_system_api_target)",
     worker1_id = "$(worker1_id)",
     worker2_id = "$(worker2_id)",
+    public_api_address_name = "$(duchy_public_api_address_name)",
+    system_api_address_name = "$(duchy_system_api_address_name)",
+)
+
+# Settings for Reporting Kubernetes deployments.
+REPORTING_K8S_SETTINGS = struct(
+    public_api_address_name = "$(reporting_public_api_address_name)",
 )
 
 # Settings for simulator Kubernetes deployments.

docs/gke/cluster-config.md

@@ -138,38 +138,3 @@ for more information.
 
 For the `dev` environment, we use
 [resource_requirements.yaml](../../src/main/k8s/dev/resource_requirements.yaml)
-
-## Reserving External IPs
-
-If you already have an IP that was assigned to a `LoadBalancer` service, you can
-reserve the IP address to make it static. You can find the assigned external IP
-using
-
-```shell
-kubectl get services
-```
-
-You can find the entry for that IP in the
-[Cloud Console](https://console.cloud.google.com/networking/addresses/list) and
-click the `RESERVE` button.
-
-### Attaching to Service
-
-If you want to ensure that a service always uses the same static load balancer
-IP, you can specify the `loadBalancerIP` field in its
-[`ServiceSpec`](https://kubernetes.io/docs/reference/kubernetes-api/service-resources/service-v1/#ServiceSpec).
-
-For example, the `ServiceSpec` for the Kingdom public API server might look
-something like
-
-```yaml
-type: LoadBalancer
-loadBalancerIP: 34.123.211.251
-selector:
-  app: v2alpha-public-api-server-app
-ports:
-  - name: grpc-port
-    port: 8443
-    protocol: TCP
-    targetPort: 8443
-```

docs/gke/duchy-deployment.md

@@ -94,6 +94,9 @@ Applying the Terraform configuration will create a new cluster. You can use the
 gcloud container clusters get-credentials worker1-duchy
 ```
 
+Applying the Terraform configuration will also create external IP resources and
+output the resource names. These will be needed in later steps.
+
 ## Build and push the container images (optional)
 
 If you aren't using
@@ -136,10 +139,12 @@ bazel build //src/main/k8s/dev:worker1_duchy.tar \
   --define google_cloud_project=halo-kingdom-demo \
   --define spanner_instance=halo-cmms \
   --define duchy_cert_id=SVVse4xWHL0 \
+  --define duchy_public_api_address_name=worker1-duchy-v2alpha \
+  --define duchy_system_api_address_name=worker1-duchy-system-v1alpha \
   --define duchy_storage_bucket=worker1-duchy \
   --define container_registry=ghcr.io \
   --define image_repo_prefix=world-federation-of-advertisers \
-  --define image_tag=0.3.0
+  --define image_tag=0.5.2
 ```
 
 Extract the generated archive to some directory. It is recommended that you
@@ -280,28 +285,6 @@ worker1-spanner-computations-server      ClusterIP    10.123.244.10  <none>
 kubernetes                               ClusterIP    10.123.240.1   <none>         443/TCP        1m
 ```
 
-## Make the Duchy accessible outside the cluster
-
-### Reserve the external IPs
-
-There are two external APIs in the duchy. The
-`worker1-requisition-fulfillment-server` (a.k.a. the public API) is called by
-the EDPs to fulfill their requisitions. The `worker1-computation-control-server`
-(a.k.a. the system API) is called by the other duchies to send computation
-related data. As you can see from the result in the previous step. Only these
-two services have external IPs. However, these external IPs are ephemeral. We
-need to reserve them such that they are stable.
-
-For example, in the halo dev instance, we have subdomains:
-
--   `public.worker1.dev.halo-cmm.org`
--   `system.worker1.dev.halo-cmm.org`
-
-The domains/subdomains are what the EDPs and other duchies use to communicate
-with the duchy.
-
-See [Reserving External IPs](cluster-config.md#reserving-external-ips)
-
 ## Q/A
 
 ### Q1. How to generate certificates/key pairs?

docs/gke/kingdom-deployment.md

@@ -113,9 +113,11 @@ values):
 bazel build //src/main/k8s/dev:kingdom.tar \
   --define google_cloud_project=halo-kingdom-demo \
   --define spanner_instance=halo-cmms \
+  --define kingdom_public_api_address_name=kingdom-v2alpha \
+  --define kingdom_system_api_address_name=kingdom-system-v1alpha \
   --define container_registry=ghcr.io \
   --define image_repo_prefix=world-federation-of-advertisers \
-  --define image_tag=0.3.0
+  --define image_tag=0.5.2
 ```
 
 Extract the generated archive to some directory. It is recommended that you
@@ -265,16 +267,6 @@ v2alpha-public-api-server LoadBalancer 10.3.255.191 34.132.87.22 8443:31300/TCP
 
 ## Make the Kingdom accessible outside the cluster
 
-### Reserve the external IPs
-
-There are two public APIs in the kingdom. The `v2alpha-public-api-server` is
-called by the EDPs, MPs and MCs. The `system-api-server` is called by the
-duchies. As you can see from the result in the previous step. Only these two
-services have external IPs. However, these external IPs are ephemeral. We need
-to reserve them such that they are stable.
-
-See [Reserving External IPs](cluster-config.md#reserving-external-ips)
-
 ### Set up DNS records
 
 In the DNS configuration for a domain you own, add `A` records to assign names

docs/gke/reporting-server-deployment.md

@@ -187,6 +187,7 @@ values):
 
 ```shell
 bazel build //src/main/k8s/dev:reporting.tar \
+  --define reporting_public_api_address_name=reporting-v1alpha \
   --define google_cloud_project=halo-cmm-dev \
   --define postgres_instance=dev-postgres \
   --define postgres_region=us-central1 \
@@ -345,13 +346,6 @@ postgres-reporting-data-server        ClusterIP      10.16.39.47    <none>
 reporting-public-api-v1alpha-server   LoadBalancer   10.16.32.255   34.135.79.68   8443:30104/TCP   8m45s
 ```
 
-## Reserve an external IP
-
-The `reporting-public-api-v1alpha-server` service has an external load balancer
-IP so that it can be accessed from outside the cluster. By default, the assigned
-IP address is ephemeral. We can reserve a static IP to make it easier to access.
-See [Reserving External IPs](cluster-config.md#reserving-external-ips).
-
 ## Appendix
 
 ### Troubleshooting

docs/gke/reporting-v2-server-deployment.md

@@ -190,6 +190,7 @@ values):
 
 ```shell
 bazel build //src/main/k8s/dev:reporting_v2.tar \
+  --define reporting_public_api_address_name=reporting-v2alpha \
   --define google_cloud_project=halo-cmm-dev \
   --define postgres_instance=dev-postgres \
   --define postgres_region=us-central1 \
@@ -359,13 +360,6 @@ NAME                       SCHEDULE     SUSPEND   ACTIVE   LAST SCHEDULE   AGE
 report-scheduling-cronjob  30 6 * * *   False     0        <none>          10m
 ```
 
-## Reserve an external IP
-
-The `reporting-v2alpha-public-api-server` service has an external load balancer
-IP so that it can be accessed from outside the cluster. By default, the assigned
-IP address is ephemeral. We can reserve a static IP to make it easier to access.
-See [Reserving External IPs](cluster-config.md#reserving-external-ips).
-
 ## Appendix
 
 ### Configuring the Report Scheduling CronJob

src/main/k8s/base.cue

@@ -330,26 +330,24 @@ objects: [ for objectSet in objectSets for object in objectSet {object}]
 	spec: {
 		selector: app: "\(metadata.name)-app"
 		ports: [...#ServicePort]
-		type?: "ClusterIP" | "LoadBalancer"
+		type?:           "ClusterIP" | "LoadBalancer"
+		loadBalancerIP?: string | null
 	}
 }
 
-#GrpcService: #Service & {
-	_name:   string
-	_system: string
-	_type:   *"ClusterIP" | "LoadBalancer"
+// K8s Service with external load balancer.
+#ExternalService: #Service & {
+	spec: type: "LoadBalancer"
+}
 
+#GrpcService: #Service & {
 	metadata: {
-		_component: _system
-
-		name: _name
 		annotations: {
 			"cloud.google.com/app-protocols":   "{\"\(#GrpcServicePort.name)\": \"HTTP2\"}"
 			"kubernetes.io/ingress.allow-http": "false"
 		}
 	}
 	spec: {
-		type: _type
 		ports: [#GrpcServicePort]
 	}
 }

src/main/k8s/dev/BUILD.bazel

@@ -7,6 +7,7 @@ load(
     "GCLOUD_SETTINGS",
     "IMAGE_REPOSITORY_SETTINGS",
     "KINGDOM_K8S_SETTINGS",
+    "REPORTING_K8S_SETTINGS",
     "SIMULATOR_K8S_SETTINGS",
 )
 load("//build/k8s:defs.bzl", "kustomization_dir")
@@ -65,6 +66,8 @@ cue_dump(
         "image_tag": IMAGE_REPOSITORY_SETTINGS.image_tag,
         "google_cloud_project": GCLOUD_SETTINGS.project,
         "spanner_instance": GCLOUD_SETTINGS.spanner_instance,
+        "public_api_address_name": KINGDOM_K8S_SETTINGS.public_api_address_name,
+        "system_api_address_name": KINGDOM_K8S_SETTINGS.system_api_address_name,
     },
     tags = ["manual"],
     deps = [
@@ -112,6 +115,8 @@ cue_dump(
         "duchy_name": "aggregator",
         "duchy_protocols_setup_config": "aggregator_protocols_setup_config.textproto",
         "secret_name": SECRET_NAME,
+        "duchy_public_api_address_name": DUCHY_K8S_SETTINGS.public_api_address_name,
+        "duchy_system_api_address_name": DUCHY_K8S_SETTINGS.system_api_address_name,
         "certificate_id": DUCHY_K8S_SETTINGS.certificate_id,
         "cloud_storage_bucket": DUCHY_K8S_SETTINGS.storage_bucket,
         "container_registry": IMAGE_REPOSITORY_SETTINGS.container_registry,
@@ -140,6 +145,8 @@ cue_dump(
         "duchy_name": "worker1",
         "duchy_protocols_setup_config": "non_aggregator_protocols_setup_config.textproto",
         "secret_name": SECRET_NAME,
+        "duchy_public_api_address_name": DUCHY_K8S_SETTINGS.public_api_address_name,
+        "duchy_system_api_address_name": DUCHY_K8S_SETTINGS.system_api_address_name,
         "certificate_id": DUCHY_K8S_SETTINGS.certificate_id,
         "cloud_storage_bucket": DUCHY_K8S_SETTINGS.storage_bucket,
         "container_registry": IMAGE_REPOSITORY_SETTINGS.container_registry,
@@ -168,6 +175,8 @@ cue_dump(
         "duchy_name": "worker2",
         "duchy_protocols_setup_config": "non_aggregator_protocols_setup_config.textproto",
         "secret_name": SECRET_NAME,
+        "duchy_public_api_address_name": DUCHY_K8S_SETTINGS.public_api_address_name,
+        "duchy_system_api_address_name": DUCHY_K8S_SETTINGS.system_api_address_name,
         "certificate_id": DUCHY_K8S_SETTINGS.certificate_id,
         "cloud_storage_bucket": DUCHY_K8S_SETTINGS.storage_bucket,
         "container_registry": IMAGE_REPOSITORY_SETTINGS.container_registry,
@@ -435,6 +444,7 @@ cue_dump(
         "container_registry": IMAGE_REPOSITORY_SETTINGS.container_registry,
         "image_repo_prefix": IMAGE_REPOSITORY_SETTINGS.repository_prefix,
         "image_tag": IMAGE_REPOSITORY_SETTINGS.image_tag,
+        "public_api_address_name": REPORTING_K8S_SETTINGS.public_api_address_name,
         "google_cloud_project": GCLOUD_SETTINGS.project,
         "postgres_instance": GCLOUD_SETTINGS.postgres_instance,
         "postgres_region": GCLOUD_SETTINGS.postgres_region,
@@ -486,6 +496,7 @@ cue_dump(
         "container_registry": IMAGE_REPOSITORY_SETTINGS.container_registry,
         "image_repo_prefix": IMAGE_REPOSITORY_SETTINGS.repository_prefix,
         "image_tag": IMAGE_REPOSITORY_SETTINGS.image_tag,
+        "public_api_address_name": REPORTING_K8S_SETTINGS.public_api_address_name,
         "google_cloud_project": GCLOUD_SETTINGS.project,
         "postgres_instance": GCLOUD_SETTINGS.postgres_instance,
         "postgres_region": GCLOUD_SETTINGS.postgres_region,

src/main/k8s/dev/base_eks.cue

@@ -47,3 +47,16 @@ package k8s
 	initialHeapSize: _ | *maxHeapSize
 	maxHeapSize:     _ | *"64M"
 }
+
+#ExternalService: {
+	_eipAllocations?: string
+
+	metadata: annotations: {
+		"service.beta.kubernetes.io/aws-load-balancer-type":            "nlb"
+		"service.beta.kubernetes.io/aws-load-balancer-scheme":          "internet-facing"
+		"service.beta.kubernetes.io/aws-load-balancer-nlb-target-type": "ip"
+		if _eipAllocations != _|_ {
+			"service.beta.kubernetes.io/aws-load-balancer-eip-allocations": _computationControlServerEips
+		}
+	}
+}

src/main/k8s/dev/base_gke.cue

@@ -111,3 +111,18 @@ package k8s
 		volumeMount: mountPath: "/etc/java"
 	}
 }
+
+#ExternalService: {
+	// Resource name of the external IP address.
+	_ipAddressName?: string
+
+	metadata: annotations: {
+		"cloud.google.com/l4-rbs": "enabled"
+		if _ipAddressName != _|_ {
+			"networking.gke.io/load-balancer-ip-addresses": _ipAddressName
+		}
+	}
+
+	// Explicitly clear deprecated field.
+	spec: loadBalancerIP: null
+}