-
Notifications
You must be signed in to change notification settings - Fork 284
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat: public network deployments #10089
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,39 @@ | ||
#!/bin/sh | ||
|
||
POD_NAME=$(echo $HOSTNAME) | ||
|
||
if [ "${NETWORK_PUBLIC}" = "true" ]; then | ||
# First try treating HOSTNAME as a pod name | ||
NODE_NAME=$(kubectl get pod $POD_NAME -n ${NAMESPACE} -o jsonpath='{.spec.nodeName}' 2>/dev/null) | ||
|
||
# If that fails, HOSTNAME might be the node name itself | ||
if [ $? -ne 0 ]; then | ||
echo "Could not find pod $POD_NAME, assuming $POD_NAME is the node name" | ||
NODE_NAME=$POD_NAME | ||
fi | ||
|
||
EXTERNAL_IP=$(kubectl get node $NODE_NAME -o jsonpath='{.status.addresses[?(@.type=="ExternalIP")].address}') | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Warning. Our AWS nodes do not have external IP addresses. So our prod deployments will likely move to GCP. |
||
|
||
if [ -z "$EXTERNAL_IP" ]; then | ||
echo "Warning: Could not find ExternalIP, falling back to InternalIP" | ||
EXTERNAL_IP=$(kubectl get node $NODE_NAME -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}') | ||
fi | ||
|
||
TCP_ADDR="${EXTERNAL_IP}:${P2P_TCP_PORT}" | ||
UDP_ADDR="${EXTERNAL_IP}:${P2P_UDP_PORT}" | ||
|
||
else | ||
# Get pod IP for non-public networks | ||
POD_IP=$(hostname -i) | ||
TCP_ADDR="${POD_IP}:${P2P_TCP_PORT}" | ||
UDP_ADDR="${POD_IP}:${P2P_UDP_PORT}" | ||
fi | ||
|
||
# Write addresses to file for sourcing | ||
echo "export P2P_TCP_ANNOUNCE_ADDR=${TCP_ADDR}" > /shared/p2p/p2p-addresses | ||
echo "export P2P_TCP_LISTEN_ADDR=0.0.0.0:${P2P_TCP_PORT}" >> /shared/p2p/p2p-addresses | ||
echo "export P2P_UDP_ANNOUNCE_ADDR=${UDP_ADDR}" >> /shared/p2p/p2p-addresses | ||
echo "export P2P_UDP_LISTEN_ADDR=0.0.0.0:${P2P_UDP_PORT}" >> /shared/p2p/p2p-addresses | ||
|
||
echo "P2P addresses configured:" | ||
cat /shared/p2p/p2p-addresses |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,88 @@ | ||
#!/bin/bash | ||
|
||
set -ex | ||
|
||
# Function to get pod and node details | ||
get_service_address() { | ||
local SERVICE_LABEL=$1 | ||
local PORT=$2 | ||
local MAX_RETRIES=30 | ||
local RETRY_INTERVAL=2 | ||
local attempt=1 | ||
|
||
# Get pod name | ||
while [ $attempt -le $MAX_RETRIES ]; do | ||
POD_NAME=$(kubectl get pods -n ${NAMESPACE} -l app=${SERVICE_LABEL} -o jsonpath='{.items[0].metadata.name}') | ||
if [ -n "$POD_NAME" ]; then | ||
break | ||
fi | ||
echo "Attempt $attempt: Waiting for ${SERVICE_LABEL} pod to be available..." >&2 | ||
sleep $RETRY_INTERVAL | ||
attempt=$((attempt + 1)) | ||
done | ||
|
||
if [ -z "$POD_NAME" ]; then | ||
echo "Error: Failed to get ${SERVICE_LABEL} pod name after $MAX_RETRIES attempts" >&2 | ||
return 1 | ||
fi | ||
echo "Pod name: [${POD_NAME}]" >&2 | ||
|
||
# Get node name | ||
attempt=1 | ||
NODE_NAME="" | ||
while [ $attempt -le $MAX_RETRIES ]; do | ||
NODE_NAME=$(kubectl get pod ${POD_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.nodeName}') | ||
if [ -n "$NODE_NAME" ]; then | ||
break | ||
fi | ||
echo "Attempt $attempt: Waiting for node name to be available..." >&2 | ||
sleep $RETRY_INTERVAL | ||
attempt=$((attempt + 1)) | ||
done | ||
|
||
if [ -z "$NODE_NAME" ]; then | ||
echo "Error: Failed to get node name after $MAX_RETRIES attempts" >&2 | ||
return 1 | ||
fi | ||
echo "Node name: ${NODE_NAME}" >&2 | ||
|
||
# Get the node's external IP | ||
NODE_IP=$(kubectl get node ${NODE_NAME} -o jsonpath='{.status.addresses[?(@.type=="ExternalIP")].address}') | ||
echo "Node IP: ${NODE_IP}" >&2 | ||
echo "http://${NODE_IP}:${PORT}" | ||
} | ||
|
||
# Configure Ethereum address | ||
if [ "${ETHEREUM_EXTERNAL_HOST}" != "" ]; then | ||
ETHEREUM_ADDR="${ETHEREUM_EXTERNAL_HOST}" | ||
elif [ "${NETWORK_PUBLIC}" = "true" ]; then | ||
ETHEREUM_ADDR=$(get_service_address "ethereum" "${ETHEREUM_PORT}") | ||
else | ||
ETHEREUM_ADDR="http://${SERVICE_NAME}-ethereum.${NAMESPACE}:${ETHEREUM_PORT}" | ||
fi | ||
|
||
# Configure Boot Node address | ||
if [ "${BOOT_NODE_EXTERNAL_HOST}" != "" ]; then | ||
BOOT_NODE_ADDR="${BOOT_NODE_EXTERNAL_HOST}" | ||
elif [ "${NETWORK_PUBLIC}" = "true" ]; then | ||
BOOT_NODE_ADDR=$(get_service_address "boot-node" "${BOOT_NODE_PORT}") | ||
else | ||
BOOT_NODE_ADDR="http://${SERVICE_NAME}-boot-node.${NAMESPACE}:${BOOT_NODE_PORT}" | ||
fi | ||
|
||
# Configure Prover Node address | ||
if [ "${PROVER_NODE_EXTERNAL_HOST}" != "" ]; then | ||
PROVER_NODE_ADDR="${PROVER_NODE_EXTERNAL_HOST}" | ||
elif [ "${NETWORK_PUBLIC}" = "true" ]; then | ||
PROVER_NODE_ADDR=$(get_service_address "prover-node" "${PROVER_NODE_PORT}") | ||
else | ||
PROVER_NODE_ADDR="http://${SERVICE_NAME}-prover-node.${NAMESPACE}:${PROVER_NODE_PORT}" | ||
fi | ||
|
||
|
||
# Write addresses to file for sourcing | ||
echo "export ETHEREUM_HOST=${ETHEREUM_ADDR}" >> /shared/config/service-addresses | ||
echo "export BOOT_NODE_HOST=${BOOT_NODE_ADDR}" >> /shared/config/service-addresses | ||
echo "export PROVER_NODE_HOST=${PROVER_NODE_ADDR}" >> /shared/config/service-addresses | ||
echo "Addresses configured:" | ||
cat /shared/config/service-addresses |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -50,37 +50,19 @@ app.kubernetes.io/name: {{ include "aztec-network.name" . }} | |
app.kubernetes.io/instance: {{ .Release.Name }} | ||
{{- end }} | ||
|
||
{{- define "aztec-network.ethereumHost" -}} | ||
{{- if .Values.ethereum.externalHost -}} | ||
http://{{ .Values.ethereum.externalHost }}:{{ .Values.ethereum.service.port }} | ||
{{- else -}} | ||
http://{{ include "aztec-network.fullname" . }}-ethereum.{{ .Release.Namespace }}:{{ .Values.ethereum.service.port }} | ||
{{- end -}} | ||
{{- end -}} | ||
|
||
|
||
{{- define "aztec-network.pxeUrl" -}} | ||
{{- if .Values.pxe.externalHost -}} | ||
http://{{ .Values.pxe.externalHost }}:{{ .Values.pxe.service.port }} | ||
{{- else -}} | ||
http://{{ include "aztec-network.fullname" . }}-pxe.{{ .Release.Namespace }}:{{ .Values.pxe.service.port }} | ||
{{- end -}} | ||
http://{{ include "aztec-network.fullname" . }}-pxe.{{ .Release.Namespace }}:{{ .Values.pxe.service.nodePort }} | ||
{{- end -}} | ||
|
||
{{- define "aztec-network.bootNodeUrl" -}} | ||
{{- if .Values.bootNode.externalTcpHost -}} | ||
http://{{ .Values.bootNode.externalTcpHost }}:{{ .Values.bootNode.service.nodePort }} | ||
{{- else -}} | ||
http://{{ include "aztec-network.fullname" . }}-boot-node-0.{{ include "aztec-network.fullname" . }}-boot-node.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.bootNode.service.nodePort }} | ||
{{- end -}} | ||
{{- end -}} | ||
|
||
{{- define "aztec-network.validatorUrl" -}} | ||
{{- if .Values.validator.externalTcpHost -}} | ||
http://{{ .Values.validator.externalTcpHost }}:{{ .Values.validator.service.nodePort }} | ||
{{- else -}} | ||
http://{{ include "aztec-network.fullname" . }}-validator.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.validator.service.nodePort }} | ||
{{- end -}} | ||
{{- end -}} | ||
|
||
{{- define "aztec-network.metricsHost" -}} | ||
http://{{ include "aztec-network.fullname" . }}-metrics.{{ .Release.Namespace }} | ||
|
@@ -123,3 +105,89 @@ http://{{ include "aztec-network.fullname" . }}-metrics.{{ .Release.Namespace }} | |
{{- end -}} | ||
{{- end -}} | ||
{{- end -}} | ||
|
||
{{/* | ||
P2P Setup Container | ||
*/}} | ||
{{- define "aztec-network.p2pSetupContainer" -}} | ||
- name: setup-p2p-addresses | ||
image: bitnami/kubectl | ||
command: | ||
- /bin/sh | ||
- -c | ||
- | | ||
cp /scripts/setup-p2p-addresses.sh /tmp/setup-p2p-addresses.sh && \ | ||
chmod +x /tmp/setup-p2p-addresses.sh && \ | ||
/tmp/setup-p2p-addresses.sh | ||
env: | ||
- name: NETWORK_PUBLIC | ||
value: "{{ .Values.network.public }}" | ||
- name: NAMESPACE | ||
value: {{ .Release.Namespace }} | ||
- name: P2P_TCP_PORT | ||
value: "{{ .Values.validator.service.p2pTcpPort }}" | ||
- name: P2P_UDP_PORT | ||
value: "{{ .Values.validator.service.p2pUdpPort }}" | ||
volumeMounts: | ||
- name: scripts | ||
mountPath: /scripts | ||
- name: p2p-addresses | ||
mountPath: /shared/p2p | ||
{{- end -}} | ||
|
||
{{/* | ||
Service Address Setup Container | ||
*/}} | ||
{{- define "aztec-network.serviceAddressSetupContainer" -}} | ||
- name: setup-service-addresses | ||
image: bitnami/kubectl | ||
command: | ||
- /bin/bash | ||
- -c | ||
- | | ||
cp /scripts/setup-service-addresses.sh /tmp/setup-service-addresses.sh && \ | ||
chmod +x /tmp/setup-service-addresses.sh && \ | ||
/tmp/setup-service-addresses.sh | ||
env: | ||
- name: NETWORK_PUBLIC | ||
value: "{{ .Values.network.public }}" | ||
- name: NAMESPACE | ||
value: {{ .Release.Namespace }} | ||
- name: EXTERNAL_ETHEREUM_HOST | ||
value: "{{ .Values.ethereum.externalHost }}" | ||
- name: ETHEREUM_PORT | ||
value: "{{ .Values.ethereum.service.port }}" | ||
- name: EXTERNAL_BOOT_NODE_HOST | ||
value: "{{ .Values.bootNode.externalHost }}" | ||
- name: BOOT_NODE_PORT | ||
value: "{{ .Values.bootNode.service.nodePort }}" | ||
- name: EXTERNAL_PROVER_NODE_HOST | ||
value: "{{ .Values.proverNode.externalHost }}" | ||
- name: PROVER_NODE_PORT | ||
value: "{{ .Values.proverNode.service.nodePort }}" | ||
- name: SERVICE_NAME | ||
value: {{ include "aztec-network.fullname" . }} | ||
volumeMounts: | ||
- name: scripts | ||
mountPath: /scripts | ||
- name: config | ||
mountPath: /shared/config | ||
{{- end -}} | ||
|
||
{{/** | ||
Anti-affinity when running in public network mode | ||
*/}} | ||
{{- define "aztec-network.publicAntiAffinity" -}} | ||
affinity: | ||
podAntiAffinity: | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This will restrict the number of validators we can run right? IIRC we mentioned the cluster having 10 nodes, so we can have a max of 10 services running? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Exactly, thanks, I meant to call that out in the description of the PR. will update it. Will also update the values.yaml to call this out explicitly. |
||
requiredDuringSchedulingIgnoredDuringExecution: | ||
- labelSelector: | ||
matchExpressions: | ||
- key: app | ||
operator: In | ||
values: | ||
- validator | ||
- boot-node | ||
- prover | ||
topologyKey: "kubernetes.io/hostname" | ||
{{- end -}} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
alias stopped working. I have no idea why.