diff --git a/provision/matchbox/talconfig.yaml b/provision/matchbox/talconfig.yaml deleted file mode 100644 index 8519b698..00000000 --- a/provision/matchbox/talconfig.yaml +++ /dev/null @@ -1,58 +0,0 @@ ---- -clusterName: servonet -endpoint: https://10.4.3.1:6443 -domain: servonet.lan - -talosVersion: v1.7.5 -kubernetesVersion: v1.30.2 - -allowSchedulingOnControlPlanes: true - -nodes: - - hostname: tom-364c - nameservers: - - 10.4.1.1 - ipAddress: "10.4.3.32" - controlPlane: true - installDisk: /dev/sda - networkInterfaces: - - interface: eth0 - dhcp: true - addresses: - - 10.4.3.32 - routes: - - network: 0.0.0.0/0 # The route's network (destination). - gateway: 10.4.1.1 # The route's gateway (if empty, creates link scope route). - metric: 1024 # The optional metric for the route. - vip: - ip: "10.4.3.1" - - hostname: tom-3a11 - ipAddress: "10.4.3.33" - controlPlane: true - installDisk: /dev/sda - networkInterfaces: - - interface: eth0 - dhcp: true - addresses: - - 10.4.3.33 - routes: - - network: 0.0.0.0/0 # The route's network (destination). - gateway: 10.4.1.1 # The route's gateway (if empty, creates link scope route). - metric: 1024 # The optional metric for the route. - vip: - ip: "10.4.3.1" - - hostname: tom-3e67 - ipAddress: "10.4.3.34" - controlPlane: true - installDisk: /dev/mmcblk0 - networkInterfaces: - - interface: eth0 - dhcp: true - addresses: - - 10.4.3.34 - routes: - - network: 0.0.0.0/0 # The route's network (destination). - gateway: 10.4.1.1 # The route's gateway (if empty, creates link scope route). - metric: 1024 # The optional metric for the route. - vip: - ip: "10.4.3.1" diff --git a/provision/servonet/bootstrap.tf b/provision/servonet/bootstrap.tf index 6607118c..c8a4b14b 100644 --- a/provision/servonet/bootstrap.tf +++ b/provision/servonet/bootstrap.tf @@ -10,5 +10,4 @@ resource "github_repository_deploy_key" "flux_deploy" { key = tls_private_key.flux.public_key_openssh read_only = "false" } - data "github_ssh_keys" "this" {} diff --git a/provision/servonet/main.tf b/provision/servonet/main.tf index 3552ca1d..2c5c04be 100644 --- a/provision/servonet/main.tf +++ b/provision/servonet/main.tf @@ -56,7 +56,6 @@ provider "talos" {} provider "github" { owner = "lambchop4prez" - token = data.vault_generic_secret.servonet.data["github_access_token"] } provider "kustomization" { diff --git a/provision/matchbox/assets/.gitignore b/provision/servonet/matchbox/assets/.gitignore similarity index 100% rename from provision/matchbox/assets/.gitignore rename to provision/servonet/matchbox/assets/.gitignore diff --git a/provision/matchbox/docker-compose.yaml b/provision/servonet/matchbox/docker-compose.yaml similarity index 58% rename from provision/matchbox/docker-compose.yaml rename to provision/servonet/matchbox/docker-compose.yaml index 8015bb76..f9350b5d 100644 --- a/provision/matchbox/docker-compose.yaml +++ b/provision/servonet/matchbox/docker-compose.yaml @@ -65,3 +65,41 @@ services: volumes: - "$PWD/assets/talos/${TALOS_VERSION}/rpi:/out:rw" - "/dev:/dev:rw" + jetson-kernel: + image: ghcr.io/siderolabs/imager:${TALOS_VERSION} + command: + - iso + - --arch + - arm64 + - --overlay-image + - ghcr.io/siderolabs/sbc-jetson:v0.1.0-beta.0@sha256:6016709654521f9a038f168651995b8c68fe73044c15ef7dadc9af72f1c8c4fb + - --overlay-name + - jetson_nano + - --output-kind + - kernel + platform: linux/arm64 + privileged: true + volumes: + - "$PWD/assets/talos/${TALOS_VERSION}/jetson:/out:rw" + - "/dev:/dev:rw" + jetson: + image: ghcr.io/siderolabs/imager:${TALOS_VERSION} + command: + - iso + - --arch + - arm64 + - --overlay-image + - ghcr.io/siderolabs/sbc-jetson:v0.1.0-beta.0@sha256:6016709654521f9a038f168651995b8c68fe73044c15ef7dadc9af72f1c8c4fb + - --overlay-name + - jetson_nano + - --system-extension-image + - ghcr.io/siderolabs/nvidia-container-toolkit:535.129.03-v1.14.6@sha256:21e7421ca1d7140fe40f60931a28bb60d0d0eaa7dd204d244888b8a791f80309 + # - --system-extension-image + # - ghcr.io/siderolabs/nvidia-open-gpu-kernel-modules:535.129.03-v1.7.0@sha256:18e8fa5925fcec86f26f4f5f6f3e57f9cdb3c5eef3b7100a24e6ac4c9977cade + - --output-kind + - initramfs + platform: linux/arm64 + privileged: true + volumes: + - "$PWD/assets/talos/${TALOS_VERSION}/jetson:/out:rw" + - "/dev:/dev:rw" diff --git a/provision/matchbox/groups/default.json b/provision/servonet/matchbox/groups/default.json similarity index 100% rename from provision/matchbox/groups/default.json rename to provision/servonet/matchbox/groups/default.json diff --git a/provision/matchbox/groups/tom-364c.json b/provision/servonet/matchbox/groups/tom-364c.json similarity index 100% rename from provision/matchbox/groups/tom-364c.json rename to provision/servonet/matchbox/groups/tom-364c.json diff --git a/provision/matchbox/groups/tom-3a11.json b/provision/servonet/matchbox/groups/tom-3a11.json similarity index 100% rename from provision/matchbox/groups/tom-3a11.json rename to provision/servonet/matchbox/groups/tom-3a11.json diff --git a/provision/matchbox/groups/tom-3e67.json b/provision/servonet/matchbox/groups/tom-3e67.json similarity index 100% rename from provision/matchbox/groups/tom-3e67.json rename to provision/servonet/matchbox/groups/tom-3e67.json diff --git a/provision/matchbox/profiles/tom-364c.json b/provision/servonet/matchbox/profiles/tom-364c.json similarity index 83% rename from provision/matchbox/profiles/tom-364c.json rename to provision/servonet/matchbox/profiles/tom-364c.json index 0b43e403..7ab52432 100644 --- a/provision/matchbox/profiles/tom-364c.json +++ b/provision/servonet/matchbox/profiles/tom-364c.json @@ -14,7 +14,7 @@ "printk.devkmsg=on", "net.ifnames=0", "talos.platform=metal", - "talos.config=http://matchbox.lan:8080/assets/clusterconfig/servonet-tom-364c.yaml" + "talos.config=http://matchbox.lan:8080/assets/servonet-tom-364c.yaml" ] } } diff --git a/provision/matchbox/profiles/tom-3a11.json b/provision/servonet/matchbox/profiles/tom-3a11.json similarity index 83% rename from provision/matchbox/profiles/tom-3a11.json rename to provision/servonet/matchbox/profiles/tom-3a11.json index 1283ed6f..df5477bd 100644 --- a/provision/matchbox/profiles/tom-3a11.json +++ b/provision/servonet/matchbox/profiles/tom-3a11.json @@ -14,7 +14,7 @@ "printk.devkmsg=on", "net.ifnames=0", "talos.platform=metal", - "talos.config=http://matchbox.lan:8080/assets/clusterconfig/servonet-tom-3a11.yaml" + "talos.config=http://matchbox.lan:8080/assets/servonet-tom-3a11.yaml" ] } } diff --git a/provision/matchbox/profiles/tom-3e67.json b/provision/servonet/matchbox/profiles/tom-3e67.json similarity index 83% rename from provision/matchbox/profiles/tom-3e67.json rename to provision/servonet/matchbox/profiles/tom-3e67.json index 46cfe843..6dc36aeb 100644 --- a/provision/matchbox/profiles/tom-3e67.json +++ b/provision/servonet/matchbox/profiles/tom-3e67.json @@ -14,7 +14,7 @@ "printk.devkmsg=on", "net.ifnames=0", "talos.platform=metal", - "talos.config=http://matchbox.lan:8080/assets/clusterconfig/servonet-tom-3e67.yaml" + "talos.config=http://matchbox.lan:8080/assets/servonet-tom-3e67.yaml" ] } } diff --git a/provision/servonet/matchbox/talconfig.yaml b/provision/servonet/matchbox/talconfig.yaml new file mode 100644 index 00000000..ec3c664d --- /dev/null +++ b/provision/servonet/matchbox/talconfig.yaml @@ -0,0 +1,157 @@ +--- +clusterName: servonet +endpoint: https://10.4.3.1:6443 +domain: servonet.lan + +talosVersion: v1.7.5 +kubernetesVersion: v1.30.2 + +allowSchedulingOnControlPlanes: true + +cniConfig: + name: none + +nodes: + - hostname: tom-364c + nameservers: + - 10.4.1.1 + ipAddress: "10.4.3.32" + controlPlane: true + installDisk: /dev/sda + networkInterfaces: + - interface: eth0 + dhcp: true + addresses: + - 10.4.3.32 + routes: + - network: 0.0.0.0/0 # The route's network (destination). + gateway: 10.4.1.1 # The route's gateway (if empty, creates link scope route). + metric: 1024 # The optional metric for the route. + vip: + ip: "10.4.3.1" + - hostname: tom-3a11 + ipAddress: "10.4.3.33" + controlPlane: true + installDisk: /dev/sda + networkInterfaces: + - interface: eth0 + dhcp: true + addresses: + - 10.4.3.33 + routes: + - network: 0.0.0.0/0 # The route's network (destination). + gateway: 10.4.1.1 # The route's gateway (if empty, creates link scope route). + metric: 1024 # The optional metric for the route. + vip: + ip: "10.4.3.1" + - hostname: tom-3e67 + ipAddress: "10.4.3.34" + controlPlane: true + installDisk: /dev/mmcblk1 + networkInterfaces: + - interface: eth0 + dhcp: true + addresses: + - 10.4.3.34 + routes: + - network: 0.0.0.0/0 # The route's network (destination). + gateway: 10.4.1.1 # The route's gateway (if empty, creates link scope route). + metric: 1024 # The optional metric for the route. + vip: + ip: "10.4.3.1" +patches: + # Enable kubePrism + - |- + machine: + features: + kubePrism: + enabled: true + port: 7445 + # Install cillium + - |- + cluster: + inlineManifests: + - name: cilium-install + contents: | + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cilium-install + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin + subjects: + - kind: ServiceAccount + name: cilium-install + namespace: kube-system + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cilium-install + namespace: kube-system + --- + apiVersion: batch/v1 + kind: Job + metadata: + name: cilium-install + namespace: kube-system + spec: + backoffLimit: 10 + template: + metadata: + labels: + app: cilium-install + spec: + restartPolicy: OnFailure + tolerations: + - operator: Exists + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + - effect: PreferNoSchedule + operator: Exists + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoExecute + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: PreferNoSchedule + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + serviceAccount: cilium-install + serviceAccountName: cilium-install + hostNetwork: true + containers: + - name: cilium-install + image: quay.io/cilium/cilium-cli-ci:latest + env: + - name: KUBERNETES_SERVICE_HOST + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: KUBERNETES_SERVICE_PORT + value: "6443" + command: + - cilium + - install + - --set ipam.mode=kubernetes + - --set kubeProxyReplacement=true + - --set securityContext.capabilities.ciliumAgent="{CHOWN,KILL,NET_ADMIN,NET_RAW,IPC_LOCK,SYS_ADMIN,SYS_RESOURCE,DAC_OVERRIDE,FOWNER,SETGID,SETUID}" + - --set securityContext.capabilities.cleanCiliumState="{NET_ADMIN,SYS_ADMIN,SYS_RESOURCE}" + - --set cgroup.autoMount.enabled=false + - --set cgroup.hostRoot=/sys/fs/cgroup + - --set k8sServiceHost=localhost + - --set k8sServicePort=7445 diff --git a/provision/matchbox/talsecret.sops.yaml b/provision/servonet/matchbox/talsecret.sops.yaml similarity index 100% rename from provision/matchbox/talsecret.sops.yaml rename to provision/servonet/matchbox/talsecret.sops.yaml diff --git a/provision/servonet/secrets.tf b/provision/servonet/secrets.tf index 47d3eab9..23015376 100644 --- a/provision/servonet/secrets.tf +++ b/provision/servonet/secrets.tf @@ -7,6 +7,3 @@ data "vault_generic_secret" "proxmox_auth" { data "vault_generic_secret" "opnsense_auth" { path = "secrets/opnsense/auth/terraform" } -data "vault_generic_secret" "servonet" { - path = "secrets/servonet" -} diff --git a/provision/servonet/variables.tf b/provision/servonet/variables.tf index d5ff1485..c0cf7dcb 100644 --- a/provision/servonet/variables.tf +++ b/provision/servonet/variables.tf @@ -94,8 +94,8 @@ variable "workers" { cores = 4 memory = 16384 storage = 32 - devices = ["tesla-p4"] - config = "gpu-worker" + devices = [] + config = null }] }