Skip to content

Commit

Permalink
refactor: Move matchbox config into servonet provisioning
Browse files Browse the repository at this point in the history
  • Loading branch information
lambchop4prez committed Aug 5, 2024
1 parent 32cb5b5 commit 54a95bd
Show file tree
Hide file tree
Showing 16 changed files with 200 additions and 68 deletions.
58 changes: 0 additions & 58 deletions provision/matchbox/talconfig.yaml

This file was deleted.

1 change: 0 additions & 1 deletion provision/servonet/bootstrap.tf
Original file line number Diff line number Diff line change
Expand Up @@ -10,5 +10,4 @@ resource "github_repository_deploy_key" "flux_deploy" {
key = tls_private_key.flux.public_key_openssh
read_only = "false"
}

data "github_ssh_keys" "this" {}
1 change: 0 additions & 1 deletion provision/servonet/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,6 @@ provider "talos" {}

provider "github" {
owner = "lambchop4prez"
token = data.vault_generic_secret.servonet.data["github_access_token"]
}

provider "kustomization" {
Expand Down
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -65,3 +65,41 @@ services:
volumes:
- "$PWD/assets/talos/${TALOS_VERSION}/rpi:/out:rw"
- "/dev:/dev:rw"
jetson-kernel:
image: ghcr.io/siderolabs/imager:${TALOS_VERSION}
command:
- iso
- --arch
- arm64
- --overlay-image
- ghcr.io/siderolabs/sbc-jetson:v0.1.0-beta.0@sha256:6016709654521f9a038f168651995b8c68fe73044c15ef7dadc9af72f1c8c4fb
- --overlay-name
- jetson_nano
- --output-kind
- kernel
platform: linux/arm64
privileged: true
volumes:
- "$PWD/assets/talos/${TALOS_VERSION}/jetson:/out:rw"
- "/dev:/dev:rw"
jetson:
image: ghcr.io/siderolabs/imager:${TALOS_VERSION}
command:
- iso
- --arch
- arm64
- --overlay-image
- ghcr.io/siderolabs/sbc-jetson:v0.1.0-beta.0@sha256:6016709654521f9a038f168651995b8c68fe73044c15ef7dadc9af72f1c8c4fb
- --overlay-name
- jetson_nano
- --system-extension-image
- ghcr.io/siderolabs/nvidia-container-toolkit:535.129.03-v1.14.6@sha256:21e7421ca1d7140fe40f60931a28bb60d0d0eaa7dd204d244888b8a791f80309
# - --system-extension-image
# - ghcr.io/siderolabs/nvidia-open-gpu-kernel-modules:535.129.03-v1.7.0@sha256:18e8fa5925fcec86f26f4f5f6f3e57f9cdb3c5eef3b7100a24e6ac4c9977cade
- --output-kind
- initramfs
platform: linux/arm64
privileged: true
volumes:
- "$PWD/assets/talos/${TALOS_VERSION}/jetson:/out:rw"
- "/dev:/dev:rw"
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
"printk.devkmsg=on",
"net.ifnames=0",
"talos.platform=metal",
"talos.config=http://matchbox.lan:8080/assets/clusterconfig/servonet-tom-364c.yaml"
"talos.config=http://matchbox.lan:8080/assets/servonet-tom-364c.yaml"
]
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
"printk.devkmsg=on",
"net.ifnames=0",
"talos.platform=metal",
"talos.config=http://matchbox.lan:8080/assets/clusterconfig/servonet-tom-3a11.yaml"
"talos.config=http://matchbox.lan:8080/assets/servonet-tom-3a11.yaml"
]
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
"printk.devkmsg=on",
"net.ifnames=0",
"talos.platform=metal",
"talos.config=http://matchbox.lan:8080/assets/clusterconfig/servonet-tom-3e67.yaml"
"talos.config=http://matchbox.lan:8080/assets/servonet-tom-3e67.yaml"
]
}
}
157 changes: 157 additions & 0 deletions provision/servonet/matchbox/talconfig.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
---
clusterName: servonet
endpoint: https://10.4.3.1:6443
domain: servonet.lan

talosVersion: v1.7.5
kubernetesVersion: v1.30.2

allowSchedulingOnControlPlanes: true

cniConfig:
name: none

nodes:
- hostname: tom-364c
nameservers:
- 10.4.1.1
ipAddress: "10.4.3.32"
controlPlane: true
installDisk: /dev/sda
networkInterfaces:
- interface: eth0
dhcp: true
addresses:
- 10.4.3.32
routes:
- network: 0.0.0.0/0 # The route's network (destination).
gateway: 10.4.1.1 # The route's gateway (if empty, creates link scope route).
metric: 1024 # The optional metric for the route.
vip:
ip: "10.4.3.1"
- hostname: tom-3a11
ipAddress: "10.4.3.33"
controlPlane: true
installDisk: /dev/sda
networkInterfaces:
- interface: eth0
dhcp: true
addresses:
- 10.4.3.33
routes:
- network: 0.0.0.0/0 # The route's network (destination).
gateway: 10.4.1.1 # The route's gateway (if empty, creates link scope route).
metric: 1024 # The optional metric for the route.
vip:
ip: "10.4.3.1"
- hostname: tom-3e67
ipAddress: "10.4.3.34"
controlPlane: true
installDisk: /dev/mmcblk1
networkInterfaces:
- interface: eth0
dhcp: true
addresses:
- 10.4.3.34
routes:
- network: 0.0.0.0/0 # The route's network (destination).
gateway: 10.4.1.1 # The route's gateway (if empty, creates link scope route).
metric: 1024 # The optional metric for the route.
vip:
ip: "10.4.3.1"
patches:
# Enable kubePrism
- |-
machine:
features:
kubePrism:
enabled: true
port: 7445
# Install cillium
- |-
cluster:
inlineManifests:
- name: cilium-install
contents: |
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: cilium-install
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: cilium-install
namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: cilium-install
namespace: kube-system
---
apiVersion: batch/v1
kind: Job
metadata:
name: cilium-install
namespace: kube-system
spec:
backoffLimit: 10
template:
metadata:
labels:
app: cilium-install
spec:
restartPolicy: OnFailure
tolerations:
- operator: Exists
- effect: NoSchedule
operator: Exists
- effect: NoExecute
operator: Exists
- effect: PreferNoSchedule
operator: Exists
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoExecute
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: PreferNoSchedule
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.kubernetes.io/control-plane
operator: Exists
serviceAccount: cilium-install
serviceAccountName: cilium-install
hostNetwork: true
containers:
- name: cilium-install
image: quay.io/cilium/cilium-cli-ci:latest
env:
- name: KUBERNETES_SERVICE_HOST
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.podIP
- name: KUBERNETES_SERVICE_PORT
value: "6443"
command:
- cilium
- install
- --set ipam.mode=kubernetes
- --set kubeProxyReplacement=true
- --set securityContext.capabilities.ciliumAgent="{CHOWN,KILL,NET_ADMIN,NET_RAW,IPC_LOCK,SYS_ADMIN,SYS_RESOURCE,DAC_OVERRIDE,FOWNER,SETGID,SETUID}"
- --set securityContext.capabilities.cleanCiliumState="{NET_ADMIN,SYS_ADMIN,SYS_RESOURCE}"
- --set cgroup.autoMount.enabled=false
- --set cgroup.hostRoot=/sys/fs/cgroup
- --set k8sServiceHost=localhost
- --set k8sServicePort=7445
File renamed without changes.
3 changes: 0 additions & 3 deletions provision/servonet/secrets.tf
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,3 @@ data "vault_generic_secret" "proxmox_auth" {
data "vault_generic_secret" "opnsense_auth" {
path = "secrets/opnsense/auth/terraform"
}
data "vault_generic_secret" "servonet" {
path = "secrets/servonet"
}
4 changes: 2 additions & 2 deletions provision/servonet/variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -94,8 +94,8 @@ variable "workers" {
cores = 4
memory = 16384
storage = 32
devices = ["tesla-p4"]
config = "gpu-worker"
devices = []
config = null
}]
}

Expand Down

0 comments on commit 54a95bd

Please sign in to comment.