diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..98a11275 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,45 @@ +{ + "name": "Home Ops", + "image": "mcr.microsoft.com/devcontainers/base:debian", + "containerEnv": { + "KUBECONFIG": "${containerWorkspaceFolder}/kubeconfig", + "SOPS_AGE_KEY_FILE": "${containerWorkspaceFolder}/.age.key" + }, + "postCreateCommand": { + "deps": "task deps" + }, + "features": { + "ghcr.io/devcontainers/features/common-utils:2": { + "configureZshAsDefaultShell": true + }, + "ghcr.io/devcontainers/features/kubectl-helm-minikube:1": { + "minikube": "none" + }, + "ghcr.io/devcontainers/features/python:1": {}, + "ghcr.io/devcontainers-contrib/features/age:1": {}, + "ghcr.io/devcontainers-contrib/features/cloudflared:1": {}, + "ghcr.io/devcontainers-contrib/features/go-task:1": {}, + "ghcr.io/devcontainers-contrib/features/sops:1": {}, + "ghcr.io/audacioustux/devcontainers/cilium:1": {}, + "ghcr.io/dhoeric/features/stern:1": {}, + "ghcr.io/eitsupi/devcontainer-features/jq-likes:2": {}, + "ghcr.io/jsburckhardt/devcontainer-features/flux:1": {}, + "ghcr.io/rio/features/kustomize:1": {} + }, + "customizations": { + "vscode": { + "settings": { + "terminal.integrated.profiles.linux": { + "bash": { + "path": "/bin/zsh" + } + }, + "terminal.integrated.defaultProfile.linux": "zsh" + }, + "extensions": [ + "redhat.ansible", + "redhat.vscode-yaml" + ] + } + } +} diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..547304ee --- /dev/null +++ b/.editorconfig @@ -0,0 +1,18 @@ +# editorconfig.org +root = true + +[*] +indent_style = space +indent_size = 2 +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true + +[Makefile] +indent_style = space +indent_size = 4 + +[*.{bash,sh}] +indent_style = space +indent_size = 4 diff --git a/.envrc b/.envrc new file mode 100644 index 00000000..6c54d515 --- /dev/null +++ b/.envrc @@ -0,0 +1,10 @@ +#shellcheck disable=SC2148,SC2155 +export KUBECONFIG="$(expand_path ./kubeconfig)" +export SOPS_AGE_KEY_FILE="$(expand_path ./age.key)" +# ansible +PATH_add "$(expand_path ./.venv/bin)" +export VIRTUAL_ENV="$(expand_path ./.venv)" +export ANSIBLE_COLLECTIONS_PATH=$(expand_path ./.venv/galaxy) +export ANSIBLE_ROLES_PATH=$(expand_path ./.venv/galaxy/ansible_roles) +export ANSIBLE_VARS_ENABLED="host_group_vars,community.sops.sops" +export K8S_AUTH_KUBECONFIG="$(expand_path ./kubeconfig)" diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..07f507f7 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +* text=auto eol=lf +*.yaml.j2 linguist-language=YAML +*.sops.* diff=sopsdiffer +*.sops.toml linguist-language=JSON diff --git a/.github/labeler.yaml b/.github/labeler.yaml new file mode 100644 index 00000000..0b474bc5 --- /dev/null +++ b/.github/labeler.yaml @@ -0,0 +1,18 @@ +--- +area/ansible: + - "ansible/**/*" + - "bootstrap/templates/ansible/**/*" + - "bootstrap/tasks/ansible/**/*" +area/github: + - ".github/**/*" +area/kubernetes: + - "kubernetes/**/*" + - "bootstrap/templates/kubernetes/**/*" + - "bootstrap/tasks/kubernetes/**/*" +area/bootstrap: + - "bootstrap/**/*" + - "bootstrap/configure.yaml" +area/addons: + - "bootstrap/templates/addons/**/*" + - "bootstrap/tasks/addons/**/*" + - "bootstrap/vars/addons.sample.yaml" diff --git a/.github/labels.yaml b/.github/labels.yaml new file mode 100644 index 00000000..cd60b061 --- /dev/null +++ b/.github/labels.yaml @@ -0,0 +1,57 @@ +--- +# Area +- name: area/ansible + color: "72ccf3" + description: >- + Changes made in the ansible directory +- name: area/github + color: "72ccf3" + description: >- + Changes made in the github directory +- name: area/kubernetes + color: "72ccf3" + description: >- + Changes made in the kubernetes directory +- name: area/addons + color: "72ccf3" + description: >- + Changes made in the addons directory +- name: area/bootstrap + color: "72ccf3" + description: >- + Changes made in the bootstrap directory +# Renovate +- name: renovate/ansible + color: "ffc300" +- name: renovate/container + color: "ffc300" +- name: renovate/github-action + color: "ffc300" +- name: renovate/github-release + color: "ffc300" +- name: renovate/helm + color: "ffc300" +# Semantic Type +- name: type/patch + color: "FFEC19" +- name: type/minor + color: "FF9800" +- name: type/major + color: "F6412D" +- name: type/break + color: "F6412D" +# Uncategorized +- name: bug + color: "ee0701" +- name: do-not-merge + color: "ee0701" +- name: docs + color: "F4D1B7" +- name: enhancement + color: "84b6eb" +- name: broken-links + color: "7B55D7" +- name: question + color: "cc317c" +- name: community + color: "0e8a16" diff --git a/.github/release.yaml b/.github/release.yaml new file mode 100644 index 00000000..1598e66b --- /dev/null +++ b/.github/release.yaml @@ -0,0 +1,4 @@ +changelog: + exclude: + authors: + - renovate diff --git a/.github/renovate.json5 b/.github/renovate.json5 new file mode 100644 index 00000000..a6946d5d --- /dev/null +++ b/.github/renovate.json5 @@ -0,0 +1,245 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "config:base", + "docker:enableMajor", + ":disableRateLimiting", + ":dependencyDashboard", + ":semanticCommits", + ":automergeDigest", + ":automergeBranch", + "helpers:pinGitHubActionDigests" + ], + "dependencyDashboard": true, + "dependencyDashboardTitle": "Renovate Dashboard 🤖", + "suppressNotifications": ["prIgnoreNotification"], + "rebaseWhen": "conflicted", + "schedule": ["on saturday"], + "flux": { + "fileMatch": [ + "(^|/)addons/.+\\.ya?ml(\\.j2)?(\\.j2)?$", + "(^|/)ansible/.+\\.ya?ml(\\.j2)?(\\.j2)?$", + "(^|/)kubernetes/.+\\.ya?ml(\\.j2)?(\\.j2)?$" + ] + }, + "helm-values": { + "fileMatch": [ + "(^|/)addons/.+\\.ya?ml(\\.j2)?(\\.j2)?$", + "(^|/)ansible/.+\\.ya?ml(\\.j2)?(\\.j2)?$", + "(^|/)kubernetes/.+\\.ya?ml(\\.j2)?(\\.j2)?$" + ] + }, + "kubernetes": { + "fileMatch": [ + "(^|/)addons/.+\\.ya?ml(\\.j2)?(\\.j2)?$", + "(^|/)ansible/.+\\.ya?ml(\\.j2)?(\\.j2)?$", + "(^|/)kubernetes/.+\\.ya?ml(\\.j2)?(\\.j2)?$" + ] + }, + "kustomize": { + "fileMatch": [ + "(^|/)kustomization\\.ya?ml(\\.j2)?$" + ] + }, + // commit message topics + "commitMessageTopic": "{{depName}}", + "commitMessageExtra": "to {{newVersion}}", + "commitMessageSuffix": "", + // package rules + "packageRules": [ + // automerge + { + "description": "Auto merge Github Actions", + "matchManagers": ["github-actions"], + "automerge": true, + "automergeType": "branch", + "ignoreTests": true, + "matchUpdateTypes": ["minor", "patch", "digest"] + }, + // groups + { + "description": "Flux Group", + "groupName": "Flux", + "matchPackagePatterns": ["flux"], + "matchDatasources": ["docker", "github-tags"], + "versioning": "semver", + "group": { + "commitMessageTopic": "{{{groupName}}} group" + }, + "separateMinorPatch": true + }, + { + "description": "System Upgrade Controller Group", + "groupName": "System Upgrade Controller", + "matchPackagePatterns": ["rancher/system-upgrade-controller"], + "matchDatasources": ["docker", "github-releases"], + "group": { + "commitMessageTopic": "{{{groupName}}} group" + }, + "separateMinorPatch": true + }, + // custom versioning + { + "description": "Use custom versioning for k3s", + "matchDatasources": ["github-releases"], + "versioning": "regex:^v(?\\d+)\\.(?\\d+)\\.(?\\d+)(?\\+k3s)(?\\d+)$", + "matchPackagePatterns": ["k3s"] + }, + // commit message topics + { + "matchDatasources": ["helm"], + "commitMessageTopic": "chart {{depName}}" + }, + { + "matchDatasources": ["docker"], + "commitMessageTopic": "image {{depName}}" + }, + // commit messages + { + "matchDatasources": ["docker"], + "matchUpdateTypes": ["major"], + "commitMessagePrefix": "feat(container)!: " + }, + { + "matchDatasources": ["docker"], + "matchUpdateTypes": ["minor"], + "semanticCommitType": "feat", + "semanticCommitScope": "container" + }, + { + "matchDatasources": ["docker"], + "matchUpdateTypes": ["patch"], + "semanticCommitType": "fix", + "semanticCommitScope": "container" + }, + { + "matchDatasources": ["docker"], + "matchUpdateTypes": ["digest"], + "semanticCommitType": "chore", + "semanticCommitScope": "container" + }, + { + "matchDatasources": ["helm"], + "matchUpdateTypes": ["major"], + "commitMessagePrefix": "feat(helm)!: " + }, + { + "matchDatasources": ["helm"], + "matchUpdateTypes": ["minor"], + "semanticCommitType": "feat", + "semanticCommitScope": "helm" + }, + { + "matchDatasources": ["helm"], + "matchUpdateTypes": ["patch"], + "semanticCommitType": "fix", + "semanticCommitScope": "helm" + }, + { + "matchDatasources": ["galaxy", "galaxy-collection"], + "matchUpdateTypes": ["major"], + "commitMessagePrefix": "feat(ansible)!: " + }, + { + "matchDatasources": ["galaxy", "galaxy-collection"], + "matchUpdateTypes": ["minor"], + "semanticCommitType": "feat", + "semanticCommitScope": "ansible" + }, + { + "matchDatasources": ["galaxy", "galaxy-collection"], + "matchUpdateTypes": ["patch"], + "semanticCommitType": "fix", + "semanticCommitScope": "ansible" + }, + { + "matchDatasources": ["github-releases", "github-tags"], + "matchUpdateTypes": ["major"], + "commitMessagePrefix": "feat(github-release)!: " + }, + { + "matchDatasources": ["github-releases", "github-tags"], + "matchUpdateTypes": ["minor"], + "semanticCommitType": "feat", + "semanticCommitScope": "github-release" + }, + { + "matchDatasources": ["github-releases", "github-tags"], + "matchUpdateTypes": ["patch"], + "semanticCommitType": "fix", + "semanticCommitScope": "github-release" + }, + { + "matchManagers": ["github-actions"], + "matchUpdateTypes": ["major"], + "commitMessagePrefix": "feat(github-action)!: " + }, + { + "matchManagers": ["github-actions"], + "matchUpdateTypes": ["minor"], + "semanticCommitType": "feat", + "semanticCommitScope": "github-action" + }, + { + "matchManagers": ["github-actions"], + "matchUpdateTypes": ["patch"], + "semanticCommitType": "fix", + "semanticCommitScope": "github-action" + }, + // labels + { + "matchUpdateTypes": ["major"], + "labels": ["type/major"] + }, + { + "matchUpdateTypes": ["minor"], + "labels": ["type/minor"] + }, + { + "matchUpdateTypes": ["patch"], + "labels": ["type/patch"] + }, + { + "matchDatasources": ["docker"], + "addLabels": ["renovate/container"] + }, + { + "matchDatasources": ["helm"], + "addLabels": ["renovate/helm"] + }, + { + "matchDatasources": ["galaxy", "galaxy-collection"], + "addLabels": ["renovate/ansible"] + }, + { + "matchDatasources": ["github-releases", "github-tags"], + "addLabels": ["renovate/github-release"] + }, + { + "matchManagers": ["github-actions"], + "addLabels": ["renovate/github-action"] + } + ], + // custom managers + "customManagers": [ + { + "customType": "regex", + "description": "Process various other dependencies", + "fileMatch": [ + "(^|/)addons/.+\\.ya?ml(\\.j2)?(\\.j2)?$", + "(^|/)ansible/.+\\.ya?ml(\\.j2)?(\\.j2)?$", + "(^|/)kubernetes/.+\\.ya?ml(\\.j2)?(\\.j2)?$" + ], + "matchStrings": [ + // Example: `k3s_release_version: "v1.27.3+k3s1"` + "datasource=(?\\S+) depName=(?\\S+)( versioning=(?\\S+))?\n.*?\"(?.*)\"\n", + // Example: `- https://github.com/rancher/system-upgrade-controller/releases/download/v0.11.0/crd.yaml` + "datasource=(?\\S+) depName=(?\\S+)( versioning=(?\\S+))?\n.*?-\\s(.*?)\/(?[^/]+)\/[^/]+\n", + // Example: apiVersion=helm.cattle.io/v1 kind=HelmChart + "datasource=(?\\S+)\n.*?repo: (?\\S+)\n.*?chart: (?\\S+)\n.*?version: (?\\S+)\n" + ], + "datasourceTemplate": "{{#if datasource}}{{{datasource}}}{{else}}github-releases{{/if}}", + "versioningTemplate": "{{#if versioning}}{{{versioning}}}{{else}}semver{{/if}}" + } + ] +} diff --git a/.github/workflows/flux-diff.yaml b/.github/workflows/flux-diff.yaml new file mode 100644 index 00000000..5fefbe76 --- /dev/null +++ b/.github/workflows/flux-diff.yaml @@ -0,0 +1,39 @@ +--- +name: "Flux Diff" + +on: + pull_request: + branches: ["main"] + paths: ["kubernetes/**.yaml"] + +jobs: + flux-diff: + name: Flux Diff + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + strategy: + matrix: + path: ["kubernetes"] + resource: ["helmrelease", "kustomization"] + steps: + - name: Diff Resources + uses: allenporter/flux-local/action/diff@19bfc6920e8964a479363bc230e6c329120ead02 # 3.2.0 + id: diff + with: + sources: home-kubernetes + path: "${{ matrix.path }}" + resource: "${{ matrix.resource }}" + + - if: ${{ steps.diff.outputs.diff != '' }} + name: Add comment + uses: mshick/add-pr-comment@7c0890544fb33b0bdd2e59467fbacb62e028a096 # v2.8.1 + with: + repo-token: "${{ secrets.GITHUB_TOKEN }}" + message-id: "${{ github.event.pull_request.number }}/${{ matrix.path }}/${{ matrix.resource }}" + message-failure: Diff was not successful + message: | + ```diff + ${{ steps.diff.outputs.diff }} + ``` diff --git a/.github/workflows/link-check.yaml b/.github/workflows/link-check.yaml new file mode 100644 index 00000000..b129b0a9 --- /dev/null +++ b/.github/workflows/link-check.yaml @@ -0,0 +1,39 @@ +--- +name: "Link Check" + +on: + workflow_dispatch: + schedule: + - cron: "0 0 * * 0" + +jobs: + link-check: + name: Link Check + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Link Checker + uses: lycheeverse/lychee-action@ec3ed119d4f44ad2673a7232460dc7dff59d2421 # v1.8.0 + id: lychee + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + + - name: Find Link Checker Issue + id: link-checker-issue + uses: micalevisk/last-issue-action@f5661581217cc78cc282d1351aa65bd8bd155003 # v2.2.1 + with: + state: open + labels: | + broken-links + + - name: Update Issue + uses: peter-evans/create-issue-from-file@433e51abf769039ee20ba1293a088ca19d573b7f # v4.0.1 + with: + title: Broken links detected 🔗 + issue-number: "${{ steps.link-checker-issue.outputs.issue-number }}" + content-filepath: ./lychee/out.md + token: "${{ secrets.GITHUB_TOKEN }}" + labels: | + broken-links diff --git a/.github/workflows/meta-labeler.yaml b/.github/workflows/meta-labeler.yaml new file mode 100644 index 00000000..193fc3f5 --- /dev/null +++ b/.github/workflows/meta-labeler.yaml @@ -0,0 +1,23 @@ +--- +name: "Meta Labeler" + +on: + workflow_dispatch: + pull_request_target: + branches: ["main"] + +permissions: + checks: write + contents: read + pull-requests: write + +jobs: + labeler: + name: Labeler + runs-on: ubuntu-latest + steps: + - name: Labeler + uses: actions/labeler@ac9175f8a1f3625fd0d4fb234536d26811351594 # v4.3.0 + with: + configuration-path: .github/labeler.yaml + repo-token: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/meta-sync-labels.yaml b/.github/workflows/meta-sync-labels.yaml new file mode 100644 index 00000000..55dccef7 --- /dev/null +++ b/.github/workflows/meta-sync-labels.yaml @@ -0,0 +1,23 @@ +--- +name: "Meta Sync labels" + +on: + workflow_dispatch: + push: + branches: ["main"] + paths: [".github/labels.yaml"] + +jobs: + labels: + name: Sync Labels + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Sync Labels + uses: EndBug/label-sync@da00f2c11fdb78e4fae44adac2fdd713778ea3e8 # v2.3.2 + with: + config-file: .github/labels.yaml + token: "${{ secrets.GITHUB_TOKEN }}" + delete-other-labels: true diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 00000000..d2d8471f --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,42 @@ +--- +name: "Release" + +on: + workflow_dispatch: + schedule: + - cron: "0 0 1 * *" + +jobs: + release: + name: Release + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Create Release + shell: bash + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + run: | + # Retrieve previous release tag + previous_tag="$(gh release list --limit 1 | awk '{ print $1 }')" + previous_major="${previous_tag%%\.*}" + previous_minor="${previous_tag#*.}" + previous_minor="${previous_minor%.*}" + previous_patch="${previous_tag##*.}" + # Determine next release tag + next_major_minor="$(date +'%Y').$(date +'%-m')" + if [[ "${previous_major}.${previous_minor}" == "${next_major_minor}" ]]; then + echo "Month release already exists for year, incrementing patch number by 1" + next_patch="$((previous_patch + 1))" + else + echo "Month release does not exist for year, setting patch number to 0" + next_patch="0" + fi + # Create release + release_tag="${next_major_minor}.${next_patch}" + gh release create "${release_tag}" \ + --repo="${GITHUB_REPOSITORY}" \ + --title="${release_tag}" \ + --generate-notes diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..4ad3e069 --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +# Trash +.DS_Store +Thumbs.db +# k8s +kubeconfig +.decrypted~*.yaml +.config.env +*.agekey +*.pub +*.key +# Ansible +.venv* +# Taskfile +.tasks diff --git a/.lycheeignore b/.lycheeignore new file mode 100644 index 00000000..8cbc880a --- /dev/null +++ b/.lycheeignore @@ -0,0 +1,2 @@ +https://dash.cloudflare.com/profile/api-tokens +https://www.mend.io/free-developer-tools/renovate/ diff --git a/.taskfiles/AnsibleTasks.yaml b/.taskfiles/AnsibleTasks.yaml new file mode 100644 index 00000000..66f63c85 --- /dev/null +++ b/.taskfiles/AnsibleTasks.yaml @@ -0,0 +1,54 @@ +--- +version: "3" + +vars: + ANSIBLE_PLAYBOOK_DIR: "{{.ANSIBLE_DIR}}/playbooks" + ANSIBLE_INVENTORY_DIR: "{{.ANSIBLE_DIR}}/inventory" + +tasks: + + prepare: + desc: Prepare all the k8s nodes for running k3s + dir: "{{.ANSIBLE_DIR}}" + cmd: ansible-playbook -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml {{.ANSIBLE_PLAYBOOK_DIR}}/cluster-prepare.yaml + + install: + desc: Install Kubernetes on the nodes + dir: "{{.ANSIBLE_DIR}}" + cmd: ansible-playbook -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml {{.ANSIBLE_PLAYBOOK_DIR}}/cluster-installation.yaml + + rollout-update: + desc: Preform operating system updates and rollout restart the cluster + dir: "{{.ANSIBLE_DIR}}" + cmd: ansible-playbook -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml {{.ANSIBLE_PLAYBOOK_DIR}}/cluster-rollout-update.yaml + + nuke: + desc: Uninstall Kubernetes on the nodes + dir: "{{.ANSIBLE_DIR}}" + interactive: true + cmd: ansible-playbook -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml {{.ANSIBLE_PLAYBOOK_DIR}}/cluster-nuke.yaml + + reboot: + desc: Reboot all the k8s nodes + dir: "{{.ANSIBLE_DIR}}" + cmd: ansible-playbook -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml {{.ANSIBLE_PLAYBOOK_DIR}}/cluster-reboot.yaml + + poweroff: + desc: Shutdown all the k8s nodes + dir: "{{.ANSIBLE_DIR}}" + cmd: ansible kubernetes -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml -a '/usr/bin/systemctl poweroff' --become + + list: + desc: List all the hosts + dir: "{{.ANSIBLE_DIR}}" + cmd: ansible all -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml --list-hosts + + ping: + desc: Ping all the hosts + dir: "{{.ANSIBLE_DIR}}" + cmd: ansible all -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml --one-line -m 'ping' + + uptime: + desc: Uptime of all the hosts + dir: "{{.ANSIBLE_DIR}}" + cmd: ansible all -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml --one-line -a 'uptime' diff --git a/.taskfiles/BrewTasks.yaml b/.taskfiles/BrewTasks.yaml new file mode 100644 index 00000000..fc8c4b1f --- /dev/null +++ b/.taskfiles/BrewTasks.yaml @@ -0,0 +1,25 @@ +--- +version: "3" + +tasks: + + deps: + desc: Install workstation dependencies with Brew + cmd: brew install {{.DEPS}} {{.CLI_ARGS}} + preconditions: + - sh: command -v brew + msg: | + Homebrew is not installed. Using MacOS, Linux or WSL? + Head over to https://brew.sh to get up and running. + vars: + DEPS: >- + age + cilium-cli + cloudflared + fluxcd/tap/flux + helm + kubernetes-cli + kustomize + sops + stern + yq diff --git a/.taskfiles/ClusterTasks.yaml b/.taskfiles/ClusterTasks.yaml new file mode 100644 index 00000000..b9a3a19f --- /dev/null +++ b/.taskfiles/ClusterTasks.yaml @@ -0,0 +1,81 @@ +--- +version: "3" + +tasks: + + verify: + desc: Verify flux meets the prerequisites + cmd: flux check --pre + + install: + desc: Install Flux into your cluster + cmds: + - kubectl apply --kustomize {{.KUBERNETES_DIR}}/bootstrap + - cat {{.SOPS_AGE_KEY_FILE}} | kubectl -n flux-system create secret generic sops-age --from-file=age.agekey=/dev/stdin + - sops --decrypt {{.KUBERNETES_DIR}}/flux/vars/cluster-secrets.sops.yaml | kubectl apply -f - + - sops --decrypt {{.KUBERNETES_DIR}}/flux/vars/cluster-secrets-user.sops.yaml | kubectl apply -f - + - kubectl apply -f {{.KUBERNETES_DIR}}/flux/vars/cluster-settings.yaml + - kubectl apply -f {{.KUBERNETES_DIR}}/flux/vars/cluster-settings-user.yaml + - kubectl apply --kustomize {{.KUBERNETES_DIR}}/flux/config + preconditions: + - sh: test -f {{.SOPS_AGE_KEY_FILE}} + msg: | + Age key file is not found. Did you forget to create it? + vars: + SOPS_AGE_KEY_FILE: "{{.ROOT_DIR}}/age.key" + + reconcile: + desc: Force update Flux to pull in changes from your Git repository + cmd: flux reconcile -n flux-system kustomization cluster --with-source + + hr-restart: + desc: Restart all failed Helm Releases + cmds: + - kubectl get hr --all-namespaces | grep False | awk '{print $2, $1}' | xargs -L1 bash -c 'flux suspend hr $0 -n $1' + - kubectl get hr --all-namespaces | grep False | awk '{print $2, $1}' | xargs -L1 bash -c 'flux resume hr $0 -n $1' + + nodes: + desc: List all the nodes in your cluster + cmd: kubectl get nodes {{.CLI_ARGS | default "-o wide"}} + + pods: + desc: List all the pods in your cluster + cmd: kubectl get pods {{.CLI_ARGS | default "-A"}} + + kustomizations: + desc: List all the kustomizations in your cluster + cmd: kubectl get kustomizations {{.CLI_ARGS | default "-A"}} + + helmreleases: + desc: List all the helmreleases in your cluster + cmd: kubectl get helmreleases {{.CLI_ARGS | default "-A"}} + + helmrepositories: + desc: List all the helmrepositories in your cluster + cmd: kubectl get helmrepositories {{.CLI_ARGS | default "-A"}} + + gitrepositories: + desc: List all the gitrepositories in your cluster + cmd: kubectl get gitrepositories {{.CLI_ARGS | default "-A"}} + + certificates: + desc: List all the certificates in your cluster + cmds: + - kubectl get certificates {{.CLI_ARGS | default "-A"}} + - kubectl get certificaterequests {{.CLI_ARGS | default "-A"}} + + ingresses: + desc: List all the ingresses in your cluster + cmd: kubectl get ingress {{.CLI_ARGS | default "-A"}} + + resources: + desc: Gather common resources in your cluster, useful when asking for support + cmds: + - task: nodes + - task: kustomizations + - task: helmreleases + - task: helmrepositories + - task: gitrepositories + - task: certificates + - task: ingresses + - task: pods diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 00000000..e7948651 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,14 @@ +{ + "recommendations": [ + "albert.TabOut", + "britesnow.vscode-toggle-quotes", + "fcrespo82.markdown-table-formatter", + "mikestead.dotenv", + "mitchdenny.ecdc", + "redhat.ansible", + "signageos.signageos-vscode-sops", + "will-stone.in-any-case", + "EditorConfig.editorconfig", + "PKief.material-icon-theme", + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..8f29572b --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,27 @@ +{ + "ansible.ansible.path": ".venv/bin/ansible", + "ansible.python.activationScript": ".venv/bin/activate", + "ansible.python.interpreterPath": ".venv/bin/python3", + "ansible.validation.enabled": true, + "ansible.validation.lint.arguments": "-c ansible/.ansible-lint", + "ansible.validation.lint.enabled": true, + "ansible.validation.lint.path": ".venv/bin/ansible-lint", + "files.associations": { + "*.json5": "jsonc", + "./ansible/**/*.yaml": "ansible", + "./ansible/**/*.sops.yaml": "yaml", + "./ansible/**/inventory/**/*.yaml": "yaml", + "./kubernetes/**/*.sops.toml": "plaintext" + }, + "sops.defaults.ageKeyFile": "age.key", + "yaml.schemas": { + "ansible": "./ansible/*.yaml", + "Kubernetes": "./kubernetes/*.yaml" + }, + "vs-kubernetes": { + "vs-kubernetes.kubeconfig": "./kubeconfig", + "vs-kubernetes.knownKubeconfigs": [ + "./kubeconfig" + ] + } +} diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..38d60cac --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 onedr0p + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 00000000..a025078c --- /dev/null +++ b/README.md @@ -0,0 +1,605 @@ +# Deploy a Kubernetes cluster backed by Flux + +Welcome to my highly opinionated template for deploying a single Kubernetes ([k3s](https://k3s.io)) cluster with [Ansible](https://www.ansible.com) and using [Flux](https://toolkit.fluxcd.io) to manage its state. + +## 👋 Introduction + +The goal of this project is to make it easy for people interested in learning Kubernetes to deploy a cluster at home and become familiar with the GitOps tool Flux. + +This template implements Flux in a way that promotes legibility and ease of use for those who are new (or relatively new) to the technology and GitOps in general. + +## ✨ Features + +- Automated, reproducible, customizable setup through Ansible templates and playbooks +- Opinionated implementation of Flux with [strong community support](https://github.com/onedr0p/flux-cluster-template/tree/main#-help) +- Encrypted secrets thanks to [SOPS](https://github.com/getsops/sops) and [Age](https://github.com/FiloSottile/age) +- Web application firewall thanks to [Cloudflare Tunnels](https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/) +- SSL certificates thanks to [Cloudflare](https://cloudflare.com) and [cert-manager](https://cert-manager.io) +- HA control plane capability thanks to [kube-vip](https://kube-vip.io) +- Next-gen networking thanks to [Cilium](https://cilium.io/) +- A [Renovate](https://www.mend.io/renovate)-ready repository with pull request diffs provided by [flux-local](https://github.com/allenporter/flux-local) +- Integrated [GitHub Actions](https://github.com/features/actions) + +... and more! + +## 📝 Pre-start checklist + +Before we get started everything below must be taken into consideration, you must... + +- [ ] have some experience with the following: Git/SCM, containers, networking and scripting. +- [ ] bring a **positive attitude** and be ready to learn and fail a lot. _The more you fail, the more you can learn from._ +- [ ] run the cluster on bare metal machines or VMs within your home network — **this is NOT designed for cloud environments**. +- [ ] have Debian 12 freshly installed on 1 or more AMD64/ARM64 bare metal machines or VMs. Each machine will be either a **control node** or a **worker node** in your cluster. +- [ ] give your nodes unrestricted internet access — **air-gapped environments won't work**. +- [ ] have a domain you can manage on Cloudflare. +- [ ] be willing to commit encrypted secrets to a public GitHub repository. +- [ ] have a DNS server that supports split DNS (e.g. Pi-Hole) deployed somewhere outside your cluster **ON** your home network. + +## 💻 Machine Preparation + +### System requirements + +📍 _k3s default behaviour is that all nodes are able to run workloads, including contol nodes. Worker nodes are therefore optional._ + +📍 _If you have 3 or more nodes it is strongly recommended to make 3 of them control nodes for a highly available control plane._ + +📍 _Ideally you will run the cluster on bare metal machines. If you intend to run your cluster on Proxmox VE, my thoughts and recommendations about that are documented [here](https://onedr0p.github.io/home-ops/notes/proxmox-considerations.html)._ + +| Role | Cores | Memory | System Disk | +|---------|----------|---------------|---------------------------| +| Control | 4 _(6*)_ | 8GB _(24GB*)_ | 100GB _(500GB*)_ SSD/NVMe | +| Worker | 4 _(6*)_ | 8GB _(24GB*)_ | 100GB _(500GB*)_ SSD/NVMe | +| _\* recommended_ | + +### Debian for AMD64 + +1. Download the latest stable release of Debian from [here](https://cdimage.debian.org/debian-cd/current/amd64/iso-dvd), then follow [this guide](https://www.linuxtechi.com/how-to-install-debian-12-step-by-step) to get it installed. Deviations from the guide: + + ```txt + Choose "Guided - use entire disk" + Choose "All files in one partition" + Delete Swap partition + Uncheck all Debian desktop environment options + ``` + +2. [Post install] Remove CD/DVD as apt source + + ```sh + su - + sed -i '/deb cdrom/d' /etc/apt/sources.list + apt update + exit + ``` + +3. [Post install] Enable sudo for your non-root user + + ```sh + su - + apt update + apt install -y sudo + usermod -aG sudo ${username} + echo "${username} ALL=(ALL) NOPASSWD:ALL" | tee /etc/sudoers.d/${username} + exit + newgrp sudo + sudo apt update + ``` + +4. [Post install] Add SSH keys (or use `ssh-copy-id` on the client that is connecting) + + 📍 _First make sure your ssh keys are up-to-date and added to your github account as [instructed](https://docs.github.com/en/authentication/connecting-to-github-with-ssh/adding-a-new-ssh-key-to-your-github-account)._ + + ```sh + mkdir -m 700 ~/.ssh + sudo apt install -y curl + curl https://github.com/${github_username}.keys > ~/.ssh/authorized_keys + chmod 600 ~/.ssh/authorized_keys + ``` + +### Debian for RasPi4 + +📍 _If you choose to use a Raspberry Pi 4 for the cluster, it is recommended to have an 8GB model. Most important is to **boot from an external SSD/NVMe** rather than an SD card. This is supported [natively](https://www.raspberrypi.com/documentation/computers/raspberry-pi.html), however if you have an early model you may need to [update the bootloader](https://www.tomshardware.com/how-to/boot-raspberry-pi-4-usb) first._ + +📍 _Be sure to check the [power requirements](https://www.raspberrypi.com/documentation/computers/raspberry-pi.html#power-supply) if using a PoE Hat and a SSD/NVMe dongle._ + +1. Download the latest stable release of Debian from [here](https://raspi.debian.net/tested-images). _**Do not** use Raspbian or DietPi or any other flavor Linux OS._ + +2. Flash the image onto an SSD/NVMe drive. + +3. Re-mount the drive to your workstation and then do the following (per the [official documentation](https://raspi.debian.net/defaults-and-settings)): + + ```txt + Open 'sysconf.txt' in a text editor and save it upon updating the information below + - Change 'root_authorized_key' to your desired public SSH key + - Change 'root_pw' to your desired root password + - Change 'hostname' to your desired hostname + ``` + +4. Connect SSD/NVMe drive to the Raspberry Pi 4 and power it on. + +5. [Post install] SSH into the device with the `root` user and then create a normal user account with `adduser ${username}` + +6. [Post install] Follow steps 3 and 4 from [Debian for AMD64](#debian-for-amd64). + +7. [Post install] Install `python3` which is needed by Ansible. + + ```sh + sudo apt install -y python3 + ``` + +## 🚀 Getting Started + +Once you have installed Debian on your nodes, there are six stages to getting a Flux-managed cluster up and runnning. + +📍 _For all stages below the commands **MUST** be ran on your personal workstation within your repository directory_ + +### 🎉 Stage 1: Create a Git repository + +1. Create a new **public** repository by clicking the big green "Use this template" button at the top of this page. + +2. Clone **your new repo** to you local workstation and `cd` into it. + +### 🌱 Stage 2: Setup your local workstation environment + +📍 _Let's get the required workstation tools installed and configured._ + +1. Install the most recent version of [task](https://taskfile.dev/) + + 📍 _See the task [installation docs](https://taskfile.dev/installation/) for other platforms_ + + ```sh + # Brew + brew install go-task + ``` + +2. Install the most recent version of [direnv](https://direnv.net/) + + 📍 _See the direnv [installation docs](https://direnv.net/docs/installation.html) for other platforms_ + + 📍 _After installing `direnv` be sure to [hook it into your shell](https://direnv.net/docs/hook.html) and after that is done run `direnv allow` while in your repos directory._ + + ```sh + # Brew + brew install direnv + ``` + +3. Setup a Python virual env and install Ansible by running the following task command. + + 📍 _This commands requires Python 3.10+ to be installed_ + + ```sh + # Platform agnostic + task deps + ``` + +4. Install the required tools: [age](https://github.com/FiloSottile/age), [flux](https://toolkit.fluxcd.io/), [cloudflared](https://github.com/cloudflare/cloudflared), [kubectl](https://kubernetes.io/docs/tasks/tools/), [sops](https://github.com/getsops/sops) + + ```sh + # Brew + task brew:deps + ``` + +### 🔧 Stage 3: Do bootstrap configuration + +📍 _Both `bootstrap/vars/config.yaml` and `bootstrap/vars/addons.yaml` files contain necessary information that is needed by bootstrap process._ + +1. Generate the `bootstrap/vars/config.yaml` and `bootstrap/vars/addons.yaml` configuration files. + + ```sh + task init + ``` + +2. Setup Age private / public key + + 📍 _Using [SOPS](https://github.com/getsops/sops) with [Age](https://github.com/FiloSottile/age) allows us to encrypt secrets and use them in Ansible and Flux._ + + 2a. Create a Age private / public key (this file is gitignored) + + ```sh + age-keygen -o age.key + ``` + + 2b. Fill out the appropriate vars in `bootstrap/vars/config.yaml` + +3. Create Cloudflare API Token + + 📍 _To use `cert-manager` with the Cloudflare DNS challenge you will need to create a API Token._ + + 3a. Head over to Cloudflare and create a API Token by going [here](https://dash.cloudflare.com/profile/api-tokens). + + 3b. Under the `API Tokens` section click the blue `Create Token` button. + + 3c. Click the blue `Use template` button for the `Edit zone DNS` template. + + 3d. Name your token something like `home-kubernetes` + + 3e. Under `Permissions`, click `+ Add More` and add each permission below: + + ```text + Zone - DNS - Edit + Account - Cloudflare Tunnel - Read + ``` + + 3f. Limit the permissions to a specific account and zone resources. + + 3g. Fill out the appropriate vars in `bootstrap/vars/config.yaml` + +4. Create Cloudflare Tunnel + + 📍 _To expose services to the internet you will need to create a [Cloudflare Tunnel](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/)._ + + 4a. Authenticate cloudflared to your domain + + ```sh + cloudflared tunnel login + ``` + + 4b. Create the tunnel + + ```sh + cloudflared tunnel create k8s + ``` + + 4c. In the `~/.cloudflared` directory there will be a json file with details you need. Ignore the `cert.pem` file. + + 4d. Fill out the appropriate vars in `bootstrap/vars/config.yaml` + +5. Complete filling out the rest of the `bootstrap/vars/config.yaml` configuration file. + + 5a. Ensure `bootstrap_acme_production_enabled` is set to `false`. + + 5b. [Optional] Update `bootstrap/vars/addons.yaml` and enable applications you would like included. + +6. Once done run the following command which will verify and generate all the files needed to continue. + + ```sh + task configure + ``` + +📍 _The configure task will create a `./ansible` directory and the following directories under `./kubernetes`._ + +```sh +📁 kubernetes # Kubernetes cluster defined as code +├─📁 bootstrap # Flux installation (not tracked by Flux) +├─📁 flux # Main Flux configuration of repository +└─📁 apps # Apps deployed into the cluster grouped by namespace +``` + +### ⚡ Stage 4: Prepare your nodes for k3s + +📍 _Here we will be running an Ansible playbook to prepare your nodes for running a Kubernetes cluster._ + +1. Ensure you are able to SSH into your nodes from your workstation using a private SSH key **without a passphrase** (for example using a SSH agent). This lets Ansible interact with your nodes. + +2. Verify Ansible can view your config + + ```sh + task ansible:list + ``` + +3. Verify Ansible can ping your nodes + + ```sh + task ansible:ping + ``` + +4. Run the Ansible prepare playbook (nodes wil reboot when done) + + ```sh + task ansible:prepare + ``` + +### ⛵ Stage 5: Use Ansible to install k3s + +📍 _Here we will be running a Ansible Playbook to install [k3s](https://k3s.io/) with [this](https://galaxy.ansible.com/xanmanning/k3s) Ansible galaxy role. If you run into problems, you can run `task ansible:nuke` to destroy the k3s cluster and start over from this point._ + +1. Verify Ansible can view your config + + ```sh + task ansible:list + ``` + +2. Verify Ansible can ping your nodes + + ```sh + task ansible:ping + ``` + +3. Install k3s with Ansible + + ```sh + task ansible:install + ``` + +4. Verify the nodes are online + + 📍 _If this command **fails** you likely haven't configured `direnv` as mentioned previously in the guide._ + + ```sh + kubectl get nodes -o wide + # NAME STATUS ROLES AGE VERSION + # k8s-0 Ready control-plane,etcd,master 1h v1.27.3+k3s1 + # k8s-1 Ready worker 1h v1.27.3+k3s1 + ``` + +5. The `kubeconfig` for interacting with your cluster should have been created in the root of your repository. + +### 🔹 Stage 6: Install Flux in your cluster + +📍 _Here we will be installing [flux](https://fluxcd.io/flux/) after some quick bootstrap steps._ + +1. Verify Flux can be installed + + ```sh + flux check --pre + # ► checking prerequisites + # ✔ kubectl 1.27.3 >=1.18.0-0 + # ✔ Kubernetes 1.27.3+k3s1 >=1.16.0-0 + # ✔ prerequisites checks passed + ``` + +2. Push you changes to git + + 📍 **Verify** all the `*.sops.yaml` and `*.sops.yaml` files under the `./ansible`, and `./kubernetes` directories are **encrypted** with SOPS + + ```sh + git add -A + git commit -m "Initial commit :rocket:" + git push + ``` + +3. Install Flux and sync the cluster to the Git repository + + ```sh + task cluster:install + # namespace/flux-system configured + # customresourcedefinition.apiextensions.k8s.io/alerts.notification.toolkit.fluxcd.io created + # ... + ``` + +4. Verify Flux components are running in the cluster + + ```sh + kubectl -n flux-system get pods -o wide + # NAME READY STATUS RESTARTS AGE + # helm-controller-5bbd94c75-89sb4 1/1 Running 0 1h + # kustomize-controller-7b67b6b77d-nqc67 1/1 Running 0 1h + # notification-controller-7c46575844-k4bvr 1/1 Running 0 1h + # source-controller-7d6875bcb4-zqw9f 1/1 Running 0 1h + ``` + +### 🎤 Verification Steps + +_Mic check, 1, 2_ - In a few moments applications should be lighting up like Christmas in July 🎄 + +1. Output all the common resources in your cluster. + + 📍 _Feel free to use the provided [cluster tasks](.taskfiles/ClusterTasks.yaml) for validation of cluster resources or continue to get familiar with the `kubectl` and `flux` CLI tools._ + + + ```sh + task cluster:resources + ``` + +2. ⚠️ It might take `cert-manager` awhile to generate certificates, this is normal so be patient. + +3. 🏆 **Congratulations** if all goes smooth you will have a Kubernetes cluster managed by Flux and your Git repository is driving the state of your cluster. + +4. 🧠 Now it's time to pause and go get some motel motor oil ☕ and admire you made it this far! + +## 📣 Post installation + +#### 🌐 Public DNS + +The `external-dns` application created in the `networking` namespace will handle creating public DNS records. By default, `echo-server` and the `flux-webhook` are the only subdomains reachable from the public internet. In order to make additional applications public you must set set the correct ingress class name and ingress annotations like in the HelmRelease for `echo-server`. + +#### 🏠 Home DNS + +`k8s_gateway` will provide DNS resolution to external Kubernetes resources (i.e. points of entry to the cluster) from any device that uses your home DNS server. For this to work, your home DNS server must be configured to forward DNS queries for `${bootstrap_cloudflare_domain}` to `${bootstrap_k8s_gateway_addr}` instead of the upstream DNS server(s) it normally uses. This is a form of **split DNS** (aka split-horizon DNS / conditional forwarding). + +📍 _Below is how to configure a Pi-hole for split DNS. Other platforms should be similar._ + +1. Apply this file on the server + + ```sh + # /etc/dnsmasq.d/99-k8s-gateway-forward.conf + server=/${bootstrap_cloudflare_domain}/${bootstrap_k8s_gateway_addr} + ``` + +2. Restart dnsmasq on the server. + +3. Query an internal-only subdomain from your workstation: `dig @${home-dns-server-ip} echo-server.${bootstrap_cloudflare_domain}`. It should resolve to `${bootstrap_internal_ingress_addr}`. + +If you're having trouble with DNS be sure to check out these two GitHub discussions: [Internal DNS](https://github.com/onedr0p/flux-cluster-template/discussions/719) and [Pod DNS resolution broken](https://github.com/onedr0p/flux-cluster-template/discussions/635). + +... Nothing working? That is expected, this is DNS after all! + +#### 📜 Certificates + +By default this template will deploy a wildcard certificate using the Let's Encrypt **staging environment**, which prevents you from getting rate-limited by the Let's Encrypt production servers if your cluster doesn't deploy properly (for example due to a misconfiguration). Once you are sure you will keep the cluster up for more than a few hours be sure to switch to the production servers as outlined in `config.yaml`. + +📍 _You will need a production certificate to reach internet-exposed applications through `cloudflared`._ + +#### 🪝 Github Webhook + +By default Flux will periodically check your git repository for changes. In order to have Flux reconcile on `git push` you must configure Github to send `push` events. + +1. Obtain the webhook path + + 📍 _Hook id and path should look like `/hook/12ebd1e363c641dc3c2e430ecf3cee2b3c7a5ac9e1234506f6f5f3ce1230e123`_ + + ```sh + kubectl -n flux-system get receiver github-receiver -o jsonpath='{.status.webhookPath}' + ``` + +2. Piece together the full URL with the webhook path appended + + ```text + https://flux-webhook.${bootstrap_cloudflare_domain}/hook/12ebd1e363c641dc3c2e430ecf3cee2b3c7a5ac9e1234506f6f5f3ce1230e123 + ``` + +3. Navigate to the settings of your repository on Github, under "Settings/Webhooks" press the "Add webhook" button. Fill in the webhook url and your `bootstrap_flux_github_webhook_token` secret and save. + +### 🤖 Renovate + +[Renovate](https://www.mend.io/renovate) is a tool that automates dependency management. It is designed to scan your repository around the clock and open PRs for out-of-date dependencies it finds. Common dependencies it can discover are Helm charts, container images, GitHub Actions, Ansible roles... even Flux itself! Merging a PR will cause Flux to apply the update to your cluster. + +To enable Renovate, click the 'Configure' button over at their [Github app page](https://github.com/apps/renovate) and select your repository. Renovate creates a "Dependency Dashboard" as an issue in your repository, giving an overview of the status of all updates. The dashboard has interactive checkboxes that let you do things like advance scheduling or reattempt update PRs you closed without merging. + +The base Renovate configuration in your repository can be viewed at [.github/renovate.json5](https://github.com/onedr0p/flux-cluster-template/blob/main/.github/renovate.json5). By default it is scheduled to be active with PRs every weekend, but you can [change the schedule to anything you want](https://docs.renovatebot.com/presets-schedule), or remove it if you want Renovate to open PRs right away. + +## 🐛 Debugging + +Below is a general guide on trying to debug an issue with an resource or application. For example, if a workload/resource is not showing up or a pod has started but in a `CrashLoopBackOff` or `Pending` state. + +1. Start by checking all Flux Kustomizations & Git Repository & OCI Repository and verify they are healthy. + + ```sh + flux get sources oci -A + flux get sources git -A + flux get ks -A + ``` + +2. Then check all the Flux Helm Releases and verify they are healthy. + + ```sh + flux get hr -A + ``` + +3. Then check the if the pod is present. + + ```sh + kubectl -n get pods -o wide + ``` + +4. Then check the logs of the pod if its there. + + ```sh + kubectl -n logs -f + # or + stern -n + ``` + +5. If a resource exists try to describe it to see what problems it might have. + + ```sh + kubectl -n describe + ``` + +6. Check the namespace events + + ```sh + kubectl -n get events --sort-by='.metadata.creationTimestamp' + ``` + +Resolving problems that you have could take some tweaking of your YAML manifests in order to get things working, other times it could be a external factor like permissions on NFS. If you are unable to figure out your problem see the help section below. + +## 👉 Help + +- Make a post in this repository's Github [Discussions](https://github.com/onedr0p/flux-cluster-template/discussions). +- Start a thread in the `support` or `flux-cluster-template` channel in the [Home Operations](https://discord.gg/home-operations) Discord server. + +## ❔ What's next + +The cluster is your oyster (or something like that). Below are some optional considerations you might want to review. + +#### Ship it + +To browse or get ideas on applications people are running, community member [@whazor](https://github.com/whazor) created [this website](https://nanne.dev/k8s-at-home-search/) as a creative way to search Flux HelmReleases across Github. + +#### Storage + +The included CSI (`local-path-provisioner`) is a great start for storage but soon you might find you need more features like replicated block storage, or to connect to a NFS/SMB/iSCSI server. If you need any of those features be sure to check out the projects like [rook-ceph](https://github.com/rook/rook), [longhorn](https://github.com/longhorn/longhorn), [openebs](https://github.com/openebs/openebs), [democratic-csi](https://github.com/democratic-csi/democratic-csi), [csi-driver-nfs](https://github.com/kubernetes-csi/csi-driver-nfs), +and [synology-csi](https://github.com/SynologyOpenSource/synology-csi). + +#### Authenticate Flux over SSH + +Authenticating Flux to your git repository has a couple benefits like using a private git repository and/or using the Flux [Image Automation Controllers](https://fluxcd.io/docs/components/image/). + +By default this template only works on a public Github repository, it is advised to keep your repository public. + +The benefits of a public repository include: + +- Debugging or asking for help, you can provide a link to a resource you are having issues with. +- Adding a topic to your repository of `k8s-at-home` to be included in the [k8s-at-home-search](https://nanne.dev/k8s-at-home-search/). This search helps people discover different configurations of Helm charts across others Flux based repositories. + +
+ Expand to read guide on adding Flux SSH authentication + +1. Generate new SSH key: + + ```sh + ssh-keygen -t ecdsa -b 521 -C "github-deploy-key" -f ./kubernetes/bootstrap/github-deploy.key -q -P "" + ``` + +2. Paste public key in the deploy keys section of your repository settings +3. Create sops secret in `./kubernetes/bootstrap/github-deploy-key.sops.yaml` with the contents of: + + ```yaml + apiVersion: v1 + kind: Secret + metadata: + name: github-deploy-key + namespace: flux-system + stringData: + # 3a. Contents of github-deploy-key + identity: | + -----BEGIN OPENSSH PRIVATE KEY----- + ... + -----END OPENSSH PRIVATE KEY----- + # 3b. Output of curl --silent https://api.github.com/meta | jq --raw-output '"github.com "+.ssh_keys[]' + known_hosts: | + github.com ssh-ed25519 ... + github.com ecdsa-sha2-nistp256 ... + github.com ssh-rsa ... + ``` + +4. Encrypt secret: + + ```sh + sops --encrypt --in-place ./kubernetes/bootstrap/github-deploy-key.sops.yaml + ``` + +5. Apply secret to cluster: + + ```sh + sops --decrypt ./kubernetes/bootstrap/github-deploy-key.sops.yaml | kubectl apply -f - + ``` + +6. Update `./kubernetes/flux/config/cluster.yaml`: + + ```yaml + apiVersion: source.toolkit.fluxcd.io/v1beta2 + kind: GitRepository + metadata: + name: home-kubernetes + namespace: flux-system + spec: + interval: 10m + # 6a: Change this to your user and repo names + url: ssh://git@github.com/$user/$repo + ref: + branch: main + secretRef: + name: github-deploy-key + ``` + +7. Commit and push changes +8. Force flux to reconcile your changes + + ```sh + flux reconcile -n flux-system kustomization cluster --with-source + ``` + +9. Verify git repository is now using SSH: + + ```sh + flux get sources git -A + ``` + +10. Optionally set your repository to Private in your repository settings. + +
+ +## 🤝 Thanks + +Big shout out to all the contributors, sponsors and everyone else who has helped on this project. diff --git a/Taskfile.yaml b/Taskfile.yaml new file mode 100644 index 00000000..60047f1e --- /dev/null +++ b/Taskfile.yaml @@ -0,0 +1,82 @@ +--- +version: "3" + +vars: + PYTHON_BIN: python3 + BOOTSTRAP_DIR: "{{.ROOT_DIR}}/bootstrap" + ANSIBLE_DIR: "{{.ROOT_DIR}}/ansible" + KUBERNETES_DIR: "{{.ROOT_DIR}}/kubernetes" + +env: + KUBECONFIG: "{{.ROOT_DIR}}/kubeconfig" + SOPS_AGE_KEY_FILE: "{{.ROOT_DIR}}/age.key" + PATH: "{{.ROOT_DIR}}/.venv/bin:$PATH" + VIRTUAL_ENV: "{{.ROOT_DIR}}/.venv" + ANSIBLE_COLLECTIONS_PATH: "{{.ROOT_DIR}}/.venv/galaxy" + ANSIBLE_ROLES_PATH: "{{.ROOT_DIR}}/.venv/galaxy/ansible_roles" + ANSIBLE_VARS_ENABLED: "host_group_vars,community.sops.sops" + K8S_AUTH_KUBECONFIG: "{{.ROOT_DIR}}/kubeconfig" + +includes: + ansible: .taskfiles/AnsibleTasks.yaml + brew: .taskfiles/BrewTasks.yaml + cluster: .taskfiles/ClusterTasks.yaml + +tasks: + + default: task -l + + deps: + desc: Create a Python virtual env and install required packages + summary: task {{.TASK}} + cmds: + - "{{.PYTHON_BIN}} -m venv {{.ROOT_DIR}}/.venv" + - .venv/bin/python3 -m pip install --upgrade pip setuptools wheel + - .venv/bin/python3 -m pip install --upgrade --requirement "{{.ROOT_DIR}}/requirements.txt" + - .venv/bin/ansible-galaxy install --role-file "{{.ROOT_DIR}}/requirements.yaml" --force + + init: + desc: Initialize configuration files + summary: task {{.TASK}} + dir: "{{.BOOTSTRAP_DIR}}" + cmds: + - cp -n vars/addons.sample.yaml vars/addons.yaml + - cp -n vars/config.sample.yaml vars/config.yaml + - cmd: echo "=== Configuration files copied ===" + silent: true + - cmd: echo "Proceed with updating the configuration files..." + silent: true + - cmd: echo "{{.BOOTSTRAP_DIR}}/vars/config.yaml" + silent: true + - cmd: echo "{{.BOOTSTRAP_DIR}}/vars/addons.yaml" + silent: true + status: + - test -f "{{.BOOTSTRAP_DIR}}/vars/addons.yaml" + - test -f "{{.BOOTSTRAP_DIR}}/vars/config.yaml" + + # TODO: Only prompt when generated directories exist + # https://github.com/go-task/task/issues/1330 + configure: + desc: Configure repository from Ansible vars + summary: task {{.TASK}} + prompt: Any conflicting config in the root kubernetes and ansible directories will be overwritten... continue? + dir: "{{.BOOTSTRAP_DIR}}" + cmd: ansible-playbook configure.yaml + env: + ANSIBLE_DISPLAY_SKIPPED_HOSTS: "false" + + update-template: + desc: Update from the upstream flux-cluster-template repository + summary: task {{.TASK}} + cmds: + - mkdir -p $(dirname {{.shafile}}) + - touch {{.shafile}} + - git remote get-url template >/dev/null 2>&1 || git remote add template git@github.com:onedr0p/flux-cluster-template + - git fetch --all + - git cherry-pick --no-commit --allow-empty $(cat {{.shafile}})..template/main + - git ls-remote template HEAD | awk '{ print $1}' > {{.shafile}} + vars: + shafile: "{{.ROOT_DIR}}/.tasks/.latest-template.sha" + preconditions: + - { msg: "Git repository not up-to-date", sh: "git diff --exit-code" } + - { msg: "Git repository not up-to-date", sh: "git diff --cached --exit-code" } diff --git a/bootstrap/configure.yaml b/bootstrap/configure.yaml new file mode 100644 index 00000000..33bfff27 --- /dev/null +++ b/bootstrap/configure.yaml @@ -0,0 +1,33 @@ +--- +- name: Cluster Installation + hosts: localhost + connection: local + vars_files: + - vars/config.yaml + - vars/addons.yaml + tasks: + - name: Get absolute path to this Git repository # noqa: command-instead-of-module + ansible.builtin.command: git rev-parse --show-toplevel + changed_when: false + check_mode: false + register: repository + failed_when: repository.rc != 0 + + - name: Set facts + ansible.builtin.set_fact: + repository_path: "{{ repository.stdout }}" + + - name: Verify configuration + ansible.builtin.include_tasks: tasks/validation/main.yaml + + - name: Template Sops configuration + ansible.builtin.include_tasks: tasks/sops/main.yaml + + - name: Template Ansible configuration + ansible.builtin.include_tasks: tasks/ansible/main.yaml + + - name: Template Kubernetes configuration + ansible.builtin.include_tasks: tasks/kubernetes/main.yaml + + - name: Template Kubernetes addon configuration + ansible.builtin.include_tasks: tasks/addons/main.yaml diff --git a/bootstrap/tasks/addons/csi_driver_nfs.yaml b/bootstrap/tasks/addons/csi_driver_nfs.yaml new file mode 100644 index 00000000..cfd0291a --- /dev/null +++ b/bootstrap/tasks/addons/csi_driver_nfs.yaml @@ -0,0 +1,34 @@ +--- +- name: Set addon facts + ansible.builtin.set_fact: + addon_name: csi-driver-nfs + addon_namespace: kube-system + +- name: Ensure directories exist for {{ addon_namespace }}/{{ addon_name }} + when: item.state == 'directory' + ansible.builtin.file: + path: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path }}" + state: directory + mode: "0755" + with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"] + +- name: Template unencrypted files for {{ addon_namespace }}/{{ addon_name }} + when: item.state == 'file' and 'sops' not in item.path + ansible.builtin.template: + src: "{{ item.src }}" + dest: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path | replace('.j2', '') }}" + mode: "0644" + with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"] + +- name: Template encrypted files for {{ addon_namespace }}/{{ addon_name }} + block: + - name: Template encrypted files + when: item.state == 'file' and 'sops' in item.path + community.sops.sops_encrypt: + path: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path | replace('.j2', '') }}" + encrypted_regex: ^(data|stringData)$ + age: ["{{ bootstrap_age_public_key }}"] + content_yaml: "{{ lookup('ansible.builtin.template', item.src) | from_yaml }}" + mode: "0644" + force: true + with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"] diff --git a/bootstrap/tasks/addons/discord_template_notifier.yaml b/bootstrap/tasks/addons/discord_template_notifier.yaml new file mode 100644 index 00000000..734065c3 --- /dev/null +++ b/bootstrap/tasks/addons/discord_template_notifier.yaml @@ -0,0 +1,34 @@ +--- +- name: Set addon facts + ansible.builtin.set_fact: + addon_name: discord-template-notifier + addon_namespace: default + +- name: Ensure directories exist for {{ addon_namespace }}/{{ addon_name }} + when: item.state == 'directory' + ansible.builtin.file: + path: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path }}" + state: directory + mode: "0755" + with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"] + +- name: Template unencrypted files for {{ addon_namespace }}/{{ addon_name }} + when: item.state == 'file' and 'sops' not in item.path + ansible.builtin.template: + src: "{{ item.src }}" + dest: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path | replace('.j2', '') }}" + mode: "0644" + with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"] + +- name: Template encrypted files for {{ addon_namespace }}/{{ addon_name }} + block: + - name: Template encrypted files + when: item.state == 'file' and 'sops' in item.path + community.sops.sops_encrypt: + path: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path | replace('.j2', '') }}" + encrypted_regex: ^(data|stringData)$ + age: ["{{ bootstrap_age_public_key }}"] + content_yaml: "{{ lookup('ansible.builtin.template', item.src) | from_yaml }}" + mode: "0644" + force: true + with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"] diff --git a/bootstrap/tasks/addons/grafana.yaml b/bootstrap/tasks/addons/grafana.yaml new file mode 100644 index 00000000..8ae41c2f --- /dev/null +++ b/bootstrap/tasks/addons/grafana.yaml @@ -0,0 +1,34 @@ +--- +- name: Set addon facts + ansible.builtin.set_fact: + addon_name: grafana + addon_namespace: monitoring + +- name: Ensure directories exist for {{ addon_namespace }}/{{ addon_name }} + when: item.state == 'directory' + ansible.builtin.file: + path: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path }}" + state: directory + mode: "0755" + with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"] + +- name: Template unencrypted files for {{ addon_namespace }}/{{ addon_name }} + when: item.state == 'file' and 'sops' not in item.path + ansible.builtin.template: + src: "{{ item.src }}" + dest: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path | replace('.j2', '') }}" + mode: "0644" + with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"] + +- name: Template encrypted files for {{ addon_namespace }}/{{ addon_name }} + block: + - name: Template encrypted files + when: item.state == 'file' and 'sops' in item.path + community.sops.sops_encrypt: + path: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path | replace('.j2', '') }}" + encrypted_regex: ^(data|stringData)$ + age: ["{{ bootstrap_age_public_key }}"] + content_yaml: "{{ lookup('ansible.builtin.template', item.src) | from_yaml }}" + mode: "0644" + force: true + with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"] diff --git a/bootstrap/tasks/addons/hajimari.yaml b/bootstrap/tasks/addons/hajimari.yaml new file mode 100644 index 00000000..c8b97610 --- /dev/null +++ b/bootstrap/tasks/addons/hajimari.yaml @@ -0,0 +1,35 @@ +--- +- name: Set addon facts + ansible.builtin.set_fact: + addon_name: hajimari + addon_namespace: default + +- name: Ensure directories exist for {{ addon_namespace }}/{{ addon_name }} + when: item.state == 'directory' + ansible.builtin.file: + path: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path }}" + state: directory + mode: "0755" + with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"] + +- name: Template unencrypted files for {{ addon_namespace }}/{{ addon_name }} + when: item.state == 'file' and 'sops' not in item.path + ansible.builtin.template: + src: "{{ item.src }}" + dest: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path | replace('.j2', '') }}" + mode: "0644" + with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"] + +# https://github.com/ansible-collections/community.sops/issues/153 +- name: Template encrypted files for {{ addon_namespace }}/{{ addon_name }} + block: + - name: Template encrypted files + when: item.state == 'file' and 'sops' in item.path + community.sops.sops_encrypt: + path: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path | replace('.j2', '') }}" + encrypted_regex: ^(data|stringData)$ + age: ["{{ bootstrap_age_public_key }}"] + content_yaml: "{{ lookup('ansible.builtin.template', item.src) | from_yaml }}" + mode: "0644" + force: true + with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"] diff --git a/bootstrap/tasks/addons/kube_prometheus_stack.yaml b/bootstrap/tasks/addons/kube_prometheus_stack.yaml new file mode 100644 index 00000000..0e7531da --- /dev/null +++ b/bootstrap/tasks/addons/kube_prometheus_stack.yaml @@ -0,0 +1,34 @@ +--- +- name: Set addon facts + ansible.builtin.set_fact: + addon_name: kube-prometheus-stack + addon_namespace: monitoring + +- name: Ensure directories exist for {{ addon_namespace }}/{{ addon_name }} + when: item.state == 'directory' + ansible.builtin.file: + path: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path }}" + state: directory + mode: "0755" + with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"] + +- name: Template unencrypted files for {{ addon_namespace }}/{{ addon_name }} + when: item.state == 'file' and 'sops' not in item.path + ansible.builtin.template: + src: "{{ item.src }}" + dest: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path | replace('.j2', '') }}" + mode: "0644" + with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"] + +- name: Template encrypted files for {{ addon_namespace }}/{{ addon_name }} + block: + - name: Template encrypted files + when: item.state == 'file' and 'sops' in item.path + community.sops.sops_encrypt: + path: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path | replace('.j2', '') }}" + encrypted_regex: ^(data|stringData)$ + age: ["{{ bootstrap_age_public_key }}"] + content_yaml: "{{ lookup('ansible.builtin.template', item.src) | from_yaml }}" + mode: "0644" + force: true + with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"] diff --git a/bootstrap/tasks/addons/kubernetes_dashboard.yaml b/bootstrap/tasks/addons/kubernetes_dashboard.yaml new file mode 100644 index 00000000..778c7967 --- /dev/null +++ b/bootstrap/tasks/addons/kubernetes_dashboard.yaml @@ -0,0 +1,34 @@ +--- +- name: Set addon facts + ansible.builtin.set_fact: + addon_name: kubernetes-dashboard + addon_namespace: monitoring + +- name: Ensure directories exist for {{ addon_namespace }}/{{ addon_name }} + when: item.state == 'directory' + ansible.builtin.file: + path: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path }}" + state: directory + mode: "0755" + with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"] + +- name: Template unencrypted files for {{ addon_namespace }}/{{ addon_name }} + when: item.state == 'file' and 'sops' not in item.path + ansible.builtin.template: + src: "{{ item.src }}" + dest: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path | replace('.j2', '') }}" + mode: "0644" + with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"] + +- name: Template encrypted files for {{ addon_namespace }}/{{ addon_name }} + block: + - name: Template encrypted files + when: item.state == 'file' and 'sops' in item.path + community.sops.sops_encrypt: + path: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path | replace('.j2', '') }}" + encrypted_regex: ^(data|stringData)$ + age: ["{{ bootstrap_age_public_key }}"] + content_yaml: "{{ lookup('ansible.builtin.template', item.src) | from_yaml }}" + mode: "0644" + force: true + with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"] diff --git a/bootstrap/tasks/addons/main.yaml b/bootstrap/tasks/addons/main.yaml new file mode 100644 index 00000000..23e4092b --- /dev/null +++ b/bootstrap/tasks/addons/main.yaml @@ -0,0 +1,33 @@ +--- + +- name: Process addon csi-driver-nfs + when: csi_driver_nfs.enabled | default(false) + ansible.builtin.include_tasks: csi_driver_nfs.yaml + +- name: Process addon hajimari + when: hajimari.enabled | default(false) + ansible.builtin.include_tasks: hajimari.yaml + +- name: Process addon grafana + when: grafana.enabled | default(false) + ansible.builtin.include_tasks: grafana.yaml + +- name: Process addon kubernetes-dashboard + when: kubernetes_dashboard.enabled | default(false) + ansible.builtin.include_tasks: kubernetes_dashboard.yaml + +- name: Process addon kube-prometheus-stack + when: kube_prometheus_stack.enabled | default(false) + ansible.builtin.include_tasks: kube_prometheus_stack.yaml + +- name: Process addon system-upgrade-controller + when: system_upgrade_controller.enabled | default(false) + ansible.builtin.include_tasks: system_upgrade_controller.yaml + +- name: Process addon weave-gitops + when: weave_gitops.enabled | default(false) + ansible.builtin.include_tasks: weave_gitops.yaml + +- name: Process addon discord-template-notifier + when: discord_template_notifier.enabled | default(false) + ansible.builtin.include_tasks: discord_template_notifier.yaml diff --git a/bootstrap/tasks/addons/system_upgrade_controller.yaml b/bootstrap/tasks/addons/system_upgrade_controller.yaml new file mode 100644 index 00000000..65e0f00c --- /dev/null +++ b/bootstrap/tasks/addons/system_upgrade_controller.yaml @@ -0,0 +1,34 @@ +--- +- name: Set addon facts + ansible.builtin.set_fact: + addon_name: system-upgrade-controller + addon_namespace: kube-system + +- name: Ensure directories exist for {{ addon_namespace }}/{{ addon_name }} + when: item.state == 'directory' + ansible.builtin.file: + path: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path }}" + state: directory + mode: "0755" + with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"] + +- name: Template unencrypted files for {{ addon_namespace }}/{{ addon_name }} + when: item.state == 'file' and 'sops' not in item.path + ansible.builtin.template: + src: "{{ item.src }}" + dest: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path | replace('.j2', '') }}" + mode: "0644" + with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"] + +- name: Template encrypted files for {{ addon_namespace }}/{{ addon_name }} + block: + - name: Template encrypted files + when: item.state == 'file' and 'sops' in item.path + community.sops.sops_encrypt: + path: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path | replace('.j2', '') }}" + encrypted_regex: ^(data|stringData)$ + age: ["{{ bootstrap_age_public_key }}"] + content_yaml: "{{ lookup('ansible.builtin.template', item.src) | from_yaml }}" + mode: "0644" + force: true + with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"] diff --git a/bootstrap/tasks/addons/weave_gitops.yaml b/bootstrap/tasks/addons/weave_gitops.yaml new file mode 100644 index 00000000..12d61240 --- /dev/null +++ b/bootstrap/tasks/addons/weave_gitops.yaml @@ -0,0 +1,34 @@ +--- +- name: Set addon facts + ansible.builtin.set_fact: + addon_name: weave-gitops + addon_namespace: flux-system + +- name: Ensure directories exist for {{ addon_namespace }}/{{ addon_name }} + when: item.state == 'directory' + ansible.builtin.file: + path: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path }}" + state: directory + mode: "0755" + with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"] + +- name: Template unencrypted files for {{ addon_namespace }}/{{ addon_name }} + when: item.state == 'file' and 'sops' not in item.path + ansible.builtin.template: + src: "{{ item.src }}" + dest: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path | replace('.j2', '') }}" + mode: "0644" + with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"] + +- name: Template encrypted files for {{ addon_namespace }}/{{ addon_name }} + block: + - name: Template encrypted files + when: item.state == 'file' and 'sops' in item.path + community.sops.sops_encrypt: + path: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path | replace('.j2', '') }}" + encrypted_regex: ^(data|stringData)$ + age: ["{{ bootstrap_age_public_key }}"] + content_yaml: "{{ lookup('ansible.builtin.template', item.src) | from_yaml }}" + mode: "0644" + force: true + with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"] diff --git a/bootstrap/tasks/ansible/main.yaml b/bootstrap/tasks/ansible/main.yaml new file mode 100644 index 00000000..5c5a286f --- /dev/null +++ b/bootstrap/tasks/ansible/main.yaml @@ -0,0 +1,39 @@ +--- +- name: Ensure Kubernetes directories exist + when: item.state == 'directory' + ansible.builtin.file: + path: "{{ repository_path }}/ansible/{{ item.path }}" + state: directory + mode: "0755" + with_community.general.filetree: ["../templates/ansible/"] + +- name: Template Ansible unencrypted files + when: item.state == 'file' and 'sops' not in item.path and '.DS_Store' not in item.path + ansible.builtin.template: + src: "{{ item.src }}" + dest: "{{ repository_path }}/ansible/{{ item.path | regex_replace('.j2$', '') }}" + mode: "0644" + with_community.general.filetree: ["../templates/ansible/"] + +- name: Template Ansible encrypted files + block: + - name: Template Ansible encrypted files + when: item.state == 'file' and 'sops' in item.path + community.sops.sops_encrypt: + path: "{{ repository_path }}/ansible/{{ item.path | replace('.j2', '') }}" + encrypted_regex: ^(data|stringData)$ + age: ["{{ bootstrap_age_public_key }}"] + content_yaml: "{{ lookup('ansible.builtin.template', item.src) | from_yaml }}" + mode: "0644" + force: true + with_community.general.filetree: ["../templates/ansible/"] + - name: Template encrypted node secrets + community.sops.sops_encrypt: + path: "{{ repository_path }}/ansible/inventory/host_vars/{{ item.name }}.sops.yaml" + age: ["{{ bootstrap_age_public_key }}"] + content_yaml: "{{ lookup('ansible.builtin.template', 'templates/node.sops.yaml.j2', template_vars=dict(password=item.password)) | from_yaml }}" + mode: "0644" + force: true + loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker | default([]) }}" + loop_control: + label: "{{ item.address }}" diff --git a/bootstrap/tasks/kubernetes/main.yaml b/bootstrap/tasks/kubernetes/main.yaml new file mode 100644 index 00000000..9c6f27ab --- /dev/null +++ b/bootstrap/tasks/kubernetes/main.yaml @@ -0,0 +1,66 @@ +--- +- name: Ensure Kubernetes directories exist + when: item.state == 'directory' + ansible.builtin.file: + path: "{{ repository_path }}/kubernetes/{{ item.path }}" + state: directory + mode: "0755" + with_community.general.filetree: ["../templates/kubernetes/"] + +- name: Template Kubernetes unencrypted files + when: + - item.state == 'file' + - "'.DS_Store' not in item.path" + - "'sops' not in item.path" + - "'cluster-settings-user.yaml.j2' not in item.path" + - "'cluster-secrets-user.yaml.j2' not in item.path" + ansible.builtin.template: + src: "{{ item.src }}" + dest: "{{ repository_path }}/kubernetes/{{ item.path | regex_replace('.j2$', '') }}" + mode: "0644" + with_community.general.filetree: ["../templates/kubernetes/"] + +- name: Check if the cluster user settings file already exists + stat: + path: "{{ repository_path }}/kubernetes/flux/vars/cluster-settings-user.yaml" + register: cluster_settings_user + +- name: Template Kubernetes user cluster settings + when: + - item.state == 'file' + - "'cluster-settings-user.yaml' in item.path" + - not cluster_settings_user.stat.exists + ansible.builtin.template: + src: "{{ item.src }}" + dest: "{{ repository_path }}/kubernetes/{{ item.path | regex_replace('.j2$', '') }}" + mode: "0644" + with_community.general.filetree: ["../templates/kubernetes/"] + +- name: Check if the cluster user secrets file already exists + stat: + path: "{{ repository_path }}/kubernetes/flux/vars/cluster-secrets-user.yaml" + register: cluster_secrets_user + +- name: Template Kubernetes user cluster secrets + when: + - item.state == 'file' + - "'cluster-secrets-user.yaml' in item.path" + - not cluster_secrets_user.stat.exists + ansible.builtin.template: + src: "{{ item.src }}" + dest: "{{ repository_path }}/kubernetes/{{ item.path | regex_replace('.j2$', '') }}" + mode: "0644" + with_community.general.filetree: ["../templates/kubernetes/"] + +- name: Template Kubernetes encrypted files + block: + - name: Template Kubernetes encrypted files + when: item.state == 'file' and 'sops' in item.path + community.sops.sops_encrypt: + path: "{{ repository_path }}/kubernetes/{{ item.path | replace('.j2', '') }}" + encrypted_regex: ^(data|stringData)$ + age: ["{{ bootstrap_age_public_key }}"] + content_yaml: "{{ lookup('ansible.builtin.template', item.src) | from_yaml }}" + mode: "0644" + force: true + with_community.general.filetree: ["../templates/kubernetes/"] diff --git a/bootstrap/tasks/sops/main.yaml b/bootstrap/tasks/sops/main.yaml new file mode 100644 index 00000000..7aaaf6b9 --- /dev/null +++ b/bootstrap/tasks/sops/main.yaml @@ -0,0 +1,6 @@ +--- +- name: Template Sops configuration file + ansible.builtin.template: + src: "templates/.sops.yaml.j2" + dest: "{{ repository_path }}/.sops.yaml" + mode: "0644" diff --git a/bootstrap/tasks/validation/age.yaml b/bootstrap/tasks/validation/age.yaml new file mode 100644 index 00000000..81d6ae8e --- /dev/null +++ b/bootstrap/tasks/validation/age.yaml @@ -0,0 +1,21 @@ +--- +- name: Query age key file + ansible.builtin.stat: + path: "{{ repository_path }}/age.key" + register: result + +- name: Check if age key file exists + ansible.builtin.assert: + that: result.stat.exists + success_msg: Age file {{ repository_path }}/age.key exists + fail_msg: Age file {{ repository_path }}/age.key does not exist + +- name: Query age key file contents + ansible.builtin.set_fact: + age_contents: "{{ lookup('file', repository_path + '/age.key') }}" + +- name: Check if age public keys match + ansible.builtin.assert: + that: bootstrap_age_public_key in age_contents + success_msg: Age public key {{ bootstrap_age_public_key }} exists + fail_msg: Age public key {{ bootstrap_age_public_key }} does not exist diff --git a/bootstrap/tasks/validation/cli.yaml b/bootstrap/tasks/validation/cli.yaml new file mode 100644 index 00000000..7e0215b9 --- /dev/null +++ b/bootstrap/tasks/validation/cli.yaml @@ -0,0 +1,9 @@ +--- +- name: Check for required CLI tools + ansible.builtin.shell: | + command -v {{ item }} >/dev/null 2>&1 + loop: [age, cloudflared, flux, sops] + changed_when: false + check_mode: false + register: result + failed_when: result.rc != 0 and result.rc != 127 diff --git a/bootstrap/tasks/validation/cloudflare.yaml b/bootstrap/tasks/validation/cloudflare.yaml new file mode 100644 index 00000000..4c2885eb --- /dev/null +++ b/bootstrap/tasks/validation/cloudflare.yaml @@ -0,0 +1,34 @@ +--- +- name: Query Cloudflare zone + ansible.builtin.uri: + url: https://api.cloudflare.com/client/v4/zones?name={{ bootstrap_cloudflare_domain }}&status=active + headers: + Authorization: Bearer {{ bootstrap_cloudflare_token }} + Content-Type: application/json + timeout: 5 + return_content: true + body_format: json + register: result + +- name: Check if Cloudflare zone exists + ansible.builtin.assert: + that: result.json.success is true + success_msg: Cloudflare zone {{ bootstrap_cloudflare_domain }} exists + fail_msg: Cloudflare zone {{ bootstrap_cloudflare_domain }} does not exist + +- name: Query Cloudflared tunnel + ansible.builtin.uri: + url: https://api.cloudflare.com/client/v4/accounts/{{ bootstrap_cloudflare_account_tag }}/cfd_tunnel/{{ bootstrap_cloudflare_tunnel_id }} + headers: + Authorization: Bearer {{ bootstrap_cloudflare_token }} + Content-Type: application/json + timeout: 5 + return_content: true + body_format: json + register: result + +- name: Check if Cloudflared tunnel exists + ansible.builtin.assert: + that: result.json.success is true + success_msg: Cloudflared tunnel {{ bootstrap_cloudflare_tunnel_id }} exists + fail_msg: Cloudflared tunnel {{ bootstrap_cloudflare_tunnel_id }} does not exist diff --git a/bootstrap/tasks/validation/github.yaml b/bootstrap/tasks/validation/github.yaml new file mode 100644 index 00000000..7ffa2952 --- /dev/null +++ b/bootstrap/tasks/validation/github.yaml @@ -0,0 +1,42 @@ +--- +- name: Query Github username + ansible.builtin.uri: + url: https://api.github.com/users/{{ bootstrap_github_username }} + timeout: 5 + return_content: true + body_format: json + register: result + +- name: Check if username exists + ansible.builtin.assert: + that: result.json.login == bootstrap_github_username + success_msg: Github user {{ bootstrap_github_username }} exists + fail_msg: Github user {{ bootstrap_github_username }} does not exist + +- name: Query Github repo + ansible.builtin.uri: + url: https://api.github.com/repos/{{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }} + timeout: 5 + return_content: true + body_format: json + register: result + +- name: Check if repo exists + ansible.builtin.assert: + that: result.json.full_name == bootstrap_github_username + '/' + bootstrap_github_repository_name + success_msg: Github repo {{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }} exists + fail_msg: Github repo {{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }} does not exist + +- name: Query Github repo branch + ansible.builtin.uri: + url: https://api.github.com/repos/{{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }}/branches/{{ bootstrap_github_repository_branch | default('main', true) }} + timeout: 5 + return_content: true + body_format: json + register: result + +- name: Check if repo branch exists + ansible.builtin.assert: + that: result.json.name == bootstrap_github_repository_branch | default('main', true) + success_msg: Github repo branch {{ bootstrap_github_repository_branch | default('main', true) }} exists + fail_msg: Github repo branch {{ bootstrap_github_repository_branch | default('main', true) }} does not exist diff --git a/bootstrap/tasks/validation/main.yaml b/bootstrap/tasks/validation/main.yaml new file mode 100644 index 00000000..38b0db19 --- /dev/null +++ b/bootstrap/tasks/validation/main.yaml @@ -0,0 +1,6 @@ +--- +- name: Verify configuration + ansible.builtin.include_tasks: "{{ task }}.yaml" + loop: [vars, age, cli, net, cloudflare, github] + loop_control: + loop_var: task diff --git a/bootstrap/tasks/validation/net.yaml b/bootstrap/tasks/validation/net.yaml new file mode 100644 index 00000000..3016d31c --- /dev/null +++ b/bootstrap/tasks/validation/net.yaml @@ -0,0 +1,165 @@ +--- +- name: Set reachable address + set_fact: + current_address: "{{ item.external_address | default(item.address) }}" + loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker | default([]) }}" + loop_control: + label: "{{ item.address }}" + +- name: Verify master node count + ansible.builtin.assert: + that: + - bootstrap_nodes.master | length > 0 + - bootstrap_nodes.master | length is odd + success_msg: Master node count {{ bootstrap_nodes.master | length }} is correct. + fail_msg: Master node count {{ bootstrap_nodes.master | length }} is not greater than 0 or is not odd. + +- name: Verify node CIDR + ansible.builtin.assert: + that: bootstrap_node_cidr is ansible.utils.ipv4 + success_msg: Node CIDR {{ bootstrap_node_cidr }} is valid. + fail_msg: Node CIDR {{ bootstrap_node_cidr }} is invalid. + +- name: Verify cluster CIDR is ipv4 OR ipv6 + when: not bootstrap_ipv6_enabled | default(false) + ansible.builtin.assert: + that: bootstrap_cluster_cidr is ansible.utils.ipv4 or bootstrap_cluster_cidr is ansible.utils.ipv6 + success_msg: Cluster CIDR {{ bootstrap_cluster_cidr }} is valid. + fail_msg: Cluster CIDR {{ bootstrap_cluster_cidr }} is invalid. + +- name: Verify service CIDR is ipv4 OR ipv6 + when: not bootstrap_ipv6_enabled | default(false) + ansible.builtin.assert: + that: bootstrap_service_cidr is ansible.utils.ipv4 or bootstrap_service_cidr is ansible.utils.ipv6 + success_msg: Service CIDR {{ bootstrap_service_cidr }} is valid. + fail_msg: Service CIDR {{ bootstrap_service_cidr }} is invalid. + +- name: Verify cluster CIDR is ipv4 AND ipv6 + when: bootstrap_ipv6_enabled | default(false) + ansible.builtin.assert: + that: > + ( + bootstrap_cluster_cidr.split(',')[0] is ansible.utils.ipv4 or + bootstrap_cluster_cidr.split(',')[1] is ansible.utils.ipv4 + ) and ( + bootstrap_cluster_cidr.split(',')[1] is ansible.utils.ipv6 or + bootstrap_cluster_cidr.split(',')[0] is ansible.utils.ipv6 + ) + success_msg: Cluster CIDR {{ bootstrap_cluster_cidr }} is valid. + fail_msg: Cluster CIDR {{ bootstrap_cluster_cidr }} is invalid. + +- name: Verify service CIDR is ipv4 AND ipv6 + when: bootstrap_ipv6_enabled | default(false) + ansible.builtin.assert: + that: > + ( + bootstrap_service_cidr.split(',')[0] is ansible.utils.ipv4 or + bootstrap_service_cidr.split(',')[1] is ansible.utils.ipv4 + ) and ( + bootstrap_service_cidr.split(',')[1] is ansible.utils.ipv6 or + bootstrap_service_cidr.split(',')[0] is ansible.utils.ipv6 + ) + success_msg: Service CIDR {{ bootstrap_service_cidr }} is valid. + fail_msg: Service CIDR {{ bootstrap_service_cidr }} is invalid. + +- name: Verify k8s_gateway + ansible.builtin.assert: + that: bootstrap_k8s_gateway_addr is ansible.utils.ipv4 + success_msg: k8s_gateway address {{ bootstrap_k8s_gateway_addr }} is valid. + fail_msg: k8s_gateway address {{ bootstrap_k8s_gateway_addr }} is invalid. + +- name: Verify k8s_gateway in node CIDR + ansible.builtin.assert: + that: bootstrap_node_cidr | ansible.utils.network_in_usable(bootstrap_k8s_gateway_addr) + success_msg: k8s_gateway address {{ bootstrap_k8s_gateway_addr }} is within {{ bootstrap_node_cidr }}. + fail_msg: k8s_gateway address {{ bootstrap_k8s_gateway_addr }} is not within {{ bootstrap_node_cidr }}. + +- name: Verify internal ingress + ansible.builtin.assert: + that: bootstrap_internal_ingress_addr is ansible.utils.ipv4 + success_msg: internal ingress address {{ bootstrap_internal_ingress_addr }} is valid. + fail_msg: internal ingress address {{ bootstrap_internal_ingress_addr }} is invalid. + +- name: Verify internal ingress in node CIDR + ansible.builtin.assert: + that: bootstrap_node_cidr | ansible.utils.network_in_usable(bootstrap_internal_ingress_addr) + success_msg: internal ingress address {{ bootstrap_internal_ingress_addr }} is within {{ bootstrap_node_cidr }}. + fail_msg: internal ingress address {{ bootstrap_internal_ingress_addr }} is not within {{ bootstrap_node_cidr }}. + +- name: Verify external ingress + ansible.builtin.assert: + that: bootstrap_external_ingress_addr is ansible.utils.ipv4 + success_msg: external ingress address {{ bootstrap_external_ingress_addr }} is valid. + fail_msg: external ingress address {{ bootstrap_external_ingress_addr }} is invalid. + +- name: Verify external ingress in node CIDR + ansible.builtin.assert: + that: bootstrap_node_cidr | ansible.utils.network_in_usable(bootstrap_external_ingress_addr) + success_msg: external ingress address {{ bootstrap_external_ingress_addr }} is within {{ bootstrap_node_cidr }}. + fail_msg: external ingress address {{ bootstrap_external_ingress_addr }} is not within {{ bootstrap_node_cidr }}. + +- name: Verify kube-vip + ansible.builtin.assert: + that: bootstrap_kube_vip_addr is ansible.utils.ipv4 + success_msg: kube-vip address {{ bootstrap_kube_vip_addr }} is valid. + fail_msg: kube-vip address {{ bootstrap_kube_vip_addr }} is invalid. + +- name: Verify kube-vip in node CIDR + ansible.builtin.assert: + that: bootstrap_node_cidr | ansible.utils.network_in_usable(bootstrap_kube_vip_addr) + success_msg: kube-vip address {{ bootstrap_kube_vip_addr }} is within {{ bootstrap_node_cidr }}. + fail_msg: kube-vip address {{ bootstrap_kube_vip_addr }} is not within {{ bootstrap_node_cidr }}. + +- name: Verify all IP addresses are unique + ansible.builtin.assert: + that: > + [ + bootstrap_k8s_gateway_addr, + bootstrap_external_ingress_addr, + bootstrap_internal_ingress_addr, + bootstrap_kube_vip_addr + ] | unique | length == 4 + success_msg: All IP addresses are unique. + fail_msg: All IP addresses are not unique. + +- name: Verify nodes are not the same IPs as k8s_gateway, ingress external/internal or kube-vip + ansible.builtin.assert: + that: item.address not in (bootstrap_k8s_gateway_addr, bootstrap_external_ingress_addr, bootstrap_internal_ingress_addr, bootstrap_kube_vip_addr) + success_msg: Node address {{ item.address }} is different than k8s_gateway, ingress-nginx or kube-vip. + fail_msg: Node address {{ item.address }} is not different than k8s_gateway, ingress-nginx or kube-vip. + quiet: true + loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker | default([]) }}" + loop_control: + label: "{{ item.address }}" + +- name: Verify nodes are ipv4 + ansible.builtin.assert: + that: item.address is ansible.utils.ipv4 + success_msg: Node address {{ item.address }} is valid. + fail_msg: Node address {{ item.address }} is invalid. + quiet: true + loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker | default([]) }}" + loop_control: + label: "{{ item.address }}" + +- name: Verify nodes are in node CIDR + ansible.builtin.assert: + that: bootstrap_node_cidr | ansible.utils.network_in_usable(item.address) + success_msg: Node address {{ item.address }} is within {{ bootstrap_node_cidr }}. + fail_msg: Node address {{ item.address }} is not within {{ bootstrap_node_cidr }}. + quiet: true + loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker | default([]) }}" + when: item.external_address is not defined + loop_control: + label: "{{ item.address }}" + +- name: Verify SSH port is reachable + ansible.builtin.wait_for: + host: "{{ current_address }}" + port: 22 + search_regex: OpenSSH + timeout: 10 + connection: local + loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker | default([]) }}" + loop_control: + label: "{{ current_address }}" diff --git a/bootstrap/tasks/validation/vars.yaml b/bootstrap/tasks/validation/vars.yaml new file mode 100644 index 00000000..9e81b7ee --- /dev/null +++ b/bootstrap/tasks/validation/vars.yaml @@ -0,0 +1,37 @@ +--- + +- name: Verify required bootstrap vars are set + ansible.builtin.assert: + that: item | default('', true) | trim != '' + success_msg: Required bootstrap var {{ item }} exists and is defined + fail_msg: Required bootstrap var {{ item }} does not exists or is not defined + loop: + - bootstrap_acme_email + - bootstrap_age_public_key + - bootstrap_cloudflare_account_tag + - bootstrap_cloudflare_domain + - bootstrap_cloudflare_token + - bootstrap_cloudflare_tunnel_id + - bootstrap_cloudflare_tunnel_secret + - bootstrap_cluster_cidr + - bootstrap_flux_github_webhook_token + - bootstrap_github_repository_name + - bootstrap_github_repository_branch + - bootstrap_github_username + - bootstrap_external_ingress_addr + - bootstrap_internal_ingress_addr + - bootstrap_ipv6_enabled + - bootstrap_k8s_gateway_addr + - bootstrap_kube_vip_addr + - bootstrap_node_cidr + - bootstrap_service_cidr + - bootstrap_timezone + +- name: Verify bootstrap node names are valid + ansible.builtin.assert: + that: item.name is match('^[a-z0-9-]+$') + success_msg: Node name {{ item.name }} is valid + fail_msg: Node name {{ item.name }} is not valid + loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker | default([]) }}" + loop_control: + label: "{{ item.name }}" diff --git a/bootstrap/templates/.sops.yaml.j2 b/bootstrap/templates/.sops.yaml.j2 new file mode 100644 index 00000000..8a86fb3b --- /dev/null +++ b/bootstrap/templates/.sops.yaml.j2 @@ -0,0 +1,16 @@ +--- +creation_rules: + - path_regex: kubernetes/.*\.sops\.ya?ml + encrypted_regex: "^(data|stringData)$" + key_groups: + - age: + - "{{ bootstrap_age_public_key }}" + - path_regex: ansible/.*\.sops\.ya?ml + key_groups: + - age: + - "{{ bootstrap_age_public_key }}" + # https://github.com/ansible-collections/community.sops/issues/153 + - path_regex: /dev/stdin + key_groups: + - age: + - "{{ bootstrap_age_public_key }}" diff --git a/bootstrap/templates/addons/csi-driver-nfs/app/helmrelease.yaml.j2 b/bootstrap/templates/addons/csi-driver-nfs/app/helmrelease.yaml.j2 new file mode 100644 index 00000000..efb42acd --- /dev/null +++ b/bootstrap/templates/addons/csi-driver-nfs/app/helmrelease.yaml.j2 @@ -0,0 +1,28 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: csi-driver-nfs +spec: + interval: 30m + chart: + spec: + chart: csi-driver-nfs + version: v4.5.0 + sourceRef: + kind: HelmRepository + name: csi-driver-nfs + namespace: flux-system + maxHistory: 2 + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + values: + externalSnapshotter: + enabled: false diff --git a/bootstrap/templates/addons/csi-driver-nfs/app/kustomization.yaml.j2 b/bootstrap/templates/addons/csi-driver-nfs/app/kustomization.yaml.j2 new file mode 100644 index 00000000..70fd2032 --- /dev/null +++ b/bootstrap/templates/addons/csi-driver-nfs/app/kustomization.yaml.j2 @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml + - ./storageclass.yaml diff --git a/bootstrap/templates/addons/csi-driver-nfs/app/storageclass.yaml.j2 b/bootstrap/templates/addons/csi-driver-nfs/app/storageclass.yaml.j2 new file mode 100644 index 00000000..d62dc847 --- /dev/null +++ b/bootstrap/templates/addons/csi-driver-nfs/app/storageclass.yaml.j2 @@ -0,0 +1,15 @@ +#jinja2: trim_blocks: True, lstrip_blocks: True +{% for item in csi_driver_nfs.storage_class %} +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ item.name }} +provisioner: nfs.csi.k8s.io +parameters: + server: {{ item.server }} + share: {{ item.share }} +reclaimPolicy: Delete +volumeBindingMode: Immediate +mountOptions: ["hard", "noatime"] +{% endfor %} diff --git a/bootstrap/templates/addons/csi-driver-nfs/ks.yaml.j2 b/bootstrap/templates/addons/csi-driver-nfs/ks.yaml.j2 new file mode 100644 index 00000000..b451eea2 --- /dev/null +++ b/bootstrap/templates/addons/csi-driver-nfs/ks.yaml.j2 @@ -0,0 +1,20 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app csi-driver-nfs + namespace: flux-system +spec: + targetNamespace: kube-system + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/apps/kube-system/csi-driver-nfs/app + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: true + interval: 30m + retryInterval: 1m + timeout: 5m diff --git a/bootstrap/templates/addons/discord-template-notifier/app/helmrelease.yaml.j2 b/bootstrap/templates/addons/discord-template-notifier/app/helmrelease.yaml.j2 new file mode 100644 index 00000000..bcd52de3 --- /dev/null +++ b/bootstrap/templates/addons/discord-template-notifier/app/helmrelease.yaml.j2 @@ -0,0 +1,63 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: discord-template-notifier +spec: + interval: 15m + chart: + spec: + chart: app-template + version: 2.3.0 + interval: 30m + sourceRef: + kind: HelmRepository + name: bjw-s + namespace: flux-system + maxHistory: 2 + install: + createNamespace: true + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + values: + controllers: + main: + annotations: + reloader.stakater.com/auto: "true" + containers: + main: + image: + repository: ghcr.io/morphy2k/rss-forwarder + tag: 0.6.1 + env: + TZ: "${TIMEZONE}" + probes: + liveness: &disabled + enabled: false + readiness: *disabled + startup: *disabled + resources: + requests: + cpu: 5m + memory: 10M + limits: + memory: 64M + service: + main: *disabled + ingress: + main: *disabled + persistence: + config: + enabled: true + type: secret + name: discord-template-notifier-secret + globalMounts: + - path: /data/config.toml + subPath: config.toml + readOnly: true diff --git a/bootstrap/templates/addons/discord-template-notifier/app/kustomization.yaml.j2 b/bootstrap/templates/addons/discord-template-notifier/app/kustomization.yaml.j2 new file mode 100644 index 00000000..95bf4747 --- /dev/null +++ b/bootstrap/templates/addons/discord-template-notifier/app/kustomization.yaml.j2 @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./secret.sops.yaml + - ./helmrelease.yaml diff --git a/bootstrap/templates/addons/discord-template-notifier/app/secret.sops.yaml.j2 b/bootstrap/templates/addons/discord-template-notifier/app/secret.sops.yaml.j2 new file mode 100644 index 00000000..12759d18 --- /dev/null +++ b/bootstrap/templates/addons/discord-template-notifier/app/secret.sops.yaml.j2 @@ -0,0 +1,14 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: discord-template-notifier-secret +type: Opaque +stringData: + config.toml: |- + [feeds.github-template] + url = "https://github.com/onedr0p/flux-cluster-template/commits/main/.atom" + interval = "10m" + retry_limit = 5 + sink.type = "discord" + sink.url = "{{ discord_template_notifier.webhook_url }}" diff --git a/bootstrap/templates/addons/discord-template-notifier/ks.yaml.j2 b/bootstrap/templates/addons/discord-template-notifier/ks.yaml.j2 new file mode 100644 index 00000000..0b333f3c --- /dev/null +++ b/bootstrap/templates/addons/discord-template-notifier/ks.yaml.j2 @@ -0,0 +1,20 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app discord-template-notifier + namespace: flux-system +spec: + targetNamespace: default + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/apps/default/discord-template-notifier/app + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m diff --git a/bootstrap/templates/addons/grafana/app/helmrelease.yaml.j2 b/bootstrap/templates/addons/grafana/app/helmrelease.yaml.j2 new file mode 100644 index 00000000..dcba10b1 --- /dev/null +++ b/bootstrap/templates/addons/grafana/app/helmrelease.yaml.j2 @@ -0,0 +1,173 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: grafana +spec: + interval: 30m + chart: + spec: + chart: grafana + version: 7.0.11 + sourceRef: + kind: HelmRepository + name: grafana + namespace: flux-system + maxHistory: 2 + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + dependsOn: + - name: local-path-provisioner + namespace: kube-system + values: + deploymentStrategy: + type: Recreate + admin: + existingSecret: grafana-admin-secret + env: + GF_EXPLORE_ENABLED: true + GF_SERVER_ROOT_URL: "https://grafana.${SECRET_DOMAIN}" + grafana.ini: + analytics: + check_for_updates: false + check_for_plugin_updates: false + reporting_enabled: false + dashboardProviders: + dashboardproviders.yaml: + apiVersion: 1 + providers: + - name: default + orgId: 1 + folder: "" + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards/default + - name: flux + orgId: 1 + folder: Flux + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards/flux + - name: kubernetes + orgId: 1 + folder: Kubernetes + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards/kubernetes + - name: nginx + orgId: 1 + folder: Nginx + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards/nginx + datasources: + datasources.yaml: + apiVersion: 1 + deleteDatasources: + - { name: Prometheus, orgId: 1 } + datasources: + - name: Prometheus + type: prometheus + uid: prometheus + access: proxy + url: http://kube-prometheus-stack-prometheus.monitoring.svc.cluster.local:9090 + jsonData: + prometheusType: Prometheus + isDefault: true + dashboards: + default: + cloudflared: + gnetId: 17457 # https://grafana.com/grafana/dashboards/17457?tab=revisions + revision: 6 + datasource: + - { name: DS_PROMETHEUS, value: Prometheus } + external-dns: + gnetId: 15038 # https://grafana.com/grafana/dashboards/15038?tab=revisions + revision: 1 + datasource: Prometheus + cert-manager: + url: https://raw.githubusercontent.com/monitoring-mixins/website/master/assets/cert-manager/dashboards/cert-manager.json + datasource: Prometheus + node-exporter-full: + gnetId: 1860 # https://grafana.com/grafana/dashboards/1860?tab=revisions + revision: 31 + datasource: Prometheus + flux: + flux-cluster: + url: https://raw.githubusercontent.com/fluxcd/flux2/main/manifests/monitoring/monitoring-config/dashboards/cluster.json + datasource: Prometheus + flux-control-plane: + url: https://raw.githubusercontent.com/fluxcd/flux2/main/manifests/monitoring/monitoring-config/dashboards/control-plane.json + datasource: Prometheus + kubernetes: + kubernetes-api-server: + url: https://raw.githubusercontent.com/dotdc/grafana-dashboards-kubernetes/master/dashboards/k8s-system-api-server.json + datasource: Prometheus + kubernetes-coredns: + url: https://raw.githubusercontent.com/dotdc/grafana-dashboards-kubernetes/master/dashboards/k8s-system-coredns.json + datasource: Prometheus + kubernetes-global: + url: https://raw.githubusercontent.com/dotdc/grafana-dashboards-kubernetes/master/dashboards/k8s-views-global.json + datasource: Prometheus + kubernetes-namespaces: + url: https://raw.githubusercontent.com/dotdc/grafana-dashboards-kubernetes/master/dashboards/k8s-views-namespaces.json + datasource: Prometheus + kubernetes-nodes: + url: https://raw.githubusercontent.com/dotdc/grafana-dashboards-kubernetes/master/dashboards/k8s-views-nodes.json + datasource: Prometheus + kubernetes-pods: + url: https://raw.githubusercontent.com/dotdc/grafana-dashboards-kubernetes/master/dashboards/k8s-views-pods.json + datasource: Prometheus + nginx: + nginx: + url: https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/grafana/dashboards/nginx.json + datasource: Prometheus + nginx-request-handling-performance: + url: https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/grafana/dashboards/request-handling-performance.json + datasource: Prometheus + sidecar: + dashboards: + enabled: true + searchNamespace: ALL + labelValue: "" + label: grafana_dashboard + folderAnnotation: grafana_folder + provider: + disableDelete: true + foldersFromFilesStructure: true + datasources: + enabled: true + searchNamespace: ALL + labelValue: "" + serviceMonitor: + enabled: true + ingress: + enabled: true + ingressClassName: internal + annotations: + hajimari.io/icon: simple-icons:grafana + hosts: + - &host "grafana.${SECRET_DOMAIN}" + tls: + - hosts: + - *host + persistence: + enabled: true + storageClassName: local-path + testFramework: + enabled: false diff --git a/bootstrap/templates/addons/grafana/app/kustomization.yaml.j2 b/bootstrap/templates/addons/grafana/app/kustomization.yaml.j2 new file mode 100644 index 00000000..95bf4747 --- /dev/null +++ b/bootstrap/templates/addons/grafana/app/kustomization.yaml.j2 @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./secret.sops.yaml + - ./helmrelease.yaml diff --git a/bootstrap/templates/addons/grafana/app/secret.sops.yaml.j2 b/bootstrap/templates/addons/grafana/app/secret.sops.yaml.j2 new file mode 100644 index 00000000..c776a17c --- /dev/null +++ b/bootstrap/templates/addons/grafana/app/secret.sops.yaml.j2 @@ -0,0 +1,8 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: grafana-admin-secret +stringData: + admin-user: admin + admin-password: "{{ grafana.password }}" diff --git a/bootstrap/templates/addons/grafana/ks.yaml.j2 b/bootstrap/templates/addons/grafana/ks.yaml.j2 new file mode 100644 index 00000000..21e71d4a --- /dev/null +++ b/bootstrap/templates/addons/grafana/ks.yaml.j2 @@ -0,0 +1,20 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app grafana + namespace: flux-system +spec: + targetNamespace: monitoring + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/apps/monitoring/grafana/app + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m diff --git a/bootstrap/templates/addons/hajimari/app/helmrelease.yaml.j2 b/bootstrap/templates/addons/hajimari/app/helmrelease.yaml.j2 new file mode 100644 index 00000000..c35dbd27 --- /dev/null +++ b/bootstrap/templates/addons/hajimari/app/helmrelease.yaml.j2 @@ -0,0 +1,65 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: hajimari +spec: + interval: 30m + chart: + spec: + chart: hajimari + version: 2.0.2 + sourceRef: + kind: HelmRepository + name: hajimari + namespace: flux-system + maxHistory: 2 + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + values: + hajimari: + title: Apps + darkTheme: espresso + alwaysTargetBlank: true + showGreeting: false + showAppGroups: false + showAppStatus: false + showBookmarkGroups: false + showGlobalBookmarks: false + showAppUrls: false + defaultEnable: true + namespaceSelector: + matchNames: + - default + - monitoring + ingress: + main: + enabled: true + ingressClassName: internal + annotations: + hajimari.io/enable: "false" + hosts: + - host: &host "hajimari.${SECRET_DOMAIN}" + paths: + - path: / + pathType: Prefix + tls: + - hosts: + - *host + podAnnotations: + configmap.reloader.stakater.com/reload: hajimari-settings + persistence: + data: + enabled: true + type: emptyDir + resources: + requests: + cpu: 100m + memory: 128M diff --git a/bootstrap/templates/addons/hajimari/app/kustomization.yaml.j2 b/bootstrap/templates/addons/hajimari/app/kustomization.yaml.j2 new file mode 100644 index 00000000..5dd7baca --- /dev/null +++ b/bootstrap/templates/addons/hajimari/app/kustomization.yaml.j2 @@ -0,0 +1,5 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml diff --git a/bootstrap/templates/addons/hajimari/ks.yaml.j2 b/bootstrap/templates/addons/hajimari/ks.yaml.j2 new file mode 100644 index 00000000..95582292 --- /dev/null +++ b/bootstrap/templates/addons/hajimari/ks.yaml.j2 @@ -0,0 +1,20 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app hajimari + namespace: flux-system +spec: + targetNamespace: default + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/apps/default/hajimari/app + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m diff --git a/bootstrap/templates/addons/kube-prometheus-stack/app/helmrelease.yaml.j2 b/bootstrap/templates/addons/kube-prometheus-stack/app/helmrelease.yaml.j2 new file mode 100644 index 00000000..4ba88042 --- /dev/null +++ b/bootstrap/templates/addons/kube-prometheus-stack/app/helmrelease.yaml.j2 @@ -0,0 +1,35 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: kube-prometheus-stack +spec: + interval: 30m + timeout: 15m + chart: + spec: + chart: kube-prometheus-stack + version: 54.2.2 + sourceRef: + kind: HelmRepository + name: prometheus-community + namespace: flux-system + maxHistory: 2 + install: + crds: CreateReplace + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + crds: CreateReplace + remediation: + retries: 3 + uninstall: + keepHistory: false + dependsOn: + - name: local-path-provisioner + namespace: kube-system + valuesFrom: + - name: kube-prometheus-stack-values + kind: ConfigMap + valuesKey: values.yaml diff --git a/bootstrap/templates/addons/kube-prometheus-stack/app/helmvalues.yaml.j2 b/bootstrap/templates/addons/kube-prometheus-stack/app/helmvalues.yaml.j2 new file mode 100644 index 00000000..60a40edc --- /dev/null +++ b/bootstrap/templates/addons/kube-prometheus-stack/app/helmvalues.yaml.j2 @@ -0,0 +1,130 @@ +#jinja2: trim_blocks: True, lstrip_blocks: True +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: kube-prometheus-stack-values +data: + values.yaml: | + crds: + enabled: true + cleanPrometheusOperatorObjectNames: true + alertmanager: + enabled: false + kube-state-metrics: + metricLabelsAllowlist: + - "deployments=[*]" + - "persistentvolumeclaims=[*]" + - "pods=[*]" + prometheus: + monitor: + enabled: true + relabelings: + - action: replace + sourceLabels: ["__meta_kubernetes_pod_node_name"] + regex: ^(.*)$ + replacement: $1 + targetLabel: kubernetes_node + kubelet: + enabled: true + serviceMonitor: + metricRelabelings: + # Remove duplicate labels provided by k3s + - action: keep + sourceLabels: ["__name__"] + regex: (apiserver_audit|apiserver_client|apiserver_delegated|apiserver_envelope|apiserver_storage|apiserver_webhooks|authentication_token|cadvisor_version|container_blkio|container_cpu|container_fs|container_last|container_memory|container_network|container_oom|container_processes|container|csi_operations|disabled_metric|get_token|go|hidden_metric|kubelet_certificate|kubelet_cgroup|kubelet_container|kubelet_containers|kubelet_cpu|kubelet_device|kubelet_graceful|kubelet_http|kubelet_lifecycle|kubelet_managed|kubelet_node|kubelet_pleg|kubelet_pod|kubelet_run|kubelet_running|kubelet_runtime|kubelet_server|kubelet_started|kubelet_volume|kubernetes_build|kubernetes_feature|machine_cpu|machine_memory|machine_nvm|machine_scrape|node_namespace|plugin_manager|prober_probe|process_cpu|process_max|process_open|process_resident|process_start|process_virtual|registered_metric|rest_client|scrape_duration|scrape_samples|scrape_series|storage_operation|volume_manager|volume_operation|workqueue)_(.+) + - action: replace + sourceLabels: ["node"] + targetLabel: instance + # Drop high cardinality labels + - action: labeldrop + regex: (uid) + - action: labeldrop + regex: (id|name) + - action: drop + sourceLabels: ["__name__"] + regex: (rest_client_request_duration_seconds_bucket|rest_client_request_duration_seconds_sum|rest_client_request_duration_seconds_count) + kubeApiServer: + enabled: true + serviceMonitor: + metricRelabelings: + # Remove duplicate labels provided by k3s + - action: keep + sourceLabels: ["__name__"] + regex: (aggregator_openapi|aggregator_unavailable|apiextensions_openapi|apiserver_admission|apiserver_audit|apiserver_cache|apiserver_cel|apiserver_client|apiserver_crd|apiserver_current|apiserver_envelope|apiserver_flowcontrol|apiserver_init|apiserver_kube|apiserver_longrunning|apiserver_request|apiserver_requested|apiserver_response|apiserver_selfrequest|apiserver_storage|apiserver_terminated|apiserver_tls|apiserver_watch|apiserver_webhooks|authenticated_user|authentication|disabled_metric|etcd_bookmark|etcd_lease|etcd_request|field_validation|get_token|go|grpc_client|hidden_metric|kube_apiserver|kubernetes_build|kubernetes_feature|node_authorizer|pod_security|process_cpu|process_max|process_open|process_resident|process_start|process_virtual|registered_metric|rest_client|scrape_duration|scrape_samples|scrape_series|serviceaccount_legacy|serviceaccount_stale|serviceaccount_valid|watch_cache|workqueue)_(.+) + # Drop high cardinality labels + - action: drop + sourceLabels: ["__name__"] + regex: (apiserver|etcd|rest_client)_request(|_sli|_slo)_duration_seconds_bucket + - action: drop + sourceLabels: ["__name__"] + regex: (apiserver_response_sizes_bucket|apiserver_watch_events_sizes_bucket) + kubeControllerManager: + enabled: true + endpoints: + {% for item in bootstrap_nodes.master %} + - {{ item.address }} + {% endfor %} + serviceMonitor: + metricRelabelings: + # Remove duplicate labels provided by k3s + - action: keep + sourceLabels: ["__name__"] + regex: "(apiserver_audit|apiserver_client|apiserver_delegated|apiserver_envelope|apiserver_storage|apiserver_webhooks|attachdetach_controller|authenticated_user|authentication|cronjob_controller|disabled_metric|endpoint_slice|ephemeral_volume|garbagecollector_controller|get_token|go|hidden_metric|job_controller|kubernetes_build|kubernetes_feature|leader_election|node_collector|node_ipam|process_cpu|process_max|process_open|process_resident|process_start|process_virtual|pv_collector|registered_metric|replicaset_controller|rest_client|retroactive_storageclass|root_ca|running_managed|scrape_duration|scrape_samples|scrape_series|service_controller|storage_count|storage_operation|ttl_after|volume_operation|workqueue)_(.+)" + kubeEtcd: + enabled: true + endpoints: + {% for item in bootstrap_nodes.master %} + - {{ item.address }} + {% endfor %} + kubeScheduler: + enabled: true + endpoints: + {% for item in bootstrap_nodes.master %} + - {{ item.address }} + {% endfor %} + serviceMonitor: + metricRelabelings: + # Remove duplicate labels provided by k3s + - action: keep + sourceLabels: ["__name__"] + regex: "(apiserver_audit|apiserver_client|apiserver_delegated|apiserver_envelope|apiserver_storage|apiserver_webhooks|authenticated_user|authentication|disabled_metric|go|hidden_metric|kubernetes_build|kubernetes_feature|leader_election|process_cpu|process_max|process_open|process_resident|process_start|process_virtual|registered_metric|rest_client|scheduler|scrape_duration|scrape_samples|scrape_series|workqueue)_(.+)" + kubeProxy: + enabled: false # Disabled due to eBPF + prometheus: + ingress: + enabled: true + ingressClassName: internal + annotations: + hajimari.io/appName: Prometheus + hajimari.io/icon: simple-icons:prometheus + pathType: Prefix + hosts: + - "prometheus.${SECRET_DOMAIN}" + tls: + - hosts: + - "prometheus.${SECRET_DOMAIN}" + prometheusSpec: + ruleSelectorNilUsesHelmValues: false + serviceMonitorSelectorNilUsesHelmValues: false + podMonitorSelectorNilUsesHelmValues: false + probeSelectorNilUsesHelmValues: false + scrapeConfigSelectorNilUsesHelmValues: false + enableAdminAPI: true + walCompression: true + retentionSize: 8GB + storageSpec: + volumeClaimTemplate: + spec: + storageClassName: local-path + resources: + requests: + storage: 10Gi + grafana: + enabled: false + forceDeployDashboards: true + sidecar: + dashboards: + multicluster: + etcd: + enabled: true diff --git a/bootstrap/templates/addons/kube-prometheus-stack/app/kustomization.yaml.j2 b/bootstrap/templates/addons/kube-prometheus-stack/app/kustomization.yaml.j2 new file mode 100644 index 00000000..dc39651b --- /dev/null +++ b/bootstrap/templates/addons/kube-prometheus-stack/app/kustomization.yaml.j2 @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmvalues.yaml + - ./helmrelease.yaml diff --git a/bootstrap/templates/addons/kube-prometheus-stack/ks.yaml.j2 b/bootstrap/templates/addons/kube-prometheus-stack/ks.yaml.j2 new file mode 100644 index 00000000..0d66c3fc --- /dev/null +++ b/bootstrap/templates/addons/kube-prometheus-stack/ks.yaml.j2 @@ -0,0 +1,20 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app kube-prometheus-stack + namespace: flux-system +spec: + targetNamespace: monitoring + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/apps/monitoring/kube-prometheus-stack/app + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m diff --git a/bootstrap/templates/addons/kubernetes-dashboard/app/helmrelease.yaml.j2 b/bootstrap/templates/addons/kubernetes-dashboard/app/helmrelease.yaml.j2 new file mode 100644 index 00000000..2ea7a893 --- /dev/null +++ b/bootstrap/templates/addons/kubernetes-dashboard/app/helmrelease.yaml.j2 @@ -0,0 +1,42 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: kubernetes-dashboard +spec: + interval: 30m + chart: + spec: + chart: kubernetes-dashboard + version: 6.0.8 + sourceRef: + kind: HelmRepository + name: kubernetes-dashboard + namespace: flux-system + maxHistory: 2 + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + values: + extraArgs: + - --enable-insecure-login + - --enable-skip-login + - --disable-settings-authorizer + ingress: + enabled: true + className: internal + annotations: + hajimari.io/icon: mdi:kubernetes + hosts: + - &host "kubernetes.${SECRET_DOMAIN}" + tls: + - hosts: + - *host + metricsScraper: + enabled: true diff --git a/bootstrap/templates/addons/kubernetes-dashboard/app/kustomization.yaml.j2 b/bootstrap/templates/addons/kubernetes-dashboard/app/kustomization.yaml.j2 new file mode 100644 index 00000000..b5b97586 --- /dev/null +++ b/bootstrap/templates/addons/kubernetes-dashboard/app/kustomization.yaml.j2 @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./rbac.yaml + - ./helmrelease.yaml diff --git a/bootstrap/templates/addons/kubernetes-dashboard/app/rbac.yaml.j2 b/bootstrap/templates/addons/kubernetes-dashboard/app/rbac.yaml.j2 new file mode 100644 index 00000000..01b56df0 --- /dev/null +++ b/bootstrap/templates/addons/kubernetes-dashboard/app/rbac.yaml.j2 @@ -0,0 +1,39 @@ +# For dashboard sign in token: +# kubectl -n monitoring get secret kubernetes-dashboard -o jsonpath='{.data.token}' | base64 -d +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubernetes-dashboard + labels: + app.kubernetes.io/managed-by: Helm + annotations: + meta.helm.sh/release-name: kubernetes-dashboard + meta.helm.sh/release-namespace: monitoring +secrets: + - name: kubernetes-dashboard +--- +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: kubernetes-dashboard + labels: + app.kubernetes.io/managed-by: Helm + annotations: + meta.helm.sh/release-name: kubernetes-dashboard + meta.helm.sh/release-namespace: monitoring + kubernetes.io/service-account.name: kubernetes-dashboard +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:kubernetes-dashboard +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: + - kind: ServiceAccount + name: kubernetes-dashboard + namespace: monitoring diff --git a/bootstrap/templates/addons/kubernetes-dashboard/ks.yaml.j2 b/bootstrap/templates/addons/kubernetes-dashboard/ks.yaml.j2 new file mode 100644 index 00000000..4ee53e92 --- /dev/null +++ b/bootstrap/templates/addons/kubernetes-dashboard/ks.yaml.j2 @@ -0,0 +1,23 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app kubernetes-dashboard + namespace: flux-system +spec: + targetNamespace: monitoring + commonMetadata: + labels: + app.kubernetes.io/name: *app + dependsOn: + - name: cert-manager + - name: metrics-server + path: ./kubernetes/apps/monitoring/kubernetes-dashboard/app + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m diff --git a/bootstrap/templates/addons/system-upgrade-controller/app/helmrelease.yaml.j2 b/bootstrap/templates/addons/system-upgrade-controller/app/helmrelease.yaml.j2 new file mode 100644 index 00000000..c897da54 --- /dev/null +++ b/bootstrap/templates/addons/system-upgrade-controller/app/helmrelease.yaml.j2 @@ -0,0 +1,103 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: &app system-upgrade-controller +spec: + interval: 30m + chart: + spec: + chart: app-template + version: 2.3.0 + sourceRef: + kind: HelmRepository + name: bjw-s + namespace: flux-system + maxHistory: 2 + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + values: + controllers: + main: + strategy: RollingUpdate + containers: + main: + image: + repository: docker.io/rancher/system-upgrade-controller + tag: v0.13.2 + env: + SYSTEM_UPGRADE_CONTROLLER_DEBUG: false + SYSTEM_UPGRADE_CONTROLLER_THREADS: 2 + SYSTEM_UPGRADE_JOB_ACTIVE_DEADLINE_SECONDS: 900 + SYSTEM_UPGRADE_JOB_BACKOFF_LIMIT: 99 + SYSTEM_UPGRADE_JOB_IMAGE_PULL_POLICY: IfNotPresent + SYSTEM_UPGRADE_JOB_KUBECTL_IMAGE: docker.io/rancher/kubectl:v1.28.4 + SYSTEM_UPGRADE_JOB_PRIVILEGED: true + SYSTEM_UPGRADE_JOB_TTL_SECONDS_AFTER_FINISH: 900 + SYSTEM_UPGRADE_PLAN_POLLING_INTERVAL: 15m + SYSTEM_UPGRADE_CONTROLLER_NAME: *app + SYSTEM_UPGRADE_CONTROLLER_NAMESPACE: + valueFrom: + fieldRef: + fieldPath: metadata.namespace + pod: + securityContext: + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + allowPrivilegeEscalation: false + seccompProfile: + type: RuntimeDefault + capabilities: + drop: ["ALL"] + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + tolerations: + - {key: CriticalAddonsOnly, operator: Exists} + - {key: node-role.kubernetes.io/master, operator: Exists, effect: NoSchedule} + - {key: node-role.kubernetes.io/controlplane, operator: Exists, effect: NoSchedule} + - {key: node-role.kubernetes.io/control-plane, operator: Exists, effect: NoSchedule} + - {key: node-role.kubernetes.io/etcd, operator: Exists, effect: NoExecute} + serviceAccount: + name: system-upgrade + service: + main: + enabled: false + persistence: + tmp: + type: emptyDir + globalMounts: + - path: /tmp + etc-ssl: + type: hostPath + hostPath: /etc/ssl + hostPathType: DirectoryOrCreate + globalMounts: + - path: /etc/ssl + readOnly: true + etc-pki: + type: hostPath + hostPath: /etc/pki + hostPathType: DirectoryOrCreate + globalMounts: + - path: /etc/pki + readOnly: true + etc-ca-certificates: + type: hostPath + hostPath: /etc/ca-certificates + hostPathType: DirectoryOrCreate + globalMounts: + - path: /etc/ca-certificates + readOnly: true diff --git a/bootstrap/templates/addons/system-upgrade-controller/app/kustomization.yaml.j2 b/bootstrap/templates/addons/system-upgrade-controller/app/kustomization.yaml.j2 new file mode 100644 index 00000000..74d66703 --- /dev/null +++ b/bootstrap/templates/addons/system-upgrade-controller/app/kustomization.yaml.j2 @@ -0,0 +1,8 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + # renovate: datasource=github-releases depName=rancher/system-upgrade-controller + - https://github.com/rancher/system-upgrade-controller/releases/download/v0.13.2/crd.yaml + - helmrelease.yaml + - rbac.yaml diff --git a/bootstrap/templates/addons/system-upgrade-controller/app/rbac.yaml.j2 b/bootstrap/templates/addons/system-upgrade-controller/app/rbac.yaml.j2 new file mode 100644 index 00000000..f531caa7 --- /dev/null +++ b/bootstrap/templates/addons/system-upgrade-controller/app/rbac.yaml.j2 @@ -0,0 +1,28 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: system-upgrade +secrets: + - name: system-upgrade +--- +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: system-upgrade + annotations: + kubernetes.io/service-account.name: system-upgrade +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system-upgrade +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: + - kind: ServiceAccount + name: system-upgrade + namespace: kube-system diff --git a/bootstrap/templates/addons/system-upgrade-controller/ks.yaml.j2 b/bootstrap/templates/addons/system-upgrade-controller/ks.yaml.j2 new file mode 100644 index 00000000..6374d702 --- /dev/null +++ b/bootstrap/templates/addons/system-upgrade-controller/ks.yaml.j2 @@ -0,0 +1,42 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app system-upgrade-controller + namespace: flux-system +spec: + targetNamespace: kube-system + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/apps/kube-system/system-upgrade-controller/app + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: true + interval: 30m + retryInterval: 1m + timeout: 5m +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app system-upgrade-controller-plans + namespace: flux-system +spec: + targetNamespace: kube-system + commonMetadata: + labels: + app.kubernetes.io/name: *app + dependsOn: + - name: system-upgrade-controller + path: ./kubernetes/apps/kube-system/system-upgrade-controller/plans + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m diff --git a/bootstrap/templates/addons/system-upgrade-controller/plans/agent.yaml.j2 b/bootstrap/templates/addons/system-upgrade-controller/plans/agent.yaml.j2 new file mode 100644 index 00000000..3a4cf75e --- /dev/null +++ b/bootstrap/templates/addons/system-upgrade-controller/plans/agent.yaml.j2 @@ -0,0 +1,18 @@ +--- +apiVersion: upgrade.cattle.io/v1 +kind: Plan +metadata: + name: agent +spec: + # renovate: datasource=github-releases depName=k3s-io/k3s + version: "v1.28.3+k3s2" + serviceAccountName: system-upgrade + concurrency: 1 + nodeSelector: + matchExpressions: + - {key: node-role.kubernetes.io/control-plane, operator: DoesNotExist} + prepare: + image: rancher/k3s-upgrade + args: ["prepare", "server"] + upgrade: + image: rancher/k3s-upgrade diff --git a/bootstrap/templates/addons/system-upgrade-controller/plans/kustomization.yaml.j2 b/bootstrap/templates/addons/system-upgrade-controller/plans/kustomization.yaml.j2 new file mode 100644 index 00000000..14d17ed1 --- /dev/null +++ b/bootstrap/templates/addons/system-upgrade-controller/plans/kustomization.yaml.j2 @@ -0,0 +1,7 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./server.yaml + - ./agent.yaml diff --git a/bootstrap/templates/addons/system-upgrade-controller/plans/server.yaml.j2 b/bootstrap/templates/addons/system-upgrade-controller/plans/server.yaml.j2 new file mode 100644 index 00000000..bf7e03b3 --- /dev/null +++ b/bootstrap/templates/addons/system-upgrade-controller/plans/server.yaml.j2 @@ -0,0 +1,23 @@ +--- +apiVersion: upgrade.cattle.io/v1 +kind: Plan +metadata: + name: server +spec: + # renovate: datasource=github-releases depName=k3s-io/k3s + version: "v1.28.3+k3s2" + serviceAccountName: system-upgrade + concurrency: 1 + cordon: true + nodeSelector: + matchExpressions: + - {key: node-role.kubernetes.io/control-plane, operator: Exists} + tolerations: + - {effect: NoSchedule, operator: Exists} + - {effect: NoExecute, operator: Exists} + - {key: node-role.kubernetes.io/control-plane, effect: NoSchedule, operator: Exists} + - {key: node-role.kubernetes.io/master, effect: NoSchedule, operator: Exists} + - {key: node-role.kubernetes.io/etcd, effect: NoExecute, operator: Exists} + - {key: CriticalAddonsOnly, operator: Exists} + upgrade: + image: rancher/k3s-upgrade diff --git a/bootstrap/templates/addons/weave-gitops/app/helmrelease.yaml.j2 b/bootstrap/templates/addons/weave-gitops/app/helmrelease.yaml.j2 new file mode 100644 index 00000000..4a82af0b --- /dev/null +++ b/bootstrap/templates/addons/weave-gitops/app/helmrelease.yaml.j2 @@ -0,0 +1,52 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: weave-gitops +spec: + interval: 30m + chart: + spec: + chart: weave-gitops + version: 4.0.35 + sourceRef: + kind: HelmRepository + name: weave-gitops + namespace: flux-system + maxHistory: 2 + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + values: + adminUser: + create: true + createSecret: false + username: admin + ingress: + enabled: true + className: internal + annotations: + hajimari.io/icon: sawtooth-wave + hosts: + - host: &host "gitops.${SECRET_DOMAIN}" + paths: + - path: / + pathType: Prefix + tls: + - hosts: + - *host + networkPolicy: + create: false + metrics: + enabled: true + rbac: + create: true + impersonationResourceNames: ["admin"] + podAnnotations: + secret.reloader.stakater.com/reload: cluster-user-auth diff --git a/bootstrap/templates/addons/weave-gitops/app/kustomization.yaml.j2 b/bootstrap/templates/addons/weave-gitops/app/kustomization.yaml.j2 new file mode 100644 index 00000000..95bf4747 --- /dev/null +++ b/bootstrap/templates/addons/weave-gitops/app/kustomization.yaml.j2 @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./secret.sops.yaml + - ./helmrelease.yaml diff --git a/bootstrap/templates/addons/weave-gitops/app/secret.sops.yaml.j2 b/bootstrap/templates/addons/weave-gitops/app/secret.sops.yaml.j2 new file mode 100644 index 00000000..22486957 --- /dev/null +++ b/bootstrap/templates/addons/weave-gitops/app/secret.sops.yaml.j2 @@ -0,0 +1,9 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: cluster-user-auth +type: Opaque +stringData: + username: admin + password: "{{ weave_gitops.password | password_hash('bcrypt', rounds=10) }}" diff --git a/bootstrap/templates/addons/weave-gitops/ks.yaml.j2 b/bootstrap/templates/addons/weave-gitops/ks.yaml.j2 new file mode 100644 index 00000000..9a2bd272 --- /dev/null +++ b/bootstrap/templates/addons/weave-gitops/ks.yaml.j2 @@ -0,0 +1,20 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app weave-gitops + namespace: flux-system +spec: + targetNamespace: flux-system + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/apps/flux-system/weave-gitops/app + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m diff --git a/bootstrap/templates/ansible/.ansible-lint.j2 b/bootstrap/templates/ansible/.ansible-lint.j2 new file mode 100644 index 00000000..59c41dc3 --- /dev/null +++ b/bootstrap/templates/ansible/.ansible-lint.j2 @@ -0,0 +1,8 @@ +skip_list: + - yaml[line-length] + - var-naming +warn_list: + - command-instead-of-shell + - deprecated-command-syntax + - experimental + - no-changed-when diff --git a/bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2 b/bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2 new file mode 100644 index 00000000..42f6fe49 --- /dev/null +++ b/bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2 @@ -0,0 +1,37 @@ +#jinja2: trim_blocks: True, lstrip_blocks: True +--- +# +# Below vars are for the xanmanning.k3s role +# ...see https://github.com/PyratLabs/ansible-role-k3s +# + +# renovate: datasource=github-releases depName=k3s-io/k3s +k3s_release_version: "v1.28.3+k3s2" +k3s_install_hard_links: true +k3s_become: true +k3s_etcd_datastore: true +k3s_use_unsupported_config: true +k3s_registration_address: "{% raw %}{{ kube_vip_addr }}{% endraw %}" +k3s_server_manifests_urls: + # Kube-vip RBAC + - url: https://raw.githubusercontent.com/kube-vip/kube-vip/main/docs/manifests/rbac.yaml + filename: kube-vip-rbac.yaml + # Essential Prometheus Operator CRDs (the rest are installed with the kube-prometheus-stack helm release) + - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml + filename: custom-prometheus-podmonitors.yaml + - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml + filename: custom-prometheus-prometheusrules.yaml + - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml + filename: custom-prometheus-scrapeconfigs.yaml + - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml + filename: custom-prometheus-servicemonitors.yaml +# /var/lib/rancher/k3s/server/manifests +k3s_server_manifests_templates: + - custom-cilium-helmchart.yaml.j2 + {% if not bootstrap_ipv6_enabled | default(false) %} + - custom-cilium-l2.yaml.j2 + {% endif %} + - custom-coredns-helmchart.yaml.j2 +# /var/lib/rancher/k3s/agent/pod-manifests +k3s_server_pod_manifests_templates: + - kube-vip-static-pod.yaml.j2 diff --git a/bootstrap/templates/ansible/inventory/group_vars/kubernetes/supplemental.yaml.j2 b/bootstrap/templates/ansible/inventory/group_vars/kubernetes/supplemental.yaml.j2 new file mode 100644 index 00000000..816dc3f5 --- /dev/null +++ b/bootstrap/templates/ansible/inventory/group_vars/kubernetes/supplemental.yaml.j2 @@ -0,0 +1,13 @@ +#jinja2: trim_blocks: True, lstrip_blocks: True +--- +timezone: "{{ bootstrap_timezone }}" +github_username: "{{ bootstrap_github_username }}" +coredns_addr: "{{ bootstrap_service_cidr.split(',')[0] | ansible.utils.nthhost(10) }}" +kube_vip_addr: "{{ bootstrap_kube_vip_addr }}" +cluster_cidr: "{{ bootstrap_cluster_cidr.split(',')[0] }}" +service_cidr: "{{ bootstrap_service_cidr.split(',')[0] }}" +node_cidr: "{{ bootstrap_node_cidr }}" +{% if bootstrap_ipv6_enabled | default(false) %} +cluster_cidr_v6: "{{ bootstrap_cluster_cidr.split(',')[1] }}" +service_cidr_v6: "{{ bootstrap_service_cidr.split(',')[1] }}" +{% endif %} diff --git a/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2 b/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2 new file mode 100644 index 00000000..0550793e --- /dev/null +++ b/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2 @@ -0,0 +1,41 @@ +#jinja2: trim_blocks: True, lstrip_blocks: True +--- +# https://rancher.com/docs/k3s/latest/en/installation/install-options/server-config/ +# https://github.com/PyratLabs/ansible-role-k3s + +k3s_control_node: true +k3s_server: +{% if bootstrap_ipv6_enabled | default(false) %} + node-ip: "{% raw %}{{ ansible_host }},{{ ansible_default_ipv6.address }}{% endraw %}" +{% else %} + node-ip: "{% raw %}{{ ansible_host }}{% endraw %}" +{% endif %} + tls-san: + - "{% raw %}{{ kube_vip_addr }}{% endraw %}" + docker: false + flannel-backend: "none" # This needs to be in quotes + disable: + - coredns # Disable coredns - replaced with Coredns Helm Chart + - flannel # Disable flannel - replaced with Cilium Helm Chart + - local-storage # Disable local-path-provisioner - installed with Flux + - metrics-server # Disable metrics-server - installed with Flux + - servicelb # Disable servicelb - replaced with Cilium Helm Chart + - traefik # Disable traefik - replaced with ingress-nginx and installed with Flux + disable-network-policy: true + disable-cloud-controller: true + disable-kube-proxy: true # Cilium uses eBPF + write-kubeconfig-mode: "644" +{% if bootstrap_ipv6_enabled | default(false) %} + cluster-cidr: "{% raw %}{{ cluster_cidr }},{{ cluster_cidr_v6 }}{% endraw %}" + service-cidr: "{% raw %}{{ service_cidr }},{{ service_cidr_v6 }}{% endraw %}" +{% else %} + cluster-cidr: "{% raw %}{{ cluster_cidr }}{% endraw %}" + service-cidr: "{% raw %}{{ service_cidr }}{% endraw %}" +{% endif %} + etcd-expose-metrics: true # Required to monitor etcd with kube-prometheus-stack + kube-controller-manager-arg: + - "bind-address=0.0.0.0" # Required to monitor kube-controller-manager with kube-prometheus-stack + kube-scheduler-arg: + - "bind-address=0.0.0.0" # Required to monitor kube-scheduler with kube-prometheus-stack + kube-apiserver-arg: + - "anonymous-auth=true" # Required for HAProxy health-checks diff --git a/bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2 b/bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2 new file mode 100644 index 00000000..687cf0dc --- /dev/null +++ b/bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2 @@ -0,0 +1,12 @@ +#jinja2: trim_blocks: True, lstrip_blocks: True +--- +# https://rancher.com/docs/k3s/latest/en/installation/install-options/agent-config/ +# https://github.com/PyratLabs/ansible-role-k3s + +k3s_control_node: false +k3s_agent: +{% if bootstrap_ipv6_enabled | default(false) %} + node-ip: "{% raw %}{{ ansible_host }},{{ ansible_default_ipv6.address }}{% endraw %}" +{% else %} + node-ip: "{% raw %}{{ ansible_host }}{% endraw %}" +{% endif %} diff --git a/bootstrap/templates/ansible/inventory/host_vars/.gitkeep.j2 b/bootstrap/templates/ansible/inventory/host_vars/.gitkeep.j2 new file mode 100644 index 00000000..e69de29b diff --git a/bootstrap/templates/ansible/inventory/hosts.yaml.j2 b/bootstrap/templates/ansible/inventory/hosts.yaml.j2 new file mode 100644 index 00000000..028c3af5 --- /dev/null +++ b/bootstrap/templates/ansible/inventory/hosts.yaml.j2 @@ -0,0 +1,28 @@ +#jinja2: trim_blocks: True, lstrip_blocks: True +--- +kubernetes: + children: + master: + hosts: + {% for item in bootstrap_nodes.master %} + {{ item.name }}: + ansible_user: {{ item.username }} + {% if item.external_address is defined %} + ansible_host: {{ item.external_address }} + {% else %} + ansible_host: {{ item.address }} + {% endif %} + {% endfor %} + {% if bootstrap_nodes.worker | default([]) | length > 0 %} + worker: + hosts: + {% for item in bootstrap_nodes.worker %} + {{ item.name }}: + ansible_user: {{ item.username }} + {% if item.external_address is defined %} + ansible_host: {{ item.external_address }} + {% else %} + ansible_host: {{ item.address }} + {% endif %} + {% endfor %} + {% endif %} diff --git a/bootstrap/templates/ansible/playbooks/cluster-installation.yaml.j2 b/bootstrap/templates/ansible/playbooks/cluster-installation.yaml.j2 new file mode 100644 index 00000000..24e502bb --- /dev/null +++ b/bootstrap/templates/ansible/playbooks/cluster-installation.yaml.j2 @@ -0,0 +1,75 @@ +#jinja2: trim_blocks: True, lstrip_blocks: True +--- +- name: Cluster Installation + hosts: all + become: true + gather_facts: true + any_errors_fatal: true + pre_tasks: + - name: Pausing for 5 seconds... + ansible.builtin.pause: + seconds: 5 + tasks: + - name: Check if cluster is installed + check_mode: false + ansible.builtin.stat: + path: /etc/rancher/k3s/config.yaml + register: k3s_installed + + - name: Ignore manifests templates and urls if the cluster is already installed + when: k3s_installed.stat.exists + ansible.builtin.set_fact: + k3s_server_manifests_templates: [] + k3s_server_manifests_urls: [] + + - name: Install Kubernetes + ansible.builtin.include_role: + name: xanmanning.k3s + public: true + vars: + k3s_state: installed + + - name: Kubeconfig + ansible.builtin.include_tasks: tasks/kubeconfig.yaml + + - name: Wait for custom manifests to rollout + when: + - k3s_primary_control_node + - (k3s_server_manifests_templates | length > 0 + or k3s_server_manifests_urls | length > 0) + kubernetes.core.k8s_info: + kubeconfig: /etc/rancher/k3s/k3s.yaml + kind: "{% raw %}{{ item.kind }}{% endraw %}" + name: "{% raw %}{{ item.name }}{% endraw %}" + namespace: "{% raw %}{{ item.namespace | default('') }}{% endraw %}" + wait: true + wait_sleep: 10 + wait_timeout: 360 + loop: + - { name: cilium, kind: HelmChart, namespace: kube-system } + - { name: coredns, kind: HelmChart, namespace: kube-system } + {% if not bootstrap_ipv6_enabled | default(false) %} + - { name: policy, kind: CiliumL2AnnouncementPolicy } + - { name: pool, kind: CiliumLoadBalancerIPPool } + {% endif %} + - { name: podmonitors.monitoring.coreos.com, kind: CustomResourceDefinition } + - { name: prometheusrules.monitoring.coreos.com, kind: CustomResourceDefinition } + - { name: scrapeconfigs.monitoring.coreos.com, kind: CustomResourceDefinition } + - { name: servicemonitors.monitoring.coreos.com, kind: CustomResourceDefinition } + + - name: Coredns + when: k3s_primary_control_node + ansible.builtin.include_tasks: tasks/coredns.yaml + + - name: Cilium + when: k3s_primary_control_node + ansible.builtin.include_tasks: tasks/cilium.yaml + + - name: Cruft + when: k3s_primary_control_node + ansible.builtin.include_tasks: tasks/cruft.yaml + + - name: Stale Containers + ansible.builtin.include_tasks: tasks/stale_containers.yaml + vars: + stale_containers_state: enabled diff --git a/bootstrap/templates/ansible/playbooks/cluster-kube-vip.yaml.j2 b/bootstrap/templates/ansible/playbooks/cluster-kube-vip.yaml.j2 new file mode 100644 index 00000000..0b207262 --- /dev/null +++ b/bootstrap/templates/ansible/playbooks/cluster-kube-vip.yaml.j2 @@ -0,0 +1,24 @@ +--- +- name: Cluster kube-vip + hosts: master + serial: 1 + become: true + gather_facts: true + any_errors_fatal: true + pre_tasks: + - name: Pausing for 5 seconds... + ansible.builtin.pause: + seconds: 5 + tasks: + - name: Ensure Kubernetes is running + ansible.builtin.include_role: + name: xanmanning.k3s + public: true + vars: + k3s_state: started + + - name: Upgrade kube-vip + ansible.builtin.template: + src: templates/kube-vip-static-pod.yaml.j2 + dest: "{% raw %}{{ k3s_server_pod_manifests_dir }}{% endraw %}/kube-vip-static-pod.yaml" + mode: preserve diff --git a/bootstrap/templates/ansible/playbooks/cluster-nuke.yaml.j2 b/bootstrap/templates/ansible/playbooks/cluster-nuke.yaml.j2 new file mode 100644 index 00000000..4400269e --- /dev/null +++ b/bootstrap/templates/ansible/playbooks/cluster-nuke.yaml.j2 @@ -0,0 +1,73 @@ +--- +- name: Cluster Nuke + hosts: all + become: true + gather_facts: true + any_errors_fatal: true + vars_prompt: + - name: nuke + prompt: |- + Are you sure you want to nuke this cluster? + Type 'YES I WANT TO DESTROY THIS CLUSTER' to proceed + default: "n" + private: false + pre_tasks: + - name: Check for confirmation + ansible.builtin.fail: + msg: Aborted nuking the cluster + when: nuke != 'YES I WANT TO DESTROY THIS CLUSTER' + + - name: Pausing for 5 seconds... + ansible.builtin.pause: + seconds: 5 + tasks: + - name: Stop Kubernetes # noqa: ignore-errors + ignore_errors: true + block: + - name: Stop Kubernetes + ansible.builtin.include_role: + name: xanmanning.k3s + public: true + vars: + k3s_state: stopped + + # https://github.com/k3s-io/docs/blob/main/docs/installation/network-options.md + - name: Networking + block: + - name: Networking | Delete Cilium links + ansible.builtin.command: + cmd: "ip link delete {% raw %}{{ item }}{% endraw %}" + removes: "/sys/class/net/{% raw %}{{ item }}{% endraw %}" + loop: ["cilium_host", "cilium_net", "cilium_vxlan"] + - name: Networking | Flush iptables + ansible.builtin.iptables: + table: "{% raw %}{{ item }}{% endraw %}" + flush: true + loop: ["filter", "nat", "mangle", "raw"] + - name: Networking | Flush ip6tables + ansible.builtin.iptables: + table: "{% raw %}{{ item }}{% endraw %}" + flush: true + ip_version: ipv6 + loop: ["filter", "nat", "mangle", "raw"] + - name: Networking | Delete CNI directory + ansible.builtin.file: + path: /etc/cni/net.d + state: absent + + - name: Uninstall Kubernetes + ansible.builtin.include_role: + name: xanmanning.k3s + public: true + vars: + k3s_state: uninstalled + + - name: Stale Containers + ansible.builtin.include_tasks: tasks/stale_containers.yaml + vars: + stale_containers_state: disabled + + - name: Reboot + ansible.builtin.reboot: + msg: Rebooting nodes + reboot_timeout: 3600 diff --git a/bootstrap/templates/ansible/playbooks/cluster-prepare.yaml.j2 b/bootstrap/templates/ansible/playbooks/cluster-prepare.yaml.j2 new file mode 100644 index 00000000..202b6fa7 --- /dev/null +++ b/bootstrap/templates/ansible/playbooks/cluster-prepare.yaml.j2 @@ -0,0 +1,138 @@ +--- +- name: Prepare System + hosts: all + become: true + gather_facts: true + any_errors_fatal: true + pre_tasks: + - name: Pausing for 5 seconds... + ansible.builtin.pause: + seconds: 5 + - name: Populate service facts + service_facts: + tasks: + - name: Locale + block: + - name: Locale | Set timezone + community.general.timezone: + name: "{% raw %}{{ timezone | default('Etc/UTC') }}{% endraw %}" + + - name: Packages + block: + - name: Packages | Install + ansible.builtin.apt: + name: apt-transport-https,ca-certificates,conntrack,curl,dirmngr,gdisk,gnupg,hdparm,htop, + iptables,iputils-ping,ipvsadm,libseccomp2,lm-sensors,neofetch,net-tools,nfs-common, + nvme-cli,open-iscsi,parted,psmisc,python3,python3-apt,python3-kubernetes,python3-yaml, + smartmontools,socat,software-properties-common,unzip,util-linux + install_recommends: false + + - name: User Configuration + block: + - name: User Configuration | SSH keys + ansible.posix.authorized_key: + user: "{% raw %}{{ ansible_user }}{% endraw %}" + key: "https://github.com/{% raw %}{{ github_username }}{% endraw %}.keys" + - name: User Configuration | Silence login + ansible.builtin.file: + dest: "{% raw %}{{ '/home/' + ansible_user if ansible_user != 'root' else '/root' }}{% endraw %}/.hushlogin" + state: touch + owner: "{% raw %}{{ ansible_user }}{% endraw %}" + group: "{% raw %}{{ ansible_user }}{% endraw %}" + mode: "0644" + modification_time: preserve + access_time: preserve + + - name: Network Configuration + notify: Reboot + block: + - name: Network Configuration | Set hostname + ansible.builtin.hostname: + name: "{% raw %}{{ inventory_hostname }}{% endraw %}" + - name: Network Configuration | Update hosts + ansible.builtin.copy: + content: | + 127.0.0.1 localhost + 127.0.1.1 {% raw %}{{ inventory_hostname }}{% endraw %} + + # The following lines are desirable for IPv6 capable hosts + ::1 localhost ip6-localhost ip6-loopback + ff02::1 ip6-allnodes + ff02::2 ip6-allrouters + dest: /etc/hosts + mode: preserve + # https://github.com/cilium/cilium/issues/18706 + - name: Network Configuration | Cilium (1) + ansible.builtin.lineinfile: + dest: /etc/systemd/networkd.conf + regexp: ManageForeignRoutingPolicyRules + line: ManageForeignRoutingPolicyRules=no + - name: Network Configuration | Cilium (2) + ansible.builtin.lineinfile: + dest: /etc/systemd/networkd.conf + regexp: ManageForeignRoutes + line: ManageForeignRoutes=no + # https://github.com/onedr0p/flux-cluster-template/discussions/635 + - name: Network Configuration | Remove /etc/resolv.conf + ansible.builtin.file: + attributes: -i + path: /etc/resolv.conf + state: absent + - name: Network Configuration | Add custom /etc/resolv.conf + ansible.builtin.copy: + attributes: +i + mode: 644 + dest: /etc/resolv.conf + content: | + search . + nameserver 1.1.1.1 + + - name: System Configuration + notify: Reboot + block: + - name: System Configuration | Neofetch + ansible.builtin.copy: + dest: /etc/profile.d/neofetch.sh + mode: "0755" + content: neofetch --config none + - name: System Configuration | Disable apparmor + when: ansible_facts.services['apparmor.service'] is defined + ansible.builtin.systemd: + name: apparmor + state: stopped + masked: true + - name: System Configuration | Disable swap + ansible.posix.mount: + name: "{% raw %}{{ item }}{% endraw %}" + fstype: swap + state: absent + loop: ["none", "swap"] + - name: System Configuration | Kernel modules (1) + community.general.modprobe: + name: "{% raw %}{{ item }}{% endraw %}" + state: present + loop: ["br_netfilter", "ceph", "ip_vs", "ip_vs_rr", "nbd", "overlay", "rbd"] + - name: System Configuration | Kernel modules (2) + ansible.builtin.copy: + dest: "/etc/modules-load.d/{% raw %}{{ item }}{% endraw %}.conf" + mode: "0644" + content: "{% raw %}{{ item }}{% endraw %}" + loop: ["br_netfilter", "ceph", "ip_vs", "ip_vs_rr", "nbd", "overlay", "rbd"] + - name: System Configuration | Sysctl + ansible.posix.sysctl: + name: "{% raw %}{{ item.key }}{% endraw %}" + value: "{% raw %}{{ item.value }}{% endraw %}" + sysctl_file: /etc/sysctl.d/99-kubernetes.conf + reload: true + with_dict: "{% raw %}{{ sysctl_config }}{% endraw %}" + vars: + sysctl_config: + fs.inotify.max_queued_events: 65536 + fs.inotify.max_user_watches: 524288 + fs.inotify.max_user_instances: 8192 + + handlers: + - name: Reboot + ansible.builtin.reboot: + msg: Rebooting nodes + reboot_timeout: 3600 diff --git a/bootstrap/templates/ansible/playbooks/cluster-reboot.yaml.j2 b/bootstrap/templates/ansible/playbooks/cluster-reboot.yaml.j2 new file mode 100644 index 00000000..4adcfe43 --- /dev/null +++ b/bootstrap/templates/ansible/playbooks/cluster-reboot.yaml.j2 @@ -0,0 +1,15 @@ +--- +- name: Reboot + hosts: all + become: true + gather_facts: true + any_errors_fatal: true + pre_tasks: + - name: Pausing for 5 seconds... + ansible.builtin.pause: + seconds: 5 + tasks: + - name: Reboot + ansible.builtin.reboot: + msg: Rebooting nodes + reboot_timeout: 3600 diff --git a/bootstrap/templates/ansible/playbooks/cluster-rollout-update.yaml.j2 b/bootstrap/templates/ansible/playbooks/cluster-rollout-update.yaml.j2 new file mode 100644 index 00000000..8791e51c --- /dev/null +++ b/bootstrap/templates/ansible/playbooks/cluster-rollout-update.yaml.j2 @@ -0,0 +1,74 @@ +--- +# https://github.com/kevincoakley/ansible-role-k8s-rolling-update +- name: Cluster rollout update + hosts: all + become: true + gather_facts: true + any_errors_fatal: true + serial: 1 + pre_tasks: + - name: Pausing for 5 seconds... + ansible.builtin.pause: + seconds: 5 + tasks: + - name: Details + ansible.builtin.command: "kubectl get node {% raw %}{{ inventory_hostname }}{% endraw %} -o json" + register: kubectl_get_node + delegate_to: "{% raw %}{{ groups['master'][0] }}{% endraw %}" + failed_when: false + changed_when: false + + - name: Update + when: + # When status.conditions[x].type == Ready then check stats.conditions[x].status for True|False + - kubectl_get_node['stdout'] | from_json | json_query("status.conditions[?type == 'Ready'].status") + # If spec.unschedulable is defined then the node is cordoned + - not (kubectl_get_node['stdout'] | from_json).spec.unschedulable is defined + block: + - name: Cordon + kubernetes.core.k8s_drain: + name: "{% raw %}{{ inventory_hostname }}{% endraw %}" + kubeconfig: /etc/rancher/k3s/k3s.yaml + state: cordon + delegate_to: "{% raw %}{{ groups['master'][0] }}{% endraw %}" + + - name: Drain + kubernetes.core.k8s_drain: + kubeconfig: /etc/rancher/k3s/k3s.yaml + state: drain + delete_options: + delete_emptydir_data: true + ignore_daemonsets: true + terminate_grace_period: 600 + wait_timeout: 900 + pod_selectors: + # Rook Ceph + - app!=rook-ceph-osd + # Longhorn + - app!=csi-attacher + - app!=csi-provisioner + delegate_to: "{% raw %}{{ groups['master'][0] }}{% endraw %}" + + - name: Update + ansible.builtin.apt: + upgrade: dist + update_cache: true + + - name: Check if reboot is required + ansible.builtin.stat: + path: /var/run/reboot-required + register: reboot_required + + - name: Reboot + when: reboot_required.stat.exists + ansible.builtin.reboot: + msg: Rebooting node + post_reboot_delay: 60 + reboot_timeout: 3600 + + - name: Uncordon + kubernetes.core.k8s_drain: + name: "{% raw %}{{ inventory_hostname }}{% endraw %}" + kubeconfig: /etc/rancher/k3s/k3s.yaml + state: uncordon + delegate_to: "{% raw %}{{ groups['master'][0] }}{% endraw %}" diff --git a/bootstrap/templates/ansible/playbooks/files/stale-containers.service.j2 b/bootstrap/templates/ansible/playbooks/files/stale-containers.service.j2 new file mode 100644 index 00000000..5136df2f --- /dev/null +++ b/bootstrap/templates/ansible/playbooks/files/stale-containers.service.j2 @@ -0,0 +1,6 @@ +[Unit] +Description=Stale containers + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/k3s crictl rmi --prune diff --git a/bootstrap/templates/ansible/playbooks/files/stale-containers.timer.j2 b/bootstrap/templates/ansible/playbooks/files/stale-containers.timer.j2 new file mode 100644 index 00000000..731885a1 --- /dev/null +++ b/bootstrap/templates/ansible/playbooks/files/stale-containers.timer.j2 @@ -0,0 +1,11 @@ +[Unit] +Description=Stale containers + +[Timer] +OnCalendar=weekly +AccuracySec=1h +Persistent=true +RandomizedDelaySec=6000 + +[Install] +WantedBy=timers.target diff --git a/bootstrap/templates/ansible/playbooks/tasks/cilium.yaml.j2 b/bootstrap/templates/ansible/playbooks/tasks/cilium.yaml.j2 new file mode 100644 index 00000000..ca242bb0 --- /dev/null +++ b/bootstrap/templates/ansible/playbooks/tasks/cilium.yaml.j2 @@ -0,0 +1,56 @@ +--- +- name: Cilium + block: + - name: Cilium | Check if Cilium HelmChart exists + kubernetes.core.k8s_info: + kubeconfig: /etc/rancher/k3s/k3s.yaml + name: cilium + kind: HelmChart + namespace: kube-system + register: cilium_helmchart + + - name: Cilium | Wait for Cilium to rollout + when: cilium_helmchart.resources | count > 0 + kubernetes.core.k8s_info: + kubeconfig: /etc/rancher/k3s/k3s.yaml + name: helm-install-cilium + kind: Job + namespace: kube-system + wait: true + wait_condition: + type: Complete + status: true + wait_timeout: 360 + + - name: Cilium | Patch the Cilium HelmChart to unmanage it + when: cilium_helmchart.resources | count > 0 + kubernetes.core.k8s_json_patch: + kubeconfig: /etc/rancher/k3s/k3s.yaml + name: cilium + kind: HelmChart + namespace: kube-system + patch: + - op: add + path: /metadata/annotations/helmcharts.helm.cattle.io~1unmanaged + value: "true" + + - name: Cilium | Delete the Cilium HelmChart CR + when: cilium_helmchart.resources | count > 0 + kubernetes.core.k8s: + kubeconfig: /etc/rancher/k3s/k3s.yaml + name: cilium + kind: HelmChart + namespace: kube-system + state: absent + + - name: Cilium | Force delete the Cilium HelmChart + when: cilium_helmchart.resources | count > 0 + kubernetes.core.k8s: + kubeconfig: /etc/rancher/k3s/k3s.yaml + name: cilium + kind: HelmChart + namespace: kube-system + state: patched + definition: + metadata: + finalizers: [] diff --git a/bootstrap/templates/ansible/playbooks/tasks/coredns.yaml.j2 b/bootstrap/templates/ansible/playbooks/tasks/coredns.yaml.j2 new file mode 100644 index 00000000..d18383a7 --- /dev/null +++ b/bootstrap/templates/ansible/playbooks/tasks/coredns.yaml.j2 @@ -0,0 +1,56 @@ +--- +- name: Coredns + block: + - name: Coredns | Check if Coredns HelmChart exists + kubernetes.core.k8s_info: + kubeconfig: /etc/rancher/k3s/k3s.yaml + name: coredns + kind: HelmChart + namespace: kube-system + register: coredns_helmchart + + - name: Coredns | Wait for Coredns to rollout + when: coredns_helmchart.resources | count > 0 + kubernetes.core.k8s_info: + kubeconfig: /etc/rancher/k3s/k3s.yaml + name: helm-install-coredns + kind: Job + namespace: kube-system + wait: true + wait_condition: + type: Complete + status: true + wait_timeout: 360 + + - name: Coredns | Patch the Coredns HelmChart to unmanage it + when: coredns_helmchart.resources | count > 0 + kubernetes.core.k8s_json_patch: + kubeconfig: /etc/rancher/k3s/k3s.yaml + name: coredns + kind: HelmChart + namespace: kube-system + patch: + - op: add + path: /metadata/annotations/helmcharts.helm.cattle.io~1unmanaged + value: "true" + + - name: Coredns | Delete the Coredns HelmChart CR + when: coredns_helmchart.resources | count > 0 + kubernetes.core.k8s: + kubeconfig: /etc/rancher/k3s/k3s.yaml + name: coredns + kind: HelmChart + namespace: kube-system + state: absent + + - name: Coredns | Force delete the Coredns HelmChart + when: coredns_helmchart.resources | count > 0 + kubernetes.core.k8s: + kubeconfig: /etc/rancher/k3s/k3s.yaml + name: coredns + kind: HelmChart + namespace: kube-system + state: patched + definition: + metadata: + finalizers: [] diff --git a/bootstrap/templates/ansible/playbooks/tasks/cruft.yaml.j2 b/bootstrap/templates/ansible/playbooks/tasks/cruft.yaml.j2 new file mode 100644 index 00000000..f22ad98a --- /dev/null +++ b/bootstrap/templates/ansible/playbooks/tasks/cruft.yaml.j2 @@ -0,0 +1,32 @@ +--- +# https://github.com/k3s-io/k3s/issues/1971 +- name: Cruft + block: + - name: Cruft | Get list of custom mantifests + ansible.builtin.find: + paths: "{% raw %}{{ k3s_server_manifests_dir }}{% endraw %}" + file_type: file + use_regex: true + patterns: ["^custom-.*"] + register: custom_manifest + + - name: Cruft | Delete custom mantifests + ansible.builtin.file: + path: "{% raw %}{{ item.path }}{% endraw %}" + state: absent + loop: "{% raw %}{{ custom_manifest.files }}{% endraw %}" + + - name: Cruft | Get list of custom addons + kubernetes.core.k8s_info: + kubeconfig: /etc/rancher/k3s/k3s.yaml + kind: Addon + register: addons_list + + - name: Cruft | Delete addons + kubernetes.core.k8s: + kubeconfig: /etc/rancher/k3s/k3s.yaml + name: "{% raw %}{{ item.metadata.name }}{% endraw %}" + kind: Addon + namespace: kube-system + state: absent + loop: "{% raw %}{{ addons_list.resources | selectattr('metadata.name', 'match', '^custom-.*') | list }}{% endraw %}" diff --git a/bootstrap/templates/ansible/playbooks/tasks/kubeconfig.yaml.j2 b/bootstrap/templates/ansible/playbooks/tasks/kubeconfig.yaml.j2 new file mode 100644 index 00000000..a3a0681a --- /dev/null +++ b/bootstrap/templates/ansible/playbooks/tasks/kubeconfig.yaml.j2 @@ -0,0 +1,26 @@ +--- +- name: Get absolute path to this Git repository # noqa: command-instead-of-module + ansible.builtin.command: git rev-parse --show-toplevel + delegate_to: localhost + become: false + run_once: true + register: repository_path + changed_when: false + check_mode: false + failed_when: repository_path.rc != 0 + +- name: Copy kubeconfig to the project directory + when: k3s_primary_control_node + ansible.builtin.fetch: + src: /etc/rancher/k3s/k3s.yaml + dest: "{% raw %}{{ repository_path.stdout }}{% endraw %}/kubeconfig" + flat: true + +- name: Update kubeconfig with the correct load balancer address + delegate_to: localhost + become: false + run_once: true + ansible.builtin.replace: + path: "{% raw %}{{ repository_path.stdout }}{% endraw %}/kubeconfig" + regexp: https://127.0.0.1:6443 + replace: "https://{% raw %}{{ k3s_registration_address }}{% endraw %}:6443" diff --git a/bootstrap/templates/ansible/playbooks/tasks/stale_containers.yaml.j2 b/bootstrap/templates/ansible/playbooks/tasks/stale_containers.yaml.j2 new file mode 100644 index 00000000..9857d6bc --- /dev/null +++ b/bootstrap/templates/ansible/playbooks/tasks/stale_containers.yaml.j2 @@ -0,0 +1,36 @@ +--- +# https://github.com/k3s-io/k3s/issues/1900 +- name: Enabled Stale containers + when: stale_containers_state == "enabled" + block: + - name: Stale containers | Create systemd unit + ansible.builtin.copy: + src: files/stale-containers.service + dest: /etc/systemd/system/stale-containers.service + owner: root + group: root + mode: "0644" + + - name: Stale containers | Create systemd timer + ansible.builtin.copy: + src: files/stale-containers.timer + dest: /etc/systemd/system/stale-containers.timer + owner: root + group: root + mode: "0644" + + - name: Stale containers | Start the systemd timer + ansible.builtin.systemd: + name: stale-containers.timer + enabled: true + daemon_reload: true + masked: false + state: started + +- name: Disable Stale containers + when: stale_containers_state == "disabled" + block: + - name: Stale containers | Mask the systemd timer + ansible.builtin.systemd: + name: stale-containers.timer + masked: true diff --git a/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2 b/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2 new file mode 100644 index 00000000..ec488a21 --- /dev/null +++ b/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2 @@ -0,0 +1,64 @@ +#jinja2: trim_blocks: True, lstrip_blocks: True +--- +# https://docs.k3s.io/helm +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: cilium + namespace: kube-system +spec: + # renovate: datasource=helm + repo: https://helm.cilium.io/ + chart: cilium + version: 1.14.4 + targetNamespace: kube-system + bootstrap: true + valuesContent: |- + autoDirectNodeRoutes: true + bpf: + masquerade: true + bgp: + enabled: false + cluster: + name: home-cluster + id: 1 + containerRuntime: + integration: containerd + socketPath: /var/run/k3s/containerd/containerd.sock + endpointRoutes: + enabled: true + hubble: + enabled: false + ipam: + mode: kubernetes + ipv4NativeRoutingCIDR: "{% raw %}{{ cluster_cidr }}{% endraw %}" + {% if bootstrap_ipv6_enabled | default(false) %} + ipv6NativeRoutingCIDR: "{% raw %}{{ cluster_cidr_v6 }}{% endraw %}" + ipv6: + enabled: true + {% endif %} + k8sServiceHost: "{% raw %}{{ kube_vip_addr }}{% endraw %}" + k8sServicePort: 6443 + kubeProxyReplacement: true + kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256 + l2announcements: + {% if bootstrap_ipv6_enabled | default(false) %} + enabled: false + {% else %} + enabled: true + # https://github.com/cilium/cilium/issues/26586 + leaseDuration: 120s + leaseRenewDeadline: 60s + leaseRetryPeriod: 1s + {% endif %} + loadBalancer: + algorithm: maglev + mode: dsr + localRedirectPolicy: true + operator: + replicas: 1 + rollOutPods: true + rollOutCiliumPods: true + securityContext: + privileged: true + tunnel: disabled diff --git a/bootstrap/templates/ansible/playbooks/templates/custom-cilium-l2.yaml.j2.j2 b/bootstrap/templates/ansible/playbooks/templates/custom-cilium-l2.yaml.j2.j2 new file mode 100644 index 00000000..51415a66 --- /dev/null +++ b/bootstrap/templates/ansible/playbooks/templates/custom-cilium-l2.yaml.j2.j2 @@ -0,0 +1,22 @@ +--- +# https://docs.cilium.io/en/latest/network/l2-announcements +apiVersion: cilium.io/v2alpha1 +kind: CiliumL2AnnouncementPolicy +metadata: + name: policy +spec: + loadBalancerIPs: true + # NOTE: This might need to be set if you have more than one active NIC on your nodes + # interfaces: + # - ^eno[0-9]+ + nodeSelector: + matchLabels: + kubernetes.io/os: linux +--- +apiVersion: cilium.io/v2alpha1 +kind: CiliumLoadBalancerIPPool +metadata: + name: pool +spec: + cidrs: + - cidr: "{% raw %}{{ node_cidr }}{% endraw %}" diff --git a/bootstrap/templates/ansible/playbooks/templates/custom-coredns-helmchart.yaml.j2.j2 b/bootstrap/templates/ansible/playbooks/templates/custom-coredns-helmchart.yaml.j2.j2 new file mode 100644 index 00000000..66811a98 --- /dev/null +++ b/bootstrap/templates/ansible/playbooks/templates/custom-coredns-helmchart.yaml.j2.j2 @@ -0,0 +1,78 @@ +#jinja2: trim_blocks: True, lstrip_blocks: True +--- +# https://docs.k3s.io/helm +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: coredns + namespace: kube-system +spec: + # renovate: datasource=helm + repo: https://coredns.github.io/helm + chart: coredns + version: 1.28.2 + targetNamespace: kube-system + bootstrap: true + valuesContent: |- + fullnameOverride: coredns + replicaCount: 1 + k8sAppLabelOverride: kube-dns + service: + name: kube-dns + clusterIP: "{% raw %}{{ coredns_addr }}{% endraw %}" + serviceAccount: + create: true + deployment: + annotations: + reloader.stakater.com/auto: "true" + servers: + - zones: + - zone: . + scheme: dns:// + use_tcp: true + port: 53 + plugins: + - name: log + - name: errors + - name: health + configBlock: |- + lameduck 5s + - name: ready + - name: kubernetes + parameters: cluster.local in-addr.arpa ip6.arpa + configBlock: |- + pods insecure + fallthrough in-addr.arpa ip6.arpa + ttl 30 + - name: prometheus + parameters: 0.0.0.0:9153 + - name: forward + parameters: . /etc/resolv.conf + - name: cache + parameters: 30 + - name: loop + - name: reload + - name: loadbalance + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app.kubernetes.io/instance: coredns diff --git a/bootstrap/templates/ansible/playbooks/templates/kube-vip-static-pod.yaml.j2.j2 b/bootstrap/templates/ansible/playbooks/templates/kube-vip-static-pod.yaml.j2.j2 new file mode 100644 index 00000000..02a94b1b --- /dev/null +++ b/bootstrap/templates/ansible/playbooks/templates/kube-vip-static-pod.yaml.j2.j2 @@ -0,0 +1,59 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: kube-vip + namespace: kube-system + labels: + app.kubernetes.io/instance: kube-vip + app.kubernetes.io/name: kube-vip +spec: + containers: + - name: kube-vip + image: ghcr.io/kube-vip/kube-vip:v0.6.3 + imagePullPolicy: IfNotPresent + args: ["manager"] + env: + - name: address + value: "{% raw %}{{ kube_vip_addr }}{% endraw %}" + - name: vip_arp + value: "true" + - name: lb_enable + value: "true" + - name: port + value: "6443" + - name: vip_cidr + value: "32" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: "false" + - name: svc_enable + value: "false" + - name: vip_leaderelection + value: "true" + - name: vip_leaseduration + value: "15" + - name: vip_renewdeadline + value: "10" + - name: vip_retryperiod + value: "2" + - name: prometheus_server + value: :2112 + securityContext: + capabilities: + add: ["NET_ADMIN", "NET_RAW"] + volumeMounts: + - mountPath: /etc/kubernetes/admin.conf + name: kubeconfig + hostAliases: + - hostnames: + - kubernetes + ip: 127.0.0.1 + hostNetwork: true + volumes: + - name: kubeconfig + hostPath: + path: /etc/rancher/k3s/k3s.yaml diff --git a/bootstrap/templates/kubernetes/apps/cert-manager/cert-manager/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/cert-manager/cert-manager/app/helmrelease.yaml.j2 new file mode 100644 index 00000000..655ebfb8 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/cert-manager/cert-manager/app/helmrelease.yaml.j2 @@ -0,0 +1,40 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: cert-manager +spec: + interval: 30m + chart: + spec: + chart: cert-manager + version: v1.13.2 + sourceRef: + kind: HelmRepository + name: jetstack + namespace: flux-system + maxHistory: 2 + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + values: + installCRDs: true + extraArgs: + - --dns01-recursive-nameservers=1.1.1.1:53,9.9.9.9:53 + - --dns01-recursive-nameservers-only + podDnsPolicy: None + podDnsConfig: + nameservers: + - "1.1.1.1" + - "9.9.9.9" + prometheus: + enabled: true + servicemonitor: + enabled: true + prometheusInstance: monitoring diff --git a/bootstrap/templates/kubernetes/apps/cert-manager/cert-manager/app/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/cert-manager/cert-manager/app/kustomization.yaml.j2 new file mode 100644 index 00000000..8ae52667 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/cert-manager/cert-manager/app/kustomization.yaml.j2 @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml + - ./prometheusrule.yaml diff --git a/bootstrap/templates/kubernetes/apps/cert-manager/cert-manager/app/prometheusrule.yaml.j2 b/bootstrap/templates/kubernetes/apps/cert-manager/cert-manager/app/prometheusrule.yaml.j2 new file mode 100644 index 00000000..a7541f6d --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/cert-manager/cert-manager/app/prometheusrule.yaml.j2 @@ -0,0 +1,61 @@ +#jinja2: trim_blocks: True +--- +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: cert-manager.rules +spec: + groups: + - name: cert-manager + rules: + - alert: CertManagerAbsent + expr: | + absent(up{job="cert-manager"}) + for: 15m + labels: + severity: critical + annotations: + description: > + New certificates will not be able to be minted, and existing ones can't be renewed until cert-manager is back. + runbook_url: https://gitlab.com/uneeq-oss/cert-manager-mixin/-/blob/master/RUNBOOK.md#certmanagerabsent + summary: "Cert Manager has dissapeared from Prometheus service discovery." + - name: certificates + rules: + - alert: CertManagerCertExpirySoon + expr: | + avg by (exported_namespace, namespace, name) (certmanager_certificate_expiration_timestamp_seconds - time()) < (21 * 24 * 3600) + for: 15m + labels: + severity: warning + annotations: + description: > + The domain that this cert covers will be unavailable after + {% raw %}{{ $value | humanizeDuration }}{% endraw %}. Clients using endpoints that this cert + protects will start to fail in {% raw %}{{ $value | humanizeDuration }}{% endraw %}. + runbook_url: https://gitlab.com/uneeq-oss/cert-manager-mixin/-/blob/master/RUNBOOK.md#certmanagercertexpirysoon + summary: | + The cert {% raw %}{{ $labels.name }}{% endraw %} is {% raw %}{{ $value | humanizeDuration }}{% endraw %} from expiry, it should have renewed over a week ago. + - alert: CertManagerCertNotReady + expr: | + max by (name, exported_namespace, namespace, condition) (certmanager_certificate_ready_status{condition!="True"} == 1) + for: 15m + labels: + severity: critical + annotations: + description: > + This certificate has not been ready to serve traffic for at least + 10m. If the cert is being renewed or there is another valid cert, the ingress + controller _may_ be able to serve that instead. + runbook_url: https://gitlab.com/uneeq-oss/cert-manager-mixin/-/blob/master/RUNBOOK.md#certmanagercertnotready + summary: "The cert {% raw %}{{ $labels.name }}{% endraw %} is not ready to serve traffic." + - alert: CertManagerHittingRateLimits + expr: | + sum by (host) (rate(certmanager_http_acme_client_request_count{status="429"}[5m])) > 0 + for: 15m + labels: + severity: critical + annotations: + description: > + Depending on the rate limit, cert-manager may be unable to generate certificates for up to a week. + runbook_url: https://gitlab.com/uneeq-oss/cert-manager-mixin/-/blob/master/RUNBOOK.md#certmanagerhittingratelimits + summary: "Cert manager hitting LetsEncrypt rate limits." diff --git a/bootstrap/templates/kubernetes/apps/cert-manager/cert-manager/issuers/issuers.yaml.j2 b/bootstrap/templates/kubernetes/apps/cert-manager/cert-manager/issuers/issuers.yaml.j2 new file mode 100644 index 00000000..1cf7148a --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/cert-manager/cert-manager/issuers/issuers.yaml.j2 @@ -0,0 +1,39 @@ +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: letsencrypt-production +spec: + acme: + server: https://acme-v02.api.letsencrypt.org/directory + email: "${SECRET_ACME_EMAIL}" + privateKeySecretRef: + name: letsencrypt-production + solvers: + - dns01: + cloudflare: + apiTokenSecretRef: + name: cert-manager-secret + key: api-token + selector: + dnsZones: + - "${SECRET_DOMAIN}" +--- +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: letsencrypt-staging +spec: + acme: + server: https://acme-staging-v02.api.letsencrypt.org/directory + email: "${SECRET_ACME_EMAIL}" + privateKeySecretRef: + name: letsencrypt-staging + solvers: + - dns01: + cloudflare: + apiTokenSecretRef: + name: cert-manager-secret + key: api-token + selector: + dnsZones: + - "${SECRET_DOMAIN}" diff --git a/bootstrap/templates/kubernetes/apps/cert-manager/cert-manager/issuers/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/cert-manager/cert-manager/issuers/kustomization.yaml.j2 new file mode 100644 index 00000000..17754be6 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/cert-manager/cert-manager/issuers/kustomization.yaml.j2 @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./secret.sops.yaml + - ./issuers.yaml diff --git a/bootstrap/templates/kubernetes/apps/cert-manager/cert-manager/issuers/secret.sops.yaml.j2 b/bootstrap/templates/kubernetes/apps/cert-manager/cert-manager/issuers/secret.sops.yaml.j2 new file mode 100644 index 00000000..d1f6e1bf --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/cert-manager/cert-manager/issuers/secret.sops.yaml.j2 @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: cert-manager-secret +stringData: + api-token: "{{ bootstrap_cloudflare_token }}" diff --git a/bootstrap/templates/kubernetes/apps/cert-manager/cert-manager/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/cert-manager/cert-manager/ks.yaml.j2 new file mode 100644 index 00000000..ea0d2af9 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/cert-manager/cert-manager/ks.yaml.j2 @@ -0,0 +1,42 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app cert-manager + namespace: flux-system +spec: + targetNamespace: cert-manager + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/apps/cert-manager/cert-manager/app + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: true + interval: 30m + retryInterval: 1m + timeout: 5m +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app cert-manager-issuers + namespace: flux-system +spec: + targetNamespace: cert-manager + commonMetadata: + labels: + app.kubernetes.io/name: *app + dependsOn: + - name: cert-manager + path: ./kubernetes/apps/cert-manager/cert-manager/issuers + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: true + interval: 30m + retryInterval: 1m + timeout: 5m diff --git a/bootstrap/templates/kubernetes/apps/cert-manager/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/cert-manager/kustomization.yaml.j2 new file mode 100644 index 00000000..a0a3e5ed --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/cert-manager/kustomization.yaml.j2 @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./namespace.yaml + - ./cert-manager/ks.yaml diff --git a/bootstrap/templates/kubernetes/apps/cert-manager/namespace.yaml.j2 b/bootstrap/templates/kubernetes/apps/cert-manager/namespace.yaml.j2 new file mode 100644 index 00000000..ed788350 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/cert-manager/namespace.yaml.j2 @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: cert-manager + labels: + kustomize.toolkit.fluxcd.io/prune: disabled diff --git a/bootstrap/templates/kubernetes/apps/default/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/default/kustomization.yaml.j2 new file mode 100644 index 00000000..23a8573e --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/default/kustomization.yaml.j2 @@ -0,0 +1,12 @@ +#jinja2: trim_blocks: True, lstrip_blocks: True +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./namespace.yaml + {% if hajimari.enabled | default(false) %} + - ./hajimari/ks.yaml + {% endif %} + {% if discord_template_notifier.enabled | default(false) %} + - ./discord-template-notifier/ks.yaml + {% endif %} diff --git a/bootstrap/templates/kubernetes/apps/default/namespace.yaml.j2 b/bootstrap/templates/kubernetes/apps/default/namespace.yaml.j2 new file mode 100644 index 00000000..f659b055 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/default/namespace.yaml.j2 @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: default + labels: + kustomize.toolkit.fluxcd.io/prune: disabled diff --git a/bootstrap/templates/kubernetes/apps/flux-system/addons/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/flux-system/addons/ks.yaml.j2 new file mode 100644 index 00000000..84f17e56 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/flux-system/addons/ks.yaml.j2 @@ -0,0 +1,20 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app flux-webhooks + namespace: flux-system +spec: + targetNamespace: flux-system + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/apps/flux-system/addons/webhooks + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: true + interval: 30m + retryInterval: 1m + timeout: 5m diff --git a/bootstrap/templates/kubernetes/apps/flux-system/addons/webhooks/github/ingress.yaml.j2 b/bootstrap/templates/kubernetes/apps/flux-system/addons/webhooks/github/ingress.yaml.j2 new file mode 100644 index 00000000..747a872d --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/flux-system/addons/webhooks/github/ingress.yaml.j2 @@ -0,0 +1,24 @@ +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: flux-webhook + annotations: + external-dns.alpha.kubernetes.io/target: "external.${SECRET_DOMAIN}" + hajimari.io/enable: "false" +spec: + ingressClassName: external + rules: + - host: &host "flux-webhook.${SECRET_DOMAIN}" + http: + paths: + - path: /hook/ + pathType: Prefix + backend: + service: + name: webhook-receiver + port: + number: 80 + tls: + - hosts: + - *host diff --git a/bootstrap/templates/kubernetes/apps/flux-system/addons/webhooks/github/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/flux-system/addons/webhooks/github/kustomization.yaml.j2 new file mode 100644 index 00000000..786e654a --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/flux-system/addons/webhooks/github/kustomization.yaml.j2 @@ -0,0 +1,7 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./secret.sops.yaml + - ./ingress.yaml + - ./receiver.yaml diff --git a/bootstrap/templates/kubernetes/apps/flux-system/addons/webhooks/github/receiver.yaml.j2 b/bootstrap/templates/kubernetes/apps/flux-system/addons/webhooks/github/receiver.yaml.j2 new file mode 100644 index 00000000..cca5931b --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/flux-system/addons/webhooks/github/receiver.yaml.j2 @@ -0,0 +1,25 @@ +--- +apiVersion: notification.toolkit.fluxcd.io/v1 +kind: Receiver +metadata: + name: github-receiver +spec: + type: github + events: + - ping + - push + secretRef: + name: github-webhook-token-secret + resources: + - apiVersion: source.toolkit.fluxcd.io/v1 + kind: GitRepository + name: home-kubernetes + namespace: flux-system + - apiVersion: kustomize.toolkit.fluxcd.io/v1 + kind: Kustomization + name: cluster + namespace: flux-system + - apiVersion: kustomize.toolkit.fluxcd.io/v1 + kind: Kustomization + name: cluster-apps + namespace: flux-system diff --git a/bootstrap/templates/kubernetes/apps/flux-system/addons/webhooks/github/secret.sops.yaml.j2 b/bootstrap/templates/kubernetes/apps/flux-system/addons/webhooks/github/secret.sops.yaml.j2 new file mode 100644 index 00000000..b4bb682c --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/flux-system/addons/webhooks/github/secret.sops.yaml.j2 @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: github-webhook-token-secret +stringData: + token: "{{ bootstrap_flux_github_webhook_token }}" diff --git a/bootstrap/templates/kubernetes/apps/flux-system/addons/webhooks/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/flux-system/addons/webhooks/kustomization.yaml.j2 new file mode 100644 index 00000000..ccd8b3eb --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/flux-system/addons/webhooks/kustomization.yaml.j2 @@ -0,0 +1,5 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./github diff --git a/bootstrap/templates/kubernetes/apps/flux-system/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/flux-system/kustomization.yaml.j2 new file mode 100644 index 00000000..bf4d7e09 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/flux-system/kustomization.yaml.j2 @@ -0,0 +1,10 @@ +#jinja2: trim_blocks: True, lstrip_blocks: True +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./namespace.yaml + - ./addons/ks.yaml + {% if weave_gitops.enabled | default(false) %} + - ./weave-gitops/ks.yaml + {% endif %} diff --git a/bootstrap/templates/kubernetes/apps/flux-system/namespace.yaml.j2 b/bootstrap/templates/kubernetes/apps/flux-system/namespace.yaml.j2 new file mode 100644 index 00000000..b48db452 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/flux-system/namespace.yaml.j2 @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: flux-system + labels: + kustomize.toolkit.fluxcd.io/prune: disabled diff --git a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/cilium-l2.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/cilium-l2.yaml.j2 new file mode 100644 index 00000000..e8eba26e --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/cilium-l2.yaml.j2 @@ -0,0 +1,22 @@ +--- +# https://docs.cilium.io/en/latest/network/l2-announcements +apiVersion: cilium.io/v2alpha1 +kind: CiliumL2AnnouncementPolicy +metadata: + name: policy +spec: + loadBalancerIPs: true + # NOTE: This might need to be set if you have more than one active NIC on your nodes + # interfaces: + # - ^eno[0-9]+ + nodeSelector: + matchLabels: + kubernetes.io/os: linux +--- +apiVersion: cilium.io/v2alpha1 +kind: CiliumLoadBalancerIPPool +metadata: + name: pool +spec: + cidrs: + - cidr: "${NODE_CIDR}" diff --git a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml.j2 new file mode 100644 index 00000000..e3afa706 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml.j2 @@ -0,0 +1,29 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: cilium +spec: + interval: 30m + chart: + spec: + chart: cilium + version: 1.14.4 + sourceRef: + kind: HelmRepository + name: cilium + namespace: flux-system + maxHistory: 2 + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + valuesFrom: + - name: cilium-values + kind: ConfigMap + valuesKey: values.yaml diff --git a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmvalues.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmvalues.yaml.j2 new file mode 100644 index 00000000..ba9fd9c3 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmvalues.yaml.j2 @@ -0,0 +1,107 @@ +#jinja2: trim_blocks: True, lstrip_blocks: True +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: cilium-values +data: + values.yaml: | + autoDirectNodeRoutes: true + bpf: + masquerade: true + bgp: + enabled: false + cluster: + name: home-cluster + id: 1 + containerRuntime: + integration: containerd + socketPath: /var/run/k3s/containerd/containerd.sock + endpointRoutes: + enabled: true + hubble: + enabled: true + metrics: + enabled: + - dns:query + - drop + - tcp + - flow + - port-distribution + - icmp + - http + serviceMonitor: + enabled: true + dashboards: + enabled: true + annotations: + grafana_folder: Cilium + relay: + enabled: true + rollOutPods: true + prometheus: + serviceMonitor: + enabled: true + ui: + enabled: true + rollOutPods: true + ingress: + enabled: true + className: internal + annotations: + hajimari.io/icon: simple-icons:cilium + hosts: + - "hubble.${SECRET_DOMAIN}" + tls: + - hosts: + - "hubble.${SECRET_DOMAIN}" + ipam: + mode: kubernetes + ipv4NativeRoutingCIDR: "${CLUSTER_CIDR}" + {% if bootstrap_ipv6_enabled | default(false) %} + ipv6NativeRoutingCIDR: "${CLUSTER_CIDR_V6}" + ipv6: + enabled: true + {% endif %} + k8sServiceHost: "${KUBE_VIP_ADDR}" + k8sServicePort: 6443 + kubeProxyReplacement: true + kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256 + l2announcements: + {% if bootstrap_ipv6_enabled | default(false) %} + enabled: false + {% else %} + enabled: true + # https://github.com/cilium/cilium/issues/26586 + leaseDuration: 120s + leaseRenewDeadline: 60s + leaseRetryPeriod: 1s + {% endif %} + loadBalancer: + algorithm: maglev + mode: dsr + localRedirectPolicy: true + operator: + replicas: 1 + rollOutPods: true + prometheus: + enabled: true + serviceMonitor: + enabled: true + dashboards: + enabled: true + annotations: + grafana_folder: Cilium + prometheus: + enabled: true + serviceMonitor: + enabled: true + trustCRDsExist: true + dashboards: + enabled: true + annotations: + grafana_folder: Cilium + rollOutCiliumPods: true + securityContext: + privileged: true + tunnel: disabled diff --git a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/kustomization.yaml.j2 new file mode 100644 index 00000000..fedf3812 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/kustomization.yaml.j2 @@ -0,0 +1,10 @@ +#jinja2: trim_blocks: True, lstrip_blocks: True +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + {% if not bootstrap_ipv6_enabled | default(false) %} + - ./cilium-l2.yaml + {% endif %} + - ./helmvalues.yaml + - ./helmrelease.yaml diff --git a/bootstrap/templates/kubernetes/apps/kube-system/cilium/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/cilium/ks.yaml.j2 new file mode 100644 index 00000000..34fbd9a9 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/kube-system/cilium/ks.yaml.j2 @@ -0,0 +1,20 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app cilium + namespace: flux-system +spec: + targetNamespace: kube-system + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/apps/kube-system/cilium/app + prune: false # never should be deleted + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m diff --git a/bootstrap/templates/kubernetes/apps/kube-system/coredns/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/coredns/app/helmrelease.yaml.j2 new file mode 100644 index 00000000..15ebe37f --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/kube-system/coredns/app/helmrelease.yaml.j2 @@ -0,0 +1,89 @@ +#jinja2: trim_blocks: True, lstrip_blocks: True +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: coredns +spec: + interval: 30m + chart: + spec: + chart: coredns + version: 1.28.2 + sourceRef: + kind: HelmRepository + name: coredns + namespace: flux-system + maxHistory: 2 + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + values: + fullnameOverride: coredns + replicaCount: 1 + k8sAppLabelOverride: kube-dns + service: + name: kube-dns + clusterIP: "${COREDNS_ADDR}" + serviceAccount: + create: true + deployment: + annotations: + reloader.stakater.com/auto: "true" + servers: + - zones: + - zone: . + scheme: dns:// + use_tcp: true + port: 53 + plugins: + - name: log + - name: errors + - name: health + configBlock: |- + lameduck 5s + - name: ready + - name: kubernetes + parameters: cluster.local in-addr.arpa ip6.arpa + configBlock: |- + pods insecure + fallthrough in-addr.arpa ip6.arpa + ttl 30 + - name: prometheus + parameters: 0.0.0.0:9153 + - name: forward + parameters: . /etc/resolv.conf + - name: cache + parameters: 30 + - name: loop + - name: reload + - name: loadbalance + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app.kubernetes.io/instance: coredns diff --git a/bootstrap/templates/kubernetes/apps/kube-system/coredns/app/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/coredns/app/kustomization.yaml.j2 new file mode 100644 index 00000000..5dd7baca --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/kube-system/coredns/app/kustomization.yaml.j2 @@ -0,0 +1,5 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml diff --git a/bootstrap/templates/kubernetes/apps/kube-system/coredns/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/coredns/ks.yaml.j2 new file mode 100644 index 00000000..bf2a537e --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/kube-system/coredns/ks.yaml.j2 @@ -0,0 +1,20 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app coredns + namespace: flux-system +spec: + targetNamespace: kube-system + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/apps/kube-system/coredns/app + prune: false # never should be deleted + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m diff --git a/bootstrap/templates/kubernetes/apps/kube-system/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/kustomization.yaml.j2 new file mode 100644 index 00000000..d53f5a57 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/kube-system/kustomization.yaml.j2 @@ -0,0 +1,18 @@ +#jinja2: trim_blocks: True, lstrip_blocks: True +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./namespace.yaml + - ./cilium/ks.yaml + - ./coredns/ks.yaml + - ./local-path-provisioner/ks.yaml + - ./metrics-server/ks.yaml + - ./reloader/ks.yaml + - ./snapshot-controller/ks.yaml + {% if csi_driver_nfs.enabled | default(false) %} + - ./csi-driver-nfs/ks.yaml + {% endif %} + {% if system_upgrade_controller.enabled | default(false) %} + - ./system-upgrade-controller/ks.yaml + {% endif %} diff --git a/bootstrap/templates/kubernetes/apps/kube-system/local-path-provisioner/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/local-path-provisioner/app/helmrelease.yaml.j2 new file mode 100644 index 00000000..a9205a15 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/kube-system/local-path-provisioner/app/helmrelease.yaml.j2 @@ -0,0 +1,70 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: local-path-provisioner +spec: + interval: 30m + chart: + spec: + chart: ./deploy/chart/local-path-provisioner + sourceRef: + kind: GitRepository + name: local-path-provisioner + namespace: flux-system + maxHistory: 2 + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + values: + helperImage: + repository: public.ecr.aws/docker/library/busybox + tag: latest + storageClass: + defaultClass: false + nodePathMap: + - node: DEFAULT_PATH_FOR_NON_LISTED_NODES + paths: ["/var/lib/rancher/k3s/storage"] + # NOTE: Do not enable Flux variable substitution on this HelmRelease + configmap: + setup: |- + #!/bin/sh + while getopts "m:s:p:" opt + do + case $opt in + p) + absolutePath=$OPTARG + ;; + s) + sizeInBytes=$OPTARG + ;; + m) + volMode=$OPTARG + ;; + esac + done + mkdir -m 0777 -p ${absolutePath} + chmod 701 ${absolutePath}/.. + teardown: |- + #!/bin/sh + while getopts "m:s:p:" opt + do + case $opt in + p) + absolutePath=$OPTARG + ;; + s) + sizeInBytes=$OPTARG + ;; + m) + volMode=$OPTARG + ;; + esac + done + rm -rf ${absolutePath} diff --git a/bootstrap/templates/kubernetes/apps/kube-system/local-path-provisioner/app/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/local-path-provisioner/app/kustomization.yaml.j2 new file mode 100644 index 00000000..5dd7baca --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/kube-system/local-path-provisioner/app/kustomization.yaml.j2 @@ -0,0 +1,5 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml diff --git a/bootstrap/templates/kubernetes/apps/kube-system/local-path-provisioner/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/local-path-provisioner/ks.yaml.j2 new file mode 100644 index 00000000..63953389 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/kube-system/local-path-provisioner/ks.yaml.j2 @@ -0,0 +1,22 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app local-path-provisioner + namespace: flux-system + labels: + substitution.flux.home.arpa/disabled: "true" +spec: + targetNamespace: kube-system + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/apps/kube-system/local-path-provisioner/app + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m diff --git a/bootstrap/templates/kubernetes/apps/kube-system/metrics-server/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/metrics-server/app/helmrelease.yaml.j2 new file mode 100644 index 00000000..b61f0405 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/kube-system/metrics-server/app/helmrelease.yaml.j2 @@ -0,0 +1,35 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: metrics-server +spec: + interval: 30m + chart: + spec: + chart: metrics-server + version: 3.11.0 + sourceRef: + kind: HelmRepository + name: metrics-server + namespace: flux-system + maxHistory: 2 + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + values: + args: + - --kubelet-insecure-tls + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + - --kubelet-use-node-status-port + - --metric-resolution=15s + metrics: + enabled: true + serviceMonitor: + enabled: true diff --git a/bootstrap/templates/kubernetes/apps/kube-system/metrics-server/app/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/metrics-server/app/kustomization.yaml.j2 new file mode 100644 index 00000000..5dd7baca --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/kube-system/metrics-server/app/kustomization.yaml.j2 @@ -0,0 +1,5 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml diff --git a/bootstrap/templates/kubernetes/apps/kube-system/metrics-server/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/metrics-server/ks.yaml.j2 new file mode 100644 index 00000000..244f53c1 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/kube-system/metrics-server/ks.yaml.j2 @@ -0,0 +1,20 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app metrics-server + namespace: flux-system +spec: + targetNamespace: kube-system + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/apps/kube-system/metrics-server/app + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m diff --git a/bootstrap/templates/kubernetes/apps/kube-system/namespace.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/namespace.yaml.j2 new file mode 100644 index 00000000..5eeb2c91 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/kube-system/namespace.yaml.j2 @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: kube-system + labels: + kustomize.toolkit.fluxcd.io/prune: disabled diff --git a/bootstrap/templates/kubernetes/apps/kube-system/reloader/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/reloader/app/helmrelease.yaml.j2 new file mode 100644 index 00000000..dc236e54 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/kube-system/reloader/app/helmrelease.yaml.j2 @@ -0,0 +1,33 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: reloader + namespace: &namespace kube-system +spec: + interval: 30m + chart: + spec: + chart: reloader + version: 1.0.52 + sourceRef: + kind: HelmRepository + name: stakater + namespace: flux-system + maxHistory: 2 + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + values: + fullnameOverride: reloader + reloader: + reloadStrategy: annotations + podMonitor: + enabled: true + namespace: *namespace diff --git a/bootstrap/templates/kubernetes/apps/kube-system/reloader/app/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/reloader/app/kustomization.yaml.j2 new file mode 100644 index 00000000..5dd7baca --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/kube-system/reloader/app/kustomization.yaml.j2 @@ -0,0 +1,5 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml diff --git a/bootstrap/templates/kubernetes/apps/kube-system/reloader/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/reloader/ks.yaml.j2 new file mode 100644 index 00000000..9aa42993 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/kube-system/reloader/ks.yaml.j2 @@ -0,0 +1,20 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app reloader + namespace: flux-system +spec: + targetNamespace: kube-system + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/apps/kube-system/reloader/app + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m diff --git a/bootstrap/templates/kubernetes/apps/kube-system/snapshot-controller/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/snapshot-controller/app/helmrelease.yaml.j2 new file mode 100644 index 00000000..ae9c9091 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/kube-system/snapshot-controller/app/helmrelease.yaml.j2 @@ -0,0 +1,33 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: snapshot-controller +spec: + interval: 30m + chart: + spec: + chart: snapshot-controller + version: 2.0.3 + sourceRef: + kind: HelmRepository + name: piraeus + namespace: flux-system + maxHistory: 2 + install: + crds: CreateReplace + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + crds: CreateReplace + remediation: + retries: 3 + uninstall: + keepHistory: false + values: + controller: + serviceMonitor: + create: true + webhook: + enabled: false diff --git a/bootstrap/templates/kubernetes/apps/kube-system/snapshot-controller/app/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/snapshot-controller/app/kustomization.yaml.j2 new file mode 100644 index 00000000..5dd7baca --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/kube-system/snapshot-controller/app/kustomization.yaml.j2 @@ -0,0 +1,5 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml diff --git a/bootstrap/templates/kubernetes/apps/kube-system/snapshot-controller/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/snapshot-controller/ks.yaml.j2 new file mode 100644 index 00000000..fff4f41a --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/kube-system/snapshot-controller/ks.yaml.j2 @@ -0,0 +1,20 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app snapshot-controller + namespace: flux-system +spec: + targetNamespace: kube-system + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/apps/kube-system/snapshot-controller/app + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m diff --git a/bootstrap/templates/kubernetes/apps/monitoring/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/monitoring/kustomization.yaml.j2 new file mode 100644 index 00000000..982a0929 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/monitoring/kustomization.yaml.j2 @@ -0,0 +1,15 @@ +#jinja2: trim_blocks: True, lstrip_blocks: True +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./namespace.yaml + {% if grafana.enabled | default(false) %} + - ./grafana/ks.yaml + {% endif %} + {% if kube_prometheus_stack.enabled | default(false) %} + - ./kube-prometheus-stack/ks.yaml + {% endif %} + {% if kubernetes_dashboard.enabled | default(false) %} + - ./kubernetes-dashboard/ks.yaml + {% endif %} diff --git a/bootstrap/templates/kubernetes/apps/monitoring/namespace.yaml.j2 b/bootstrap/templates/kubernetes/apps/monitoring/namespace.yaml.j2 new file mode 100644 index 00000000..ef4dd87a --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/monitoring/namespace.yaml.j2 @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: monitoring + labels: + kustomize.toolkit.fluxcd.io/prune: disabled diff --git a/bootstrap/templates/kubernetes/apps/networking/cloudflared/app/configs/config.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/cloudflared/app/configs/config.yaml.j2 new file mode 100644 index 00000000..fdb39f3d --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/networking/cloudflared/app/configs/config.yaml.j2 @@ -0,0 +1,14 @@ +--- +originRequest: + http2Origin: true + +ingress: + - hostname: "${SECRET_DOMAIN}" + service: https://nginx-external-controller.networking.svc.cluster.local:443 + originRequest: + originServerName: "external.${SECRET_DOMAIN}" + - hostname: "*.${SECRET_DOMAIN}" + service: https://nginx-external-controller.networking.svc.cluster.local:443 + originRequest: + originServerName: "external.${SECRET_DOMAIN}" + - service: http_status:404 diff --git a/bootstrap/templates/kubernetes/apps/networking/cloudflared/app/dnsendpoint.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/cloudflared/app/dnsendpoint.yaml.j2 new file mode 100644 index 00000000..43d7d7b2 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/networking/cloudflared/app/dnsendpoint.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: externaldns.k8s.io/v1alpha1 +kind: DNSEndpoint +metadata: + name: cloudflared +spec: + endpoints: + - dnsName: "external.${SECRET_DOMAIN}" + recordType: CNAME + targets: ["${SECRET_CLOUDFLARE_TUNNEL_ID}.cfargotunnel.com"] diff --git a/bootstrap/templates/kubernetes/apps/networking/cloudflared/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/cloudflared/app/helmrelease.yaml.j2 new file mode 100644 index 00000000..cac9615c --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/networking/cloudflared/app/helmrelease.yaml.j2 @@ -0,0 +1,103 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: cloudflared +spec: + interval: 30m + chart: + spec: + chart: app-template + version: 2.3.0 + sourceRef: + kind: HelmRepository + name: bjw-s + namespace: flux-system + maxHistory: 2 + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + values: + controllers: + main: + replicas: 2 + strategy: RollingUpdate + annotations: + reloader.stakater.com/auto: "true" + containers: + main: + image: + repository: docker.io/cloudflare/cloudflared + tag: 2023.10.0 + env: + NO_AUTOUPDATE: "true" + TUNNEL_CRED_FILE: /etc/cloudflared/creds/credentials.json + TUNNEL_METRICS: 0.0.0.0:8080 + TUNNEL_TRANSPORT_PROTOCOL: quic + TUNNEL_POST_QUANTUM: true + TUNNEL_ID: + valueFrom: + secretKeyRef: + name: cloudflared-secret + key: TUNNEL_ID + args: + - tunnel + - --config + - /etc/cloudflared/config/config.yaml + - run + - "$(TUNNEL_ID)" + probes: + liveness: &probes + enabled: true + custom: true + spec: + httpGet: + path: /ready + port: &port 8080 + initialDelaySeconds: 0 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + readiness: *probes + startup: + enabled: false + resources: + requests: + cpu: 5m + memory: 128M + limits: + memory: 256M + pod: + securityContext: + runAsUser: 568 + runAsGroup: 568 + service: + main: + ports: + http: + port: *port + serviceMonitor: + main: + enabled: true + persistence: + config: + enabled: true + type: configMap + name: cloudflared-configmap + globalMounts: + - path: /etc/cloudflared/config/config.yaml + subPath: config.yaml + readOnly: true + creds: + type: secret + name: cloudflared-secret + globalMounts: + - path: /etc/cloudflared/creds/credentials.json + subPath: credentials.json + readOnly: true diff --git a/bootstrap/templates/kubernetes/apps/networking/cloudflared/app/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/cloudflared/app/kustomization.yaml.j2 new file mode 100644 index 00000000..891a864a --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/networking/cloudflared/app/kustomization.yaml.j2 @@ -0,0 +1,13 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./dnsendpoint.yaml + - ./secret.sops.yaml + - ./helmrelease.yaml +configMapGenerator: + - name: cloudflared-configmap + files: + - ./configs/config.yaml +generatorOptions: + disableNameSuffixHash: true diff --git a/bootstrap/templates/kubernetes/apps/networking/cloudflared/app/secret.sops.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/cloudflared/app/secret.sops.yaml.j2 new file mode 100644 index 00000000..3e7dc1ac --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/networking/cloudflared/app/secret.sops.yaml.j2 @@ -0,0 +1,13 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: cloudflared-secret +stringData: + TUNNEL_ID: "{{ bootstrap_cloudflare_tunnel_id }}" + credentials.json: | + { + "AccountTag": "{{ bootstrap_cloudflare_account_tag }}", + "TunnelSecret": "{{ bootstrap_cloudflare_tunnel_secret }}", + "TunnelID": "{{ bootstrap_cloudflare_tunnel_id }}" + } diff --git a/bootstrap/templates/kubernetes/apps/networking/cloudflared/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/cloudflared/ks.yaml.j2 new file mode 100644 index 00000000..629e1108 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/networking/cloudflared/ks.yaml.j2 @@ -0,0 +1,22 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app cloudflared + namespace: flux-system +spec: + targetNamespace: networking + commonMetadata: + labels: + app.kubernetes.io/name: *app + dependsOn: + - name: external-dns + path: ./kubernetes/apps/networking/cloudflared/app + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m diff --git a/bootstrap/templates/kubernetes/apps/networking/echo-server/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/echo-server/app/helmrelease.yaml.j2 new file mode 100644 index 00000000..b76b99e7 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/networking/echo-server/app/helmrelease.yaml.j2 @@ -0,0 +1,85 @@ +#jinja2: trim_blocks: True +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: echo-server +spec: + interval: 30m + chart: + spec: + chart: app-template + version: 2.3.0 + sourceRef: + kind: HelmRepository + name: bjw-s + namespace: flux-system + maxHistory: 2 + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + values: + controllers: + main: + replicas: 2 + strategy: RollingUpdate + containers: + main: + image: + repository: docker.io/jmalloc/echo-server + tag: 0.3.6 + env: + PORT: &port 8080 + probes: + liveness: &probes + enabled: true + custom: true + spec: + httpGet: + path: /health + port: *port + initialDelaySeconds: 0 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + readiness: *probes + startup: + enabled: false + resources: + requests: + cpu: 5m + memory: 10M + limits: + memory: 64M + pod: + securityContext: + runAsUser: 568 + runAsGroup: 568 + service: + main: + ports: + http: + port: *port + ingress: + main: + enabled: true + className: external + annotations: + external-dns.alpha.kubernetes.io/target: "external.${SECRET_DOMAIN}" + hajimari.io/icon: video-input-antenna + hosts: + - host: &host "{% raw %}{{ .Release.Name }}{% endraw %}.${SECRET_DOMAIN}" + paths: + - path: / + service: + name: main + port: http + tls: + - hosts: + - *host diff --git a/bootstrap/templates/kubernetes/apps/networking/echo-server/app/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/echo-server/app/kustomization.yaml.j2 new file mode 100644 index 00000000..5dd7baca --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/networking/echo-server/app/kustomization.yaml.j2 @@ -0,0 +1,5 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml diff --git a/bootstrap/templates/kubernetes/apps/networking/echo-server/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/echo-server/ks.yaml.j2 new file mode 100644 index 00000000..a2c643c6 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/networking/echo-server/ks.yaml.j2 @@ -0,0 +1,20 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app echo-server + namespace: flux-system +spec: + targetNamespace: networking + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/apps/networking/echo-server/app + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m diff --git a/bootstrap/templates/kubernetes/apps/networking/external-dns/app/dnsendpoint-crd.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/external-dns/app/dnsendpoint-crd.yaml.j2 new file mode 100644 index 00000000..9254f89d --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/networking/external-dns/app/dnsendpoint-crd.yaml.j2 @@ -0,0 +1,93 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.5.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-sigs/external-dns/pull/2007" + creationTimestamp: null + name: dnsendpoints.externaldns.k8s.io +spec: + group: externaldns.k8s.io + names: + kind: DNSEndpoint + listKind: DNSEndpointList + plural: dnsendpoints + singular: dnsendpoint + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: DNSEndpointSpec defines the desired state of DNSEndpoint + properties: + endpoints: + items: + description: Endpoint is a high-level way of a connection between a service and an IP + properties: + dnsName: + description: The hostname of the DNS record + type: string + labels: + additionalProperties: + type: string + description: Labels stores labels defined for the Endpoint + type: object + providerSpecific: + description: ProviderSpecific stores provider specific config + items: + description: ProviderSpecificProperty holds the name and value of a configuration which is specific to individual DNS providers + properties: + name: + type: string + value: + type: string + type: object + type: array + recordTTL: + description: TTL for the record + format: int64 + type: integer + recordType: + description: RecordType type of record, e.g. CNAME, A, SRV, TXT etc + type: string + setIdentifier: + description: Identifier to distinguish multiple records with the same name and type (e.g. Route53 records with routing policies other than 'simple') + type: string + targets: + description: The targets the DNS record points to + items: + type: string + type: array + type: object + type: array + type: object + status: + description: DNSEndpointStatus defines the observed state of DNSEndpoint + properties: + observedGeneration: + description: The generation observed by the external-dns controller. + format: int64 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/bootstrap/templates/kubernetes/apps/networking/external-dns/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/external-dns/app/helmrelease.yaml.j2 new file mode 100644 index 00000000..9c1d93ce --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/networking/external-dns/app/helmrelease.yaml.j2 @@ -0,0 +1,48 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: &app external-dns +spec: + interval: 30m + chart: + spec: + chart: external-dns + version: 1.13.1 + sourceRef: + kind: HelmRepository + name: external-dns + namespace: flux-system + maxHistory: 2 + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + values: + fullnameOverride: *app + provider: cloudflare + env: + - name: CF_API_TOKEN + valueFrom: + secretKeyRef: + name: external-dns-secret + key: api-token + extraArgs: + - --ingress-class=external + - --cloudflare-proxied + - --crd-source-apiversion=externaldns.k8s.io/v1alpha1 + - --crd-source-kind=DNSEndpoint + policy: sync + sources: ["crd", "ingress"] + txtPrefix: k8s. + txtOwnerId: default + domainFilters: ["${SECRET_DOMAIN}"] + serviceMonitor: + enabled: true + podAnnotations: + secret.reloader.stakater.com/reload: external-dns-secret diff --git a/bootstrap/templates/kubernetes/apps/networking/external-dns/app/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/external-dns/app/kustomization.yaml.j2 new file mode 100644 index 00000000..069449ad --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/networking/external-dns/app/kustomization.yaml.j2 @@ -0,0 +1,7 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./dnsendpoint-crd.yaml + - ./secret.sops.yaml + - ./helmrelease.yaml diff --git a/bootstrap/templates/kubernetes/apps/networking/external-dns/app/secret.sops.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/external-dns/app/secret.sops.yaml.j2 new file mode 100644 index 00000000..075967d4 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/networking/external-dns/app/secret.sops.yaml.j2 @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: external-dns-secret +stringData: + api-token: "{{ bootstrap_cloudflare_token }}" diff --git a/bootstrap/templates/kubernetes/apps/networking/external-dns/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/external-dns/ks.yaml.j2 new file mode 100644 index 00000000..20a25912 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/networking/external-dns/ks.yaml.j2 @@ -0,0 +1,20 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app external-dns + namespace: flux-system +spec: + targetNamespace: networking + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/apps/networking/external-dns/app + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: true + interval: 30m + retryInterval: 1m + timeout: 5m diff --git a/bootstrap/templates/kubernetes/apps/networking/k8s-gateway/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/k8s-gateway/app/helmrelease.yaml.j2 new file mode 100644 index 00000000..33c3c2db --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/networking/k8s-gateway/app/helmrelease.yaml.j2 @@ -0,0 +1,35 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: k8s-gateway +spec: + interval: 30m + chart: + spec: + chart: k8s-gateway + version: 2.1.0 + sourceRef: + kind: HelmRepository + name: k8s-gateway + namespace: flux-system + maxHistory: 2 + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + values: + fullnameOverride: k8s-gateway + domain: "${SECRET_DOMAIN}" + ttl: 1 + service: + type: LoadBalancer + port: 53 + annotations: + io.cilium/lb-ipam-ips: "{{ bootstrap_k8s_gateway_addr }}" + externalTrafficPolicy: Cluster diff --git a/bootstrap/templates/kubernetes/apps/networking/k8s-gateway/app/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/k8s-gateway/app/kustomization.yaml.j2 new file mode 100644 index 00000000..5dd7baca --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/networking/k8s-gateway/app/kustomization.yaml.j2 @@ -0,0 +1,5 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml diff --git a/bootstrap/templates/kubernetes/apps/networking/k8s-gateway/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/k8s-gateway/ks.yaml.j2 new file mode 100644 index 00000000..8fee22f0 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/networking/k8s-gateway/ks.yaml.j2 @@ -0,0 +1,20 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app k8s-gateway + namespace: flux-system +spec: + targetNamespace: networking + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/apps/networking/k8s-gateway/app + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m diff --git a/bootstrap/templates/kubernetes/apps/networking/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/kustomization.yaml.j2 new file mode 100644 index 00000000..4ad531cd --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/networking/kustomization.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./namespace.yaml + - ./cloudflared/ks.yaml + - ./echo-server/ks.yaml + - ./external-dns/ks.yaml + - ./k8s-gateway/ks.yaml + - ./nginx/ks.yaml diff --git a/bootstrap/templates/kubernetes/apps/networking/namespace.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/namespace.yaml.j2 new file mode 100644 index 00000000..b9e4a416 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/networking/namespace.yaml.j2 @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: networking + labels: + kustomize.toolkit.fluxcd.io/prune: disabled diff --git a/bootstrap/templates/kubernetes/apps/networking/nginx/certificates/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/nginx/certificates/kustomization.yaml.j2 new file mode 100644 index 00000000..81f57a48 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/networking/nginx/certificates/kustomization.yaml.j2 @@ -0,0 +1,9 @@ +#jinja2: trim_blocks: True, lstrip_blocks: True +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./staging.yaml + {% if bootstrap_acme_production_enabled | default(false) %} + - ./production.yaml + {% endif %} diff --git a/bootstrap/templates/kubernetes/apps/networking/nginx/certificates/production.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/nginx/certificates/production.yaml.j2 new file mode 100644 index 00000000..b5afdf41 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/networking/nginx/certificates/production.yaml.j2 @@ -0,0 +1,14 @@ +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: "${SECRET_DOMAIN/./-}-production" +spec: + secretName: "${SECRET_DOMAIN/./-}-production-tls" + issuerRef: + name: letsencrypt-production + kind: ClusterIssuer + commonName: "${SECRET_DOMAIN}" + dnsNames: + - "${SECRET_DOMAIN}" + - "*.${SECRET_DOMAIN}" diff --git a/bootstrap/templates/kubernetes/apps/networking/nginx/certificates/staging.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/nginx/certificates/staging.yaml.j2 new file mode 100644 index 00000000..9c869425 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/networking/nginx/certificates/staging.yaml.j2 @@ -0,0 +1,14 @@ +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: "${SECRET_DOMAIN/./-}-staging" +spec: + secretName: "${SECRET_DOMAIN/./-}-staging-tls" + issuerRef: + name: letsencrypt-staging + kind: ClusterIssuer + commonName: "${SECRET_DOMAIN}" + dnsNames: + - "${SECRET_DOMAIN}" + - "*.${SECRET_DOMAIN}" diff --git a/bootstrap/templates/kubernetes/apps/networking/nginx/external/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/nginx/external/helmrelease.yaml.j2 new file mode 100644 index 00000000..593d5609 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/networking/nginx/external/helmrelease.yaml.j2 @@ -0,0 +1,98 @@ +#jinja2: trim_blocks: True, lstrip_blocks: True +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: nginx-external + namespace: &namespace networking +spec: + interval: 30m + chart: + spec: + chart: ingress-nginx + version: 4.7.1 + sourceRef: + kind: HelmRepository + name: ingress-nginx + namespace: flux-system + maxHistory: 2 + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + dependsOn: + - name: cloudflared + namespace: networking + values: + fullnameOverride: nginx-external + controller: + replicaCount: 1 + service: + annotations: + external-dns.alpha.kubernetes.io/hostname: "external.${SECRET_DOMAIN}" + io.cilium/lb-ipam-ips: "{{ bootstrap_external_ingress_addr }}" + externalTrafficPolicy: Cluster + ingressClassResource: + name: external + default: false + controllerValue: k8s.io/external + admissionWebhooks: + objectSelector: + matchExpressions: + - key: ingress-class + operator: In + values: ["external"] + config: + client-body-buffer-size: 100M + client-body-timeout: 120 + client-header-timeout: 120 + enable-brotli: "true" + enable-real-ip: "true" + hsts-max-age: 31449600 + keep-alive-requests: 10000 + keep-alive: 120 + log-format-escape-json: "true" + log-format-upstream: > + {"time": "$time_iso8601", "remote_addr": "$proxy_protocol_addr", "x_forwarded_for": "$proxy_add_x_forwarded_for", + "request_id": "$req_id", "remote_user": "$remote_user", "bytes_sent": $bytes_sent, "request_time": $request_time, + "status": $status, "vhost": "$host", "request_proto": "$server_protocol", "path": "$uri", "request_query": "$args", + "request_length": $request_length, "duration": $request_time, "method": "$request_method", "http_referrer": "$http_referer", + "http_user_agent": "$http_user_agent"} + proxy-body-size: 0 + proxy-buffer-size: 16k + ssl-protocols: TLSv1.3 TLSv1.2 + metrics: + enabled: true + serviceMonitor: + enabled: true + namespace: *namespace + namespaceSelector: + any: true + extraArgs: + {% if bootstrap_acme_production_enabled | default(false) %} + default-ssl-certificate: "networking/${SECRET_DOMAIN/./-}-production-tls" + {% else %} + default-ssl-certificate: "networking/${SECRET_DOMAIN/./-}-staging-tls" + {% endif %} + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: nginx-external + app.kubernetes.io/component: controller + resources: + requests: + cpu: 10m + memory: 250Mi + limits: + memory: 500Mi + defaultBackend: + enabled: false diff --git a/bootstrap/templates/kubernetes/apps/networking/nginx/external/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/nginx/external/kustomization.yaml.j2 new file mode 100644 index 00000000..5dd7baca --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/networking/nginx/external/kustomization.yaml.j2 @@ -0,0 +1,5 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml diff --git a/bootstrap/templates/kubernetes/apps/networking/nginx/internal/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/nginx/internal/helmrelease.yaml.j2 new file mode 100644 index 00000000..bb71b372 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/networking/nginx/internal/helmrelease.yaml.j2 @@ -0,0 +1,95 @@ +#jinja2: trim_blocks: True, lstrip_blocks: True +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: nginx-internal + namespace: &namespace networking +spec: + interval: 30m + chart: + spec: + chart: ingress-nginx + version: 4.7.1 + sourceRef: + kind: HelmRepository + name: ingress-nginx + namespace: flux-system + maxHistory: 2 + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + values: + fullnameOverride: nginx-internal + controller: + replicaCount: 1 + service: + annotations: + external-dns.alpha.kubernetes.io/hostname: "internal.${SECRET_DOMAIN}" + io.cilium/lb-ipam-ips: "{{ bootstrap_internal_ingress_addr }}" + externalTrafficPolicy: Cluster + ingressClassResource: + name: internal + default: true + controllerValue: k8s.io/internal + admissionWebhooks: + objectSelector: + matchExpressions: + - key: ingress-class + operator: In + values: ["internal"] + config: + client-body-buffer-size: 100M + client-body-timeout: 120 + client-header-timeout: 120 + enable-brotli: "true" + enable-real-ip: "true" + hsts-max-age: 31449600 + keep-alive-requests: 10000 + keep-alive: 120 + log-format-escape-json: "true" + log-format-upstream: > + {"time": "$time_iso8601", "remote_addr": "$proxy_protocol_addr", "x_forwarded_for": "$proxy_add_x_forwarded_for", + "request_id": "$req_id", "remote_user": "$remote_user", "bytes_sent": $bytes_sent, "request_time": $request_time, + "status": $status, "vhost": "$host", "request_proto": "$server_protocol", "path": "$uri", "request_query": "$args", + "request_length": $request_length, "duration": $request_time, "method": "$request_method", "http_referrer": "$http_referer", + "http_user_agent": "$http_user_agent"} + proxy-body-size: 0 + proxy-buffer-size: 16k + ssl-protocols: TLSv1.3 TLSv1.2 + metrics: + enabled: true + serviceMonitor: + enabled: true + namespace: *namespace + namespaceSelector: + any: true + extraArgs: + {% if bootstrap_acme_production_enabled | default(false) %} + default-ssl-certificate: "networking/${SECRET_DOMAIN/./-}-production-tls" + {% else %} + default-ssl-certificate: "networking/${SECRET_DOMAIN/./-}-staging-tls" + {% endif %} + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: nginx-internal + app.kubernetes.io/component: controller + resources: + requests: + cpu: 10m + memory: 250Mi + limits: + memory: 500Mi + defaultBackend: + enabled: false diff --git a/bootstrap/templates/kubernetes/apps/networking/nginx/internal/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/nginx/internal/kustomization.yaml.j2 new file mode 100644 index 00000000..5dd7baca --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/networking/nginx/internal/kustomization.yaml.j2 @@ -0,0 +1,5 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml diff --git a/bootstrap/templates/kubernetes/apps/networking/nginx/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/networking/nginx/ks.yaml.j2 new file mode 100644 index 00000000..9cb24b81 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/networking/nginx/ks.yaml.j2 @@ -0,0 +1,66 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app nginx-certificates + namespace: flux-system +spec: + targetNamespace: networking + commonMetadata: + labels: + app.kubernetes.io/name: *app + dependsOn: + - name: cert-manager-issuers + path: ./kubernetes/apps/networking/nginx/certificates + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: true + interval: 30m + retryInterval: 1m + timeout: 5m +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app nginx-external + namespace: flux-system +spec: + targetNamespace: networking + commonMetadata: + labels: + app.kubernetes.io/name: *app + dependsOn: + - name: nginx-certificates + path: ./kubernetes/apps/networking/nginx/external + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app nginx-internal + namespace: flux-system +spec: + targetNamespace: networking + commonMetadata: + labels: + app.kubernetes.io/name: *app + dependsOn: + - name: nginx-certificates + path: ./kubernetes/apps/networking/nginx/internal + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m diff --git a/bootstrap/templates/kubernetes/apps/volsync/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/volsync/kustomization.yaml.j2 new file mode 100644 index 00000000..7b6ea1df --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/volsync/kustomization.yaml.j2 @@ -0,0 +1,8 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + # Pre Flux-Kustomizations + - ./namespace.yaml + # Flux-Kustomizations + - ./volsync/ks.yaml diff --git a/bootstrap/templates/kubernetes/apps/volsync/namespace.yaml.j2 b/bootstrap/templates/kubernetes/apps/volsync/namespace.yaml.j2 new file mode 100644 index 00000000..b2062055 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/volsync/namespace.yaml.j2 @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: volsync + labels: + kustomize.toolkit.fluxcd.io/prune: disabled diff --git a/bootstrap/templates/kubernetes/apps/volsync/volsync/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/volsync/volsync/app/helmrelease.yaml.j2 new file mode 100644 index 00000000..f32ef7a8 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/volsync/volsync/app/helmrelease.yaml.j2 @@ -0,0 +1,42 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: volsync +spec: + interval: 30m + chart: + spec: + chart: volsync + version: 0.8.0 + sourceRef: + kind: HelmRepository + name: backube + namespace: flux-system + maxHistory: 2 + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + dependsOn: + - name: snapshot-controller + namespace: kube-system + values: + manageCRDs: true + image: + # https://github.com/backube/volsync/issues/828 + repository: &image ghcr.io/onedr0p/volsync + tag: &tag 0.8.0 + rclone: + repository: *image + tag: *tag + restic: + repository: *image + tag: *tag + metrics: + disableAuth: true diff --git a/bootstrap/templates/kubernetes/apps/volsync/volsync/app/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/volsync/volsync/app/kustomization.yaml.j2 new file mode 100644 index 00000000..8ae52667 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/volsync/volsync/app/kustomization.yaml.j2 @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml + - ./prometheusrule.yaml diff --git a/bootstrap/templates/kubernetes/apps/volsync/volsync/app/prometheusrule.yaml.j2 b/bootstrap/templates/kubernetes/apps/volsync/volsync/app/prometheusrule.yaml.j2 new file mode 100644 index 00000000..9d677c5f --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/volsync/volsync/app/prometheusrule.yaml.j2 @@ -0,0 +1,23 @@ +--- +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: volsync +spec: + groups: + - name: volsync.rules + rules: + - alert: VolSyncComponentAbsent + annotations: + summary: VolSync component has disappeared from Prometheus target discovery. + expr: absent(up{job="volsync-metrics"}) + for: 15m + labels: + severity: critical + - alert: VolSyncVolumeOutOfSync + annotations: + summary: "{% raw %}{{ $labels.obj_namespace }}/{{ $labels.obj_name }} volume is out of sync.{% endraw %}" + expr: volsync_volume_out_of_sync == 1 + for: 15m + labels: + severity: critical diff --git a/bootstrap/templates/kubernetes/apps/volsync/volsync/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/volsync/volsync/ks.yaml.j2 new file mode 100644 index 00000000..9913e31e --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/volsync/volsync/ks.yaml.j2 @@ -0,0 +1,20 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app volsync + namespace: flux-system +spec: + targetNamespace: volsync + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/apps/volsync/volsync/app + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m diff --git a/bootstrap/templates/kubernetes/bootstrap/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/bootstrap/kustomization.yaml.j2 new file mode 100644 index 00000000..c1b5060c --- /dev/null +++ b/bootstrap/templates/kubernetes/bootstrap/kustomization.yaml.j2 @@ -0,0 +1,18 @@ +# IMPORTANT: This file is not tracked by flux and should never be. Its +# purpose is to only install the Flux components and CRDs into your cluster. +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - github.com/fluxcd/flux2/manifests/install?ref=v2.1.2 +patches: + # Remove the default network policies + - patch: |- + $patch: delete + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + name: not-used + target: + group: networking.k8s.io + kind: NetworkPolicy diff --git a/bootstrap/templates/kubernetes/flux/apps.yaml.j2 b/bootstrap/templates/kubernetes/flux/apps.yaml.j2 new file mode 100644 index 00000000..d557f828 --- /dev/null +++ b/bootstrap/templates/kubernetes/flux/apps.yaml.j2 @@ -0,0 +1,52 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: cluster-apps + namespace: flux-system +spec: + interval: 30m + path: ./kubernetes/apps + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + decryption: + provider: sops + secretRef: + name: sops-age + postBuild: + substituteFrom: + - kind: ConfigMap + name: cluster-settings + - kind: Secret + name: cluster-secrets + - kind: ConfigMap + name: cluster-settings-user + - kind: Secret + name: cluster-secrets-user + patches: + - patch: |- + apiVersion: kustomize.toolkit.fluxcd.io/v1 + kind: Kustomization + metadata: + name: not-used + spec: + decryption: + provider: sops + secretRef: + name: sops-age + postBuild: + substituteFrom: + - kind: ConfigMap + name: cluster-settings + - kind: Secret + name: cluster-secrets + - kind: ConfigMap + name: cluster-settings-user + - kind: Secret + name: cluster-secrets-user + target: + group: kustomize.toolkit.fluxcd.io + kind: Kustomization + labelSelector: substitution.flux.home.arpa/disabled notin (true) diff --git a/bootstrap/templates/kubernetes/flux/config/cluster.yaml.j2 b/bootstrap/templates/kubernetes/flux/config/cluster.yaml.j2 new file mode 100644 index 00000000..ecd2060c --- /dev/null +++ b/bootstrap/templates/kubernetes/flux/config/cluster.yaml.j2 @@ -0,0 +1,40 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: GitRepository +metadata: + name: home-kubernetes + namespace: flux-system +spec: + interval: 30m + ref: + branch: {{ bootstrap_github_repository_branch | default('main', true) }} + url: "https://github.com/{{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }}.git" + ignore: | + # exclude all + /* + # include kubernetes directory + !/kubernetes +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: cluster + namespace: flux-system +spec: + interval: 30m + path: ./kubernetes/flux + prune: true + wait: false + sourceRef: + kind: GitRepository + name: home-kubernetes + decryption: + provider: sops + secretRef: + name: sops-age + postBuild: + substituteFrom: + - kind: ConfigMap + name: cluster-settings + - kind: Secret + name: cluster-secrets diff --git a/bootstrap/templates/kubernetes/flux/config/flux.yaml.j2 b/bootstrap/templates/kubernetes/flux/config/flux.yaml.j2 new file mode 100644 index 00000000..18b35080 --- /dev/null +++ b/bootstrap/templates/kubernetes/flux/config/flux.yaml.j2 @@ -0,0 +1,86 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: OCIRepository +metadata: + name: flux-manifests + namespace: flux-system +spec: + interval: 10m + url: oci://ghcr.io/fluxcd/flux-manifests + ref: + tag: v2.1.2 +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: flux + namespace: flux-system +spec: + interval: 10m + path: ./ + prune: true + wait: true + sourceRef: + kind: OCIRepository + name: flux-manifests + patches: + # Remove the network policies that does not work with k3s + - patch: | + $patch: delete + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + name: not-used + target: + group: networking.k8s.io + kind: NetworkPolicy + # Increase the number of reconciliations that can be performed in parallel and bump the resources limits + # https://fluxcd.io/flux/cheatsheets/bootstrap/#increase-the-number-of-workers + - patch: | + - op: add + path: /spec/template/spec/containers/0/args/- + value: --concurrent=8 + - op: add + path: /spec/template/spec/containers/0/args/- + value: --kube-api-qps=500 + - op: add + path: /spec/template/spec/containers/0/args/- + value: --kube-api-burst=1000 + - op: add + path: /spec/template/spec/containers/0/args/- + value: --requeue-dependency=5s + target: + kind: Deployment + name: (kustomize-controller|helm-controller|source-controller) + - patch: | + apiVersion: apps/v1 + kind: Deployment + metadata: + name: not-used + spec: + template: + spec: + containers: + - name: manager + resources: + limits: + cpu: 2000m + memory: 2Gi + target: + kind: Deployment + name: (kustomize-controller|helm-controller|source-controller) + # Enable Helm near OOM detection + # https://fluxcd.io/flux/cheatsheets/bootstrap/#enable-helm-near-oom-detection + - patch: | + - op: add + path: /spec/template/spec/containers/0/args/- + value: --feature-gates=OOMWatch=true + - op: add + path: /spec/template/spec/containers/0/args/- + value: --oom-watch-memory-threshold=95 + - op: add + path: /spec/template/spec/containers/0/args/- + value: --oom-watch-interval=500ms + target: + kind: Deployment + name: helm-controller diff --git a/bootstrap/templates/kubernetes/flux/config/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/flux/config/kustomization.yaml.j2 new file mode 100644 index 00000000..ef231746 --- /dev/null +++ b/bootstrap/templates/kubernetes/flux/config/kustomization.yaml.j2 @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./flux.yaml + - ./cluster.yaml diff --git a/bootstrap/templates/kubernetes/flux/repositories/git/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/flux/repositories/git/kustomization.yaml.j2 new file mode 100644 index 00000000..bb78ecf0 --- /dev/null +++ b/bootstrap/templates/kubernetes/flux/repositories/git/kustomization.yaml.j2 @@ -0,0 +1,5 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./local-path-provisioner.yaml diff --git a/bootstrap/templates/kubernetes/flux/repositories/git/local-path-provisioner.yaml.j2 b/bootstrap/templates/kubernetes/flux/repositories/git/local-path-provisioner.yaml.j2 new file mode 100644 index 00000000..25d4b147 --- /dev/null +++ b/bootstrap/templates/kubernetes/flux/repositories/git/local-path-provisioner.yaml.j2 @@ -0,0 +1,16 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: GitRepository +metadata: + name: local-path-provisioner + namespace: flux-system +spec: + interval: 30m + url: https://github.com/rancher/local-path-provisioner + ref: + tag: v0.0.26 + ignore: | + # exclude all + /* + # include kubernetes directory + !/deploy/chart/local-path-provisioner diff --git a/bootstrap/templates/kubernetes/flux/repositories/helm/backube.yaml.j2 b/bootstrap/templates/kubernetes/flux/repositories/helm/backube.yaml.j2 new file mode 100644 index 00000000..348802dc --- /dev/null +++ b/bootstrap/templates/kubernetes/flux/repositories/helm/backube.yaml.j2 @@ -0,0 +1,9 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: backube + namespace: flux-system +spec: + interval: 1h + url: https://backube.github.io/helm-charts/ diff --git a/bootstrap/templates/kubernetes/flux/repositories/helm/bitnami.yaml.j2 b/bootstrap/templates/kubernetes/flux/repositories/helm/bitnami.yaml.j2 new file mode 100644 index 00000000..eca160a1 --- /dev/null +++ b/bootstrap/templates/kubernetes/flux/repositories/helm/bitnami.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: bitnami + namespace: flux-system +spec: + type: oci + interval: 5m + url: oci://registry-1.docker.io/bitnamicharts diff --git a/bootstrap/templates/kubernetes/flux/repositories/helm/bjw-s.yaml.j2 b/bootstrap/templates/kubernetes/flux/repositories/helm/bjw-s.yaml.j2 new file mode 100644 index 00000000..df0c6474 --- /dev/null +++ b/bootstrap/templates/kubernetes/flux/repositories/helm/bjw-s.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: bjw-s + namespace: flux-system +spec: + type: oci + interval: 5m + url: oci://ghcr.io/bjw-s/helm diff --git a/bootstrap/templates/kubernetes/flux/repositories/helm/cilium.yaml.j2 b/bootstrap/templates/kubernetes/flux/repositories/helm/cilium.yaml.j2 new file mode 100644 index 00000000..51c65d69 --- /dev/null +++ b/bootstrap/templates/kubernetes/flux/repositories/helm/cilium.yaml.j2 @@ -0,0 +1,9 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: cilium + namespace: flux-system +spec: + interval: 1h + url: https://helm.cilium.io diff --git a/bootstrap/templates/kubernetes/flux/repositories/helm/coredns.yaml.j2 b/bootstrap/templates/kubernetes/flux/repositories/helm/coredns.yaml.j2 new file mode 100644 index 00000000..e3a16bd1 --- /dev/null +++ b/bootstrap/templates/kubernetes/flux/repositories/helm/coredns.yaml.j2 @@ -0,0 +1,9 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: coredns + namespace: flux-system +spec: + interval: 1h + url: https://coredns.github.io/helm diff --git a/bootstrap/templates/kubernetes/flux/repositories/helm/csi-driver-nfs.yaml.j2 b/bootstrap/templates/kubernetes/flux/repositories/helm/csi-driver-nfs.yaml.j2 new file mode 100644 index 00000000..b48140d7 --- /dev/null +++ b/bootstrap/templates/kubernetes/flux/repositories/helm/csi-driver-nfs.yaml.j2 @@ -0,0 +1,9 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: csi-driver-nfs + namespace: flux-system +spec: + interval: 1h + url: https://raw.githubusercontent.com/kubernetes-csi/csi-driver-nfs/master/charts diff --git a/bootstrap/templates/kubernetes/flux/repositories/helm/external-dns.yaml.j2 b/bootstrap/templates/kubernetes/flux/repositories/helm/external-dns.yaml.j2 new file mode 100644 index 00000000..b76b9662 --- /dev/null +++ b/bootstrap/templates/kubernetes/flux/repositories/helm/external-dns.yaml.j2 @@ -0,0 +1,9 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: external-dns + namespace: flux-system +spec: + interval: 1h + url: https://kubernetes-sigs.github.io/external-dns diff --git a/bootstrap/templates/kubernetes/flux/repositories/helm/grafana.yaml.j2 b/bootstrap/templates/kubernetes/flux/repositories/helm/grafana.yaml.j2 new file mode 100644 index 00000000..5c3939d5 --- /dev/null +++ b/bootstrap/templates/kubernetes/flux/repositories/helm/grafana.yaml.j2 @@ -0,0 +1,9 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: grafana + namespace: flux-system +spec: + interval: 1h + url: https://grafana.github.io/helm-charts diff --git a/bootstrap/templates/kubernetes/flux/repositories/helm/hajimari.yaml.j2 b/bootstrap/templates/kubernetes/flux/repositories/helm/hajimari.yaml.j2 new file mode 100644 index 00000000..e246f09b --- /dev/null +++ b/bootstrap/templates/kubernetes/flux/repositories/helm/hajimari.yaml.j2 @@ -0,0 +1,9 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: hajimari + namespace: flux-system +spec: + interval: 1h + url: https://hajimari.io diff --git a/bootstrap/templates/kubernetes/flux/repositories/helm/ingress-nginx.yaml.j2 b/bootstrap/templates/kubernetes/flux/repositories/helm/ingress-nginx.yaml.j2 new file mode 100644 index 00000000..4dcf5eea --- /dev/null +++ b/bootstrap/templates/kubernetes/flux/repositories/helm/ingress-nginx.yaml.j2 @@ -0,0 +1,9 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: ingress-nginx + namespace: flux-system +spec: + interval: 1h + url: https://kubernetes.github.io/ingress-nginx diff --git a/bootstrap/templates/kubernetes/flux/repositories/helm/jetstack.yaml.j2 b/bootstrap/templates/kubernetes/flux/repositories/helm/jetstack.yaml.j2 new file mode 100644 index 00000000..d7e64ffc --- /dev/null +++ b/bootstrap/templates/kubernetes/flux/repositories/helm/jetstack.yaml.j2 @@ -0,0 +1,9 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: jetstack + namespace: flux-system +spec: + interval: 1h + url: https://charts.jetstack.io/ diff --git a/bootstrap/templates/kubernetes/flux/repositories/helm/k8s-gateway.yaml.j2 b/bootstrap/templates/kubernetes/flux/repositories/helm/k8s-gateway.yaml.j2 new file mode 100644 index 00000000..a18177eb --- /dev/null +++ b/bootstrap/templates/kubernetes/flux/repositories/helm/k8s-gateway.yaml.j2 @@ -0,0 +1,9 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: k8s-gateway + namespace: flux-system +spec: + interval: 1h + url: https://ori-edge.github.io/k8s_gateway/ diff --git a/bootstrap/templates/kubernetes/flux/repositories/helm/kubernetes-dashboard.yaml.j2 b/bootstrap/templates/kubernetes/flux/repositories/helm/kubernetes-dashboard.yaml.j2 new file mode 100644 index 00000000..d63e74b7 --- /dev/null +++ b/bootstrap/templates/kubernetes/flux/repositories/helm/kubernetes-dashboard.yaml.j2 @@ -0,0 +1,9 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: kubernetes-dashboard + namespace: flux-system +spec: + interval: 1h + url: https://kubernetes.github.io/dashboard/ diff --git a/bootstrap/templates/kubernetes/flux/repositories/helm/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/flux/repositories/helm/kustomization.yaml.j2 new file mode 100644 index 00000000..df330874 --- /dev/null +++ b/bootstrap/templates/kubernetes/flux/repositories/helm/kustomization.yaml.j2 @@ -0,0 +1,22 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./backube.yaml + - ./bitnami.yaml + - ./bjw-s.yaml + - ./cilium.yaml + - ./coredns.yaml + - ./csi-driver-nfs.yaml + - ./external-dns.yaml + - ./grafana.yaml + - ./hajimari.yaml + - ./ingress-nginx.yaml + - ./jetstack.yaml + - ./k8s-gateway.yaml + - ./kubernetes-dashboard.yaml + - ./metrics-server.yaml + - ./piraeus.yaml + - ./prometheus-community.yaml + - ./stakater.yaml + - ./weave-gitops.yaml diff --git a/bootstrap/templates/kubernetes/flux/repositories/helm/metrics-server.yaml.j2 b/bootstrap/templates/kubernetes/flux/repositories/helm/metrics-server.yaml.j2 new file mode 100644 index 00000000..57e7aa0c --- /dev/null +++ b/bootstrap/templates/kubernetes/flux/repositories/helm/metrics-server.yaml.j2 @@ -0,0 +1,9 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: metrics-server + namespace: flux-system +spec: + interval: 1h + url: https://kubernetes-sigs.github.io/metrics-server diff --git a/bootstrap/templates/kubernetes/flux/repositories/helm/piraeus.yaml.j2 b/bootstrap/templates/kubernetes/flux/repositories/helm/piraeus.yaml.j2 new file mode 100644 index 00000000..cc285faa --- /dev/null +++ b/bootstrap/templates/kubernetes/flux/repositories/helm/piraeus.yaml.j2 @@ -0,0 +1,9 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: piraeus + namespace: flux-system +spec: + interval: 1h + url: https://piraeus.io/helm-charts/ diff --git a/bootstrap/templates/kubernetes/flux/repositories/helm/prometheus-community.yaml.j2 b/bootstrap/templates/kubernetes/flux/repositories/helm/prometheus-community.yaml.j2 new file mode 100644 index 00000000..a97a3d44 --- /dev/null +++ b/bootstrap/templates/kubernetes/flux/repositories/helm/prometheus-community.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: prometheus-community + namespace: flux-system +spec: + type: oci + interval: 5m + url: oci://ghcr.io/prometheus-community/charts diff --git a/bootstrap/templates/kubernetes/flux/repositories/helm/stakater.yaml.j2 b/bootstrap/templates/kubernetes/flux/repositories/helm/stakater.yaml.j2 new file mode 100644 index 00000000..1846e8ae --- /dev/null +++ b/bootstrap/templates/kubernetes/flux/repositories/helm/stakater.yaml.j2 @@ -0,0 +1,9 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: stakater + namespace: flux-system +spec: + interval: 1h + url: https://stakater.github.io/stakater-charts diff --git a/bootstrap/templates/kubernetes/flux/repositories/helm/weave-gitops.yaml.j2 b/bootstrap/templates/kubernetes/flux/repositories/helm/weave-gitops.yaml.j2 new file mode 100644 index 00000000..f325c18b --- /dev/null +++ b/bootstrap/templates/kubernetes/flux/repositories/helm/weave-gitops.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: weave-gitops + namespace: flux-system +spec: + type: oci + interval: 5m + url: oci://ghcr.io/weaveworks/charts diff --git a/bootstrap/templates/kubernetes/flux/repositories/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/flux/repositories/kustomization.yaml.j2 new file mode 100644 index 00000000..cb57fda1 --- /dev/null +++ b/bootstrap/templates/kubernetes/flux/repositories/kustomization.yaml.j2 @@ -0,0 +1,7 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./git + - ./helm + # - ./oci diff --git a/bootstrap/templates/kubernetes/flux/repositories/oci/.gitkeep b/bootstrap/templates/kubernetes/flux/repositories/oci/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/bootstrap/templates/kubernetes/flux/vars/cluster-secrets-user.sops.yaml.j2 b/bootstrap/templates/kubernetes/flux/vars/cluster-secrets-user.sops.yaml.j2 new file mode 100644 index 00000000..281accfb --- /dev/null +++ b/bootstrap/templates/kubernetes/flux/vars/cluster-secrets-user.sops.yaml.j2 @@ -0,0 +1,8 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: cluster-secrets-user + namespace: flux-system +stringData: + SECRET_PLACEHOLDER: "secret-value" diff --git a/bootstrap/templates/kubernetes/flux/vars/cluster-secrets.sops.yaml.j2 b/bootstrap/templates/kubernetes/flux/vars/cluster-secrets.sops.yaml.j2 new file mode 100644 index 00000000..54f10d93 --- /dev/null +++ b/bootstrap/templates/kubernetes/flux/vars/cluster-secrets.sops.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: cluster-secrets + namespace: flux-system +stringData: + SECRET_DOMAIN: "{{ bootstrap_cloudflare_domain }}" + SECRET_ACME_EMAIL: "{{ bootstrap_acme_email }}" + SECRET_CLOUDFLARE_TUNNEL_ID: "{{ bootstrap_cloudflare_tunnel_id }}" diff --git a/bootstrap/templates/kubernetes/flux/vars/cluster-settings-user.yaml.j2 b/bootstrap/templates/kubernetes/flux/vars/cluster-settings-user.yaml.j2 new file mode 100644 index 00000000..7b817616 --- /dev/null +++ b/bootstrap/templates/kubernetes/flux/vars/cluster-settings-user.yaml.j2 @@ -0,0 +1,8 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: cluster-settings-user + namespace: flux-system +data: + SETTINGS_PLACEHOLDER: "settings-value" diff --git a/bootstrap/templates/kubernetes/flux/vars/cluster-settings.yaml.j2 b/bootstrap/templates/kubernetes/flux/vars/cluster-settings.yaml.j2 new file mode 100644 index 00000000..511c2bd2 --- /dev/null +++ b/bootstrap/templates/kubernetes/flux/vars/cluster-settings.yaml.j2 @@ -0,0 +1,18 @@ +#jinja2: trim_blocks: True, lstrip_blocks: True +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: cluster-settings + namespace: flux-system +data: + TIMEZONE: "{{ bootstrap_timezone }}" + COREDNS_ADDR: "{{ bootstrap_service_cidr.split(',')[0] | ansible.utils.nthhost(10) }}" + KUBE_VIP_ADDR: "{{ bootstrap_kube_vip_addr }}" + CLUSTER_CIDR: "{{ bootstrap_cluster_cidr.split(',')[0] }}" + SERVICE_CIDR: "{{ bootstrap_service_cidr.split(',')[0] }}" + NODE_CIDR: "{{ bootstrap_node_cidr }}" + {% if bootstrap_ipv6_enabled | default(false) %} + CLUSTER_CIDR_V6: "{{ bootstrap_cluster_cidr.split(',')[1] }}" + SERVICE_CIDR_V6: "{{ bootstrap_service_cidr.split(',')[1] }}" + {% endif %} diff --git a/bootstrap/templates/kubernetes/flux/vars/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/flux/vars/kustomization.yaml.j2 new file mode 100644 index 00000000..dd93387a --- /dev/null +++ b/bootstrap/templates/kubernetes/flux/vars/kustomization.yaml.j2 @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./cluster-settings.yaml + - ./cluster-settings-user.yaml + - ./cluster-secrets.sops.yaml + - ./cluster-secrets-user.sops.yaml diff --git a/bootstrap/templates/node.sops.yaml.j2 b/bootstrap/templates/node.sops.yaml.j2 new file mode 100644 index 00000000..e538fd2e --- /dev/null +++ b/bootstrap/templates/node.sops.yaml.j2 @@ -0,0 +1,2 @@ +--- +ansible_become_pass: "{{ password }}" diff --git a/bootstrap/vars/.gitignore b/bootstrap/vars/.gitignore new file mode 100644 index 00000000..35ba105e --- /dev/null +++ b/bootstrap/vars/.gitignore @@ -0,0 +1,2 @@ +addons.yaml +config.yaml diff --git a/bootstrap/vars/addons.sample.yaml b/bootstrap/vars/addons.sample.yaml new file mode 100644 index 00000000..83ab4826 --- /dev/null +++ b/bootstrap/vars/addons.sample.yaml @@ -0,0 +1,51 @@ +--- +# +# Addons configuration - addons.yaml is gitignored +# + +# https://github.com/toboshii/hajimari +hajimari: + enabled: false + +# https://github.com/grafana/grafana +grafana: + enabled: false + # password: # password for `admin` user + +# https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack +kube_prometheus_stack: + enabled: false + +# https://github.com/kubernetes/dashboard +kubernetes_dashboard: + enabled: false + # Password can be obtained by running the following command once it is deployed: + # kubectl -n monitoring get secret kubernetes-dashboard -o jsonpath='{.data.token}' | base64 -d + +# https://github.com/weaveworks/weave-gitops +weave_gitops: + enabled: false + # password: # password for `admin` user + +# https://github.com/kubernetes-csi/csi-driver-nfs +csi_driver_nfs: + enabled: false + storage_class: + # - name: # name of the storage class (must match [a-z0-9-]+) + # server: # dns name or ip address of nfs server + # share: # exported share (path) + # ... + +# https://github.com/rancher/system-upgrade-controller +system_upgrade_controller: + # WARNING: Only enable this if you also track the version of k3s in the + # ansible configuration files. Running ansible against an already provisioned + # cluster with this enabled might cause your cluster to be downgraded. + enabled: false + +# https://github.com/morphy2k/rss-forwarder +discord_template_notifier: + # Will post commits from the template repository to the specified discord channel + # so it's easier to keep track of changes. + enabled: false + webhook_url: # Discord webhook url diff --git a/bootstrap/vars/config.sample.yaml b/bootstrap/vars/config.sample.yaml new file mode 100644 index 00000000..7361a432 --- /dev/null +++ b/bootstrap/vars/config.sample.yaml @@ -0,0 +1,75 @@ +--- +# +# Bootstrap configuration - config.yaml is gitignored +# + +# Github username (e.g. onedr0p) +bootstrap_github_username: +# Github repository (e.g. flux-cluster-template) +bootstrap_github_repository_name: +# Github repository branch (e.g. main) +bootstrap_github_repository_branch: main +# Age Public Key (e.g. age15uzrw396e67z9wdzsxzdk7ka0g2gr3l460e0slaea563zll3hdfqwqxdta) +bootstrap_age_public_key: +# Choose your timezone (e.g. America/New_York) +bootstrap_timezone: Etc/UTC +# Email you want to be associated with the ACME account (used for TLS certificates via letsencrypt.org) +bootstrap_acme_email: +# Use the ACME production env when requesting the wildcard certificate, +# the default here is `false` which means ACME staging env will be used instead. +# This is to prevent being rate-limited. Update this option to `true` when you +# have verified the staging certificate works and then re-run `task configure` +# and push your changes to Github. +bootstrap_acme_production_enabled: false + +# Flux github webhook token (openssl rand -hex 12) +bootstrap_flux_github_webhook_token: + +# Cloudflare domain +bootstrap_cloudflare_domain: +# Cloudflare API Token (not API Key) +bootstrap_cloudflare_token: +# Cloudflared Account Tag (cat ~/.cloudflared/*.json | jq -r .AccountTag) +bootstrap_cloudflare_account_tag: +# Cloudflared Tunnel Secret (cat ~/.cloudflared/*.json | jq -r .TunnelSecret) +bootstrap_cloudflare_tunnel_secret: +# Cloudflared Tunnel ID (cat ~/.cloudflared/*.json | jq -r .TunnelID) +bootstrap_cloudflare_tunnel_id: + +# CIDR your nodes are on (e.g. 192.168.1.0/24) +bootstrap_node_cidr: +# The IP address to use with kube-vip, choose an available IP in your nodes network that is not being used +bootstrap_kube_vip_addr: +# The Load balancer IP for k8s_gateway, choose an available IP in your nodes network that is not being used +bootstrap_k8s_gateway_addr: +# The Load balancer IP for external ingress, choose an available IP in your nodes network that is not being used +bootstrap_external_ingress_addr: +# The Load balancer IP for internal ingress, choose an available IP in your nodes network that is not being used +bootstrap_internal_ingress_addr: + +# Keep the next three options default unless you know what you are doing +# (Advanced) Enable ipv6 +bootstrap_ipv6_enabled: false +# (Advanced) For ipv6 use format 10.42.0.0/16,fd78:c889:47fb:10::/60 +# /60 IPv6 block is enough for 16 nodes +bootstrap_cluster_cidr: 10.42.0.0/16 +# (Advanced) For ipv6 use format 10.43.0.0/16,fd78:c889:47fb:e0::/112 +bootstrap_service_cidr: 10.43.0.0/16 + +# Node information +bootstrap_nodes: + # Use only 1, 3 or more odd master nodes, recommended is 3 + master: + # - name: # name of the master node (must match [a-z0-9-]+) + # address: # ip address of the master node + # username: # ssh username of the master node + # password: # password of ssh username for the master node + # # external_address: # Only use when SSH is not reachable on the IP specified in 'address' field + # ... + worker: # set to [] or omit if no workers are needed + # - name: # name of the worker node (must match [a-z0-9-]+) + # address: # ip address of the worker node + # username: # ssh username of the worker node + # password: # password of ssh username for the worker node + # # external_address: # Only use when SSH is not reachable on the IP specified in 'address' field + # ... diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..af848aee --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +ansible==9.0.1 +ansible-lint==6.22.1 +# https://github.com/pyca/bcrypt/issues/684 +bcrypt==4.0.1 +jmespath==1.0.1 +netaddr==0.9.0 +openshift==0.13.2 +passlib==1.7.4 diff --git a/requirements.yaml b/requirements.yaml new file mode 100644 index 00000000..6aa38fe8 --- /dev/null +++ b/requirements.yaml @@ -0,0 +1,16 @@ +--- +collections: + - name: ansible.posix + version: 1.5.4 + - name: ansible.utils + version: 3.0.0 + - name: community.general + version: 8.0.2 + - name: community.sops + version: 1.6.7 + - name: kubernetes.core + version: 3.0.0 +roles: + - name: xanmanning.k3s + src: https://github.com/PyratLabs/ansible-role-k3s + version: v3.4.2