From ed97560276da23f1c9d10645bbd5ca66224b9046 Mon Sep 17 00:00:00 2001 From: Nandaja Varma Date: Thu, 9 Jun 2022 12:01:17 +0000 Subject: [PATCH] [installer] Add upgrade test werft flag --- .werft/build.ts | 78 +++++++------ .werft/installer-tests.ts | 55 +++++++-- .werft/jobs/build/job-config.ts | 88 +++++++------- .../jobs/build/self-hosted-upgrade-tests.ts | 38 ++++++ .werft/k3s-installer-tests.yaml | 33 +++++- .werft/run-sh-upgrade-tests-gke.yaml | 110 ++++++++++++++++++ install/infra/terraform/k3s/main.tf | 2 +- .../terraform/tools/external-dns/main.tf | 4 +- .../terraform/tools/external-dns/variables.tf | 7 +- install/tests/Makefile | 18 ++- install/tests/main.tf | 3 +- install/tests/manifests/kots-config.yaml | 3 + 12 files changed, 341 insertions(+), 98 deletions(-) create mode 100644 .werft/jobs/build/self-hosted-upgrade-tests.ts create mode 100644 .werft/run-sh-upgrade-tests-gke.yaml diff --git a/.werft/build.ts b/.werft/build.ts index 01273df7995b57..adf04cde273ff1 100644 --- a/.werft/build.ts +++ b/.werft/build.ts @@ -1,75 +1,81 @@ -import * as fs from 'fs'; -import { SpanStatusCode } from '@opentelemetry/api'; -import { Werft } from './util/werft'; -import { reportBuildFailureInSlack } from './util/slack'; -import * as Tracing from './observability/tracing' -import * as VM from './vm/vm' -import { buildAndPublish } from './jobs/build/build-and-publish'; -import { validateChanges } from './jobs/build/validate-changes'; -import { prepare } from './jobs/build/prepare'; -import { deployToPreviewEnvironment } from './jobs/build/deploy-to-preview-environment'; -import { triggerIntegrationTests } from './jobs/build/trigger-integration-tests'; -import { jobConfig } from './jobs/build/job-config'; -import { typecheckWerftJobs } from './jobs/build/typecheck-werft-jobs'; +import * as fs from "fs"; +import { SpanStatusCode } from "@opentelemetry/api"; +import { Werft } from "./util/werft"; +import { reportBuildFailureInSlack } from "./util/slack"; +import * as Tracing from "./observability/tracing"; +import * as VM from "./vm/vm"; +import { buildAndPublish } from "./jobs/build/build-and-publish"; +import { validateChanges } from "./jobs/build/validate-changes"; +import { prepare } from "./jobs/build/prepare"; +import { deployToPreviewEnvironment } from "./jobs/build/deploy-to-preview-environment"; +import { triggerIntegrationTests } from "./jobs/build/trigger-integration-tests"; +import { triggerUpgradeTests } from "./jobs/build/self-hosted-upgrade-tests"; +import { jobConfig } from "./jobs/build/job-config"; +import { typecheckWerftJobs } from "./jobs/build/typecheck-werft-jobs"; // Will be set once tracing has been initialized -let werft: Werft -const context: any = JSON.parse(fs.readFileSync('context.json').toString()); +let werft: Werft; +const context: any = JSON.parse(fs.readFileSync("context.json").toString()); Tracing.initialize() .then(() => { - werft = new Werft("build") + werft = new Werft("build"); }) .then(() => run(context)) .catch((err) => { werft.rootSpan.setStatus({ code: SpanStatusCode.ERROR, - message: err - }) + message: err, + }); - console.log('Error', err) + console.log("Error", err); if (context.Repository.ref === "refs/heads/main") { reportBuildFailureInSlack(context, err).catch((error: Error) => { - console.error("Failed to send message to Slack", error) + console.error("Failed to send message to Slack", error); }); } // Explicitly not using process.exit as we need to flush tracing, see tracing.js - process.exitCode = 1 + process.exitCode = 1; }) .finally(() => { - werft.phase("Stop kubectl port forwards", "Stopping kubectl port forwards") - VM.stopKubectlPortForwards() + werft.phase("Stop kubectl port forwards", "Stopping kubectl port forwards"); + VM.stopKubectlPortForwards(); - werft.phase("Flushing telemetry", "Flushing telemetry before stopping job") - werft.endAllSpans() - }) + werft.phase("Flushing telemetry", "Flushing telemetry before stopping job"); + werft.endAllSpans(); + }); async function run(context: any) { - const config = jobConfig(werft, context) + const config = jobConfig(werft, context); - await validateChanges(werft, config) - await prepare(werft, config) - await typecheckWerftJobs(werft) - await buildAndPublish(werft, config) + await validateChanges(werft, config); + await prepare(werft, config); + if (config.withUpgradeTests) { + // this will trigger an upgrade test on a self-hosted gitpod instance on a new cluster. + await triggerUpgradeTests(werft, config, context.Owner); + return; + } + await typecheckWerftJobs(werft); + await buildAndPublish(werft, config); if (config.noPreview) { werft.phase("deploy", "not deploying"); console.log("no-preview or publish-release is set"); - return + return; } try { - await deployToPreviewEnvironment(werft, config) + await deployToPreviewEnvironment(werft, config); } catch (e) { // We currently don't support concurrent deployments to the same preview environment. // Until we do we don't want errors to mark the main build as failed. if (config.mainBuild) { - return + return; } - throw e + throw e; } - await triggerIntegrationTests(werft, config, context.Owner) + await triggerIntegrationTests(werft, config, context.Owner); } diff --git a/.werft/installer-tests.ts b/.werft/installer-tests.ts index ba66df6bef78ff..7832d746c4c443 100644 --- a/.werft/installer-tests.ts +++ b/.werft/installer-tests.ts @@ -4,7 +4,10 @@ import { Werft } from "./util/werft"; const testConfig: string = process.argv.length > 2 ? process.argv[2] : "STANDARD_K3S_TEST"; // we can provide the version of the gitpod to install (eg: 2022.4.2) -const version: string = process.argv.length > 3 ? process.argv[3] : ""; +// "-" is the default value which will install the latest version +const version: string = process.argv.length > 3 ? process.argv[3] : "-"; + +const channel: string = process.argv.length > 4 ? process.argv[4] : "unstable"; const makefilePath: string = join("install/tests"); @@ -41,14 +44,19 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = { makeTarget: "managed-dns", description: "Sets up external-dns & cloudDNS config", }, + GENERATE_KOTS_CONFIG: { + phase: "generate-kots-config", + makeTarget: "generate-kots-config", + description: `Generate KOTS Config file`, + }, INSTALL_GITPOD_IGNORE_PREFLIGHTS: { phase: "install-gitpod-without-preflights", - makeTarget: `kots-install channel=unstable version=${version} preflights=false`, // this is a bit of a hack, for now we pass params like this + makeTarget: `kots-install channel=${channel} version=${version} preflights=false`, // this is a bit of a hack, for now we pass params like this description: "Install gitpod using kots community edition without preflights", }, INSTALL_GITPOD: { phase: "install-gitpod", - makeTarget: `kots-install channel=unstable version=${version} preflights=true`, + makeTarget: `kots-install channel=${channel} version=${version} preflights=true`, description: "Install gitpod using kots community edition", }, CHECK_INSTALLATION: { @@ -57,6 +65,11 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = { makeTarget: "check-gitpod-installation", description: "Check gitpod installation", }, + KOTS_UPGRADE: { + phase: "kots-upgrade", + makeTarget: "kots-uprgade", + description: "Upgrade Gitpod installation to latest version using KOTS CLI", + }, RUN_INTEGRATION_TESTS: { phase: "run-integration-tests", makeTarget: "run-tests", @@ -88,6 +101,7 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = { "STANDARD_GKE_CLUSTER", "CERT_MANAGER", "GCP_MANAGED_DNS", + "GENERATE_KOTS_CONFIG", "INSTALL_GITPOD", "CHECK_INSTALLATION", "RUN_INTEGRATION_TESTS", @@ -95,6 +109,20 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = { "DESTROY", ], }, + STANDARD_GKE_UPGRADE_TEST: { + DESCRIPTION: `Deploy Gitpod on GKE, and test upgrade from ${version} to latest version`, + PHASES: [ + "STANDARD_GKE_CLUSTER", + "CERT_MANAGER", + "GCP_MANAGED_DNS", + "GENERATE_KOTS_CONFIG", + "INSTALL_GITPOD", + "CHECK_INSTALLATION", + "KOTS_UPGRADE", + "CHECK_INSTALLATION", + "DESTROY", + ], + }, STANDARD_K3S_TEST: { DESCRIPTION: "Deploy Gitpod on a K3s cluster, created on a GCP instance," + @@ -102,7 +130,8 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = { PHASES: [ "STANDARD_K3S_CLUSTER_ON_GCP", "CERT_MANAGER", - "INSTALL_GITPOD_IGNORE_PREFLIGHTS", + "GENERATE_KOTS_CONFIG", + "INSTALL_GITPOD", "CHECK_INSTALLATION", "RUN_INTEGRATION_TESTS", "RESULTS", @@ -113,8 +142,9 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = { DESCRIPTION: "Create a SH Gitpod preview environment on a K3s cluster, created on a GCP instance", PHASES: [ "STANDARD_K3S_CLUSTER_ON_GCP", - "GCP_MANAGED_DNS", - "INSTALL_GITPOD_IGNORE_PREFLIGHTS", + "CERT_MANAGER", + "GENERATE_KOTS_CONFIG", + "INSTALL_GITPOD", "CHECK_INSTALLATION", "RESULTS", ], @@ -144,8 +174,9 @@ function getKubeconfig() { export async function installerTests(config: TestConfig) { console.log(config.DESCRIPTION); for (let phase of config.PHASES) { - const phaseSteps = INFRA_PHASES[phase]; - const ret = callMakeTargets(phaseSteps.phase, phaseSteps.description, phaseSteps.makeTarget); + const args = phase.split(" "); + const phaseSteps = INFRA_PHASES[args[0]]; + const ret = callMakeTargets(phaseSteps.phase, phaseSteps.description, phaseSteps.makeTarget, args.slice(1)); if (ret) { // there is not point in continuing if one stage fails // TODO: maybe add failable, phases @@ -154,8 +185,8 @@ export async function installerTests(config: TestConfig) { } } -function callMakeTargets(phase: string, description: string, makeTarget: string) { - werft.phase(phase, description); +function callMakeTargets(phase: string, description: string, makeTarget: string, args: string[]) { + werft.phase(phase, `${description} ${args}`); const response = exec(`make -C ${makefilePath} ${makeTarget}`, { slice: "call-make-target", dontCheckRc: true }); @@ -170,6 +201,10 @@ function callMakeTargets(phase: string, description: string, makeTarget: string) return response.code; } +function sample(stages: string[]): string { + return stages[Math.floor(Math.random() * stages.length)]; +} + function cleanup() { const phase = "destroy-infrastructure"; werft.phase(phase, "Destroying all the created resources"); diff --git a/.werft/jobs/build/job-config.ts b/.werft/jobs/build/job-config.ts index 791c8691ea367c..f0dd5d7c798dbd 100644 --- a/.werft/jobs/build/job-config.ts +++ b/.werft/jobs/build/job-config.ts @@ -3,32 +3,34 @@ import { Werft } from "../../util/werft"; import { previewNameFromBranchName } from "../../util/preview"; export interface JobConfig { - analytics: string + analytics: string; buildConfig: any; - cleanSlateDeployment: boolean - coverageOutput: string + cleanSlateDeployment: boolean; + coverageOutput: string; dontTest: boolean; dynamicCPULimits: boolean; - installEELicense: boolean - localAppVersion: string + installEELicense: boolean; + localAppVersion: string; mainBuild: boolean; noPreview: boolean; publishRelease: boolean; - publishToJBMarketplace: string - publishToNpm: string + publishToJBMarketplace: string; + publishToNpm: string; publishToKots: boolean; - retag: string + retag: string; storage: string; version: string; - withContrib: boolean + withContrib: boolean; withIntegrationTests: boolean; - withObservability: boolean - withPayment: boolean - withVM: boolean + withUpgradeTests: boolean; + fromVersion: string; + withObservability: boolean; + withPayment: boolean; + withVM: boolean; workspaceFeatureFlags: string[]; - previewEnvironment: PreviewEnvironmentConfig, - repository: Repository - observability: Observability + previewEnvironment: PreviewEnvironmentConfig; + repository: Repository; + observability: Observability; } export interface PreviewEnvironmentConfig { @@ -45,14 +47,14 @@ export interface Repository { export interface Observability { // The branch of gitpod-io/observability to use - branch: string + branch: string; } export function jobConfig(werft: Werft, context: any): JobConfig { - const sliceId = 'Parsing job configuration' - werft.phase('Job configuration') - werft.log(sliceId , "Parsing the job configuration") - const version = parseVersion(context) + const sliceId = "Parsing job configuration"; + werft.phase("Job configuration"); + werft.log(sliceId, "Parsing the job configuration"); + const version = parseVersion(context); const repo = `${context.Repository.host}/${context.Repository.owner}/${context.Repository.repo}`; const mainBuild = repo === "github.com/gitpod-io/gitpod" && context.Repository.ref.includes("refs/heads/main"); @@ -64,7 +66,7 @@ export function jobConfig(werft: Werft, context: any): JobConfig { if (!raw) { return []; } - return raw.split(",").map(e => e.trim()); + return raw.split(",").map((e) => e.trim()); })(); const coverageOutput = exec("mktemp -d", { silent: true }).stdout.trim(); @@ -76,13 +78,15 @@ export function jobConfig(werft: Werft, context: any): JobConfig { const noPreview = ("no-preview" in buildConfig && buildConfig["no-preview"] !== "false") || publishRelease; const storage = buildConfig["storage"] || ""; const withIntegrationTests = "with-integration-tests" in buildConfig && !mainBuild; + const withUpgradeTests = "with-upgrade-tests" in buildConfig && !mainBuild; + const fromVersion = withUpgradeTests ? buildConfig["from-version"] : ""; const publishToNpm = "publish-to-npm" in buildConfig || mainBuild; const publishToJBMarketplace = "publish-to-jb-marketplace" in buildConfig || mainBuild; const publishToKots = "publish-to-kots" in buildConfig; const analytics = buildConfig["analytics"]; - const localAppVersion = mainBuild || ("with-localapp-version" in buildConfig) ? version : "unknown"; - const retag = ("with-retag" in buildConfig) ? "" : "--dont-retag"; - const cleanSlateDeployment = mainBuild || ("with-clean-slate-deployment" in buildConfig); + const localAppVersion = mainBuild || "with-localapp-version" in buildConfig ? version : "unknown"; + const retag = "with-retag" in buildConfig ? "" : "--dont-retag"; + const cleanSlateDeployment = mainBuild || "with-clean-slate-deployment" in buildConfig; const installEELicense = !("without-ee-license" in buildConfig) || mainBuild; const withPayment = "with-payment" in buildConfig && !mainBuild; const withObservability = "with-observability" in buildConfig && !mainBuild; @@ -91,7 +95,7 @@ export function jobConfig(werft: Werft, context: any): JobConfig { repo: context.Repository.repo, ref: context.Repository.ref, branch: context.Repository.ref, - } + }; const refsPrefix = "refs/heads/"; if (repository.branch.startsWith(refsPrefix)) { repository.branch = repository.branch.substring(refsPrefix.length); @@ -99,16 +103,16 @@ export function jobConfig(werft: Werft, context: any): JobConfig { const withoutVM = "without-vm" in buildConfig; const withVM = !withoutVM || mainBuild; - const previewName = previewNameFromBranchName(repository.branch) + const previewName = previewNameFromBranchName(repository.branch); const previewEnvironmentNamespace = withVM ? `default` : `staging-${previewName}`; const previewEnvironment = { destname: previewName, - namespace: previewEnvironmentNamespace - } + namespace: previewEnvironmentNamespace, + }; const observability: Observability = { - branch: context.Annotations.withObservabilityBranch || "main" - } + branch: context.Annotations.withObservabilityBranch || "main", + }; const jobConfig = { analytics, @@ -117,6 +121,7 @@ export function jobConfig(werft: Werft, context: any): JobConfig { coverageOutput, dontTest, dynamicCPULimits, + fromVersion, installEELicense, localAppVersion, mainBuild, @@ -135,21 +140,24 @@ export function jobConfig(werft: Werft, context: any): JobConfig { withIntegrationTests, withObservability, withPayment, + withUpgradeTests, withVM, workspaceFeatureFlags, - } + }; werft.log("job config", JSON.stringify(jobConfig)); - const globalAttributes = Object.fromEntries(Object.entries(jobConfig).map((kv) => { - const [key, value] = kv - return [`werft.job.config.${key}`, value] - })) - globalAttributes['werft.job.config.branch'] = context.Repository.ref - werft.addAttributes(globalAttributes) + const globalAttributes = Object.fromEntries( + Object.entries(jobConfig).map((kv) => { + const [key, value] = kv; + return [`werft.job.config.${key}`, value]; + }), + ); + globalAttributes["werft.job.config.branch"] = context.Repository.ref; + werft.addAttributes(globalAttributes); - werft.done(sliceId) + werft.done(sliceId); - return jobConfig + return jobConfig; } function parseVersion(context: any) { @@ -163,5 +171,5 @@ function parseVersion(context: any) { if (version.substr(0, PREFIX_TO_STRIP.length) === PREFIX_TO_STRIP) { version = version.substr(PREFIX_TO_STRIP.length); } - return version + return version; } diff --git a/.werft/jobs/build/self-hosted-upgrade-tests.ts b/.werft/jobs/build/self-hosted-upgrade-tests.ts new file mode 100644 index 00000000000000..5c735000dcc8cc --- /dev/null +++ b/.werft/jobs/build/self-hosted-upgrade-tests.ts @@ -0,0 +1,38 @@ +import { exec } from "../../util/shell"; +import { Werft } from "../../util/werft"; +import { JobConfig } from "./job-config"; + +const phases = { + TRIGGER_UPGRADE_TESTS: "trigger self-hosted upgrade tests", +}; + +/** + * Trigger self hosted upgrade tests + */ +export async function triggerUpgradeTests(werft: Werft, config: JobConfig, username: string) { + werft.phase(phases.TRIGGER_UPGRADE_TESTS, "Trigger upgrade tests on self-hosted gitpod"); + + if (!config.withUpgradeTests || !config.fromVersion) { + werft.log(phases.TRIGGER_UPGRADE_TESTS, "Skipped upgrade tests"); + werft.done(phases.TRIGGER_UPGRADE_TESTS); + return; + } + + try { + exec(`git config --global user.name "${username}"`); + const annotation = `-a fromVersion=${config.fromVersion}`; + exec( + `WERFT_CREDENTIAL_HELPER="" KUBECONFIG=/workspace/gitpod/kubeconfigs/core-dev werft run --remote-job-path .werft/run-sh-upgrade-tests-gke.yaml ${annotation} github`, + { + slice: phases.TRIGGER_UPGRADE_TESTS, + }, + ).trim(); + + werft.done(phases.TRIGGER_UPGRADE_TESTS); + } catch (err) { + if (!config.mainBuild) { + werft.fail(phases.TRIGGER_UPGRADE_TESTS, err); + } + exec("exit 0"); + } +} diff --git a/.werft/k3s-installer-tests.yaml b/.werft/k3s-installer-tests.yaml index e275daa4448291..19b5bdfe601847 100644 --- a/.werft/k3s-installer-tests.yaml +++ b/.werft/k3s-installer-tests.yaml @@ -1,4 +1,17 @@ # debug using `werft run github -f -s .werft/installer-tests.ts -j .werft/k3s-installer-tests.yaml -a debug=true` +args: +- name: destroy + desc: "Should the setup be destroyed or not" + required: false + default: true +- name: channel + desc: "Replicated channel to use" + required: false + default: "" +- name: version + desc: "Version of gitpod to install initially" + required: false + default: "" pod: serviceAccount: werft affinity: @@ -63,7 +76,25 @@ pod: (cd .werft && yarn install && mv node_modules ..) | werft log slice prep printf '{{ toJson . }}' > context.json - npx ts-node .werft/installer-tests.ts "STANDARD_K3S_TEST" + export DESTROY="{{ .Annotations.destroy }}" + + if [ $DESTROY == "false" ]; then + export STAGE=STANDARD_K3S_PREVIEW + else + export STAGE=STANDARD_K3S_TEST + fi + + export VERSION="{{ .Annotations.version }}" + if [[ "$VERSION" == "" ]]; then + VERSION="" + fi + + export CHANNEL="{{ .Annotations.channel }}" + if [[ "$CHANNEL" == "" ]]; then + CHANNEL="" + fi + + npx ts-node .werft/installer-tests.ts ${STAGE} ${VERSION} ${CHANNEL} # The bit below makes this a cron job plugins: cron: "15 3 * * *" diff --git a/.werft/run-sh-upgrade-tests-gke.yaml b/.werft/run-sh-upgrade-tests-gke.yaml new file mode 100644 index 00000000000000..d4f5f48501ae68 --- /dev/null +++ b/.werft/run-sh-upgrade-tests-gke.yaml @@ -0,0 +1,110 @@ +# debug using `werft run github -f -s .werft/installer-tests.ts -j .werft/run-sh-upgrade-tests-gke.yaml -a debug=true` +args: +- name: fromVersion + desc: "The version from which you want to test upgrading to the latest" + required: true +- name: channel + desc: "Replicated channel to use" + required: false +pod: + serviceAccount: werft + nodeSelector: + dev/workload: builds + imagePullSecrets: + - name: eu-gcr-io-pull-secret + volumes: + - name: gcp-sa + secret: + secretName: gcp-sa-gitpod-dev-deployer + - name: config + emptyDir: {} + - name: sh-playground-sa-perm + secret: + secretName: sh-playground-sa-perm + - name: sh-playground-dns-perm + secret: + secretName: sh-playground-dns-perm + initContainers: + - name: gcloud + image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:af-add-gh-cli-to-dev-image.0 + workingDir: /workspace + imagePullPolicy: IfNotPresent + volumeMounts: + - name: gcp-sa + mountPath: /mnt/secrets/gcp-sa + readOnly: true + - name: config + mountPath: /config + readOnly: false + command: + - bash + - -c + - | + + echo "[prep] preparing config." + + gcloud auth activate-service-account --key-file /mnt/secrets/gcp-sa/service-account.json + cp -R /home/gitpod/.config/gcloud /config/gcloud + cp /home/gitpod/.kube/config /config/kubeconfig + + echo "[prep] copied config..." + + securityContext: + runAsUser: 0 + containers: + - name: nightly-test + image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:af-add-gh-cli-to-dev-image.0 + workingDir: /workspace + imagePullPolicy: Always + volumeMounts: + - name: sh-playground-sa-perm + mountPath: /mnt/secrets/sh-playground-sa-perm + - name: sh-playground-dns-perm # this sa is used for the DNS management + mountPath: /mnt/secrets/sh-playground-dns-perm + env: + - name: config + mountPath: /config + readOnly: false + - name: WERFT_HOST + value: "werft.werft.svc.cluster.local:7777" + - name: GOOGLE_APPLICATION_CREDENTIALS + value: "/mnt/secrets/sh-playground-sa-perm/sh-sa.json" + - name: TF_VAR_sa_creds + value: "/mnt/secrets/sh-playground-sa-perm/sh-sa.json" + - name: TF_VAR_dns_sa_creds + value: "/mnt/secrets/sh-playground-dns-perm/sh-dns-sa.json" + - name: WERFT_K8S_NAMESPACE + value: "werft" + - name: WERFT_K8S_LABEL + value: "component=werft" + - name: NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + command: + - bash + - -c + - | + sleep 1 + set -Eeuo pipefail + + sudo chown -R gitpod:gitpod /workspace + sudo apt update && apt install gettext-base + export WERFT_CREDENTIAL_HELPER=/workspace/dev/preview/werft-credential-helper.sh + + export TF_VAR_TEST_ID=$(echo $RANDOM | md5sum | head -c 5; echo) + + (cd .werft && yarn install && mv node_modules ..) | werft log slice prep + printf '{{ toJson . }}' > context.json + + FROM_VERSION="{{ .Annotations.fromVersion }}" + + export CHANNEL="{{ .Annotations.channel }}" + if [[ "$CHANNEL" == "" ]]; then + CHANNEL="beta" + fi + + npx ts-node .werft/installer-tests.ts "STANDARD_GKE_UPGRADE_TEST" ${FROM_VERSION} ${CHANNEL} +# The bit below makes this a cron job +# plugins: +# cron: "15 4 * * *" diff --git a/install/infra/terraform/k3s/main.tf b/install/infra/terraform/k3s/main.tf index bc6790a0b0c2ec..784be84a8eb2ed 100644 --- a/install/infra/terraform/k3s/main.tf +++ b/install/infra/terraform/k3s/main.tf @@ -27,7 +27,7 @@ provider "google" { } resource "google_service_account" "gcp_instance" { - account_id = "gcp-k3s-compute" + account_id = "sa-${var.name}" display_name = "Service Account" } diff --git a/install/infra/terraform/tools/external-dns/main.tf b/install/infra/terraform/tools/external-dns/main.tf index c3968e68cf265e..bb719f259abc5e 100644 --- a/install/infra/terraform/tools/external-dns/main.tf +++ b/install/infra/terraform/tools/external-dns/main.tf @@ -68,7 +68,7 @@ resource "helm_release" "external-dns" { value = "external-dns" } set { - name = "policy" - value = "sync" + name = "txt-owner-id" + value = var.txt_owner_id } } diff --git a/install/infra/terraform/tools/external-dns/variables.tf b/install/infra/terraform/tools/external-dns/variables.tf index b7ff458562b68d..86046e5029bc2f 100644 --- a/install/infra/terraform/tools/external-dns/variables.tf +++ b/install/infra/terraform/tools/external-dns/variables.tf @@ -15,9 +15,14 @@ variable "gcp_region" { variable "gcp_zone" { description = "Google cloud Zone to perform operations in" - default = "europe-west1-b" + default = "europe-west1-d" } variable "credentials" { description = "Path to the JSON file storing Google service account credentials" } + +variable "txt_owner_id" { + description = "A unique value for the external-dns setup" + default = "gitpod" +} diff --git a/install/tests/Makefile b/install/tests/Makefile index 931afcbf6181c4..f45bf30303ad04 100644 --- a/install/tests/Makefile +++ b/install/tests/Makefile @@ -56,6 +56,12 @@ get-kubeconfig: ${KUBECONFIG} KOTS_KONFIG := "./manifests/kots-config.yaml" + +.PHONY: +## generate-kots-config: Generate the kots config based on test config +generate-kots-config: + envsubst < ${KOTS_KONFIG} > tmp_config.yml + license_community_beta := "../licenses/Community (Beta).yaml" license_community_stable := "../licenses/Community.yaml" license_community_unstable := "../licenses/Community (Unstable).yaml" @@ -68,9 +74,8 @@ channel ?= unstable version ?= - kots-install: version-flag = $(if $(version:-=),--app-version-label=$(version),) kots-install: preflight-flag = $(if $(preflights:true=),--skip-preflights,) -kots-install: license-file = $(license_community_$(channel)) +kots-install: license-file = $(if $(license_community_$(channel)),$(license_community_$(channel)),"../licenses/$(channel).yaml") kots-install: install-kots-cli - envsubst < ${KOTS_KONFIG} > tmp_config.yml kubectl kots install gitpod/${channel} \ --skip-rbac-check ${version-flag} ${preflight-flag} \ --namespace gitpod --kubeconfig=${KUBECONFIG} \ @@ -92,6 +97,10 @@ check-gitpod-installation: delete-cm-setup check-kots-app check-env-sub-domain run-tests: ./tests.sh ${KUBECONFIG} +kots-uprgade: + @echo "Upgrade gitpod KOTS app to latest" + kubectl kots upstream upgrade --kubeconfig=${KUBECONFIG} gitpod -n gitpod --deploy + cleanup: terraform workspace select $(TF_VAR_TEST_ID) which ${KUBECONFIG} && terraform destroy -target=module.externaldns -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file" @@ -101,11 +110,8 @@ cleanup: get-results: @echo "If you have gotten this far, it means your setup succeeded" - @echo - @echo "URL of your setup is: "https://$(TF_VAR_TEST_ID).gitpod-self-hosted.com"" - @echo + @echo "The IP address of you setup is "$(TF_VAR_TEST_ID).gitpod-self-hosted.com"" @echo "Following is the KUBECONFIG you can use to connect to the cluster:" - @echo @cat ${KUBECONFIG} list-state: diff --git a/install/tests/main.tf b/install/tests/main.tf index beef1a67194862..8e2e4949b2d503 100644 --- a/install/tests/main.tf +++ b/install/tests/main.tf @@ -24,7 +24,7 @@ module "gke" { credentials = var.sa_creds kubeconfig = var.kubeconfig region = "europe-west1" - zone = "europe-west1-b" + zone = "europe-west1-d" } module "k3s" { @@ -55,4 +55,5 @@ module "externaldns" { kubeconfig = var.kubeconfig credentials = var.dns_sa_creds + txt_owner_id = var.TEST_ID } diff --git a/install/tests/manifests/kots-config.yaml b/install/tests/manifests/kots-config.yaml index 7e68ea33566f06..f6f7fd0383a899 100644 --- a/install/tests/manifests/kots-config.yaml +++ b/install/tests/manifests/kots-config.yaml @@ -5,3 +5,6 @@ spec: domain: value: "${TF_VAR_TEST_ID}.gitpod-self-hosted.com" data: "domain" + ssh_gateway: + value: "1" + data: "ssh_gateway"