Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

remove core-dev preview environment options #10795

Merged
merged 2 commits into from
Jun 28, 2022
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 0 additions & 6 deletions .werft/build.yaml
Original file line number Diff line number Diff line change
@@ -23,9 +23,6 @@ pod:
- name: gcp-sa-release
secret:
secretName: gcp-sa-gitpod-release-deployer
- name: gpsh-coredev-license
secret:
secretName: gpsh-coredev-license
- name: prometheus-remote-write-auth
secret:
secretName: prometheus-remote-write-auth
@@ -95,9 +92,6 @@ pod:
- name: gcp-sa-release
liam-j-bennett marked this conversation as resolved.
Show resolved Hide resolved
mountPath: /mnt/secrets/gcp-sa-release
readOnly: true
- name: gpsh-coredev-license
mountPath: /mnt/secrets/gpsh-coredev
readOnly: true
- name: gpsh-harvester-license
mountPath: /mnt/secrets/gpsh-harvester
readOnly: true
247 changes: 82 additions & 165 deletions .werft/jobs/build/deploy-to-preview-environment.ts

Large diffs are not rendered by default.

5 changes: 2 additions & 3 deletions .werft/jobs/build/installer/installer.ts
Original file line number Diff line number Diff line change
@@ -33,7 +33,6 @@ export type InstallerOptions = {
deploymentNamespace: string
analytics: Analytics
withEELicense: boolean
withVM: boolean
workspaceFeatureFlags: string[]
gitpodDaemonsetPorts: GitpodDaemonsetPorts
smithToken: string
@@ -239,7 +238,7 @@ EOF`)
private configureLicense(slice: string): void {
if (this.options.withEELicense) {
// Previews in core-dev and harvester use different domain, which requires different licenses.
exec(`cp /mnt/secrets/gpsh-${this.options.withVM ? 'harvester' : 'coredev'}/license /tmp/license`, { slice: slice });
exec(`cp /mnt/secrets/gpsh-harvester/license /tmp/license`, { slice: slice });
// post-process.sh looks for /tmp/license, and if it exists, adds it to the configmap
} else {
exec(`touch /tmp/license`, { slice: slice });
@@ -279,7 +278,7 @@ EOF`)

private process(slice: string): void {
const nodepoolIndex = getNodePoolIndex(this.options.deploymentNamespace);
const flags = this.options.withVM ? "WITH_VM=true " : ""
const flags = "WITH_VM=true "

exec(`${flags}./.werft/jobs/build/installer/post-process.sh ${this.options.gitpodDaemonsetPorts.registryFacade} ${this.options.gitpodDaemonsetPorts.wsDaemon} ${nodepoolIndex} ${this.options.previewName} ${this.options.smithToken}`, { slice: slice });
}
8 changes: 2 additions & 6 deletions .werft/jobs/build/job-config.ts
Original file line number Diff line number Diff line change
@@ -26,7 +26,6 @@ export interface JobConfig {
fromVersion: string;
withObservability: boolean;
withPayment: boolean;
withVM: boolean;
workspaceFeatureFlags: string[];
previewEnvironment: PreviewEnvironmentConfig;
repository: Repository;
@@ -100,11 +99,9 @@ export function jobConfig(werft: Werft, context: any): JobConfig {
if (repository.branch.startsWith(refsPrefix)) {
repository.branch = repository.branch.substring(refsPrefix.length);
}
const withoutVM = "without-vm" in buildConfig;
const withVM = !withoutVM || mainBuild;

const previewName = previewNameFromBranchName(repository.branch);
const previewEnvironmentNamespace = withVM ? `default` : `staging-${previewName}`;
const previewName = previewNameFromBranchName(repository.branch)
const previewEnvironmentNamespace = `default`;
const previewEnvironment = {
destname: previewName,
namespace: previewEnvironmentNamespace,
@@ -141,7 +138,6 @@ export function jobConfig(werft: Werft, context: any): JobConfig {
withObservability,
withPayment,
withUpgradeTests,
withVM,
workspaceFeatureFlags,
};

96 changes: 51 additions & 45 deletions .werft/jobs/build/prepare.ts
Original file line number Diff line number Diff line change
@@ -1,105 +1,111 @@
import { previewNameFromBranchName } from '../../util/preview';
import { exec } from '../../util/shell';
import { previewNameFromBranchName } from "../../util/preview";
import { exec } from "../../util/shell";
import { Werft } from "../../util/werft";
import * as VM from '../../vm/vm'
import * as VM from "../../vm/vm";
import { CORE_DEV_KUBECONFIG_PATH, GCLOUD_SERVICE_ACCOUNT_PATH, HARVESTER_KUBECONFIG_PATH } from "./const";
import { issueMetaCerts } from './deploy-to-preview-environment';
import { JobConfig } from './job-config';
import * as Manifests from '../../vm/manifests';
import { issueMetaCerts } from "./deploy-to-preview-environment";
import { JobConfig } from "./job-config";
import * as Manifests from "../../vm/manifests";

const phaseName = "prepare";
const prepareSlices = {
CONFIGURE_CORE_DEV: "Configuring core-dev access.",
BOOT_VM: "Booting VM.",
ISSUE_CERTIFICATES: "Issuing certificates for the preview."
}
ISSUE_CERTIFICATES: "Issuing certificates for the preview.",
};

export async function prepare(werft: Werft, config: JobConfig) {
werft.phase(phaseName);
try {
werft.log(prepareSlices.CONFIGURE_CORE_DEV, prepareSlices.CONFIGURE_CORE_DEV)
activateCoreDevServiceAccount()
configureDocker()
configureStaticClustersAccess()
werft.done(prepareSlices.CONFIGURE_CORE_DEV)

await issueCertificate(werft, config)
decideHarvesterVMCreation(werft, config)
werft.log(prepareSlices.CONFIGURE_CORE_DEV, prepareSlices.CONFIGURE_CORE_DEV);
activateCoreDevServiceAccount();
configureDocker();
configureStaticClustersAccess();
werft.done(prepareSlices.CONFIGURE_CORE_DEV);

await issueCertificate(werft, config);
decideHarvesterVMCreation(werft, config);
} catch (err) {
werft.fail(phaseName, err);
}
werft.done(phaseName);
}

function activateCoreDevServiceAccount() {
const rc = exec(`gcloud auth activate-service-account --key-file "${GCLOUD_SERVICE_ACCOUNT_PATH}"`, { slice: prepareSlices.CONFIGURE_CORE_DEV }).code;
const rc = exec(`gcloud auth activate-service-account --key-file "${GCLOUD_SERVICE_ACCOUNT_PATH}"`, {
slice: prepareSlices.CONFIGURE_CORE_DEV,
}).code;

if (rc != 0) {
throw new Error("Failed to activate core-dev service account.")
throw new Error("Failed to activate core-dev service account.");
}
}

function configureDocker() {
const rcDocker = exec("gcloud auth configure-docker --quiet", { slice: prepareSlices.CONFIGURE_CORE_DEV }).code;
const rcDockerRegistry = exec("gcloud auth configure-docker europe-docker.pkg.dev --quiet", { slice: prepareSlices.CONFIGURE_CORE_DEV }).code;
const rcDockerRegistry = exec("gcloud auth configure-docker europe-docker.pkg.dev --quiet", {
slice: prepareSlices.CONFIGURE_CORE_DEV,
}).code;

if (rcDocker != 0 || rcDockerRegistry != 0) {
throw new Error("Failed to configure docker with gcloud.")
throw new Error("Failed to configure docker with gcloud.");
}
}

function configureStaticClustersAccess() {
const rcCoreDev = exec(`KUBECONFIG=${CORE_DEV_KUBECONFIG_PATH} gcloud container clusters get-credentials core-dev --zone europe-west1-b --project gitpod-core-dev`, { slice: prepareSlices.CONFIGURE_CORE_DEV }).code;
const rcCoreDev = exec(
`KUBECONFIG=${CORE_DEV_KUBECONFIG_PATH} gcloud container clusters get-credentials core-dev --zone europe-west1-b --project gitpod-core-dev`,
{ slice: prepareSlices.CONFIGURE_CORE_DEV },
).code;
if (rcCoreDev != 0) {
throw new Error("Failed to get core-dev kubeconfig credentials.")
throw new Error("Failed to get core-dev kubeconfig credentials.");
}

const rcHarvester = exec(`cp /mnt/secrets/harvester-kubeconfig/harvester-kubeconfig.yml ${HARVESTER_KUBECONFIG_PATH}`, { slice: prepareSlices.CONFIGURE_CORE_DEV }).code;
const rcHarvester = exec(
`cp /mnt/secrets/harvester-kubeconfig/harvester-kubeconfig.yml ${HARVESTER_KUBECONFIG_PATH}`,
{ slice: prepareSlices.CONFIGURE_CORE_DEV },
).code;

if (rcHarvester != 0) {
throw new Error("Failed to get Harvester kubeconfig credentials.")
throw new Error("Failed to get Harvester kubeconfig credentials.");
}
}

async function issueCertificate(werft: Werft, config: JobConfig) {
const certName = config.withVM ? `harvester-${previewNameFromBranchName(config.repository.branch)}` : `staging-${previewNameFromBranchName(config.repository.branch)}`
const domain = config.withVM ? `${config.previewEnvironment.destname}.preview.gitpod-dev.com` : `${config.previewEnvironment.destname}.staging.gitpod-dev.com`;
const certName = `harvester-${previewNameFromBranchName(config.repository.branch)}`;
const domain = `${config.previewEnvironment.destname}.preview.gitpod-dev.com`;

werft.log(prepareSlices.ISSUE_CERTIFICATES, prepareSlices.ISSUE_CERTIFICATES)
await issueMetaCerts(werft, certName, "certs", domain, config.withVM, prepareSlices.ISSUE_CERTIFICATES)
werft.done(prepareSlices.ISSUE_CERTIFICATES)
werft.log(prepareSlices.ISSUE_CERTIFICATES, prepareSlices.ISSUE_CERTIFICATES);
await issueMetaCerts(werft, certName, "certs", domain, prepareSlices.ISSUE_CERTIFICATES);
werft.done(prepareSlices.ISSUE_CERTIFICATES);
}

function decideHarvesterVMCreation(werft: Werft, config: JobConfig) {
if (shouldCreateVM(config)) {
createVM(werft, config)
} else {
werft.currentPhaseSpan.setAttribute("preview.created_vm", false)
}
if (config.withVM) {
applyLoadBalancer({ name: config.previewEnvironment.destname })
createVM(werft, config);
}
werft.done(prepareSlices.BOOT_VM)
applyLoadBalancer({ name: config.previewEnvironment.destname });
werft.done(prepareSlices.BOOT_VM);
}

function shouldCreateVM(config: JobConfig) {
return config.withVM && config.withPreview && (
!VM.vmExists({ name: config.previewEnvironment.destname }) ||
config.cleanSlateDeployment
)
return (
config.withPreview &&
(!VM.vmExists({ name: config.previewEnvironment.destname }) || config.cleanSlateDeployment)
);
}

// createVM only triggers the VM creation.
// Readiness is not guaranted.
function createVM(werft: Werft, config: JobConfig) {
if (config.cleanSlateDeployment) {
werft.log(prepareSlices.BOOT_VM, "Cleaning previously created VM")
VM.deleteVM({ name: config.previewEnvironment.destname })
werft.log(prepareSlices.BOOT_VM, "Cleaning previously created VM");
VM.deleteVM({ name: config.previewEnvironment.destname });
}

werft.log(prepareSlices.BOOT_VM, 'Creating VM')
VM.startVM({ name: config.previewEnvironment.destname })
werft.currentPhaseSpan.setAttribute("preview.created_vm", true)
werft.log(prepareSlices.BOOT_VM, "Creating VM");
VM.startVM({ name: config.previewEnvironment.destname });
werft.currentPhaseSpan.setAttribute("preview.created_vm", true);
}

function applyLoadBalancer(option: { name: string }) {
22 changes: 8 additions & 14 deletions .werft/observability/monitoring-satellite.ts
Original file line number Diff line number Diff line change
@@ -12,7 +12,6 @@ type MonitoringSatelliteInstallerOptions = {
previewName: string;
previewDomain: string;
stackdriverServiceAccount: any;
withVM: boolean;
};

const sliceName = "observability";
@@ -29,7 +28,6 @@ export class MonitoringSatelliteInstaller {
branch,
satelliteNamespace,
stackdriverServiceAccount,
withVM,
previewDomain,
previewName,
nodeExporterPort,
@@ -68,7 +66,7 @@ export class MonitoringSatelliteInstaller {
domain: '${previewDomain}',
nodeExporterPort: ${nodeExporterPort},
},
${withVM ? "" : "nodeAffinity: { nodeSelector: { 'gitpod.io/workload_services': 'true' }, },"}
${"nodeAffinity: { nodeSelector: { 'gitpod.io/workload_services': 'true' }, },"}
liam-j-bennett marked this conversation as resolved.
Show resolved Hide resolved
stackdriver: {
defaultProject: '${stackdriverServiceAccount.project_id}',
clientEmail: '${stackdriverServiceAccount.client_email}',
@@ -104,9 +102,7 @@ export class MonitoringSatelliteInstaller {

werft.log(sliceName, "rendering YAML files");
exec(jsonnetRenderCmd, { silent: true });
if (withVM) {
this.postProcessManifests();
}
this.postProcessManifests();

this.ensureCorrectInstallationOrder()
this.deployGitpodServiceMonitors();
@@ -153,14 +149,12 @@ export class MonitoringSatelliteInstaller {

// core-dev is just too unstable for node-exporter
// we don't guarantee that it will run at all
if (this.options.withVM) {
checks.push(
exec(
`kubectl --kubeconfig ${kubeconfigPath} rollout status -n ${satelliteNamespace} daemonset node-exporter`,
{ slice: sliceName, async: true },
),
);
}
checks.push(
exec(
`kubectl --kubeconfig ${kubeconfigPath} rollout status -n ${satelliteNamespace} daemonset node-exporter`,
{ slice: sliceName, async: true },
),
);

await Promise.all(checks);
}
Loading