diff --git a/cmd/app.go b/cmd/app.go index 5d1ebc04..14f2a9b6 100644 --- a/cmd/app.go +++ b/cmd/app.go @@ -113,6 +113,7 @@ func createBuilder(provider AppStartArguments) *builder.ProcessorBuilder { processor.NewConfigRegionsProcessorFactory(), processor.NewConfigStorageClassesProcessorFactory(), processor.NewSourceProjectGetProcessorFactory(provider.SourceGCPProjectProvider, provider.TargetPrincipalForProjectProvider), + processor.NewTrashcanCleanUpProcessorFactory(provider.TargetPrincipalForProjectProvider, provider.SecretProvider), ) } diff --git a/cron.yaml b/cron.yaml index 4242251b..f60c641c 100644 --- a/cron.yaml +++ b/cron.yaml @@ -20,6 +20,9 @@ cron: - description: "check backup status" url: /api/tasks/check_backups_status schedule: every 15 minutes from 00:08 to 23:58 + - description: "cleanup trashcans" + url: /api/tasks/cleanup_trashcans + schedule: every 60 minutes from 00:08 to 23:58 - description: "check app health status" url: /_ah/health schedule: every 1 minutes \ No newline at end of file diff --git a/frontend/src/components/BackupViewDialog.vue b/frontend/src/components/BackupViewDialog.vue index 6b9b459e..e5834a4a 100644 --- a/frontend/src/components/BackupViewDialog.vue +++ b/frontend/src/components/BackupViewDialog.vue @@ -1,11 +1,13 @@ + diff --git a/frontend/src/components/common/ConfirmDialog.vue b/frontend/src/components/common/ConfirmDialog.vue new file mode 100644 index 00000000..7fa58743 --- /dev/null +++ b/frontend/src/components/common/ConfirmDialog.vue @@ -0,0 +1,100 @@ + + + + + diff --git a/frontend/src/models/api/core/ApiError.ts b/frontend/src/models/api/core/ApiError.ts index d6b8fcc3..ec7b16af 100644 --- a/frontend/src/models/api/core/ApiError.ts +++ b/frontend/src/models/api/core/ApiError.ts @@ -1,4 +1,4 @@ -/* generated using openapi-typescript-codegen -- do no edit */ +/* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ diff --git a/frontend/src/models/api/core/ApiRequestOptions.ts b/frontend/src/models/api/core/ApiRequestOptions.ts index c19adcc9..93143c3c 100644 --- a/frontend/src/models/api/core/ApiRequestOptions.ts +++ b/frontend/src/models/api/core/ApiRequestOptions.ts @@ -1,4 +1,4 @@ -/* generated using openapi-typescript-codegen -- do no edit */ +/* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ diff --git a/frontend/src/models/api/core/ApiResult.ts b/frontend/src/models/api/core/ApiResult.ts index ad8fef2b..ee1126e2 100644 --- a/frontend/src/models/api/core/ApiResult.ts +++ b/frontend/src/models/api/core/ApiResult.ts @@ -1,4 +1,4 @@ -/* generated using openapi-typescript-codegen -- do no edit */ +/* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ diff --git a/frontend/src/models/api/core/CancelablePromise.ts b/frontend/src/models/api/core/CancelablePromise.ts index 55fef851..d70de929 100644 --- a/frontend/src/models/api/core/CancelablePromise.ts +++ b/frontend/src/models/api/core/CancelablePromise.ts @@ -1,4 +1,4 @@ -/* generated using openapi-typescript-codegen -- do no edit */ +/* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ @@ -51,7 +51,7 @@ export class CancelablePromise implements Promise { return; } this.#isResolved = true; - this.#resolve?.(value); + if (this.#resolve) this.#resolve(value); }; const onReject = (reason?: any): void => { @@ -59,7 +59,7 @@ export class CancelablePromise implements Promise { return; } this.#isRejected = true; - this.#reject?.(reason); + if (this.#reject) this.#reject(reason); }; const onCancel = (cancelHandler: () => void): void => { @@ -85,9 +85,9 @@ export class CancelablePromise implements Promise { }); } - get [Symbol.toStringTag]() { - return "Cancellable Promise"; - } + get [Symbol.toStringTag]() { + return "Cancellable Promise"; + } public then( onFulfilled?: ((value: T) => TResult1 | PromiseLike) | null, @@ -122,7 +122,7 @@ export class CancelablePromise implements Promise { } } this.#cancelHandlers.length = 0; - this.#reject?.(new CancelError('Request aborted')); + if (this.#reject) this.#reject(new CancelError('Request aborted')); } public get isCancelled(): boolean { diff --git a/frontend/src/models/api/core/OpenAPI.ts b/frontend/src/models/api/core/OpenAPI.ts index 03d495a4..2745d347 100644 --- a/frontend/src/models/api/core/OpenAPI.ts +++ b/frontend/src/models/api/core/OpenAPI.ts @@ -1,4 +1,4 @@ -/* generated using openapi-typescript-codegen -- do no edit */ +/* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ diff --git a/frontend/src/models/api/core/request.ts b/frontend/src/models/api/core/request.ts index b018a07c..f83d7119 100644 --- a/frontend/src/models/api/core/request.ts +++ b/frontend/src/models/api/core/request.ts @@ -1,4 +1,4 @@ -/* generated using openapi-typescript-codegen -- do no edit */ +/* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ @@ -137,10 +137,12 @@ export const resolve = async (options: ApiRequestOptions, resolver?: T | Reso }; export const getHeaders = async (config: OpenAPIConfig, options: ApiRequestOptions): Promise => { - const token = await resolve(options, config.TOKEN); - const username = await resolve(options, config.USERNAME); - const password = await resolve(options, config.PASSWORD); - const additionalHeaders = await resolve(options, config.HEADERS); + const [token, username, password, additionalHeaders] = await Promise.all([ + resolve(options, config.TOKEN), + resolve(options, config.USERNAME), + resolve(options, config.PASSWORD), + resolve(options, config.HEADERS), + ]); const headers = Object.entries({ Accept: 'application/json', @@ -162,7 +164,7 @@ export const getHeaders = async (config: OpenAPIConfig, options: ApiRequestOptio headers['Authorization'] = `Basic ${credentials}`; } - if (options.body) { + if (options.body !== undefined) { if (options.mediaType) { headers['Content-Type'] = options.mediaType; } else if (isBlob(options.body)) { diff --git a/frontend/src/models/api/index.ts b/frontend/src/models/api/index.ts index b15c82f0..88f02195 100644 --- a/frontend/src/models/api/index.ts +++ b/frontend/src/models/api/index.ts @@ -1,4 +1,4 @@ -/* generated using openapi-typescript-codegen -- do no edit */ +/* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ @@ -25,6 +25,7 @@ export { Role } from './models/Role'; export type { SnapshotOptions } from './models/SnapshotOptions'; export type { SourceProject } from './models/SourceProject'; export type { TargetOptions } from './models/TargetOptions'; +export { TrashcanCleanupStatus } from './models/TrashcanCleanupStatus'; export type { UpdateRequest } from './models/UpdateRequest'; export type { UserResponse } from './models/UserResponse'; diff --git a/frontend/src/models/api/models/AvailabilityClass.ts b/frontend/src/models/api/models/AvailabilityClass.ts index 0853601c..d9d37fc7 100644 --- a/frontend/src/models/api/models/AvailabilityClass.ts +++ b/frontend/src/models/api/models/AvailabilityClass.ts @@ -1,8 +1,7 @@ -/* generated using openapi-typescript-codegen -- do no edit */ +/* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ - export enum AvailabilityClass { /** * A1 Irrelevant - A recovery test SHOULD be conducted after changes to the backup process. diff --git a/frontend/src/models/api/models/Backup.ts b/frontend/src/models/api/models/Backup.ts index d833df4e..5f94c3ee 100644 --- a/frontend/src/models/api/models/Backup.ts +++ b/frontend/src/models/api/models/Backup.ts @@ -1,8 +1,7 @@ -/* generated using openapi-typescript-codegen -- do no edit */ +/* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ - import type { AvailabilityClass } from './AvailabilityClass'; import type { BackupStatus } from './BackupStatus'; import type { BackupStrategy } from './BackupStrategy'; @@ -15,7 +14,7 @@ import type { RecoveryPointObjective } from './RecoveryPointObjective'; import type { RecoveryTimeObjective } from './RecoveryTimeObjective'; import type { SnapshotOptions } from './SnapshotOptions'; import type { TargetOptions } from './TargetOptions'; - +import type { TrashcanCleanupStatus } from './TrashcanCleanupStatus'; export type Backup = { id?: string; type?: BackupType; @@ -38,5 +37,8 @@ export type Backup = { data_availability_class?: AvailabilityClass; recovery_point_objective?: RecoveryPointObjective; recovery_time_objective?: RecoveryTimeObjective; + trashcan_cleanup_status?: TrashcanCleanupStatus; + trashcan_cleanup_error_message?: string; + trashcan_cleanup_last_scheduled_time?: string; }; diff --git a/frontend/src/models/api/models/BackupStatus.ts b/frontend/src/models/api/models/BackupStatus.ts index e3655d9d..7014b894 100644 --- a/frontend/src/models/api/models/BackupStatus.ts +++ b/frontend/src/models/api/models/BackupStatus.ts @@ -1,8 +1,7 @@ -/* generated using openapi-typescript-codegen -- do no edit */ +/* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ - export enum BackupStatus { NOT_STARTED = 'NotStarted', PREPARED = 'Prepared', diff --git a/frontend/src/models/api/models/BackupStrategy.ts b/frontend/src/models/api/models/BackupStrategy.ts index d6190305..444cf6d9 100644 --- a/frontend/src/models/api/models/BackupStrategy.ts +++ b/frontend/src/models/api/models/BackupStrategy.ts @@ -1,8 +1,7 @@ -/* generated using openapi-typescript-codegen -- do no edit */ +/* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ - export enum BackupStrategy { SNAPSHOT = 'Snapshot', MIRROR = 'Mirror', diff --git a/frontend/src/models/api/models/BackupType.ts b/frontend/src/models/api/models/BackupType.ts index b2d4b1f3..d7085c0d 100644 --- a/frontend/src/models/api/models/BackupType.ts +++ b/frontend/src/models/api/models/BackupType.ts @@ -1,8 +1,7 @@ -/* generated using openapi-typescript-codegen -- do no edit */ +/* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ - export enum BackupType { BIG_QUERY = 'BigQuery', CLOUD_STORAGE = 'CloudStorage', diff --git a/frontend/src/models/api/models/BigQueryOptions.ts b/frontend/src/models/api/models/BigQueryOptions.ts index 867f9263..1b3726ce 100644 --- a/frontend/src/models/api/models/BigQueryOptions.ts +++ b/frontend/src/models/api/models/BigQueryOptions.ts @@ -1,8 +1,7 @@ -/* generated using openapi-typescript-codegen -- do no edit */ +/* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ - export type BigQueryOptions = { dataset?: string; table?: Array; diff --git a/frontend/src/models/api/models/CreateRequest.ts b/frontend/src/models/api/models/CreateRequest.ts index 9629e218..d27d7def 100644 --- a/frontend/src/models/api/models/CreateRequest.ts +++ b/frontend/src/models/api/models/CreateRequest.ts @@ -1,8 +1,7 @@ -/* generated using openapi-typescript-codegen -- do no edit */ +/* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ - import type { BackupStrategy } from './BackupStrategy'; import type { BackupType } from './BackupType'; import type { BigQueryOptions } from './BigQueryOptions'; @@ -12,7 +11,6 @@ import type { RecoveryPointObjective } from './RecoveryPointObjective'; import type { RecoveryTimeObjective } from './RecoveryTimeObjective'; import type { SnapshotOptions } from './SnapshotOptions'; import type { TargetOptions } from './TargetOptions'; - export type CreateRequest = { type?: BackupType; strategy?: BackupStrategy; @@ -24,6 +22,5 @@ export type CreateRequest = { gcs_options?: GCSOptions; recovery_point_objective?: RecoveryPointObjective; recovery_time_objective?: RecoveryTimeObjective; - status?: string; }; diff --git a/frontend/src/models/api/models/GCSOptions.ts b/frontend/src/models/api/models/GCSOptions.ts index 6bacca74..858a4478 100644 --- a/frontend/src/models/api/models/GCSOptions.ts +++ b/frontend/src/models/api/models/GCSOptions.ts @@ -1,8 +1,7 @@ -/* generated using openapi-typescript-codegen -- do no edit */ +/* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ - export type GCSOptions = { bucket?: string; include_prefixes?: Array; diff --git a/frontend/src/models/api/models/Job.ts b/frontend/src/models/api/models/Job.ts index 99d618f3..1a314f18 100644 --- a/frontend/src/models/api/models/Job.ts +++ b/frontend/src/models/api/models/Job.ts @@ -1,10 +1,8 @@ -/* generated using openapi-typescript-codegen -- do no edit */ +/* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ - import type { JobStatus } from './JobStatus'; - export type Job = { id?: string; backup_id?: string; diff --git a/frontend/src/models/api/models/JobStatus.ts b/frontend/src/models/api/models/JobStatus.ts index b0007969..9b69993f 100644 --- a/frontend/src/models/api/models/JobStatus.ts +++ b/frontend/src/models/api/models/JobStatus.ts @@ -1,8 +1,7 @@ -/* generated using openapi-typescript-codegen -- do no edit */ +/* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ - export enum JobStatus { NOT_SCHEDULED = 'NotScheduled', SCHEDULED = 'Scheduled', diff --git a/frontend/src/models/api/models/MirrorOptions.ts b/frontend/src/models/api/models/MirrorOptions.ts index 32bbbe25..2aa0cecf 100644 --- a/frontend/src/models/api/models/MirrorOptions.ts +++ b/frontend/src/models/api/models/MirrorOptions.ts @@ -1,8 +1,7 @@ -/* generated using openapi-typescript-codegen -- do no edit */ +/* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ - export type MirrorOptions = { lifetime_in_days?: number; }; diff --git a/frontend/src/models/api/models/RecoveryPointObjective.ts b/frontend/src/models/api/models/RecoveryPointObjective.ts index 28b6948f..d40bcca3 100644 --- a/frontend/src/models/api/models/RecoveryPointObjective.ts +++ b/frontend/src/models/api/models/RecoveryPointObjective.ts @@ -1,8 +1,7 @@ -/* generated using openapi-typescript-codegen -- do no edit */ +/* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ - /** * RPO - minimal frequency a backup must be conducted (hours) */ diff --git a/frontend/src/models/api/models/RecoveryTimeObjective.ts b/frontend/src/models/api/models/RecoveryTimeObjective.ts index 84a2d32d..2e5f0b63 100644 --- a/frontend/src/models/api/models/RecoveryTimeObjective.ts +++ b/frontend/src/models/api/models/RecoveryTimeObjective.ts @@ -1,8 +1,7 @@ -/* generated using openapi-typescript-codegen -- do no edit */ +/* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ - /** * RTO - the recovery process time duration needed to restore data from backup storage to project/service (minutes) */ diff --git a/frontend/src/models/api/models/RestoreResponse.ts b/frontend/src/models/api/models/RestoreResponse.ts index 929b5509..5f1aa3b6 100644 --- a/frontend/src/models/api/models/RestoreResponse.ts +++ b/frontend/src/models/api/models/RestoreResponse.ts @@ -1,8 +1,7 @@ -/* generated using openapi-typescript-codegen -- do no edit */ +/* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ - export type RestoreResponse = { backup_id?: string; actions?: Array<{ diff --git a/frontend/src/models/api/models/Role.ts b/frontend/src/models/api/models/Role.ts index bdf5dd0a..42b07ea3 100644 --- a/frontend/src/models/api/models/Role.ts +++ b/frontend/src/models/api/models/Role.ts @@ -1,8 +1,7 @@ -/* generated using openapi-typescript-codegen -- do no edit */ +/* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ - export enum Role { NONE = 'none', VIEWER = 'viewer', diff --git a/frontend/src/models/api/models/SnapshotOptions.ts b/frontend/src/models/api/models/SnapshotOptions.ts index 67246736..698d1046 100644 --- a/frontend/src/models/api/models/SnapshotOptions.ts +++ b/frontend/src/models/api/models/SnapshotOptions.ts @@ -1,8 +1,7 @@ -/* generated using openapi-typescript-codegen -- do no edit */ +/* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ - export type SnapshotOptions = { lifetime_in_days?: number; frequency_in_hours?: number; diff --git a/frontend/src/models/api/models/SourceProject.ts b/frontend/src/models/api/models/SourceProject.ts index 0bccb7ff..f16e469e 100644 --- a/frontend/src/models/api/models/SourceProject.ts +++ b/frontend/src/models/api/models/SourceProject.ts @@ -1,10 +1,8 @@ -/* generated using openapi-typescript-codegen -- do no edit */ +/* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ - import type { AvailabilityClass } from './AvailabilityClass'; - export type SourceProject = { data_owner?: string; availability_class?: AvailabilityClass; diff --git a/frontend/src/models/api/models/TargetOptions.ts b/frontend/src/models/api/models/TargetOptions.ts index a06552e3..687e9487 100644 --- a/frontend/src/models/api/models/TargetOptions.ts +++ b/frontend/src/models/api/models/TargetOptions.ts @@ -1,8 +1,7 @@ -/* generated using openapi-typescript-codegen -- do no edit */ +/* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ - export type TargetOptions = { region?: string; dual_region?: string; diff --git a/frontend/src/models/api/models/TrashcanCleanupStatus.ts b/frontend/src/models/api/models/TrashcanCleanupStatus.ts new file mode 100644 index 00000000..2f03f397 --- /dev/null +++ b/frontend/src/models/api/models/TrashcanCleanupStatus.ts @@ -0,0 +1,9 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export enum TrashcanCleanupStatus { + NOOP = 'Noop', + SCHEDULED = 'Scheduled', + ERROR = 'Error', +} diff --git a/frontend/src/models/api/models/UpdateRequest.ts b/frontend/src/models/api/models/UpdateRequest.ts index 3c91ba0e..da3e525e 100644 --- a/frontend/src/models/api/models/UpdateRequest.ts +++ b/frontend/src/models/api/models/UpdateRequest.ts @@ -1,12 +1,10 @@ -/* generated using openapi-typescript-codegen -- do no edit */ +/* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ - import type { BackupStatus } from './BackupStatus'; import type { RecoveryPointObjective } from './RecoveryPointObjective'; import type { RecoveryTimeObjective } from './RecoveryTimeObjective'; - export type UpdateRequest = { backup_id?: string; status?: BackupStatus; diff --git a/frontend/src/models/api/models/UserResponse.ts b/frontend/src/models/api/models/UserResponse.ts index 25e432ef..de986b08 100644 --- a/frontend/src/models/api/models/UserResponse.ts +++ b/frontend/src/models/api/models/UserResponse.ts @@ -1,10 +1,8 @@ -/* generated using openapi-typescript-codegen -- do no edit */ +/* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ - import type { Role } from './Role'; - export type UserResponse = { User?: { Email?: string; diff --git a/frontend/src/models/api/services/DefaultService.ts b/frontend/src/models/api/services/DefaultService.ts index 35c3792e..fe64dd0f 100644 --- a/frontend/src/models/api/services/DefaultService.ts +++ b/frontend/src/models/api/services/DefaultService.ts @@ -1,4 +1,4 @@ -/* generated using openapi-typescript-codegen -- do no edit */ +/* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ @@ -13,13 +13,10 @@ import type { SourceProject } from '../models/SourceProject'; import type { TargetOptions } from '../models/TargetOptions'; import type { UpdateRequest } from '../models/UpdateRequest'; import type { UserResponse } from '../models/UserResponse'; - import type { CancelablePromise } from '../core/CancelablePromise'; import { OpenAPI } from '../core/OpenAPI'; import { request as __request } from '../core/request'; - export class DefaultService { - /** * Get current user * @returns UserResponse OK @@ -34,7 +31,6 @@ export class DefaultService { }, }); } - /** * Get all backups * @param project Project ID @@ -57,7 +53,6 @@ export class DefaultService { }, }); } - /** * Create a new backup * @param requestBody @@ -77,7 +72,6 @@ export class DefaultService { }, }); } - /** * Update a backup * @param requestBody @@ -97,7 +91,6 @@ export class DefaultService { }, }); } - /** * Get a backup * @param backupId Backup ID @@ -126,7 +119,6 @@ export class DefaultService { }, }); } - /** * Calculate backup costs * @param requestBody @@ -163,7 +155,6 @@ export class DefaultService { }, }); } - /** * Checks backup compliance level * @param requestBody @@ -199,7 +190,6 @@ export class DefaultService { }, }); } - /** * Restore a backup * @param backupId Backup ID @@ -225,7 +215,6 @@ export class DefaultService { }, }); } - /** * Get all available backup regions * @returns any OK @@ -242,7 +231,6 @@ export class DefaultService { }, }); } - /** * Get all available backup storge classes * @returns any OK @@ -259,7 +247,6 @@ export class DefaultService { }, }); } - /** * Get all datasets * @param projectId Project ID @@ -282,7 +269,6 @@ export class DefaultService { }, }); } - /** * Get all buckets * @param projectId Project ID @@ -305,7 +291,6 @@ export class DefaultService { }, }); } - /** * Get source project * @param projectId Project ID @@ -328,7 +313,6 @@ export class DefaultService { }, }); } - /** * Run a task * @param task Task name @@ -349,5 +333,24 @@ export class DefaultService { }, }); } - + /** + * Clean up trashcan for backup sink + * @param backupId Backup ID + * @returns void + * @throws ApiError + */ + public static postTrashcansCleanUp( + backupId: string, + ): CancelablePromise { + return __request(OpenAPI, { + method: 'POST', + url: '/trashcans/{backupId}/clean_up', + path: { + 'backupId': backupId, + }, + errors: { + 400: `Bad Request`, + }, + }); + } } diff --git a/pkg/builder/processor_builder.go b/pkg/builder/processor_builder.go index 76e9b2aa..6890751d 100644 --- a/pkg/builder/processor_builder.go +++ b/pkg/builder/processor_builder.go @@ -22,6 +22,7 @@ type ProcessorBuilder struct { datasetListingProcessorFactory processor.DatasetListingProcessorFactory configRegionsProcessorFactory processor.ConfigRegionsProcessorFactory configStorageClassesProcessorFactory processor.ConfigStorageClassesProcessorFactory + trashcanCleanUpProcessorFactory processor.TrashcanCleanUpProcessorFactory } // NewProcessorBuilder created a new ProcessorBuilder @@ -38,7 +39,7 @@ func NewProcessorBuilder( configRegionsProcessorFactory processor.ConfigRegionsProcessorFactory, configStorageClassesProcessorFactory processor.ConfigStorageClassesProcessorFactory, sourceProjectGetProcessorFactory processor.SourceProjectGetProcessorFactory, -) *ProcessorBuilder { + trashcanCleanUpProcessorFactory processor.TrashcanCleanUpProcessorFactory) *ProcessorBuilder { return &ProcessorBuilder{ creatingProcessorFactory: creatingProcessorFactory, gettingProcessorFactory: gettingProcessorFactory, @@ -52,6 +53,7 @@ func NewProcessorBuilder( configRegionsProcessorFactory: configRegionsProcessorFactory, configStorageClassesProcessorFactory: configStorageClassesProcessorFactory, sourceProjectGetProcessorFactory: sourceProjectGetProcessorFactory, + trashcanCleanUpProcessorFactory: trashcanCleanUpProcessorFactory, } } @@ -138,3 +140,10 @@ func (p *ProcessorBuilder) ProcessorForConfigStorageClasses(ctx context.Context) } return p.configStorageClassesProcessorFactory.CreateProcessor(ctx) } + +func (p *ProcessorBuilder) ProcessorForTrashcanCleanUp(ctx context.Context) (processor.Operation[requestobjects.TrashcanCleanUpRequest, requestobjects.TrashcanCleanUpResponse], error) { + if p.trashcanCleanUpProcessorFactory == nil { + return nil, errors.New("factory not found") + } + return p.trashcanCleanUpProcessorFactory.CreateProcessor(ctx) +} diff --git a/pkg/http/actions/trashcan_cleanup.go b/pkg/http/actions/trashcan_cleanup.go new file mode 100644 index 00000000..d6179a79 --- /dev/null +++ b/pkg/http/actions/trashcan_cleanup.go @@ -0,0 +1,36 @@ +package actions + +import ( + "github.com/gorilla/mux" + "github.com/ottogroup/penelope/pkg/builder" + "github.com/ottogroup/penelope/pkg/requestobjects" + "go.opencensus.io/trace" + "net/http" +) + +type TrashcanCleanUp struct { + processorBuilder *builder.ProcessorBuilder +} + +func NewTrashcanCleanUp(processorBuilder *builder.ProcessorBuilder) *TrashcanCleanUp { + return &TrashcanCleanUp{processorBuilder: processorBuilder} +} + +// ServeHTTP will handle TrashcanCleanUp operation +func (tc *TrashcanCleanUp) ServeHTTP(w http.ResponseWriter, r *http.Request) { + ctx, span := trace.StartSpan(r.Context(), "TrashcanCleanUp.ServeHTTP") + defer span.End() + + backupID, exist := mux.Vars(r)["backup_id"] + if !exist { + msg := "Bad request missing parameter: backup_id" + prepareResponse(w, msg, msg, http.StatusBadRequest) + return + } + + var request = requestobjects.TrashcanCleanUpRequest{ + BackupID: backupID, + } + + handleRequestByProcessor(ctx, w, r, request, http.StatusNoContent, tc.processorBuilder.ProcessorForTrashcanCleanUp) +} diff --git a/pkg/http/auth/rbac.go b/pkg/http/auth/rbac.go index f221136c..3fc1ee61 100644 --- a/pkg/http/auth/rbac.go +++ b/pkg/http/auth/rbac.go @@ -17,7 +17,7 @@ func CheckRequestIsAllowed(principal *model.Principal, requestType requestobject switch requestType { case requestobjects.Updating: isAllowed = matchRole(rbacRole, model.Owner) - case requestobjects.Creating: + case requestobjects.Creating, requestobjects.Cleanup: isAllowed = matchRole(rbacRole, model.Owner) case requestobjects.Getting, requestobjects.Listing, requestobjects.Restoring, requestobjects.Calculating, requestobjects.DatasetListing, requestobjects.BucketListing, requestobjects.SourceProjectGet: diff --git a/pkg/http/mock/fixtures.go b/pkg/http/mock/fixtures.go index 9981cd5a..0907355b 100644 --- a/pkg/http/mock/fixtures.go +++ b/pkg/http/mock/fixtures.go @@ -2,6 +2,7 @@ package mock import ( "bytes" + "fmt" "net/http" "strconv" "strings" @@ -23,6 +24,11 @@ var ( // ImpersonationHTTPMock request ImpersonationHTTPMock = NewMockedHTTPRequest("POST", "/v1/projects/-/serviceAccounts/.*:generateAccessToken", impersonateResponse) + ListObjectsHTTPMockFunc = func(bucketName string) MockedHTTPRequest { + path := fmt.Sprintf("/storage/v1/b/%s/o", bucketName) + return NewMockedHTTPRequest("GET", path, listObjectsResponse) + } + // ObjectsExistsHTTPMock request ObjectsExistsHTTPMock = NewMockedHTTPRequest("GET", "/storage/v1/b/.*/o", objectsExistsResponse) // SinkNotExistsHTTPMock request @@ -295,6 +301,74 @@ Content-Type: application/json; charset=UTF-8 ] } } +` + + listObjectsResponse = `HTTP/1.1 200 +Content-Type: application/json; charset=UTF-8 +{ + "kind": "storage#objects", + "items": [ + { + "kind": "storage#object", + "id": "uuid-5678-123456/.trashcan_uuid-5678-123456//1724709325384603", + "selfLink": "https://www.googleapis.com/storage/v1/b/uuid-5678-123456/o/.trashcan_uuid-5678-123456%2F", + "mediaLink": "https://content-storage.googleapis.com/download/storage/v1/b/uuid-5678-123456/o/.trashcan_uuid-5678-123456%2F?generation=1724709325384603&alt=media", + "name": ".trashcan_uuid-5678-123456/", + "bucket": "uuid-5678-123456", + "generation": "1724709325384603", + "metageneration": "1", + "contentType": "text/plain", + "storageClass": "STANDARD", + "size": "0", + "md5Hash": "1B2M2Y8AsgTpgAmY7PhCfg==", + "crc32c": "AAAAAA==", + "etag": "CJvXuPXSk4gDEAE=", + "temporaryHold": false, + "eventBasedHold": false, + "timeCreated": "2024-08-26T21:55:25.425Z", + "updated": "2024-08-26T21:55:25.425Z", + "timeStorageClassUpdated": "2024-08-26T21:55:25.425Z" + }, + { + "kind": "storage#object", + "id": "uuid-5678-123456/.trashcan_uuid-5678-123456/THIS_TRASHCAN_CONTAINS_DELETED_OBJECTS_FROM_SOURCE/1724709388143440", + "selfLink": "https://www.googleapis.com/storage/v1/b/uuid-5678-123456/o/.trashcan_uuid-5678-123456%2FTHIS_TRASHCAN_CONTAINS_DELETED_OBJECTS_FROM_SOURCE", + "mediaLink": "https://content-storage.googleapis.com/download/storage/v1/b/uuid-5678-123456/o/.trashcan_uuid-5678-123456%2FTHIS_TRASHCAN_CONTAINS_DELETED_OBJECTS_FROM_SOURCE?generation=1724709388143440&alt=media", + "name": ".trashcan_uuid-5678-123456/THIS_TRASHCAN_CONTAINS_DELETED_OBJECTS_FROM_SOURCE", + "bucket": "uuid-5678-123456", + "generation": "1724709388143440", + "metageneration": "1", + "contentType": "application/octet-stream", + "storageClass": "STANDARD", + "size": "0", + "md5Hash": "1B2M2Y8AsgTpgAmY7PhCfg==", + "crc32c": "AAAAAA==", + "etag": "CNCWr5PTk4gDEAE=", + "timeCreated": "2024-08-26T21:56:28.169Z", + "updated": "2024-08-26T21:56:28.169Z", + "timeStorageClassUpdated": "2024-08-26T21:56:28.169Z" + }, + { + "kind": "storage#object", + "id": "uuid-5678-123456/.trashcan_uuid-5678-123456/file.txt/1724709421446103", + "selfLink": "https://www.googleapis.com/storage/v1/b/uuid-5678-123456/o/.trashcan_uuid-5678-123456%2Ffile.txt", + "mediaLink": "https://content-storage.googleapis.com/download/storage/v1/b/uuid-5678-123456/o/.trashcan_uuid-5678-123456%2Ffile.txt?generation=1724709421446103&alt=media", + "name": ".trashcan_uuid-5678-123456/file.txt", + "bucket": "uuid-5678-123456", + "generation": "1724709421446103", + "metageneration": "1", + "contentType": "text/plain", + "storageClass": "STANDARD", + "size": "12", + "md5Hash": "dKoTNf4rcU8FwuSLAem5Lw==", + "crc32c": "Z1bzyw==", + "etag": "CNfnn6PTk4gDEAE=", + "timeCreated": "2024-08-26T21:57:01.471Z", + "updated": "2024-08-26T21:57:01.471Z", + "timeStorageClassUpdated": "2024-08-26T21:57:01.471Z" + } + ] +} ` ) diff --git a/pkg/http/rest/api.go b/pkg/http/rest/api.go index cc191f0c..7ca04271 100644 --- a/pkg/http/rest/api.go +++ b/pkg/http/rest/api.go @@ -97,6 +97,12 @@ func createEndpoints(processorBuilder *builder.ProcessorBuilder, tokenSourceProv actions.NewGettingBackupHandler(processorBuilder).ServeHTTP, []string{http.MethodGet}, ), + newAPIEndpoint( + fmt.Sprintf("%s/{backup_id}/clean_up", trashcansPath), + true, + actions.NewTrashcanCleanUp(processorBuilder).ServeHTTP, + []string{http.MethodPost}, + ), newAPIEndpoint( backupPath, true, diff --git a/pkg/http/rest/api_test.go b/pkg/http/rest/api_test.go index 62894e63..e9ac3d96 100644 --- a/pkg/http/rest/api_test.go +++ b/pkg/http/rest/api_test.go @@ -33,7 +33,7 @@ var mockSourceTokenProvider = &mockSourceGCPProjectProvider{ DataOwner: "john.doe", } -func (m *mockSourceGCPProjectProvider) GetSourceGCPProject(ctxIn context.Context, gcpProjectID string) (provider.SourceGCPProject, error) { +func (m *mockSourceGCPProjectProvider) GetSourceGCPProject(_ context.Context, _ string) (provider.SourceGCPProject, error) { return provider.SourceGCPProject{ AvailabilityClass: m.AvailabilityClass, DataOwner: m.DataOwner, @@ -54,6 +54,7 @@ func createBuilder(backupProvider provider.SinkGCPProjectProvider, tokenSourcePr nil, nil, nil, + nil, ) } @@ -63,20 +64,21 @@ func restAPIFactoryWithStubFactory(tokenSourceProvider impersonate.TargetPrincip if err != nil { panic(fmt.Errorf("error creating AuthenticationMiddleware: %s", err)) } - app := NewRestAPI(builder.NewProcessorBuilder( - &StubFactory[requestobjects.CreateRequest, requestobjects.BackupResponse]{DefaultValue: requestobjects.BackupResponse{}}, - &StubFactory[requestobjects.GetRequest, requestobjects.BackupResponse]{DefaultValue: requestobjects.BackupResponse{}}, - &StubFactory[requestobjects.ListRequest, requestobjects.ListingResponse]{DefaultValue: requestobjects.ListingResponse{}}, - &StubFactory[requestobjects.UpdateRequest, requestobjects.UpdateResponse]{DefaultValue: requestobjects.UpdateResponse{}}, - &StubFactory[requestobjects.RestoreRequest, requestobjects.RestoreResponse]{DefaultValue: requestobjects.RestoreResponse{}}, - &StubFactory[requestobjects.CalculateRequest, requestobjects.CalculatedResponse]{DefaultValue: requestobjects.CalculatedResponse{}}, - &StubFactory[requestobjects.ComplianceRequest, requestobjects.ComplianceResponse]{DefaultValue: requestobjects.ComplianceResponse{}}, - &StubFactory[requestobjects.BucketListRequest, requestobjects.BucketListResponse]{DefaultValue: requestobjects.BucketListResponse{}}, - &StubFactory[requestobjects.DatasetListRequest, requestobjects.DatasetListResponse]{DefaultValue: requestobjects.DatasetListResponse{}}, - &StubFactory[requestobjects.EmptyRequest, requestobjects.RegionsListResponse]{DefaultValue: requestobjects.RegionsListResponse{}}, - &StubFactory[requestobjects.EmptyRequest, requestobjects.StorageClassListResponse]{DefaultValue: requestobjects.StorageClassListResponse{}}, - &StubFactory[requestobjects.SourceProjectGetRequest, requestobjects.SourceProjectGetResponse]{DefaultValue: requestobjects.SourceProjectGetResponse{}}, - ), authenticationMiddleware, tokenSourceProvider, credentialProvider) + app := NewRestAPI( + builder.NewProcessorBuilder( + &StubFactory[requestobjects.CreateRequest, requestobjects.BackupResponse]{DefaultValue: requestobjects.BackupResponse{}}, + &StubFactory[requestobjects.GetRequest, requestobjects.BackupResponse]{DefaultValue: requestobjects.BackupResponse{}}, + &StubFactory[requestobjects.ListRequest, requestobjects.ListingResponse]{DefaultValue: requestobjects.ListingResponse{}}, + &StubFactory[requestobjects.UpdateRequest, requestobjects.UpdateResponse]{DefaultValue: requestobjects.UpdateResponse{}}, + &StubFactory[requestobjects.RestoreRequest, requestobjects.RestoreResponse]{DefaultValue: requestobjects.RestoreResponse{}}, + &StubFactory[requestobjects.CalculateRequest, requestobjects.CalculatedResponse]{DefaultValue: requestobjects.CalculatedResponse{}}, + &StubFactory[requestobjects.ComplianceRequest, requestobjects.ComplianceResponse]{DefaultValue: requestobjects.ComplianceResponse{}}, + &StubFactory[requestobjects.BucketListRequest, requestobjects.BucketListResponse]{DefaultValue: requestobjects.BucketListResponse{}}, + &StubFactory[requestobjects.DatasetListRequest, requestobjects.DatasetListResponse]{DefaultValue: requestobjects.DatasetListResponse{}}, + &StubFactory[requestobjects.EmptyRequest, requestobjects.RegionsListResponse]{DefaultValue: requestobjects.RegionsListResponse{}}, + &StubFactory[requestobjects.EmptyRequest, requestobjects.StorageClassListResponse]{DefaultValue: requestobjects.StorageClassListResponse{}}, + &StubFactory[requestobjects.SourceProjectGetRequest, requestobjects.SourceProjectGetResponse]{DefaultValue: requestobjects.SourceProjectGetResponse{}}, nil, + ), authenticationMiddleware, tokenSourceProvider, credentialProvider) return httptest.NewServer(authenticationMiddleware.AddAuthentication(app.ServeHTTP)) } @@ -166,7 +168,7 @@ type StubProcessor[T, R any] struct { DefaultValue R } -func (p *StubProcessor[T, R]) Process(ctxIn context.Context, args *processor.Argument[T]) (R, error) { +func (p *StubProcessor[T, R]) Process(_ context.Context, _ *processor.Argument[T]) (R, error) { return p.DefaultValue, nil } @@ -174,7 +176,7 @@ type StubFactory[T, R any] struct { DefaultValue R } -func (s *StubFactory[T, R]) CreateProcessor(ctxIn context.Context) (processor.Operation[T, R], error) { +func (s *StubFactory[T, R]) CreateProcessor(_ context.Context) (processor.Operation[T, R], error) { return &StubProcessor[T, R]{DefaultValue: s.DefaultValue}, nil } diff --git a/pkg/http/rest/endpoint.go b/pkg/http/rest/endpoint.go index 1fcd3310..559b56bc 100644 --- a/pkg/http/rest/endpoint.go +++ b/pkg/http/rest/endpoint.go @@ -8,6 +8,7 @@ import ( const apiRootAppPath = "/api/" const healthCheckPath = "/_ah/" const backupPath = "backups" +const trashcansPath = "trashcans" const bucketsPath = "buckets" const datasetsPath = "datasets" const restorePath = "restore" diff --git a/pkg/processor/helpers.go b/pkg/processor/helpers.go index b6cb2749..e32d41e3 100644 --- a/pkg/processor/helpers.go +++ b/pkg/processor/helpers.go @@ -53,15 +53,18 @@ func mapBackupToResponse(backup *repository.Backup, jobs []*repository.Job, sour } return requestobjects.BackupResponse{ - ID: backup.ID, - Sink: backup.SinkOptions.Sink, - Status: status.String(), - SinkProject: backup.SinkOptions.TargetProject, - CreatedTimestamp: formatTime(backup.CreatedTimestamp), - UpdatedTimestamp: formatTime(backup.UpdatedTimestamp), - DeletedTimestamp: formatTime(backup.DeletedTimestamp), - DataOwner: sourceGCPProject.DataOwner, - DataAvailabilityClass: sourceGCPProject.AvailabilityClass, + ID: backup.ID, + Sink: backup.SinkOptions.Sink, + Status: status.String(), + SinkProject: backup.SinkOptions.TargetProject, + CreatedTimestamp: formatTime(backup.CreatedTimestamp), + UpdatedTimestamp: formatTime(backup.UpdatedTimestamp), + DeletedTimestamp: formatTime(backup.DeletedTimestamp), + DataOwner: sourceGCPProject.DataOwner, + DataAvailabilityClass: sourceGCPProject.AvailabilityClass, + TrashcanCleanupStatus: backup.TrashcanCleanup.Status.String(), + TrashcanCleanupErrorMessage: backup.TrashcanCleanup.ErrorMessage, + TrashcanCleanupLastScheduledTime: formatTime(backup.TrashcanCleanup.LastScheduled), CreateRequest: requestobjects.CreateRequest{ Type: backup.Type.String(), Strategy: backup.Strategy.String(), diff --git a/pkg/processor/schedule_processor_test.go b/pkg/processor/schedule_processor_test.go index 22e74a29..d81c3d64 100644 --- a/pkg/processor/schedule_processor_test.go +++ b/pkg/processor/schedule_processor_test.go @@ -461,6 +461,10 @@ type stubGcsClient struct { fDeleteObjectsErr error } +func (g *stubGcsClient) DeleteObjectWithPrefix(ctxIn context.Context, bucket string, objectPrefixName string) error { + panic("implement me") +} + func (g *stubGcsClient) GetProject(ctxIn context.Context, projectID string) (*resourcemanagerpb.Project, error) { panic("implement me") } diff --git a/pkg/processor/trashcan_clean_up_processor_factory.go b/pkg/processor/trashcan_clean_up_processor_factory.go new file mode 100644 index 00000000..bee3fc70 --- /dev/null +++ b/pkg/processor/trashcan_clean_up_processor_factory.go @@ -0,0 +1,84 @@ +package processor + +import ( + "context" + "fmt" + "github.com/go-pg/pg/v10" + "github.com/golang/glog" + "github.com/ottogroup/penelope/pkg/http/auth" + "github.com/ottogroup/penelope/pkg/http/impersonate" + "github.com/ottogroup/penelope/pkg/repository" + "github.com/ottogroup/penelope/pkg/requestobjects" + "github.com/ottogroup/penelope/pkg/secret" + "github.com/pkg/errors" + "go.opencensus.io/trace" +) + +func NewTrashcanCleanUpProcessorFactory(tokenSourceProvider impersonate.TargetPrincipalForProjectProvider, credentialsProvider secret.SecretProvider) TrashcanCleanUpProcessorFactory { + return &trashcanCleanUpProcessorFactory{ + tokenSourceProvider: tokenSourceProvider, + credentialsProvider: credentialsProvider, + } +} + +type TrashcanCleanUpProcessorFactory interface { + CreateProcessor(ctxIn context.Context) (Operation[requestobjects.TrashcanCleanUpRequest, requestobjects.TrashcanCleanUpResponse], error) +} + +// TrashcanCleanUpProcessorFactory create Process for TrashcanCleanUp +type trashcanCleanUpProcessorFactory struct { + tokenSourceProvider impersonate.TargetPrincipalForProjectProvider + credentialsProvider secret.SecretProvider +} + +func (p *trashcanCleanUpProcessorFactory) CreateProcessor(ctxIn context.Context) (Operation[requestobjects.TrashcanCleanUpRequest, requestobjects.TrashcanCleanUpResponse], error) { + ctx, span := trace.StartSpan(ctxIn, "(*trashcanCleanUpProcessorFactory).CreateProcessor") + defer span.End() + + backupRepository, err := repository.NewBackupRepository(ctx, p.credentialsProvider) + if err != nil { + glog.Error(err) + return &trashcanCleanUpProcessor{}, err + } + + return &trashcanCleanUpProcessor{ + backupRepository: backupRepository, + tokenSourceProvider: p.tokenSourceProvider, + }, nil +} + +type trashcanCleanUpProcessor struct { + backupRepository repository.BackupRepository + tokenSourceProvider impersonate.TargetPrincipalForProjectProvider +} + +func (p *trashcanCleanUpProcessor) Process(ctxIn context.Context, args *Argument[requestobjects.TrashcanCleanUpRequest]) (requestobjects.TrashcanCleanUpResponse, error) { + ctx, span := trace.StartSpan(ctxIn, "(*trashcanCleanUpProcessor).Process") + defer span.End() + + var request = &args.Request + + backup, err := p.backupRepository.GetBackup(ctx, request.BackupID) + if err != nil { + if err == pg.ErrNoRows { + return requestobjects.TrashcanCleanUpResponse{}, requestobjects.ApiError{ + Code: 404, + Message: fmt.Sprintf("no backup with id %q found", request.BackupID), + } + } + return requestobjects.TrashcanCleanUpResponse{}, errors.Wrapf(err, "get backup failed %s", request.BackupID) + } + + if !auth.CheckRequestIsAllowed(args.Principal, requestobjects.Cleanup, backup.SourceProject) { + return requestobjects.TrashcanCleanUpResponse{}, fmt.Errorf("%s is not allowed for user %q on project %q", requestobjects.Cleanup.String(), args.Principal.User.Email, backup.TargetProject) + } + + err = p.backupRepository.MarkTrashcanCleanup(ctx, backup.ID, repository.TrashcanCleanup{ + Status: repository.ScheduledTrashcanCleanupStatus, + }) + if err != nil { + return requestobjects.TrashcanCleanUpResponse{}, errors.Wrapf(err, "mark trashcan cleanup status failed %s", backup.ID) + } + + return requestobjects.TrashcanCleanUpResponse{}, nil +} diff --git a/pkg/repository/backup_repository.go b/pkg/repository/backup_repository.go index ec507b54..97aad7ea 100644 --- a/pkg/repository/backup_repository.go +++ b/pkg/repository/backup_repository.go @@ -47,6 +47,8 @@ type BackupRepository interface { GetExpiredBigQueryMirrorRevisions(ctxIn context.Context, maxRevisionLifetimeInWeeks int) ([]*MirrorRevision, error) GetBigQueryOneShotSnapshots(ctxIn context.Context, status BackupStatus) ([]*Backup, error) GetScheduledBackups(context.Context, BackupType) ([]*Backup, error) + GetBackupsByCleanupTrashcanStatus(ctx context.Context, status TrashcanCleanupStatus) ([]*Backup, error) + MarkTrashcanCleanup(ctx context.Context, id string, trashcanCleanup TrashcanCleanup) error } // defaultBackupRepository implements BackupRepository @@ -55,6 +57,56 @@ type defaultBackupRepository struct { ctx context.Context } +func (d *defaultBackupRepository) MarkTrashcanCleanup(ctx context.Context, id string, trashcanCleanup TrashcanCleanup) error { + _, span := trace.StartSpan(ctx, "(*defaultBackupRepository).MarkTrashcanCleanup") + defer span.End() + + columns := []string{ + "trashcan_cleanup_status", + "trashcan_cleanup_error_message", + "trashcan_cleanup_start_running_timestamp", + } + + backup := &Backup{ + ID: id, + TrashcanCleanup: trashcanCleanup, + } + + if !trashcanCleanup.LastScheduled.IsZero() { + columns = append(columns, "trashcan_cleanup_last_scheduled_timestamp") + } + + _, err := d.storageService.DB().Model(backup). + Column(columns...). + WherePK(). + Update() + + if err != nil { + logQueryError("MarkTrashcanCleanup", err) + return fmt.Errorf("error during executing updating backup statemant: %s", err) + } + + return nil +} + +func (d *defaultBackupRepository) GetBackupsByCleanupTrashcanStatus(ctx context.Context, status TrashcanCleanupStatus) ([]*Backup, error) { + _, span := trace.StartSpan(ctx, "(*defaultBackupRepository).GetBackupsByCleanupTrashcanStatus") + defer span.End() + + var backups []*Backup + + err := d.storageService.DB().Model(&backups). + Where("trashcan_cleanup_status = ?", status). + Select() + + if err != nil { + logQueryError("GetBackupsByCleanupTrashcanStatus", err) + return backups, fmt.Errorf("error during executing get backup by status statement: %s", err) + } + + return backups, nil +} + // NewBackupRepository return instance of BackupRepository func NewBackupRepository(ctxIn context.Context, credentialsProvider secret.SecretProvider) (BackupRepository, error) { ctx, span := trace.StartSpan(ctxIn, "NewBackupRepository") diff --git a/pkg/repository/entities.go b/pkg/repository/entities.go index 31686ac0..45302807 100644 --- a/pkg/repository/entities.go +++ b/pkg/repository/entities.go @@ -55,6 +55,7 @@ type Backup struct { BackupOptions EntityAudit MirrorOptions + TrashcanCleanup } // GetTrashcanPath give a patho to object moved into trashcan @@ -110,6 +111,14 @@ type SnapshotOptions struct { FrequencyInHours uint `pg:"snapshot_frequency_in_hours,use_zero"` } +// TrashcanCleanup status of trashcan cleanup +type TrashcanCleanup struct { + Status TrashcanCleanupStatus `pg:"trashcan_cleanup_status"` + ErrorMessage string `pg:"trashcan_cleanup_error_message"` + LastScheduled time.Time `pg:"trashcan_cleanup_last_scheduled_timestamp"` + StartRunningTimestamp time.Time `pg:"trashcan_cleanup_start_running_timestamp"` +} + // MirrorOptions strategy backup options type MirrorOptions struct { LifetimeInDays uint `pg:"mirror_lifetime_in_days,use_zero"` diff --git a/pkg/repository/memory/backup_repository.go b/pkg/repository/memory/backup_repository.go index 07b47896..63246647 100644 --- a/pkg/repository/memory/backup_repository.go +++ b/pkg/repository/memory/backup_repository.go @@ -13,7 +13,26 @@ type BackupRepository struct { backups []*repository.Backup } -// UpdateBackupStatus is not implemented +func (r *BackupRepository) MarkTrashcanCleanup(_ context.Context, id string, status repository.TrashcanCleanup) error { + for _, backup := range r.backups { + if backup.ID == id { + backup.TrashcanCleanup = status + return nil + } + } + return nil +} + +func (r *BackupRepository) GetBackupsByCleanupTrashcanStatus(_ context.Context, status repository.TrashcanCleanupStatus) ([]*repository.Backup, error) { + var backups []*repository.Backup + for _, backup := range r.backups { + if backup.TrashcanCleanup.Status == status { + backups = append(backups, backup) + } + } + return backups, nil +} + func (r *BackupRepository) UpdateBackup(ctxIn context.Context, updateFields repository.UpdateFields) error { _, span := trace.StartSpan(ctxIn, "(*BackupRepository).UpdateBackupStatus") defer span.End() diff --git a/pkg/repository/objects.go b/pkg/repository/objects.go index b1a27602..e463e3f9 100644 --- a/pkg/repository/objects.go +++ b/pkg/repository/objects.go @@ -13,6 +13,13 @@ type BackupStatus string // JobStatus for backup type JobStatus string +// TrashcanCleanupStatus status for scheduled cleanup of trashcan +type TrashcanCleanupStatus string + +func (s TrashcanCleanupStatus) String() string { + return string(s) +} + // Operation for a backup type Operation string @@ -95,6 +102,17 @@ const ( Delete Operation = "Delete" ) +const ( + // NoopCleanupTrashcanCleanupStatus no operation + NoopCleanupTrashcanCleanupStatus TrashcanCleanupStatus = "Noop" + // ScheduledTrashcanCleanupStatus scheduled cleanup + ScheduledTrashcanCleanupStatus TrashcanCleanupStatus = "Scheduled" + // ErrorCleanupTrashcanCleanupStatus error during cleanup + ErrorCleanupTrashcanCleanupStatus TrashcanCleanupStatus = "Error" + // InProgressCleanupTrashcanCleanupStatus cleanup in progress + InProgressCleanupTrashcanCleanupStatus TrashcanCleanupStatus = "InProgress" +) + // Strategies for a backups var Strategies = []Strategy{Snapshot, Mirror} diff --git a/pkg/requestobjects/backup_request.go b/pkg/requestobjects/backup_request.go index e839db08..c58ba9f1 100644 --- a/pkg/requestobjects/backup_request.go +++ b/pkg/requestobjects/backup_request.go @@ -139,6 +139,10 @@ type BackupResponse struct { Jobs []JobResponse `json:"jobs,omitempty"` JobsTotal uint64 `json:"jobs_total,omitempty"` + + TrashcanCleanupStatus string `json:"trashcan_cleanup_status,omitempty"` + TrashcanCleanupErrorMessage string `json:"trashcan_cleanup_error_message,omitempty"` + TrashcanCleanupLastScheduledTime string `json:"trashcan_cleanup_last_scheduled_time,omitempty"` } // JobResponse get backup job details @@ -282,3 +286,10 @@ type ProjectSinkComplianceRequest struct { type ProjectSinkComplianceResponse struct { Checks []ProjectSinkComplianceCheck `json:"checks"` } + +type TrashcanCleanUpRequest struct { + BackupID string `json:"backup_id"` +} + +type TrashcanCleanUpResponse struct { +} diff --git a/pkg/requestobjects/types.go b/pkg/requestobjects/types.go index 3f1092d4..a006927d 100644 --- a/pkg/requestobjects/types.go +++ b/pkg/requestobjects/types.go @@ -26,6 +26,8 @@ const ( BucketListing RequestType = "BuckeListing" // SourceProjectGet - get Source Project for given project ID SourceProjectGet RequestType = "SourceProjectGet" + // Cleanup - cleanup trash can for a backup + Cleanup RequestType = "Cleanup" ) func (s RequestType) String() string { diff --git a/pkg/service/gcs/gcs_client.go b/pkg/service/gcs/gcs_client.go index 82af21a1..31fd25ae 100644 --- a/pkg/service/gcs/gcs_client.go +++ b/pkg/service/gcs/gcs_client.go @@ -46,6 +46,7 @@ type CloudStorageClient interface { Close(ctxIn context.Context) UpdateBucket(ctxIn context.Context, bucket string, lifetimeInDays uint, archiveTTM uint) error GetBucketDetails(ctxIn context.Context, bucket string) (*storage.BucketAttrs, error) + DeleteObjectWithPrefix(ctxIn context.Context, bucket string, objectPrefixName string) error } // CloudStorageClientFactory creates a CloudStorageClient with the credentails for a specified project @@ -60,6 +61,27 @@ type defaultGcsClient struct { projectClient *resourcemanager.ProjectsClient } +func (c *defaultGcsClient) DeleteObjectWithPrefix(ctxIn context.Context, bucket string, objectPrefixName string) error { + ctx, span := trace.StartSpan(ctxIn, "(*defaultGcsClient).DeleteObjectWithPrefix") + defer span.End() + + objectIterator := c.client.Bucket(bucket).Objects(ctx, &storage.Query{Prefix: objectPrefixName}) + for { + objAttrs, err := objectIterator.Next() + if errors.Is(err, iterator.Done) { + break + } + if err != nil { + return errors.Wrap(err, fmt.Sprintf("DeleteObjectWithPrefix failed for bucket %s, prefix %s", bucket, objectPrefixName)) + } + if err := c.client.Bucket(bucket).Object(objAttrs.Name).Delete(ctx); err != nil { + return errors.Wrap(err, fmt.Sprintf("DeleteObjectWithPrefix failed for bucket %s, prefix %s", bucket, objectPrefixName)) + } + } + + return nil +} + func (c *defaultGcsClient) GetProject(ctxIn context.Context, projectID string) (*resourcemanagerpb.Project, error) { _, span := trace.StartSpan(ctxIn, "(*defaultGcsClient).GetProject") defer span.End() diff --git a/pkg/service/gcs/gcs_mock.go b/pkg/service/gcs/gcs_mock.go index b3bcd851..eb7da131 100644 --- a/pkg/service/gcs/gcs_mock.go +++ b/pkg/service/gcs/gcs_mock.go @@ -16,6 +16,10 @@ type MockGcsClient struct { ObjectContent []byte } +func (c *MockGcsClient) DeleteObjectWithPrefix(ctxIn context.Context, bucket string, objectPrefixName string) error { + panic("implement me") +} + func (c *MockGcsClient) GetProject(ctxIn context.Context, projectID string) (*resourcemanagerpb.Project, error) { panic("implement me") } diff --git a/pkg/tasks/cleanup_trashcans_service.go b/pkg/tasks/cleanup_trashcans_service.go new file mode 100644 index 00000000..57c20ae4 --- /dev/null +++ b/pkg/tasks/cleanup_trashcans_service.go @@ -0,0 +1,103 @@ +package tasks + +import ( + "context" + "fmt" + "github.com/golang/glog" + "github.com/ottogroup/penelope/pkg/http/impersonate" + "github.com/ottogroup/penelope/pkg/repository" + "github.com/ottogroup/penelope/pkg/secret" + "github.com/ottogroup/penelope/pkg/service/gcs" + "go.opencensus.io/trace" + "strings" + "time" +) + +func newCleanupTrashcansService(ctxIn context.Context, tokenSourceProvider impersonate.TargetPrincipalForProjectProvider, credentialsProvider secret.SecretProvider) (*cleanupTrashcansService, error) { + ctx, span := trace.StartSpan(ctxIn, "newPrepareBackupJobsService") + defer span.End() + + backupRepository, err := repository.NewBackupRepository(ctx, credentialsProvider) + if err != nil { + return nil, err + } + + return &cleanupTrashcansService{tokenSourceProvider: tokenSourceProvider, backupRepository: backupRepository}, nil +} + +type cleanupTrashcansService struct { + tokenSourceProvider impersonate.TargetPrincipalForProjectProvider + backupRepository repository.BackupRepository +} + +func (s *cleanupTrashcansService) Run(ctxIn context.Context) { + ctx, span := trace.StartSpan(ctxIn, "(*cleanupTrashcansService).Run") + defer span.End() + + backups, err := s.backupRepository.GetBackupsByCleanupTrashcanStatus(ctx, repository.ScheduledTrashcanCleanupStatus) + if err != nil { + glog.Errorf("could not get list of backups scheduled to cleanup trashcan: %s", err) + } + + for _, backup := range backups { + gcsClient, err := gcs.NewCloudStorageClient(ctx, s.tokenSourceProvider, backup.TargetProject) + if err != nil { + glog.Errorf("could not create new CloudStorageClient: %s", err) + return + } + defer gcsClient.Close(ctx) + + err = s.backupRepository.MarkTrashcanCleanup(ctx, backup.ID, repository.TrashcanCleanup{ + Status: repository.InProgressCleanupTrashcanCleanupStatus, + StartRunningTimestamp: time.Now(), + }) + if err != nil { + return + } + + trashcanPath := backup.GetTrashcanPath() + if strings.EqualFold(trashcanPath, "") { + errMsg := fmt.Sprintf("trashcan path is empty for backup with id %s", backup.ID) + glog.Errorf(errMsg) + err = s.backupRepository.MarkTrashcanCleanup(ctx, backup.ID, repository.TrashcanCleanup{ + Status: repository.ErrorCleanupTrashcanCleanupStatus, + ErrorMessage: errMsg, + }) + if err != nil { + glog.Errorf("could not mark trashcan cleanup status to %s: %s", repository.ErrorCleanupTrashcanCleanupStatus, err) + } + return + } + + err = gcsClient.DeleteObjectWithPrefix(ctx, backup.Sink, trashcanPath) + if err != nil { + errMsg := fmt.Sprintf("could not delete objects in trashcan for backup with id %s: %s", backup.ID, err) + glog.Errorf(errMsg) + err = s.backupRepository.MarkTrashcanCleanup(ctx, backup.ID, repository.TrashcanCleanup{ + Status: repository.ErrorCleanupTrashcanCleanupStatus, + ErrorMessage: errMsg, + }) + if err != nil { + glog.Errorf("could not mark trashcan cleanup status to %s: %s", repository.ErrorCleanupTrashcanCleanupStatus, err) + } + return + } + + err = gcsClient.CreateObject(ctx, backup.Sink, fmt.Sprintf("%s/THIS_TRASHCAN_CONTAINS_DELETED_OBJECTS_FROM_SOURCE", trashcanPath), "") + if err != nil { + glog.Errorf("could not create THIS_TRASHCAN_CONTAINS_DELETED_OBJECTS_FROM_SOURCE object in trashcan: %s", err) + return + } + + err = s.backupRepository.MarkTrashcanCleanup(ctx, backup.ID, repository.TrashcanCleanup{ + Status: repository.NoopCleanupTrashcanCleanupStatus, + LastScheduled: time.Now(), + }) + if err != nil { + glog.Errorf("could not mark trashcan cleanup status to %s: %s", repository.NoopCleanupTrashcanCleanupStatus, err) + return + } + + glog.Infof("trashcan cleanup for backup completed: %s", backup.ID) + } +} diff --git a/pkg/tasks/cleanup_trashcans_service_test.go b/pkg/tasks/cleanup_trashcans_service_test.go new file mode 100644 index 00000000..a892c26b --- /dev/null +++ b/pkg/tasks/cleanup_trashcans_service_test.go @@ -0,0 +1,116 @@ +package tasks + +import ( + "context" + "fmt" + "github.com/ottogroup/penelope/pkg/http/mock" + "github.com/ottogroup/penelope/pkg/provider" + "github.com/ottogroup/penelope/pkg/repository" + "github.com/ottogroup/penelope/pkg/secret" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "strings" + "testing" +) + +func TestCleanupTrashcansService_Schedule(t *testing.T) { + httpMockHandler.Start() + defer httpMockHandler.Stop() + + ctx := context.Background() + service, err := newCleanupTrashcansService(ctx, provider.NewDefaultImpersonatedTokenConfigProvider(), secret.NewEnvSecretProvider()) + require.NoError(t, err) + + backupRepository, err := repository.NewBackupRepository(ctx, secret.NewEnvSecretProvider()) + require.NoErrorf(t, err, "backupRepository should be instantiate") + + backup, err := backupRepository.AddBackup(ctx, scheduleTrashcanBackup()) + require.NoError(t, err, "should add new backup") + defer func() { _ = deleteBackup(backup.ID) }() + + httpMockHandler.Register(mock.ListObjectsHTTPMockFunc(backup.Sink)) + + _, stdErr, err := captureStderr(func() { + service.Run(ctx) + }) + + fmt.Println(stdErr) + + cleanupTrashcanBucket, err := backupRepository.GetBackup(ctx, backup.ID) + assert.Equal(t, repository.NoopCleanupTrashcanCleanupStatus, cleanupTrashcanBucket.TrashcanCleanup.Status) + + require.NoError(t, err) + logMsg := "trashcan cleanup for backup completed:" + if !strings.Contains(strings.TrimSpace(stdErr), logMsg) { + t.Errorf("Run should write log message %q but it logged\n\t%s", logMsg, stdErr) + } +} + +func TestCleanupTrashcansService_Noop(t *testing.T) { + httpMockHandler.Start() + defer httpMockHandler.Stop() + + ctx := context.Background() + service, err := newCleanupTrashcansService(ctx, provider.NewDefaultImpersonatedTokenConfigProvider(), secret.NewEnvSecretProvider()) + require.NoError(t, err) + + backupRepository, err := repository.NewBackupRepository(ctx, secret.NewEnvSecretProvider()) + require.NoErrorf(t, err, "backupRepository should be instantiate") + + backup, err := backupRepository.AddBackup(ctx, noopTrashcanBackup()) + require.NoError(t, err, "should add new backup") + defer func() { _ = deleteBackup(backup.ID) }() + + _, stdErr, err := captureStderr(func() { + service.Run(ctx) + }) + + cleanupTrashcanBucket, err := backupRepository.GetBackup(ctx, backup.ID) + assert.Equal(t, repository.NoopCleanupTrashcanCleanupStatus, cleanupTrashcanBucket.TrashcanCleanup.Status) + + require.NoError(t, err) + logMsg := "" + if !strings.Contains(strings.TrimSpace(stdErr), logMsg) { + t.Errorf("Run should write log message %q but it logged\n\t%s", logMsg, stdErr) + } +} + +func scheduleTrashcanBackup() *repository.Backup { + return &repository.Backup{ + ID: "somerandom-id", + Status: repository.Prepared, + TrashcanCleanup: repository.TrashcanCleanup{Status: repository.ScheduledTrashcanCleanupStatus}, + SourceProject: "local-ability", + Strategy: repository.Snapshot, + Type: repository.BigQuery, + SinkOptions: repository.SinkOptions{ + TargetProject: "local-ability-backup", + Sink: "uuid-5678-123456", + Region: "europe-west1", + StorageClass: "NEARLINE", + }, + BackupOptions: repository.BackupOptions{ + BigQueryOptions: repository.BigQueryOptions{"demo_delete_me_backup_target", []string{"gcp_billing_budget_amount_plan"}, []string{}}, + }, + } +} + +func noopTrashcanBackup() *repository.Backup { + return &repository.Backup{ + ID: "somerandom-id", + Status: repository.Prepared, + TrashcanCleanup: repository.TrashcanCleanup{Status: repository.NoopCleanupTrashcanCleanupStatus}, + SourceProject: "local-ability", + Strategy: repository.Snapshot, + Type: repository.BigQuery, + SinkOptions: repository.SinkOptions{ + TargetProject: "local-ability-backup", + Sink: "uuid-5678-123456", + Region: "europe-west1", + StorageClass: "NEARLINE", + }, + BackupOptions: repository.BackupOptions{ + BigQueryOptions: repository.BigQueryOptions{"demo_delete_me_backup_target", []string{"gcp_billing_budget_amount_plan"}, []string{}}, + }, + } +} diff --git a/pkg/tasks/tasks_multiplexer.go b/pkg/tasks/tasks_multiplexer.go index 37306e02..528f03ea 100644 --- a/pkg/tasks/tasks_multiplexer.go +++ b/pkg/tasks/tasks_multiplexer.go @@ -26,6 +26,8 @@ const ( CheckJobsStuck = "check_jobs_stuck" // CheckSinkProjectCompliance is handled by task that checks if backups are still immutable CheckSinkProjectCompliance = "check_sink_project_compliance" + // CleanupTrashcans is handled by task that cleans up trashcan + CleanupTrashcans = "cleanup_trashcans" ) // TaskRunner runs tasks @@ -91,6 +93,13 @@ func RunTask(task string, tokenSourceProvider impersonate.TargetPrincipalForProj return } service.Run(ctx) + case CleanupTrashcans: + service, err := newCleanupTrashcansService(ctx, tokenSourceProvider, credentialsProvider) + if err != nil { + glog.Errorf("could not instantiate new CleanupTrashcansService: %s", err) + return + } + service.Run(ctx) default: glog.Warningf("no Service found for action: %s", task) } diff --git a/resources/migrations/V1.0.4__add_cleanup_trashcan_for_backup.sql b/resources/migrations/V1.0.4__add_cleanup_trashcan_for_backup.sql new file mode 100644 index 00000000..981cf5a8 --- /dev/null +++ b/resources/migrations/V1.0.4__add_cleanup_trashcan_for_backup.sql @@ -0,0 +1,8 @@ +ALTER TABLE backups + ADD trashcan_cleanup_status TEXT DEFAULT 'Noop'; +ALTER TABLE backups + ADD trashcan_cleanup_error_message TEXT DEFAULT NULL; +ALTER TABLE backups + ADD trashcan_cleanup_last_scheduled_timestamp TIMESTAMP DEFAULT NULL; +ALTER TABLE backups + ADD trashcan_cleanup_start_running_timestamp TIMESTAMP DEFAULT NULL; \ No newline at end of file diff --git a/resources/openapi.yaml b/resources/openapi.yaml index a22fd5a3..d3041a96 100644 --- a/resources/openapi.yaml +++ b/resources/openapi.yaml @@ -353,6 +353,21 @@ paths: description: Created '403': description: Forbidden + /trashcans/{backupId}/clean_up: + post: + summary: Clean up trashcan for backup sink + parameters: + - in: path + name: backupId + schema: + type: string + required: true + description: Backup ID + responses: + '204': + description: OK + '400': + description: Bad Request components: schemas: UserResponse: @@ -422,6 +437,13 @@ components: $ref: '#/components/schemas/RecoveryPointObjective' recovery_time_objective: $ref: '#/components/schemas/RecoveryTimeObjective' + trashcan_cleanup_status: + $ref: '#/components/schemas/TrashcanCleanupStatus' + trashcan_cleanup_error_message: + type: string + trashcan_cleanup_last_scheduled_time: + type: string + format: date-time Job: type: object properties: @@ -579,6 +601,12 @@ components: - Snapshot - Mirror - Oneshot + TrashcanCleanupStatus: + type: string + enum: + - Noop + - Scheduled + - Error BackupStatus: type: string enum: