From 3aba5f12fca034be82425124bcf0c01591ac91c1 Mon Sep 17 00:00:00 2001 From: Bohdan Shulha <b.shulha@pm.me> Date: Tue, 16 Jul 2024 15:28:59 +0200 Subject: [PATCH] feat: #65 allow to backup volumes --- app/Console/Commands/DispatchBackupTask.php | 115 ++++++++++++++++++ .../ServiceExec/ServiceExecCompleted.php | 9 ++ .../ServiceExec/ServiceExecFailed.php | 9 ++ .../UploadS3File/UploadS3FileCompleted.php | 9 ++ .../UploadS3File/UploadS3FileFailed.php | 9 ++ app/Http/Controllers/ServiceController.php | 10 +- app/Http/Controllers/SwarmController.php | 4 +- app/Http/Controllers/SwarmTaskController.php | 7 +- app/Models/DeploymentData/BackupSchedule.php | 18 +++ app/Models/DeploymentData/BackupVolume.php | 18 +++ app/Models/DeploymentData/CronPreset.php | 9 ++ app/Models/DeploymentData/Process.php | 57 +++++++-- app/Models/DeploymentData/Volume.php | 4 +- app/Models/NodeTaskGroupType.php | 1 + app/Models/NodeTaskType.php | 18 +++ .../NodeTasks/ServiceExec/ServiceExecMeta.php | 21 ++++ .../ServiceExec/ServiceExecResult.php | 20 +++ .../UploadS3File/UploadS3FileMeta.php | 21 ++++ .../UploadS3File/UploadS3FileResult.php | 19 +++ app/Models/SwarmData.php | 4 +- app/Traits/HasOwningTeam.php | 4 +- bootstrap/app.php | 32 +++++ ..._update_deployments_add_volume_backups.php | 58 +++++++++ resources/js/Components/BackupSchedule.vue | 45 +++++++ .../js/Components/Service/ComponentBlock.vue | 2 +- .../js/Pages/Nodes/Partials/S3Storages.vue | 2 +- resources/js/Pages/Services/Create.vue | 10 +- .../Services/Partials/DeploymentData.vue | 81 +++++++++++- resources/js/Pages/Services/Show.vue | 3 +- 29 files changed, 594 insertions(+), 25 deletions(-) create mode 100644 app/Console/Commands/DispatchBackupTask.php create mode 100644 app/Events/NodeTasks/ServiceExec/ServiceExecCompleted.php create mode 100644 app/Events/NodeTasks/ServiceExec/ServiceExecFailed.php create mode 100644 app/Events/NodeTasks/UploadS3File/UploadS3FileCompleted.php create mode 100644 app/Events/NodeTasks/UploadS3File/UploadS3FileFailed.php create mode 100644 app/Models/DeploymentData/BackupSchedule.php create mode 100644 app/Models/DeploymentData/BackupVolume.php create mode 100644 app/Models/DeploymentData/CronPreset.php create mode 100644 app/Models/NodeTasks/ServiceExec/ServiceExecMeta.php create mode 100644 app/Models/NodeTasks/ServiceExec/ServiceExecResult.php create mode 100644 app/Models/NodeTasks/UploadS3File/UploadS3FileMeta.php create mode 100644 app/Models/NodeTasks/UploadS3File/UploadS3FileResult.php create mode 100644 database/migrations/2024_07_14_115507_update_deployments_add_volume_backups.php create mode 100644 resources/js/Components/BackupSchedule.vue diff --git a/app/Console/Commands/DispatchBackupTask.php b/app/Console/Commands/DispatchBackupTask.php new file mode 100644 index 0000000..cae9425 --- /dev/null +++ b/app/Console/Commands/DispatchBackupTask.php @@ -0,0 +1,115 @@ +<?php + +namespace App\Console\Commands; + +use App\Models\Node; +use App\Models\NodeTask; +use App\Models\NodeTaskGroupType; +use App\Models\NodeTasks\ServiceExec\ServiceExecMeta; +use App\Models\NodeTaskType; +use App\Models\Service; +use Exception; +use Illuminate\Console\Command; +use Illuminate\Support\Facades\DB; + +class DispatchBackupTask extends Command +{ + /** + * The name and signature of the console command. + * + * @var string + */ + protected $signature = 'app:dispatch-backup-task {--service-id=} {--process=} {--volume=}'; + + /** + * The console command description. + * + * @var string + */ + protected $description = 'Adds a task to create Volume Backup on a Node.'; + + /** + * Execute the console command. + */ + public function handle(): void + { + DB::transaction(function () { + $this->dispatchBackupTask(); + }); + } + + /** + * @throws Exception + */ + protected function dispatchBackupTask(): void + { + /* @var Service $service */ + $service = Service::findOrFail($this->option('service-id')); + + $deployment = $service->latestDeployment; + + $process = $deployment->data->findProcess($this->option('process')); + if ($process === null) { + throw new Exception("Could not find process {$this->option('process')} in deployment {$deployment->id}."); + } + + $volume = $process->findVolume($this->option('volume')); + if ($volume === null) { + throw new Exception("Could not find volume {$this->option('volume')} in process {$process->name}."); + } + + $node = Node::findOrFail($deployment->data->placementNodeId); + + $taskGroup = $node->taskGroups()->create([ + 'swarm_id' => $node->swarm_id, + 'node_id' => $node->id, + 'type' => NodeTaskGroupType::CreateBackup, + 'invoker_id' => $deployment->latestTaskGroup->invoker_id, + 'team_id' => $service->team_id, + ]); + + $date = now()->format('Y-m-d_His'); + $backupFileName = dockerize_name("svc-{$service->id}-{$process->name}-{$volume->name}-{$date}") . '.tar.gz'; + $archivePath = "{$process->backupVolume->path}/$backupFileName"; + $backupCommand = "tar czfv $archivePath -C {$volume->path} ."; + + $s3Storage = $node->swarm->data->findS3Storage($volume->backupSchedule->s3StorageId); + if ($s3Storage === null) { + throw new Exception("Could not find S3 storage {$volume->backupSchedule->s3StorageId} in swarm {$node->swarm_id}."); + } + + $taskGroup->tasks()->createMany([ + [ + 'type' => NodeTaskType::ServiceExec, + 'meta' => [ + 'serviceId' => $service->id, + 'command' => $backupCommand, + ], + 'payload' => [ + 'ProcessName' => $process->dockerName, + 'ExecSpec' => [ + 'AttachStdout' => true, + 'AttachStderr' => true, + 'Cmd' => ['sh', '-c', $backupCommand], + ] + ] + ], + [ + 'type' => NodeTaskType::UploadS3File, + 'meta' => [ + 'serviceId' => $service->id, + ], + 'payload' => [ + 'S3StorageConfigName' => $s3Storage->dockerName, + 'VolumeSpec' => [ + 'Type' => 'volume', + 'Source' => $process->backupVolume->dockerName, + 'Target' => $process->backupVolume->path, + ], + 'SrcFilePath' => $archivePath, + 'DestFilePath' => $s3Storage->pathPrefix . '/' . $backupFileName, + ], + ] + ]); + } +} diff --git a/app/Events/NodeTasks/ServiceExec/ServiceExecCompleted.php b/app/Events/NodeTasks/ServiceExec/ServiceExecCompleted.php new file mode 100644 index 0000000..ba385a4 --- /dev/null +++ b/app/Events/NodeTasks/ServiceExec/ServiceExecCompleted.php @@ -0,0 +1,9 @@ +<?php + +namespace App\Events\NodeTasks\ServiceExec; + +use App\Events\NodeTasks\BaseTaskEvent; + +class ServiceExecCompleted extends BaseTaskEvent +{ +} diff --git a/app/Events/NodeTasks/ServiceExec/ServiceExecFailed.php b/app/Events/NodeTasks/ServiceExec/ServiceExecFailed.php new file mode 100644 index 0000000..3137a1c --- /dev/null +++ b/app/Events/NodeTasks/ServiceExec/ServiceExecFailed.php @@ -0,0 +1,9 @@ +<?php + +namespace App\Events\NodeTasks\ServiceExec; + +use App\Events\NodeTasks\BaseTaskEvent; + +class ServiceExecFailed extends BaseTaskEvent +{ +} diff --git a/app/Events/NodeTasks/UploadS3File/UploadS3FileCompleted.php b/app/Events/NodeTasks/UploadS3File/UploadS3FileCompleted.php new file mode 100644 index 0000000..b09c4f0 --- /dev/null +++ b/app/Events/NodeTasks/UploadS3File/UploadS3FileCompleted.php @@ -0,0 +1,9 @@ +<?php + +namespace App\Events\NodeTasks\UploadS3File; + +use App\Events\NodeTasks\BaseTaskEvent; + +class UploadS3FileCompleted extends BaseTaskEvent +{ +} diff --git a/app/Events/NodeTasks/UploadS3File/UploadS3FileFailed.php b/app/Events/NodeTasks/UploadS3File/UploadS3FileFailed.php new file mode 100644 index 0000000..3d5d5af --- /dev/null +++ b/app/Events/NodeTasks/UploadS3File/UploadS3FileFailed.php @@ -0,0 +1,9 @@ +<?php + +namespace App\Events\NodeTasks\UploadS3File; + +use App\Events\NodeTasks\BaseTaskEvent; + +class UploadS3FileFailed extends BaseTaskEvent +{ +} diff --git a/app/Http/Controllers/ServiceController.php b/app/Http/Controllers/ServiceController.php index 9dd08af..5b4c748 100644 --- a/app/Http/Controllers/ServiceController.php +++ b/app/Http/Controllers/ServiceController.php @@ -40,6 +40,7 @@ public function create() $networks = count($swarms) ? $swarms[0]->networks : []; $nodes = count($swarms) ? $swarms[0]->nodes : []; $dockerRegistries = count($swarms) ? $swarms[0]->data->registries : []; + $s3Storages = count($swarms) ? $swarms[0]->data->s3Storages : []; $deploymentData = DeploymentData::make([ 'networkName' => count($networks) ? $networks[0]->name : null, @@ -50,7 +51,8 @@ public function create() 'networks' => $networks, 'nodes' => $nodes, 'deploymentData' => $deploymentData, - 'dockerRegistries' => $dockerRegistries + 'dockerRegistries' => $dockerRegistries, + 's3Storages' => $s3Storages, ]); } @@ -82,13 +84,15 @@ public function show(Service $service) $networks = $service->swarm->networks; $nodes = $service->swarm->nodes; - $dockerRegistries = $service->swarm->data->registries ; + $dockerRegistries = $service->swarm->data->registries; + $s3Storages = $service->swarm->data->s3Storages; return Inertia::render('Services/Show', [ 'service' => $service, 'networks' => $networks, 'nodes' => $nodes, - 'dockerRegistries' => $dockerRegistries + 'dockerRegistries' => $dockerRegistries, + 's3Storages' => $s3Storages, ]); } diff --git a/app/Http/Controllers/SwarmController.php b/app/Http/Controllers/SwarmController.php index 68e6f05..33d70ed 100644 --- a/app/Http/Controllers/SwarmController.php +++ b/app/Http/Controllers/SwarmController.php @@ -163,11 +163,9 @@ public function updateS3Storages(Swarm $swarm, Request $request) $tasks = []; foreach ($swarmData->s3Storages as $s3Storage) { - $previous = $s3Storage->dockerName ? $swarm->data->findS3Storage($s3Storage->dockerName) : null; + $previous = $swarm->data->findS3Storage($s3Storage->id); if ($previous) { if ($s3Storage->sameAs($previous)) { - $s3Storage->dockerName = $previous->dockerName; - continue; } } diff --git a/app/Http/Controllers/SwarmTaskController.php b/app/Http/Controllers/SwarmTaskController.php index 9f6f52d..752a142 100644 --- a/app/Http/Controllers/SwarmTaskController.php +++ b/app/Http/Controllers/SwarmTaskController.php @@ -19,6 +19,7 @@ use App\Models\Swarm; use App\Models\SwarmData; use Illuminate\Support\Facades\DB; +use Illuminate\Support\Str; class SwarmTaskController extends Controller { @@ -31,6 +32,8 @@ public function initCluster(InitClusterFormRequest $request) 'data' => SwarmData::validateAndCreate([ 'registriesRev' => 0, 'registries' => [], + 's3StoragesRev' => 0, + 's3Storages' => [], ]), ]); @@ -121,7 +124,7 @@ public function initCluster(InitClusterFormRequest $request) 'data' => DeploymentData::validateAndCreate([ 'networkName' => $network->docker_name, 'internalDomain' => 'caddy.ptah.local', - 'placementNodeId' => null, + 'placementNodeId' => $node->id, 'processes' => [ [ 'name' => 'svc', @@ -155,10 +158,12 @@ public function initCluster(InitClusterFormRequest $request) 'secretFiles' => [], 'volumes' => [ [ + 'id' => 'volume-' . Str::random(11), 'name' => 'data', 'path' => '/data', ], [ + 'id' => 'volume-' . Str::random(11), 'name' => 'config', 'path' => '/config', ] diff --git a/app/Models/DeploymentData/BackupSchedule.php b/app/Models/DeploymentData/BackupSchedule.php new file mode 100644 index 0000000..0b4fece --- /dev/null +++ b/app/Models/DeploymentData/BackupSchedule.php @@ -0,0 +1,18 @@ +<?php + +namespace App\Models\DeploymentData; + +use Spatie\LaravelData\Data; + +class BackupSchedule extends Data +{ + public function __construct( + public CronPreset $preset, + public string $s3StorageId, + // TODO: !!! validate CRON expr + public string $expr, + ) + { + + } +} diff --git a/app/Models/DeploymentData/BackupVolume.php b/app/Models/DeploymentData/BackupVolume.php new file mode 100644 index 0000000..495e8e1 --- /dev/null +++ b/app/Models/DeploymentData/BackupVolume.php @@ -0,0 +1,18 @@ +<?php + +namespace App\Models\DeploymentData; + +use Spatie\LaravelData\Data; + +class BackupVolume extends Data +{ + public function __construct( + public string $id, + public string $name, + public ?string $dockerName, + public string $path, + ) + { + + } +} \ No newline at end of file diff --git a/app/Models/DeploymentData/CronPreset.php b/app/Models/DeploymentData/CronPreset.php new file mode 100644 index 0000000..69798d3 --- /dev/null +++ b/app/Models/DeploymentData/CronPreset.php @@ -0,0 +1,9 @@ +<?php + +namespace App\Models\DeploymentData; + +enum CronPreset: string +{ + case Daily = 'daily'; + case Custom = 'custom'; +} diff --git a/app/Models/DeploymentData/Process.php b/app/Models/DeploymentData/Process.php index 61461ae..9044381 100644 --- a/app/Models/DeploymentData/Process.php +++ b/app/Models/DeploymentData/Process.php @@ -11,6 +11,7 @@ use App\Models\NodeTasks\UpdateService\UpdateServiceMeta; use App\Models\NodeTaskType; use App\Rules\RequiredIfArrayHas; +use Illuminate\Support\Str; use Spatie\LaravelData\Attributes\DataCollectionOf; use Spatie\LaravelData\Attributes\Validation\Enum; use Spatie\LaravelData\Attributes\Validation\Rule; @@ -43,6 +44,7 @@ public function __construct( #[DataCollectionOf(Volume::class)] /* @var Volume[] */ public array $volumes, + public ?BackupVolume $backupVolume, public int $replicas, #[DataCollectionOf(NodePort::class)] /* @var NodePort[] */ @@ -59,6 +61,12 @@ public function __construct( { } + + public function findVolume(string $dockerName): ?Volume + { + return collect($this->volumes)->first(fn(Volume $volume) => $volume->dockerName === $dockerName); + } + public function findConfigFile(string $path): ?ConfigFile { return collect($this->configFiles)->first(fn(ConfigFile $file) => $file->path === $path); @@ -205,6 +213,46 @@ public function asNodeTasks(Deployment $deployment): array 'serviceName' => $deployment->service->name, ]; + $volumes = $this->volumes; + + $mounts = collect($volumes) + ->map(fn(Volume $volume) => [ + 'Type' => 'volume', + 'Source' => $volume->dockerName, + 'Target' => $volume->path, + 'VolumeOptions' => [ + 'Labels' => dockerize_labels([ + 'id' => $volume->id, + ...$labels, + ]), + ] + ]) + ->toArray(); + + // TODO: if (has volumes with backups enabled OR has a Backup Script defined) + if (count($this->volumes)) { + if ($this->backupVolume == null) { + $this->backupVolume = BackupVolume::validateAndCreate([ + 'id' => 'backups-' . Str::random(11), + 'name' => 'backups', + 'dockerName' => $this->makeResourceName('/ptah/backups'), + 'path' => '/ptah/backups', + ]); + } + + $mounts[] = [ + 'Type' => 'volume', + 'Source' => $this->backupVolume->dockerName, + 'Target' => $this->backupVolume->path, + 'VolumeOptions' => [ + 'Labels' => dockerize_labels([ + 'id' => $this->backupVolume->id, + ...$labels, + ]), + ] + ]; + } + // FIXME: this is going to work wrong if the initial deployment is pending. // Don't allow to schedule deployments if the service has not been created yet? // This code is duplicated in the next block @@ -228,14 +276,7 @@ public function asNodeTasks(Deployment $deployment): array 'Args' => $args, 'Hostname' => "dpl-{$deployment->id}.{$internalDomain}", 'Env' => collect($this->envVars)->map(fn(EnvVar $var) => "{$var->name}={$var->value}")->toArray(), - 'Mounts' => collect($this->volumes)->map(fn(Volume $volume) => [ - 'Type' => 'volume', - 'Source' => $volume->dockerName, - 'Target' => $volume->path, - 'VolumeOptions' => [ - 'Labels' => $labels, - ] - ])->toArray(), + 'Mounts' => $mounts, 'Hosts' => [ $internalDomain, ], diff --git a/app/Models/DeploymentData/Volume.php b/app/Models/DeploymentData/Volume.php index 6047c4f..c1c6813 100644 --- a/app/Models/DeploymentData/Volume.php +++ b/app/Models/DeploymentData/Volume.php @@ -7,9 +7,11 @@ class Volume extends Data { public function __construct( + public string $id, public string $name, public ?string $dockerName, - public string $path + public string $path, + public ?BackupSchedule $backupSchedule ) { diff --git a/app/Models/NodeTaskGroupType.php b/app/Models/NodeTaskGroupType.php index bf3b0f0..23d1df9 100644 --- a/app/Models/NodeTaskGroupType.php +++ b/app/Models/NodeTaskGroupType.php @@ -11,4 +11,5 @@ enum NodeTaskGroupType: int case SelfUpgrade = 4; case UpdateDockerRegistries = 5; case UpdateS3Storages = 6; + case CreateBackup = 7; } diff --git a/app/Models/NodeTaskType.php b/app/Models/NodeTaskType.php index 607de75..b2d855b 100644 --- a/app/Models/NodeTaskType.php +++ b/app/Models/NodeTaskType.php @@ -30,12 +30,16 @@ use App\Events\NodeTasks\PullDockerImage\PullDockerImageFailed; use App\Events\NodeTasks\RebuildCaddyConfig\ApplyCaddyConfigCompleted; use App\Events\NodeTasks\RebuildCaddyConfig\ApplyCaddyConfigFailed; +use App\Events\NodeTasks\ServiceExec\ServiceExecCompleted; +use App\Events\NodeTasks\ServiceExec\ServiceExecFailed; use App\Events\NodeTasks\UpdateAgentSymlink\UpdateAgentSymlinkCompleted; use App\Events\NodeTasks\UpdateAgentSymlink\UpdateAgentSymlinkFailed; use App\Events\NodeTasks\UpdateNode\UpdateCurrentNodeCompleted; use App\Events\NodeTasks\UpdateNode\UpdateCurrentNodeFailed; use App\Events\NodeTasks\UpdateService\UpdateServiceCompleted; use App\Events\NodeTasks\UpdateService\UpdateServiceFailed; +use App\Events\NodeTasks\UploadS3File\UploadS3FileCompleted; +use App\Events\NodeTasks\UploadS3File\UploadS3FileFailed; use App\Models\NodeTasks\ApplyCaddyConfig\ApplyCaddyConfigMeta; use App\Models\NodeTasks\ApplyCaddyConfig\ApplyCaddyConfigResult; use App\Models\NodeTasks\CheckRegistryAuth\CheckRegistryAuthMeta; @@ -64,12 +68,16 @@ use App\Models\NodeTasks\InitSwarm\InitSwarmResult; use App\Models\NodeTasks\PullDockerImage\PullDockerImageMeta; use App\Models\NodeTasks\PullDockerImage\PullDockerImageResult; +use App\Models\NodeTasks\ServiceExec\ServiceExecMeta; +use App\Models\NodeTasks\ServiceExec\ServiceExecResult; use App\Models\NodeTasks\UpdateAgentSymlink\UpdateAgentSymlinkMeta; use App\Models\NodeTasks\UpdateAgentSymlink\UpdateAgentSymlinkResult; use App\Models\NodeTasks\UpdateCurrentNode\UpdateCurrentNodeMeta; use App\Models\NodeTasks\UpdateCurrentNode\UpdateCurrentNodeResult; use App\Models\NodeTasks\UpdateService\UpdateServiceMeta; use App\Models\NodeTasks\UpdateService\UpdateServiceResult; +use App\Models\NodeTasks\UploadS3File\UploadS3FileMeta; +use App\Models\NodeTasks\UploadS3File\UploadS3FileResult; // Mb use dynamic class names? $class = "{$this->name}Payload"; ?? enum NodeTaskType: int @@ -91,6 +99,8 @@ enum NodeTaskType: int case PullDockerImage = 14; case CreateS3Storage = 15; case CheckS3Storage = 16; + case ServiceExec = 17; + case UploadS3File = 18; public function meta(): string { @@ -112,6 +122,8 @@ public function meta(): string self::PullDockerImage => PullDockerImageMeta::class, self::CreateS3Storage => CreateS3StorageMeta::class, self::CheckS3Storage => CheckS3StorageMeta::class, + self::ServiceExec => ServiceExecMeta::class, + self::UploadS3File => UploadS3FileMeta::class }; } @@ -135,6 +147,8 @@ public function result(): string self::PullDockerImage => PullDockerImageResult::class, self::CreateS3Storage => CreateS3StorageResult::class, self::CheckS3Storage => CheckS3StorageResult::class, + self::ServiceExec => ServiceExecResult::class, + self::UploadS3File => UploadS3FileResult::class }; } @@ -158,6 +172,8 @@ public function completed(): string self::PullDockerImage => PullDockerImageCompleted::class, self::CreateS3Storage => CreateS3StorageCompleted::class, self::CheckS3Storage => CheckS3StorageCompleted::class, + self::ServiceExec => ServiceExecCompleted::class, + self::UploadS3File => UploadS3FileCompleted::class }; } @@ -181,6 +197,8 @@ public function failed(): string self::PullDockerImage => PullDockerImageFailed::class, self::CreateS3Storage => CreateS3StorageFailed::class, self::CheckS3Storage => CheckS3StorageFailed::class, + self::ServiceExec => ServiceExecFailed::class, + self::UploadS3File => UploadS3FileFailed::class }; } } diff --git a/app/Models/NodeTasks/ServiceExec/ServiceExecMeta.php b/app/Models/NodeTasks/ServiceExec/ServiceExecMeta.php new file mode 100644 index 0000000..129525e --- /dev/null +++ b/app/Models/NodeTasks/ServiceExec/ServiceExecMeta.php @@ -0,0 +1,21 @@ +<?php + +namespace App\Models\NodeTasks\ServiceExec; + +use App\Models\NodeTasks\AbstractTaskMeta; + +class ServiceExecMeta extends AbstractTaskMeta +{ + public function __construct( + public int $serviceId, + public string $command, + ) + { + // + } + + public function formattedHtml(): string + { + return "Execute <code>$this->command</code> on the service <code>$this->serviceId</code>."; + } +} diff --git a/app/Models/NodeTasks/ServiceExec/ServiceExecResult.php b/app/Models/NodeTasks/ServiceExec/ServiceExecResult.php new file mode 100644 index 0000000..7087970 --- /dev/null +++ b/app/Models/NodeTasks/ServiceExec/ServiceExecResult.php @@ -0,0 +1,20 @@ +<?php + +namespace App\Models\NodeTasks\ServiceExec; + +use App\Models\NodeTasks\AbstractTaskResult; + +class ServiceExecResult extends AbstractTaskResult +{ + public function __construct( + public array $output + ) + { + // + } + + public function formattedHtml(): string + { + return join('<br>', $this->output); + } +} diff --git a/app/Models/NodeTasks/UploadS3File/UploadS3FileMeta.php b/app/Models/NodeTasks/UploadS3File/UploadS3FileMeta.php new file mode 100644 index 0000000..877da96 --- /dev/null +++ b/app/Models/NodeTasks/UploadS3File/UploadS3FileMeta.php @@ -0,0 +1,21 @@ +<?php + +namespace App\Models\NodeTasks\UploadS3File; + +use App\Models\NodeTasks\AbstractTaskMeta; + +class UploadS3FileMeta extends AbstractTaskMeta +{ + public function __construct( + public string $serviceId, + public string $destPath, + ) + { + // + } + + public function formattedHtml(): string + { + return "Upload file to S3 Storage: {$this->destPath}"; + } +} diff --git a/app/Models/NodeTasks/UploadS3File/UploadS3FileResult.php b/app/Models/NodeTasks/UploadS3File/UploadS3FileResult.php new file mode 100644 index 0000000..976dbfc --- /dev/null +++ b/app/Models/NodeTasks/UploadS3File/UploadS3FileResult.php @@ -0,0 +1,19 @@ +<?php + +namespace App\Models\NodeTasks\UploadS3File; + +use App\Models\NodeTasks\AbstractTaskResult; + +class UploadS3FileResult extends AbstractTaskResult +{ + public function __construct( + ) + { + // + } + + public function formattedHtml(): string + { + return "UploadS3File - Task Result"; + } +} diff --git a/app/Models/SwarmData.php b/app/Models/SwarmData.php index ea49c82..8176269 100644 --- a/app/Models/SwarmData.php +++ b/app/Models/SwarmData.php @@ -30,10 +30,10 @@ public function findRegistry(string $dockerName): ?DockerRegistry ->first(); } - public function findS3Storage(string $dockerName): ?S3Storage + public function findS3Storage(string $id): ?S3Storage { return collect($this->s3Storages) - ->filter(fn (S3Storage $s3Storage) => $s3Storage->dockerName === $dockerName) + ->filter(fn (S3Storage $s3Storage) => $s3Storage->id === $id) ->first(); } } diff --git a/app/Traits/HasOwningTeam.php b/app/Traits/HasOwningTeam.php index b5347f9..a5cac4b 100644 --- a/app/Traits/HasOwningTeam.php +++ b/app/Traits/HasOwningTeam.php @@ -9,7 +9,9 @@ trait HasOwningTeam { protected static function bootHasOwningTeam(): void { - static::addGlobalScope(new TeamScope()); + if (!app()->runningInConsole()) { + static::addGlobalScope(new TeamScope()); + } } public function team() diff --git a/bootstrap/app.php b/bootstrap/app.php index c4a8bc4..1eaf84d 100644 --- a/bootstrap/app.php +++ b/bootstrap/app.php @@ -1,8 +1,11 @@ <?php use ApiNodes\Http\Middleware\AgentTokenAuth; +use App\Console\Commands\DispatchBackupTask; use App\Http\Middleware\HandleInertiaRequests; use App\Jobs\CheckAgentUpdates; +use App\Models\DeploymentData\CronPreset; +use App\Models\Service; use Illuminate\Console\Scheduling\Schedule; use Illuminate\Foundation\Application; use Illuminate\Foundation\Configuration\Exceptions; @@ -40,6 +43,35 @@ ->withSchedule(function (Schedule $schedule) { $schedule->job(CheckAgentUpdates::class) ->everyMinute() + ->onOneServer() ->withoutOverlapping(); + + Service::withoutGlobalScopes()->with(['latestDeployment' => fn ($query) => $query->withoutGlobalScopes()])->chunk(100, function ( + /* @var Service[] $services */ + $services + ) use ($schedule) { + foreach ($services as $service) { + foreach ($service->latestDeployment->data->processes as $process) { + if ($process->replicas === 0) { + continue; + } + foreach ($process->volumes as $volume) { + $backupSchedule = $volume->backupSchedule; + if ($backupSchedule === null) { + continue; + } + + $schedule + ->command(DispatchBackupTask::class, [ + 'serviceId' => $service->id, + 'volumeId' => $volume->id, + ]) + ->cron($backupSchedule->expr) + ->onOneServer() + ->withoutOverlapping(); + } + } + } + }); }) ->create(); diff --git a/database/migrations/2024_07_14_115507_update_deployments_add_volume_backups.php b/database/migrations/2024_07_14_115507_update_deployments_add_volume_backups.php new file mode 100644 index 0000000..179b343 --- /dev/null +++ b/database/migrations/2024_07_14_115507_update_deployments_add_volume_backups.php @@ -0,0 +1,58 @@ +<?php + +use Illuminate\Database\Migrations\Migration; +use Illuminate\Database\Schema\Blueprint; +use Illuminate\Support\Facades\DB; +use Illuminate\Support\Facades\Schema; + +return new class extends Migration +{ + /** + * Run the migrations. + */ + public function up(): void + { + DB::update(" + UPDATE deployments + SET data = jsonb_set( + data, + '{processes}', + ( + SELECT jsonb_agg( + jsonb_set( + process, + '{volumes}', + COALESCE( + ( + SELECT jsonb_agg( + volume_item || jsonb_build_object( + 'id', + concat('volume-', substr(md5(random()::text), 1, 11)) + ) || jsonb_build_object( + 'backupSchedule', + '{\"preset\": \"cron-disabled\"}'::jsonb + ) + ) + FROM jsonb_array_elements(process->'volumes') AS volume_item + ), + '[]'::jsonb + ) + ) || jsonb_build_object( + 'backupVolume', + CONCAT('{\"id\":\"backups-', substr(md5(random()::text), 1, 11) ,'\",\"name\":\"backups\",\"path\":\"/ptah/backups\",\"dockerName\":\"svc_', deployments.service_id,'_', (select name from services where id = deployments.service_id),'_svc_ptah_backups\",\"backupSchedule\":{\"expr\":null,\"preset\":\"cron-disabled\"}}')::jsonb + ) + ) + FROM jsonb_array_elements(data->'processes') AS process + ) + ); + "); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + // + } +}; diff --git a/resources/js/Components/BackupSchedule.vue b/resources/js/Components/BackupSchedule.vue new file mode 100644 index 0000000..18084c2 --- /dev/null +++ b/resources/js/Components/BackupSchedule.vue @@ -0,0 +1,45 @@ +<script setup lang="ts"> +import Select from "@/Components/Select.vue"; +import FormField from "@/Components/FormField.vue"; +import TextInput from "@/Components/TextInput.vue"; +import {effect} from "vue"; + +const model = defineModel(); + +const props = defineProps({ + 's3Storages': Array, + 'errors': Object, +}) + +effect(() => { + if (model.value.preset === 'daily') { + model.value.expr = '0 0 * * *'; + } +}); +</script> + +<template> + <FormField class="col-span-2" :error="props.errors?.['preset']"> + <template #label>Backup Schedule</template> + + <Select v-model="model.preset"> + <option value="daily">Every Day</option> + </Select> + </FormField> + + <FormField v-if="model.preset !== 'cron-disabled'" class="col-span-2" :error="props.errors?.['s3StorageId']"> + <template #label>Storage</template> + + <Select v-model="model.s3StorageId"> + <option v-for="s3Storage in props.s3Storages" :value="s3Storage.id">{{ s3Storage.name }}</option> + </Select> + </FormField> + +<!-- <input v-if="model.preset === 'daily'" type="hidden" v-model="model.expr" value="0 0 * * *" />--> + +<!-- <FormField v-if="model.preset === 'custom'" class="col-span-2">--> +<!-- <template #label>Expression</template>--> + +<!-- <TextInput v-model="model.expr" />--> +<!-- </FormField>--> +</template> \ No newline at end of file diff --git a/resources/js/Components/Service/ComponentBlock.vue b/resources/js/Components/Service/ComponentBlock.vue index 51b2d1d..9aa3c9e 100644 --- a/resources/js/Components/Service/ComponentBlock.vue +++ b/resources/js/Components/Service/ComponentBlock.vue @@ -15,7 +15,7 @@ defineModel(); <div v-for="(item, index) in modelValue"> <hr class="my-2" v-if="index > 0" /> - <div class="grid grid-cols-6 gap-4"> + <div class="grid grid-cols-6 gap-4" v-auto-animate> <slot :item="modelValue[index]" :$index="index" /> </div> diff --git a/resources/js/Pages/Nodes/Partials/S3Storages.vue b/resources/js/Pages/Nodes/Partials/S3Storages.vue index 75e5dfb..c5f6cd9 100644 --- a/resources/js/Pages/Nodes/Partials/S3Storages.vue +++ b/resources/js/Pages/Nodes/Partials/S3Storages.vue @@ -58,7 +58,7 @@ const submitForm = () => { Endpoint </template> - <TextInput v-model="item.endpoint" class="w-full" placeholder="https://backups.nyc3.digitaloceanspaces.com" /> + <TextInput v-model="item.endpoint" class="w-full" placeholder="nyc3.digitaloceanspaces.com" /> </FormField> <FormField class="col-span-2" :error="form.errors[`s3Storages.${$index}.region`]"> diff --git a/resources/js/Pages/Services/Create.vue b/resources/js/Pages/Services/Create.vue index 111b4b0..95e3064 100644 --- a/resources/js/Pages/Services/Create.vue +++ b/resources/js/Pages/Services/Create.vue @@ -21,6 +21,7 @@ const props = defineProps({ 'nodes': Array, 'deploymentData': Object, 'dockerRegistries': Array, + 's3Storages': Array, }) const form = useForm({ @@ -67,7 +68,14 @@ const createService = () => { </template> </ActionSection> - <DeploymentData v-model="form.deploymentData" :networks="networks" :nodes="nodes" :errors="form.errors" :service-name="form.name" :docker-registries="$props.dockerRegistries" /> + <DeploymentData v-model="form.deploymentData" + :networks="networks" + :nodes="nodes" + :errors="form.errors" + :service-name="form.name" + :docker-registries="props.dockerRegistries" + :s3-storages="props.s3Storages" + /> <div class="flex justify-end"> <PrimaryButton :class="{ 'opacity-25': form.processing }" :disabled="form.processing"> diff --git a/resources/js/Pages/Services/Partials/DeploymentData.vue b/resources/js/Pages/Services/Partials/DeploymentData.vue index 1259f54..5a505ee 100644 --- a/resources/js/Pages/Services/Partials/DeploymentData.vue +++ b/resources/js/Pages/Services/Partials/DeploymentData.vue @@ -14,7 +14,7 @@ import TextArea from "@/Components/TextArea.vue"; import {computed, effect, nextTick, reactive, ref} from "vue"; import ServiceDetailsForm from "@/Pages/Services/Partials/ServiceDetailsForm.vue"; import TabItem from "@/Components/Tabs/TabItem.vue"; -import {FwbTooltip} from "flowbite-vue"; +import {FwbToggle, FwbTooltip} from "flowbite-vue"; import ProcessTabs from "@/Pages/Services/Partials/ProcessTabs.vue"; import DangerButton from "@/Components/DangerButton.vue"; import DialogModal from "@/Components/DialogModal.vue"; @@ -22,6 +22,7 @@ import AddComponentButton from "@/Components/Service/AddComponentButton.vue"; import RemoveComponentButton from "@/Components/Service/RemoveComponentButton.vue"; import ComponentBlock from "@/Components/Service/ComponentBlock.vue"; import FormFieldGrid from "@/Components/FormFieldGrid.vue"; +import BackupSchedule from "@/Components/BackupSchedule.vue"; const model = defineModel() @@ -31,6 +32,7 @@ const props = defineProps({ 'nodes': Array, 'serviceName': String | undefined, 'dockerRegistries': Array, + 's3Storages': Array, 'errors': Object, }); @@ -72,7 +74,14 @@ const addSecretFile = () => { } const addVolume = () => { - model.value.processes[state.selectedProcessIndex['volumes']].volumes.push({id: makeId('volume'), name: '', path: ''}); + model.value.processes[state.selectedProcessIndex['volumes']].volumes.push({ + id: makeId('volume'), + name: '', + path: '', + backupSchedule: { + preset: 'cron-disabled', + } + }); } const addPort = () => { @@ -168,6 +177,18 @@ effect(() => { : null; }); +const toggleVolumeBackups = (volume) => { + if (volume.backupSchedule === null) { + volume.backupSchedule = { + preset: 'daily', + s3StorageName: props.s3Storages[0].dockerName, + expr: '0 0 * * *', + } + } else { + volume.backupSchedule = null; + } +} + const processRemoveInput = ref(); const processRemoval = reactive({ @@ -200,6 +221,10 @@ const submitProcessRemoval = () => { closeProcessRemovalModal(); } } + +const extractFieldErrors = (basePath) => { + return Object.fromEntries(Object.keys(props.errors).filter((key) => key.startsWith(basePath)).map((key) => [key.substring(basePath.length + 1), props.errors[key]])); +} </script> <template> @@ -698,6 +723,58 @@ const submitProcessRemoval = () => { </SecondaryButton> </div> </FormField> + <ComponentBlock v-else v-model="model.processes[state.selectedProcessIndex['volumes']].volumes" v-slot="{ item, $index }" @remove="model.processes[state.selectedProcessIndex['volumes']].volumes.splice($event, 1)"> + <FormField + :error="props.errors[`processes.${state.selectedProcessIndex['volumes']}.volumes.${$index}.name`] || props.errors[`processes.${state.selectedProcessIndex['volumes']}.volumes.${$index}.path`]" + class="col-span-2"> + + <template #label>Volume Name</template> + + <TextInput v-model="item.name" class="w-full" placeholder="Name"/> + </FormField> + + <FormField + :error="props.errors[`processes.${state.selectedProcessIndex['volumes']}.volumes.${$index}.path`]" + class="col-span-3"> + + <template #label>Mount Path</template> + + <TextInput v-model="item.path" class="w-full" placeholder="Path"/> + </FormField> + + <FormField class="col-span-1"> + <template #label v-if="props.s3Storages.length > 0">Enable Backups</template> + + <template #label v-else> + <FwbTooltip trigger="hover"> + <template #trigger> + <div class="flex"> + Backup + + <svg class="ms-1 w-4 h-4 text-blue-600 dark:text-white" aria-hidden="true" xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24"> + <path stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M9.529 9.988a2.502 2.502 0 1 1 5 .191A2.441 2.441 0 0 1 12 12.582V14m-.01 3.008H12M21 12a9 9 0 1 1-18 0 9 9 0 0 1 18 0Z"/> + </svg> + </div> + </template> + + <template #content> + Backups can't be enabled - no S3 Storages configured + </template> + </FwbTooltip> + </template> + + <div class="pt-2 select-none"> + <FwbToggle :model-value="item.backupSchedule !== null" :disabled="props.s3Storages.length === 0" :class="props.s3Storages.length === 0 ? 'cursor-not-allowed opacity-40' : ''" @change="toggleVolumeBackups(item)" /> + </div> + </FormField> + + <BackupSchedule + v-if="item.backupSchedule" + v-model="item.backupSchedule" + :errors="extractFieldErrors(`processes.${state.selectedProcessIndex['volumes']}.volumes.${$index}.backupSchedule`)" + :s3-storages="props.s3Storages" + /> + </ComponentBlock> </template> <template #actions> diff --git a/resources/js/Pages/Services/Show.vue b/resources/js/Pages/Services/Show.vue index 88c9831..fa8153b 100644 --- a/resources/js/Pages/Services/Show.vue +++ b/resources/js/Pages/Services/Show.vue @@ -17,6 +17,7 @@ import ActionSection from "@/Components/ActionSection.vue"; const props = defineProps({ service: Object, dockerRegistries: Array, + s3Storages: Array, }) const serviceForm = useForm({ @@ -95,7 +96,7 @@ const closeDeletionModal = () => { </FormSection> <form @submit.prevent="deploy"> - <DeploymentData v-model="deploymentForm" :errors="deploymentForm.errors" :docker-registries="$props.dockerRegistries" /> + <DeploymentData v-model="deploymentForm" :errors="deploymentForm.errors" :docker-registries="props.dockerRegistries" :s3-storages="props.s3Storages" /> <div class="flex justify-end"> <PrimaryButton :class="{ 'opacity-25': deploymentForm.processing }" :disabled="deploymentForm.processing">