From 3aba5f12fca034be82425124bcf0c01591ac91c1 Mon Sep 17 00:00:00 2001
From: Bohdan Shulha <b.shulha@pm.me>
Date: Tue, 16 Jul 2024 15:28:59 +0200
Subject: [PATCH] feat: #65 allow to backup volumes

---
 app/Console/Commands/DispatchBackupTask.php   | 115 ++++++++++++++++++
 .../ServiceExec/ServiceExecCompleted.php      |   9 ++
 .../ServiceExec/ServiceExecFailed.php         |   9 ++
 .../UploadS3File/UploadS3FileCompleted.php    |   9 ++
 .../UploadS3File/UploadS3FileFailed.php       |   9 ++
 app/Http/Controllers/ServiceController.php    |  10 +-
 app/Http/Controllers/SwarmController.php      |   4 +-
 app/Http/Controllers/SwarmTaskController.php  |   7 +-
 app/Models/DeploymentData/BackupSchedule.php  |  18 +++
 app/Models/DeploymentData/BackupVolume.php    |  18 +++
 app/Models/DeploymentData/CronPreset.php      |   9 ++
 app/Models/DeploymentData/Process.php         |  57 +++++++--
 app/Models/DeploymentData/Volume.php          |   4 +-
 app/Models/NodeTaskGroupType.php              |   1 +
 app/Models/NodeTaskType.php                   |  18 +++
 .../NodeTasks/ServiceExec/ServiceExecMeta.php |  21 ++++
 .../ServiceExec/ServiceExecResult.php         |  20 +++
 .../UploadS3File/UploadS3FileMeta.php         |  21 ++++
 .../UploadS3File/UploadS3FileResult.php       |  19 +++
 app/Models/SwarmData.php                      |   4 +-
 app/Traits/HasOwningTeam.php                  |   4 +-
 bootstrap/app.php                             |  32 +++++
 ..._update_deployments_add_volume_backups.php |  58 +++++++++
 resources/js/Components/BackupSchedule.vue    |  45 +++++++
 .../js/Components/Service/ComponentBlock.vue  |   2 +-
 .../js/Pages/Nodes/Partials/S3Storages.vue    |   2 +-
 resources/js/Pages/Services/Create.vue        |  10 +-
 .../Services/Partials/DeploymentData.vue      |  81 +++++++++++-
 resources/js/Pages/Services/Show.vue          |   3 +-
 29 files changed, 594 insertions(+), 25 deletions(-)
 create mode 100644 app/Console/Commands/DispatchBackupTask.php
 create mode 100644 app/Events/NodeTasks/ServiceExec/ServiceExecCompleted.php
 create mode 100644 app/Events/NodeTasks/ServiceExec/ServiceExecFailed.php
 create mode 100644 app/Events/NodeTasks/UploadS3File/UploadS3FileCompleted.php
 create mode 100644 app/Events/NodeTasks/UploadS3File/UploadS3FileFailed.php
 create mode 100644 app/Models/DeploymentData/BackupSchedule.php
 create mode 100644 app/Models/DeploymentData/BackupVolume.php
 create mode 100644 app/Models/DeploymentData/CronPreset.php
 create mode 100644 app/Models/NodeTasks/ServiceExec/ServiceExecMeta.php
 create mode 100644 app/Models/NodeTasks/ServiceExec/ServiceExecResult.php
 create mode 100644 app/Models/NodeTasks/UploadS3File/UploadS3FileMeta.php
 create mode 100644 app/Models/NodeTasks/UploadS3File/UploadS3FileResult.php
 create mode 100644 database/migrations/2024_07_14_115507_update_deployments_add_volume_backups.php
 create mode 100644 resources/js/Components/BackupSchedule.vue

diff --git a/app/Console/Commands/DispatchBackupTask.php b/app/Console/Commands/DispatchBackupTask.php
new file mode 100644
index 0000000..cae9425
--- /dev/null
+++ b/app/Console/Commands/DispatchBackupTask.php
@@ -0,0 +1,115 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Models\Node;
+use App\Models\NodeTask;
+use App\Models\NodeTaskGroupType;
+use App\Models\NodeTasks\ServiceExec\ServiceExecMeta;
+use App\Models\NodeTaskType;
+use App\Models\Service;
+use Exception;
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\DB;
+
+class DispatchBackupTask extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'app:dispatch-backup-task {--service-id=} {--process=} {--volume=}';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Adds a task to create Volume Backup on a Node.';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle(): void
+    {
+        DB::transaction(function () {
+            $this->dispatchBackupTask();
+        });
+    }
+
+    /**
+     * @throws Exception
+     */
+    protected function dispatchBackupTask(): void
+    {
+        /* @var Service $service */
+        $service = Service::findOrFail($this->option('service-id'));
+
+        $deployment = $service->latestDeployment;
+
+        $process = $deployment->data->findProcess($this->option('process'));
+        if ($process === null) {
+            throw new Exception("Could not find process {$this->option('process')} in deployment {$deployment->id}.");
+        }
+
+        $volume = $process->findVolume($this->option('volume'));
+        if ($volume === null) {
+            throw new Exception("Could not find volume {$this->option('volume')} in process {$process->name}.");
+        }
+
+        $node = Node::findOrFail($deployment->data->placementNodeId);
+
+        $taskGroup = $node->taskGroups()->create([
+            'swarm_id' => $node->swarm_id,
+            'node_id' => $node->id,
+            'type' => NodeTaskGroupType::CreateBackup,
+            'invoker_id' => $deployment->latestTaskGroup->invoker_id,
+            'team_id' => $service->team_id,
+        ]);
+
+        $date = now()->format('Y-m-d_His');
+        $backupFileName = dockerize_name("svc-{$service->id}-{$process->name}-{$volume->name}-{$date}") . '.tar.gz';
+        $archivePath = "{$process->backupVolume->path}/$backupFileName";
+        $backupCommand = "tar czfv $archivePath -C {$volume->path} .";
+
+        $s3Storage = $node->swarm->data->findS3Storage($volume->backupSchedule->s3StorageId);
+        if ($s3Storage === null) {
+            throw new Exception("Could not find S3 storage {$volume->backupSchedule->s3StorageId} in swarm {$node->swarm_id}.");
+        }
+
+        $taskGroup->tasks()->createMany([
+            [
+                'type' => NodeTaskType::ServiceExec,
+                'meta' => [
+                    'serviceId' => $service->id,
+                    'command' => $backupCommand,
+                ],
+                'payload' => [
+                    'ProcessName' => $process->dockerName,
+                    'ExecSpec' => [
+                        'AttachStdout' => true,
+                        'AttachStderr' => true,
+                        'Cmd' => ['sh', '-c', $backupCommand],
+                    ]
+                ]
+            ],
+            [
+                'type' => NodeTaskType::UploadS3File,
+                'meta' => [
+                    'serviceId' => $service->id,
+                ],
+                'payload' => [
+                    'S3StorageConfigName' => $s3Storage->dockerName,
+                    'VolumeSpec' => [
+                        'Type' => 'volume',
+                        'Source' => $process->backupVolume->dockerName,
+                        'Target' => $process->backupVolume->path,
+                    ],
+                    'SrcFilePath' => $archivePath,
+                    'DestFilePath' => $s3Storage->pathPrefix . '/' . $backupFileName,
+                ],
+            ]
+        ]);
+    }
+}
diff --git a/app/Events/NodeTasks/ServiceExec/ServiceExecCompleted.php b/app/Events/NodeTasks/ServiceExec/ServiceExecCompleted.php
new file mode 100644
index 0000000..ba385a4
--- /dev/null
+++ b/app/Events/NodeTasks/ServiceExec/ServiceExecCompleted.php
@@ -0,0 +1,9 @@
+<?php
+
+namespace App\Events\NodeTasks\ServiceExec;
+
+use App\Events\NodeTasks\BaseTaskEvent;
+
+class ServiceExecCompleted extends BaseTaskEvent
+{
+}
diff --git a/app/Events/NodeTasks/ServiceExec/ServiceExecFailed.php b/app/Events/NodeTasks/ServiceExec/ServiceExecFailed.php
new file mode 100644
index 0000000..3137a1c
--- /dev/null
+++ b/app/Events/NodeTasks/ServiceExec/ServiceExecFailed.php
@@ -0,0 +1,9 @@
+<?php
+
+namespace App\Events\NodeTasks\ServiceExec;
+
+use App\Events\NodeTasks\BaseTaskEvent;
+
+class ServiceExecFailed extends BaseTaskEvent
+{
+}
diff --git a/app/Events/NodeTasks/UploadS3File/UploadS3FileCompleted.php b/app/Events/NodeTasks/UploadS3File/UploadS3FileCompleted.php
new file mode 100644
index 0000000..b09c4f0
--- /dev/null
+++ b/app/Events/NodeTasks/UploadS3File/UploadS3FileCompleted.php
@@ -0,0 +1,9 @@
+<?php
+
+namespace App\Events\NodeTasks\UploadS3File;
+
+use App\Events\NodeTasks\BaseTaskEvent;
+
+class UploadS3FileCompleted extends BaseTaskEvent
+{
+}
diff --git a/app/Events/NodeTasks/UploadS3File/UploadS3FileFailed.php b/app/Events/NodeTasks/UploadS3File/UploadS3FileFailed.php
new file mode 100644
index 0000000..3d5d5af
--- /dev/null
+++ b/app/Events/NodeTasks/UploadS3File/UploadS3FileFailed.php
@@ -0,0 +1,9 @@
+<?php
+
+namespace App\Events\NodeTasks\UploadS3File;
+
+use App\Events\NodeTasks\BaseTaskEvent;
+
+class UploadS3FileFailed extends BaseTaskEvent
+{
+}
diff --git a/app/Http/Controllers/ServiceController.php b/app/Http/Controllers/ServiceController.php
index 9dd08af..5b4c748 100644
--- a/app/Http/Controllers/ServiceController.php
+++ b/app/Http/Controllers/ServiceController.php
@@ -40,6 +40,7 @@ public function create()
         $networks = count($swarms) ? $swarms[0]->networks : [];
         $nodes = count($swarms) ? $swarms[0]->nodes : [];
         $dockerRegistries = count($swarms) ? $swarms[0]->data->registries : [];
+        $s3Storages = count($swarms) ? $swarms[0]->data->s3Storages : [];
 
         $deploymentData = DeploymentData::make([
             'networkName' => count($networks) ? $networks[0]->name : null,
@@ -50,7 +51,8 @@ public function create()
             'networks' => $networks,
             'nodes' => $nodes,
             'deploymentData' => $deploymentData,
-            'dockerRegistries' => $dockerRegistries
+            'dockerRegistries' => $dockerRegistries,
+            's3Storages' => $s3Storages,
         ]);
     }
 
@@ -82,13 +84,15 @@ public function show(Service $service)
 
         $networks = $service->swarm->networks;
         $nodes = $service->swarm->nodes;
-        $dockerRegistries = $service->swarm->data->registries ;
+        $dockerRegistries = $service->swarm->data->registries;
+        $s3Storages = $service->swarm->data->s3Storages;
 
         return Inertia::render('Services/Show', [
             'service' => $service,
             'networks' => $networks,
             'nodes' => $nodes,
-            'dockerRegistries' => $dockerRegistries
+            'dockerRegistries' => $dockerRegistries,
+            's3Storages' => $s3Storages,
         ]);
     }
 
diff --git a/app/Http/Controllers/SwarmController.php b/app/Http/Controllers/SwarmController.php
index 68e6f05..33d70ed 100644
--- a/app/Http/Controllers/SwarmController.php
+++ b/app/Http/Controllers/SwarmController.php
@@ -163,11 +163,9 @@ public function updateS3Storages(Swarm $swarm, Request $request)
             $tasks = [];
 
             foreach ($swarmData->s3Storages as $s3Storage) {
-                $previous = $s3Storage->dockerName ? $swarm->data->findS3Storage($s3Storage->dockerName) : null;
+                $previous = $swarm->data->findS3Storage($s3Storage->id);
                 if ($previous) {
                     if ($s3Storage->sameAs($previous)) {
-                        $s3Storage->dockerName = $previous->dockerName;
-
                         continue;
                     }
                 }
diff --git a/app/Http/Controllers/SwarmTaskController.php b/app/Http/Controllers/SwarmTaskController.php
index 9f6f52d..752a142 100644
--- a/app/Http/Controllers/SwarmTaskController.php
+++ b/app/Http/Controllers/SwarmTaskController.php
@@ -19,6 +19,7 @@
 use App\Models\Swarm;
 use App\Models\SwarmData;
 use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Str;
 
 class SwarmTaskController extends Controller
 {
@@ -31,6 +32,8 @@ public function initCluster(InitClusterFormRequest $request)
                 'data' => SwarmData::validateAndCreate([
                     'registriesRev' => 0,
                     'registries' => [],
+                    's3StoragesRev' => 0,
+                    's3Storages' => [],
                 ]),
             ]);
 
@@ -121,7 +124,7 @@ public function initCluster(InitClusterFormRequest $request)
                 'data' => DeploymentData::validateAndCreate([
                     'networkName' => $network->docker_name,
                     'internalDomain' => 'caddy.ptah.local',
-                    'placementNodeId' => null,
+                    'placementNodeId' => $node->id,
                     'processes' => [
                         [
                             'name' => 'svc',
@@ -155,10 +158,12 @@ public function initCluster(InitClusterFormRequest $request)
                             'secretFiles' => [],
                             'volumes' => [
                                 [
+                                    'id' => 'volume-' . Str::random(11),
                                     'name' => 'data',
                                     'path' => '/data',
                                 ],
                                 [
+                                    'id' => 'volume-' . Str::random(11),
                                     'name' => 'config',
                                     'path' => '/config',
                                 ]
diff --git a/app/Models/DeploymentData/BackupSchedule.php b/app/Models/DeploymentData/BackupSchedule.php
new file mode 100644
index 0000000..0b4fece
--- /dev/null
+++ b/app/Models/DeploymentData/BackupSchedule.php
@@ -0,0 +1,18 @@
+<?php
+
+namespace App\Models\DeploymentData;
+
+use Spatie\LaravelData\Data;
+
+class BackupSchedule extends Data
+{
+    public function __construct(
+        public CronPreset $preset,
+        public string     $s3StorageId,
+        // TODO: !!! validate CRON expr
+        public string     $expr,
+    )
+    {
+
+    }
+}
diff --git a/app/Models/DeploymentData/BackupVolume.php b/app/Models/DeploymentData/BackupVolume.php
new file mode 100644
index 0000000..495e8e1
--- /dev/null
+++ b/app/Models/DeploymentData/BackupVolume.php
@@ -0,0 +1,18 @@
+<?php
+
+namespace App\Models\DeploymentData;
+
+use Spatie\LaravelData\Data;
+
+class BackupVolume extends Data
+{
+    public function __construct(
+        public string $id,
+        public string $name,
+        public ?string $dockerName,
+        public string $path,
+    )
+    {
+
+    }
+}
\ No newline at end of file
diff --git a/app/Models/DeploymentData/CronPreset.php b/app/Models/DeploymentData/CronPreset.php
new file mode 100644
index 0000000..69798d3
--- /dev/null
+++ b/app/Models/DeploymentData/CronPreset.php
@@ -0,0 +1,9 @@
+<?php
+
+namespace App\Models\DeploymentData;
+
+enum CronPreset: string
+{
+    case Daily = 'daily';
+    case Custom = 'custom';
+}
diff --git a/app/Models/DeploymentData/Process.php b/app/Models/DeploymentData/Process.php
index 61461ae..9044381 100644
--- a/app/Models/DeploymentData/Process.php
+++ b/app/Models/DeploymentData/Process.php
@@ -11,6 +11,7 @@
 use App\Models\NodeTasks\UpdateService\UpdateServiceMeta;
 use App\Models\NodeTaskType;
 use App\Rules\RequiredIfArrayHas;
+use Illuminate\Support\Str;
 use Spatie\LaravelData\Attributes\DataCollectionOf;
 use Spatie\LaravelData\Attributes\Validation\Enum;
 use Spatie\LaravelData\Attributes\Validation\Rule;
@@ -43,6 +44,7 @@ public function __construct(
         #[DataCollectionOf(Volume::class)]
                            /* @var Volume[] */
         public array  $volumes,
+        public ?BackupVolume $backupVolume,
         public int    $replicas,
         #[DataCollectionOf(NodePort::class)]
         /* @var NodePort[] */
@@ -59,6 +61,12 @@ public function __construct(
     {
 
     }
+
+    public function findVolume(string $dockerName): ?Volume
+    {
+        return collect($this->volumes)->first(fn(Volume $volume) => $volume->dockerName === $dockerName);
+    }
+
     public function findConfigFile(string $path): ?ConfigFile
     {
         return collect($this->configFiles)->first(fn(ConfigFile $file) => $file->path === $path);
@@ -205,6 +213,46 @@ public function asNodeTasks(Deployment $deployment): array
             'serviceName' => $deployment->service->name,
         ];
 
+        $volumes = $this->volumes;
+
+        $mounts = collect($volumes)
+            ->map(fn(Volume $volume) => [
+                'Type' => 'volume',
+                'Source' => $volume->dockerName,
+                'Target' => $volume->path,
+                'VolumeOptions' => [
+                    'Labels' => dockerize_labels([
+                        'id' => $volume->id,
+                        ...$labels,
+                    ]),
+                ]
+            ])
+            ->toArray();
+
+        // TODO: if (has volumes with backups enabled OR has a Backup Script defined)
+        if (count($this->volumes)) {
+            if ($this->backupVolume == null) {
+                $this->backupVolume = BackupVolume::validateAndCreate([
+                    'id' => 'backups-' . Str::random(11),
+                    'name' => 'backups',
+                    'dockerName' => $this->makeResourceName('/ptah/backups'),
+                    'path' => '/ptah/backups',
+                ]);
+            }
+
+            $mounts[] = [
+                'Type' => 'volume',
+                'Source' => $this->backupVolume->dockerName,
+                'Target' => $this->backupVolume->path,
+                'VolumeOptions' => [
+                    'Labels' => dockerize_labels([
+                        'id' => $this->backupVolume->id,
+                        ...$labels,
+                    ]),
+                ]
+            ];
+        }
+
         // FIXME: this is going to work wrong if the initial deployment is pending.
         //   Don't allow to schedule deployments if the service has not been created yet?
         //   This code is duplicated in the next block
@@ -228,14 +276,7 @@ public function asNodeTasks(Deployment $deployment): array
                             'Args' => $args,
                             'Hostname' => "dpl-{$deployment->id}.{$internalDomain}",
                             'Env' => collect($this->envVars)->map(fn(EnvVar $var) => "{$var->name}={$var->value}")->toArray(),
-                            'Mounts' => collect($this->volumes)->map(fn(Volume $volume) => [
-                                'Type' => 'volume',
-                                'Source' => $volume->dockerName,
-                                'Target' => $volume->path,
-                                'VolumeOptions' => [
-                                    'Labels' => $labels,
-                                ]
-                            ])->toArray(),
+                            'Mounts' => $mounts,
                             'Hosts' => [
                                 $internalDomain,
                             ],
diff --git a/app/Models/DeploymentData/Volume.php b/app/Models/DeploymentData/Volume.php
index 6047c4f..c1c6813 100644
--- a/app/Models/DeploymentData/Volume.php
+++ b/app/Models/DeploymentData/Volume.php
@@ -7,9 +7,11 @@
 class Volume extends Data
 {
     public function __construct(
+        public string $id,
         public string $name,
         public ?string $dockerName,
-        public string $path
+        public string $path,
+        public ?BackupSchedule $backupSchedule
     )
     {
 
diff --git a/app/Models/NodeTaskGroupType.php b/app/Models/NodeTaskGroupType.php
index bf3b0f0..23d1df9 100644
--- a/app/Models/NodeTaskGroupType.php
+++ b/app/Models/NodeTaskGroupType.php
@@ -11,4 +11,5 @@ enum NodeTaskGroupType: int
     case SelfUpgrade = 4;
     case UpdateDockerRegistries = 5;
     case UpdateS3Storages = 6;
+    case CreateBackup = 7;
 }
diff --git a/app/Models/NodeTaskType.php b/app/Models/NodeTaskType.php
index 607de75..b2d855b 100644
--- a/app/Models/NodeTaskType.php
+++ b/app/Models/NodeTaskType.php
@@ -30,12 +30,16 @@
 use App\Events\NodeTasks\PullDockerImage\PullDockerImageFailed;
 use App\Events\NodeTasks\RebuildCaddyConfig\ApplyCaddyConfigCompleted;
 use App\Events\NodeTasks\RebuildCaddyConfig\ApplyCaddyConfigFailed;
+use App\Events\NodeTasks\ServiceExec\ServiceExecCompleted;
+use App\Events\NodeTasks\ServiceExec\ServiceExecFailed;
 use App\Events\NodeTasks\UpdateAgentSymlink\UpdateAgentSymlinkCompleted;
 use App\Events\NodeTasks\UpdateAgentSymlink\UpdateAgentSymlinkFailed;
 use App\Events\NodeTasks\UpdateNode\UpdateCurrentNodeCompleted;
 use App\Events\NodeTasks\UpdateNode\UpdateCurrentNodeFailed;
 use App\Events\NodeTasks\UpdateService\UpdateServiceCompleted;
 use App\Events\NodeTasks\UpdateService\UpdateServiceFailed;
+use App\Events\NodeTasks\UploadS3File\UploadS3FileCompleted;
+use App\Events\NodeTasks\UploadS3File\UploadS3FileFailed;
 use App\Models\NodeTasks\ApplyCaddyConfig\ApplyCaddyConfigMeta;
 use App\Models\NodeTasks\ApplyCaddyConfig\ApplyCaddyConfigResult;
 use App\Models\NodeTasks\CheckRegistryAuth\CheckRegistryAuthMeta;
@@ -64,12 +68,16 @@
 use App\Models\NodeTasks\InitSwarm\InitSwarmResult;
 use App\Models\NodeTasks\PullDockerImage\PullDockerImageMeta;
 use App\Models\NodeTasks\PullDockerImage\PullDockerImageResult;
+use App\Models\NodeTasks\ServiceExec\ServiceExecMeta;
+use App\Models\NodeTasks\ServiceExec\ServiceExecResult;
 use App\Models\NodeTasks\UpdateAgentSymlink\UpdateAgentSymlinkMeta;
 use App\Models\NodeTasks\UpdateAgentSymlink\UpdateAgentSymlinkResult;
 use App\Models\NodeTasks\UpdateCurrentNode\UpdateCurrentNodeMeta;
 use App\Models\NodeTasks\UpdateCurrentNode\UpdateCurrentNodeResult;
 use App\Models\NodeTasks\UpdateService\UpdateServiceMeta;
 use App\Models\NodeTasks\UpdateService\UpdateServiceResult;
+use App\Models\NodeTasks\UploadS3File\UploadS3FileMeta;
+use App\Models\NodeTasks\UploadS3File\UploadS3FileResult;
 
 // Mb use dynamic class names? $class = "{$this->name}Payload"; ??
 enum NodeTaskType: int
@@ -91,6 +99,8 @@ enum NodeTaskType: int
     case PullDockerImage = 14;
     case CreateS3Storage = 15;
     case CheckS3Storage = 16;
+    case ServiceExec = 17;
+    case UploadS3File = 18;
 
     public function meta(): string
     {
@@ -112,6 +122,8 @@ public function meta(): string
             self::PullDockerImage => PullDockerImageMeta::class,
             self::CreateS3Storage => CreateS3StorageMeta::class,
             self::CheckS3Storage => CheckS3StorageMeta::class,
+            self::ServiceExec => ServiceExecMeta::class,
+            self::UploadS3File => UploadS3FileMeta::class
         };
     }
 
@@ -135,6 +147,8 @@ public function result(): string
             self::PullDockerImage => PullDockerImageResult::class,
             self::CreateS3Storage => CreateS3StorageResult::class,
             self::CheckS3Storage => CheckS3StorageResult::class,
+            self::ServiceExec => ServiceExecResult::class,
+            self::UploadS3File => UploadS3FileResult::class
         };
     }
 
@@ -158,6 +172,8 @@ public function completed(): string
             self::PullDockerImage => PullDockerImageCompleted::class,
             self::CreateS3Storage => CreateS3StorageCompleted::class,
             self::CheckS3Storage => CheckS3StorageCompleted::class,
+            self::ServiceExec => ServiceExecCompleted::class,
+            self::UploadS3File => UploadS3FileCompleted::class
         };
     }
 
@@ -181,6 +197,8 @@ public function failed(): string
             self::PullDockerImage => PullDockerImageFailed::class,
             self::CreateS3Storage => CreateS3StorageFailed::class,
             self::CheckS3Storage => CheckS3StorageFailed::class,
+            self::ServiceExec => ServiceExecFailed::class,
+            self::UploadS3File => UploadS3FileFailed::class
         };
     }
 }
diff --git a/app/Models/NodeTasks/ServiceExec/ServiceExecMeta.php b/app/Models/NodeTasks/ServiceExec/ServiceExecMeta.php
new file mode 100644
index 0000000..129525e
--- /dev/null
+++ b/app/Models/NodeTasks/ServiceExec/ServiceExecMeta.php
@@ -0,0 +1,21 @@
+<?php
+
+namespace App\Models\NodeTasks\ServiceExec;
+
+use App\Models\NodeTasks\AbstractTaskMeta;
+
+class ServiceExecMeta extends AbstractTaskMeta
+{
+    public function __construct(
+        public int $serviceId,
+        public string $command,
+    )
+    {
+        //
+    }
+
+    public function formattedHtml(): string
+    {
+        return "Execute <code>$this->command</code> on the service <code>$this->serviceId</code>.";
+    }
+}
diff --git a/app/Models/NodeTasks/ServiceExec/ServiceExecResult.php b/app/Models/NodeTasks/ServiceExec/ServiceExecResult.php
new file mode 100644
index 0000000..7087970
--- /dev/null
+++ b/app/Models/NodeTasks/ServiceExec/ServiceExecResult.php
@@ -0,0 +1,20 @@
+<?php
+
+namespace App\Models\NodeTasks\ServiceExec;
+
+use App\Models\NodeTasks\AbstractTaskResult;
+
+class ServiceExecResult extends AbstractTaskResult
+{
+    public function __construct(
+        public array $output
+    )
+    {
+        //
+    }
+
+    public function formattedHtml(): string
+    {
+        return join('<br>', $this->output);
+    }
+}
diff --git a/app/Models/NodeTasks/UploadS3File/UploadS3FileMeta.php b/app/Models/NodeTasks/UploadS3File/UploadS3FileMeta.php
new file mode 100644
index 0000000..877da96
--- /dev/null
+++ b/app/Models/NodeTasks/UploadS3File/UploadS3FileMeta.php
@@ -0,0 +1,21 @@
+<?php
+
+namespace App\Models\NodeTasks\UploadS3File;
+
+use App\Models\NodeTasks\AbstractTaskMeta;
+
+class UploadS3FileMeta extends AbstractTaskMeta
+{
+    public function __construct(
+        public string $serviceId,
+        public string $destPath,
+    )
+    {
+        //
+    }
+
+    public function formattedHtml(): string
+    {
+        return "Upload file to S3 Storage: {$this->destPath}";
+    }
+}
diff --git a/app/Models/NodeTasks/UploadS3File/UploadS3FileResult.php b/app/Models/NodeTasks/UploadS3File/UploadS3FileResult.php
new file mode 100644
index 0000000..976dbfc
--- /dev/null
+++ b/app/Models/NodeTasks/UploadS3File/UploadS3FileResult.php
@@ -0,0 +1,19 @@
+<?php
+
+namespace App\Models\NodeTasks\UploadS3File;
+
+use App\Models\NodeTasks\AbstractTaskResult;
+
+class UploadS3FileResult extends AbstractTaskResult
+{
+    public function __construct(
+    )
+    {
+        //
+    }
+
+    public function formattedHtml(): string
+    {
+        return "UploadS3File - Task Result";
+    }
+}
diff --git a/app/Models/SwarmData.php b/app/Models/SwarmData.php
index ea49c82..8176269 100644
--- a/app/Models/SwarmData.php
+++ b/app/Models/SwarmData.php
@@ -30,10 +30,10 @@ public function findRegistry(string $dockerName): ?DockerRegistry
             ->first();
     }
 
-    public function findS3Storage(string $dockerName): ?S3Storage
+    public function findS3Storage(string $id): ?S3Storage
     {
         return collect($this->s3Storages)
-            ->filter(fn (S3Storage $s3Storage) => $s3Storage->dockerName === $dockerName)
+            ->filter(fn (S3Storage $s3Storage) => $s3Storage->id === $id)
             ->first();
     }
 }
diff --git a/app/Traits/HasOwningTeam.php b/app/Traits/HasOwningTeam.php
index b5347f9..a5cac4b 100644
--- a/app/Traits/HasOwningTeam.php
+++ b/app/Traits/HasOwningTeam.php
@@ -9,7 +9,9 @@ trait HasOwningTeam
 {
     protected static function bootHasOwningTeam(): void
     {
-        static::addGlobalScope(new TeamScope());
+        if (!app()->runningInConsole()) {
+            static::addGlobalScope(new TeamScope());
+        }
     }
 
     public function team()
diff --git a/bootstrap/app.php b/bootstrap/app.php
index c4a8bc4..1eaf84d 100644
--- a/bootstrap/app.php
+++ b/bootstrap/app.php
@@ -1,8 +1,11 @@
 <?php
 
 use ApiNodes\Http\Middleware\AgentTokenAuth;
+use App\Console\Commands\DispatchBackupTask;
 use App\Http\Middleware\HandleInertiaRequests;
 use App\Jobs\CheckAgentUpdates;
+use App\Models\DeploymentData\CronPreset;
+use App\Models\Service;
 use Illuminate\Console\Scheduling\Schedule;
 use Illuminate\Foundation\Application;
 use Illuminate\Foundation\Configuration\Exceptions;
@@ -40,6 +43,35 @@
     ->withSchedule(function (Schedule $schedule) {
         $schedule->job(CheckAgentUpdates::class)
             ->everyMinute()
+            ->onOneServer()
             ->withoutOverlapping();
+
+        Service::withoutGlobalScopes()->with(['latestDeployment' => fn ($query) => $query->withoutGlobalScopes()])->chunk(100, function (
+            /* @var Service[] $services */
+            $services
+        ) use ($schedule) {
+            foreach ($services as $service) {
+                foreach ($service->latestDeployment->data->processes as $process) {
+                    if ($process->replicas === 0) {
+                        continue;
+                    }
+                    foreach ($process->volumes as $volume) {
+                        $backupSchedule = $volume->backupSchedule;
+                        if ($backupSchedule === null) {
+                            continue;
+                        }
+
+                        $schedule
+                            ->command(DispatchBackupTask::class, [
+                                'serviceId' => $service->id,
+                                'volumeId' => $volume->id,
+                            ])
+                            ->cron($backupSchedule->expr)
+                            ->onOneServer()
+                            ->withoutOverlapping();
+                    }
+                }
+            }
+        });
     })
     ->create();
diff --git a/database/migrations/2024_07_14_115507_update_deployments_add_volume_backups.php b/database/migrations/2024_07_14_115507_update_deployments_add_volume_backups.php
new file mode 100644
index 0000000..179b343
--- /dev/null
+++ b/database/migrations/2024_07_14_115507_update_deployments_add_volume_backups.php
@@ -0,0 +1,58 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        DB::update("
+            UPDATE deployments
+            SET data = jsonb_set(
+              data,
+              '{processes}',
+              (
+                SELECT jsonb_agg(
+                  jsonb_set(
+                    process,
+                    '{volumes}',
+                    COALESCE(
+                      (
+                        SELECT jsonb_agg(
+                          volume_item || jsonb_build_object(
+                            'id', 
+                            concat('volume-', substr(md5(random()::text), 1, 11))
+                          ) || jsonb_build_object(
+                            'backupSchedule',
+                            '{\"preset\": \"cron-disabled\"}'::jsonb
+                          )
+                        )
+                        FROM jsonb_array_elements(process->'volumes') AS volume_item
+                      ),
+                      '[]'::jsonb
+                    )
+                  ) || jsonb_build_object(
+                    'backupVolume',
+                    CONCAT('{\"id\":\"backups-', substr(md5(random()::text), 1, 11) ,'\",\"name\":\"backups\",\"path\":\"/ptah/backups\",\"dockerName\":\"svc_', deployments.service_id,'_', (select name from services where id = deployments.service_id),'_svc_ptah_backups\",\"backupSchedule\":{\"expr\":null,\"preset\":\"cron-disabled\"}}')::jsonb
+                  )
+                )
+                FROM jsonb_array_elements(data->'processes') AS process
+              )
+            );
+        ");
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        //
+    }
+};
diff --git a/resources/js/Components/BackupSchedule.vue b/resources/js/Components/BackupSchedule.vue
new file mode 100644
index 0000000..18084c2
--- /dev/null
+++ b/resources/js/Components/BackupSchedule.vue
@@ -0,0 +1,45 @@
+<script setup lang="ts">
+import Select from "@/Components/Select.vue";
+import FormField from "@/Components/FormField.vue";
+import TextInput from "@/Components/TextInput.vue";
+import {effect} from "vue";
+
+const model = defineModel();
+
+const props = defineProps({
+  's3Storages': Array,
+  'errors': Object,
+})
+
+effect(() => {
+  if (model.value.preset === 'daily') {
+    model.value.expr = '0 0 * * *';
+  }
+});
+</script>
+
+<template>
+  <FormField class="col-span-2" :error="props.errors?.['preset']">
+    <template #label>Backup Schedule</template>
+
+    <Select v-model="model.preset">
+      <option value="daily">Every Day</option>
+    </Select>
+  </FormField>
+
+  <FormField v-if="model.preset !== 'cron-disabled'" class="col-span-2" :error="props.errors?.['s3StorageId']">
+    <template #label>Storage</template>
+
+    <Select v-model="model.s3StorageId">
+      <option v-for="s3Storage in props.s3Storages" :value="s3Storage.id">{{ s3Storage.name }}</option>
+    </Select>
+  </FormField>
+
+<!--  <input v-if="model.preset === 'daily'" type="hidden" v-model="model.expr" value="0 0 * * *" />-->
+
+<!--  <FormField v-if="model.preset === 'custom'" class="col-span-2">-->
+<!--    <template #label>Expression</template>-->
+
+<!--    <TextInput v-model="model.expr" />-->
+<!--  </FormField>-->
+</template>
\ No newline at end of file
diff --git a/resources/js/Components/Service/ComponentBlock.vue b/resources/js/Components/Service/ComponentBlock.vue
index 51b2d1d..9aa3c9e 100644
--- a/resources/js/Components/Service/ComponentBlock.vue
+++ b/resources/js/Components/Service/ComponentBlock.vue
@@ -15,7 +15,7 @@ defineModel();
     <div v-for="(item, index) in modelValue">
       <hr class="my-2" v-if="index > 0" />
 
-      <div class="grid grid-cols-6 gap-4">
+      <div class="grid grid-cols-6 gap-4" v-auto-animate>
         <slot :item="modelValue[index]" :$index="index" />
       </div>
 
diff --git a/resources/js/Pages/Nodes/Partials/S3Storages.vue b/resources/js/Pages/Nodes/Partials/S3Storages.vue
index 75e5dfb..c5f6cd9 100644
--- a/resources/js/Pages/Nodes/Partials/S3Storages.vue
+++ b/resources/js/Pages/Nodes/Partials/S3Storages.vue
@@ -58,7 +58,7 @@ const submitForm = () => {
             Endpoint
           </template>
 
-          <TextInput v-model="item.endpoint" class="w-full" placeholder="https://backups.nyc3.digitaloceanspaces.com" />
+          <TextInput v-model="item.endpoint" class="w-full" placeholder="nyc3.digitaloceanspaces.com" />
         </FormField>
 
         <FormField class="col-span-2" :error="form.errors[`s3Storages.${$index}.region`]">
diff --git a/resources/js/Pages/Services/Create.vue b/resources/js/Pages/Services/Create.vue
index 111b4b0..95e3064 100644
--- a/resources/js/Pages/Services/Create.vue
+++ b/resources/js/Pages/Services/Create.vue
@@ -21,6 +21,7 @@ const props = defineProps({
   'nodes': Array,
   'deploymentData': Object,
   'dockerRegistries': Array,
+  's3Storages': Array,
 })
 
 const form = useForm({
@@ -67,7 +68,14 @@ const createService = () => {
           </template>
         </ActionSection>
 
-        <DeploymentData v-model="form.deploymentData" :networks="networks" :nodes="nodes" :errors="form.errors" :service-name="form.name" :docker-registries="$props.dockerRegistries" />
+        <DeploymentData v-model="form.deploymentData"
+                        :networks="networks"
+                        :nodes="nodes"
+                        :errors="form.errors"
+                        :service-name="form.name"
+                        :docker-registries="props.dockerRegistries"
+                        :s3-storages="props.s3Storages"
+        />
 
         <div class="flex justify-end">
           <PrimaryButton :class="{ 'opacity-25': form.processing }" :disabled="form.processing">
diff --git a/resources/js/Pages/Services/Partials/DeploymentData.vue b/resources/js/Pages/Services/Partials/DeploymentData.vue
index 1259f54..5a505ee 100644
--- a/resources/js/Pages/Services/Partials/DeploymentData.vue
+++ b/resources/js/Pages/Services/Partials/DeploymentData.vue
@@ -14,7 +14,7 @@ import TextArea from "@/Components/TextArea.vue";
 import {computed, effect, nextTick, reactive, ref} from "vue";
 import ServiceDetailsForm from "@/Pages/Services/Partials/ServiceDetailsForm.vue";
 import TabItem from "@/Components/Tabs/TabItem.vue";
-import {FwbTooltip} from "flowbite-vue";
+import {FwbToggle, FwbTooltip} from "flowbite-vue";
 import ProcessTabs from "@/Pages/Services/Partials/ProcessTabs.vue";
 import DangerButton from "@/Components/DangerButton.vue";
 import DialogModal from "@/Components/DialogModal.vue";
@@ -22,6 +22,7 @@ import AddComponentButton from "@/Components/Service/AddComponentButton.vue";
 import RemoveComponentButton from "@/Components/Service/RemoveComponentButton.vue";
 import ComponentBlock from "@/Components/Service/ComponentBlock.vue";
 import FormFieldGrid from "@/Components/FormFieldGrid.vue";
+import BackupSchedule from "@/Components/BackupSchedule.vue";
 
 const model = defineModel()
 
@@ -31,6 +32,7 @@ const props = defineProps({
   'nodes': Array,
   'serviceName': String | undefined,
   'dockerRegistries': Array,
+  's3Storages': Array,
   'errors': Object,
 });
 
@@ -72,7 +74,14 @@ const addSecretFile = () => {
 }
 
 const addVolume = () => {
-  model.value.processes[state.selectedProcessIndex['volumes']].volumes.push({id: makeId('volume'), name: '', path: ''});
+  model.value.processes[state.selectedProcessIndex['volumes']].volumes.push({
+    id: makeId('volume'),
+    name: '',
+    path: '',
+    backupSchedule: {
+      preset: 'cron-disabled',
+    }
+  });
 }
 
 const addPort = () => {
@@ -168,6 +177,18 @@ effect(() => {
       : null;
 });
 
+const toggleVolumeBackups = (volume) => {
+  if (volume.backupSchedule === null) {
+    volume.backupSchedule = {
+      preset: 'daily',
+      s3StorageName: props.s3Storages[0].dockerName,
+      expr: '0 0 * * *',
+    }
+  } else {
+    volume.backupSchedule = null;
+  }
+}
+
 const processRemoveInput = ref();
 
 const processRemoval = reactive({
@@ -200,6 +221,10 @@ const submitProcessRemoval = () => {
     closeProcessRemovalModal();
   }
 }
+
+const extractFieldErrors = (basePath) => {
+  return Object.fromEntries(Object.keys(props.errors).filter((key) => key.startsWith(basePath)).map((key) => [key.substring(basePath.length + 1), props.errors[key]]));
+}
 </script>
 
 <template>
@@ -698,6 +723,58 @@ const submitProcessRemoval = () => {
           </SecondaryButton>
         </div>
       </FormField>
+      <ComponentBlock v-else v-model="model.processes[state.selectedProcessIndex['volumes']].volumes" v-slot="{ item, $index }" @remove="model.processes[state.selectedProcessIndex['volumes']].volumes.splice($event, 1)">
+        <FormField
+                   :error="props.errors[`processes.${state.selectedProcessIndex['volumes']}.volumes.${$index}.name`] || props.errors[`processes.${state.selectedProcessIndex['volumes']}.volumes.${$index}.path`]"
+                   class="col-span-2">
+
+            <template #label>Volume Name</template>
+
+            <TextInput v-model="item.name" class="w-full" placeholder="Name"/>
+        </FormField>
+
+        <FormField
+                   :error="props.errors[`processes.${state.selectedProcessIndex['volumes']}.volumes.${$index}.path`]"
+                   class="col-span-3">
+
+          <template #label>Mount Path</template>
+
+          <TextInput v-model="item.path" class="w-full" placeholder="Path"/>
+        </FormField>
+
+        <FormField class="col-span-1">
+          <template #label v-if="props.s3Storages.length > 0">Enable Backups</template>
+
+          <template #label v-else>
+            <FwbTooltip trigger="hover">
+              <template #trigger>
+                <div class="flex">
+                  Backup
+
+                  <svg class="ms-1 w-4 h-4 text-blue-600 dark:text-white" aria-hidden="true" xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24">
+                    <path stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M9.529 9.988a2.502 2.502 0 1 1 5 .191A2.441 2.441 0 0 1 12 12.582V14m-.01 3.008H12M21 12a9 9 0 1 1-18 0 9 9 0 0 1 18 0Z"/>
+                  </svg>
+                </div>
+              </template>
+
+              <template #content>
+                Backups can't be enabled - no S3 Storages configured
+              </template>
+            </FwbTooltip>
+          </template>
+
+          <div class="pt-2 select-none">
+            <FwbToggle :model-value="item.backupSchedule !== null" :disabled="props.s3Storages.length === 0" :class="props.s3Storages.length === 0 ? 'cursor-not-allowed opacity-40' : ''" @change="toggleVolumeBackups(item)" />
+          </div>
+        </FormField>
+
+        <BackupSchedule
+            v-if="item.backupSchedule"
+            v-model="item.backupSchedule"
+            :errors="extractFieldErrors(`processes.${state.selectedProcessIndex['volumes']}.volumes.${$index}.backupSchedule`)"
+            :s3-storages="props.s3Storages"
+        />
+      </ComponentBlock>
     </template>
 
     <template #actions>
diff --git a/resources/js/Pages/Services/Show.vue b/resources/js/Pages/Services/Show.vue
index 88c9831..fa8153b 100644
--- a/resources/js/Pages/Services/Show.vue
+++ b/resources/js/Pages/Services/Show.vue
@@ -17,6 +17,7 @@ import ActionSection from "@/Components/ActionSection.vue";
 const props = defineProps({
   service: Object,
   dockerRegistries: Array,
+  s3Storages: Array,
 })
 
 const serviceForm = useForm({
@@ -95,7 +96,7 @@ const closeDeletionModal = () => {
     </FormSection>
 
     <form @submit.prevent="deploy">
-      <DeploymentData v-model="deploymentForm" :errors="deploymentForm.errors" :docker-registries="$props.dockerRegistries" />
+      <DeploymentData v-model="deploymentForm" :errors="deploymentForm.errors" :docker-registries="props.dockerRegistries" :s3-storages="props.s3Storages" />
 
       <div class="flex justify-end">
         <PrimaryButton :class="{ 'opacity-25': deploymentForm.processing }" :disabled="deploymentForm.processing">