feat: #225, #226 new workers definition ui and scheduled workers

ptah-sh · Oct 15, 2024 · 1f585b7 · 1f585b7
1 parent 61d392c
commit 1f585b7
Show file tree

Hide file tree

Showing 44 changed files with 1,201 additions and 964 deletions.
diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml
@@ -80,7 +80,13 @@ jobs:
           service: 'ptah-server-prod'
           processes: |
             - name: svc
-              dockerImage: ghcr.io/ptah-sh/ptah-server:${{ needs.release-please.outputs.tag_name }}
+              workers:
+                - name: main
+                  dockerImage: ghcr.io/ptah-sh/ptah-server:${{ needs.release-please.outputs.tag_name }}
+                - name: scheduler
+                  dockerImage: ghcr.io/ptah-sh/ptah-server:${{ needs.release-please.outputs.tag_name }}
+                - name: queue
+                  dockerImage: ghcr.io/ptah-sh/ptah-server:${{ needs.release-please.outputs.tag_name }}
               envVars:
                 - name: SENTRY_RELEASE
                   value: ${{ needs.release-please.outputs.tag_name }}
diff --git a/api-nodes/Http/Controllers/MetricsController.php b/api-nodes/Http/Controllers/MetricsController.php
@@ -88,10 +88,6 @@ public function __invoke(Request $request, Logger $log, Node $node)
             ],
         ];
 
-        $log->info('Services:', [
-            'services' => $services,
-        ]);
-
         foreach ($request->all() as $metricsDoc) {
             if ($metricsDoc === null) {
                 continue;

diff --git a/app/Console/Commands/DispatchProcessBackupTask.php b/app/Console/Commands/DispatchProcessBackupTask.php
diff --git a/app/Console/Commands/DispatchVolumeBackupTask.php b/app/Console/Commands/DispatchVolumeBackupTask.php
diff --git a/app/Console/Commands/ExecuteScheduledWorker.php b/app/Console/Commands/ExecuteScheduledWorker.php
@@ -0,0 +1,100 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Models\Node;
+use App\Models\NodeTaskGroup;
+use App\Models\NodeTaskGroupType;
+use App\Models\NodeTaskType;
+use App\Models\Service;
+use Exception;
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\DB;
+
+class ExecuteScheduledWorker extends Command
+{
+    protected $signature = 'app:workers:execute {--service-id=} {--process=} {--worker=}';
+
+    protected $description = 'Execute a scheduled worker';
+
+    public function handle()
+    {
+        DB::transaction(function () {
+            $this->executeWorker();
+        });
+    }
+
+    protected function executeWorker(): void
+    {
+        /* @var Service $service */
+        $service = Service::findOrFail($this->option('service-id'));
+
+        $deployment = $service->latestDeployment;
+
+        $process = $deployment->data->findProcess($this->option('process'));
+        if ($process === null) {
+            throw new Exception("Could not find process {$this->option('process')} in deployment {$deployment->id}.");
+        }
+
+        $worker = $process->findWorker($this->option('worker'));
+        if ($worker === null) {
+            throw new Exception("Could not find worker {$this->option('worker')} in process {$process->name}.");
+        }
+
+        $node = $process->placementNodeId ? Node::findOrFail($process->placementNodeId) : null;
+
+        $taskGroup = NodeTaskGroup::create([
+            'type' => NodeTaskGroupType::LaunchService,
+            'swarm_id' => $service->swarm_id,
+            'node_id' => $node->id,
+            'invoker_id' => $deployment->latestTaskGroup->invoker_id,
+            'team_id' => $service->team_id,
+        ]);
+
+        $tasks = [];
+
+        $tasks = [
+            ...$tasks,
+            ...$worker->asNodeTasks($deployment, $process, desiredReplicas: 1),
+        ];
+
+        if ($worker->backupOptions) {
+            $s3Storage = $node->swarm->data->findS3Storage($worker->backupOptions->s3StorageId);
+            if ($s3Storage === null) {
+                throw new Exception("Could not find S3 storage {$worker->backupOptions->s3StorageId} in swarm {$node->swarm_id}.");
+            }
+
+            $archiveFormat = $worker->backupOptions->archive?->format->value;
+
+            $date = now()->format('Y-m-d_His');
+
+            $ext = $archiveFormat ? ".$archiveFormat" : '';
+            $backupFilePath = "/{$service->slug}/{$process->name}/{$worker->name}/{$service->slug}-{$process->name}-{$worker->name}-{$date}$ext";
+
+            $tasks[] = [
+                'type' => NodeTaskType::UploadS3File,
+                'meta' => [
+                    'serviceId' => $service->id,
+                    'destPath' => $backupFilePath,
+                ],
+                'payload' => [
+                    'Archive' => [
+                        'Enabled' => $worker->backupOptions->archive !== null,
+                        'Format' => $archiveFormat,
+                    ],
+                    'S3StorageConfigName' => $s3Storage->dockerName,
+                    'VolumeSpec' => [
+                        'Type' => 'volume',
+                        'Source' => $worker->backupOptions->backupVolume->dockerName,
+                        'Target' => $worker->backupOptions->backupVolume->path,
+                    ],
+                    'SrcFilePath' => $worker->backupOptions->backupVolume->path,
+                    'DestFilePath' => $backupFilePath,
+                    'RemoveSrcFile' => true,
+                ],
+            ];
+        }
+
+        $taskGroup->tasks()->createMany($tasks);
+    }
+}