From 05e7ca5f4f775f5206de4f51c232885ab9ea9816 Mon Sep 17 00:00:00 2001 From: Nicolas Brassard Date: Mon, 30 Sep 2024 16:51:54 -0400 Subject: [PATCH] (feat) Add TUS support --- README.md | 17 ++- composer.json | 2 +- src/client/CloudflareVideoStreamClient.php | 65 ++++++++- src/jobs/PollVideoJob.php | 8 +- src/jobs/TusUploadVideoJob.php | 157 +++++++++++++++++++++ src/jobs/UploadVideoJob.php | 48 +++++-- 6 files changed, 271 insertions(+), 26 deletions(-) create mode 100644 src/jobs/TusUploadVideoJob.php diff --git a/README.md b/README.md index 3717f5d..9427856 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ > This plugin offers a easy way to upload your videos assets from Craft CMS to Cloudflare stream. -Installation: +## Installation 1) Install with composer @@ -16,7 +16,7 @@ composer require deuxhuithuit/craft-cloudflare-stream craft plugin/install cloudflare-stream ``` -3) Add your account id and api token in the settings. +3) Add your account id and api token in the settings. You can (and should) use env vars. 4) Create a video stream Field and add it to your Asset data model. @@ -85,14 +85,23 @@ query MyQuery { } ``` -8) You can also mass re-upload everything via Craft's cli +8) You can also mass re-upload everything via Craft's cli. Please note that this creates _news_ streams. ```sh ./craft cloudflare-stream/reupload ``` This extension uses Craft's Queue system, so make sure it works properly. -Please make sure that Craft's max upload limit is also properly set. + +## Dealing with large files + +Videos can be quite large, which makes uploading them a little trickier. +Please make sure that Craft's and php's max upload limit are properly set. + +If you still want to deal with larger file than what your host allows, +you might want to try the [Chunked Upload plugin](https://plugins.craftcms.com/chunked-uploads) +to bypass your host restrictions. The plugin will use TUS when appropriated. The chunk size can +be set by setting the `CF_STREAM_TUS_CHUNK_SIZE` env var to a value in bytes, in the 5 to 200 Mb range. Made with ❤️ in Montréal. diff --git a/composer.json b/composer.json index 5498d8d..64e03af 100644 --- a/composer.json +++ b/composer.json @@ -2,7 +2,7 @@ "name": "deuxhuithuit/craft-cloudflare-stream", "description": "Upload your videos assets to Cloudflare Stream from Craft CMS assets interface", "type": "craft-plugin", - "version": "2.0.0", + "version": "2.1.0.alpha", "keywords": [ "cloudflare", "stream", diff --git a/src/client/CloudflareVideoStreamClient.php b/src/client/CloudflareVideoStreamClient.php index 0c91906..8eb1578 100644 --- a/src/client/CloudflareVideoStreamClient.php +++ b/src/client/CloudflareVideoStreamClient.php @@ -51,7 +51,7 @@ public function uploadVideoByUrl(string $videoUrl, string $videoName, ?string $v if ($uploadRes->getStatusCode() !== 200) { return [ 'error' => 'Error uploading video', - 'message' => $uploadRes->getBody(), + 'message' => $uploadRes->getBody()->getContents(), ]; } @@ -63,8 +63,9 @@ public function uploadVideoByUrl(string $videoUrl, string $videoName, ?string $v public function uploadVideoByPath(string $videoPath, string $videoFilename) { $client = new GuzzleHttp\Client(); - $fullPath = rtrim($videoPath, '/') . '/' . $videoFilename; - // Guzzle might close the file for us... + $fullPath = $this->createSafeFullPath($videoPath, $videoFilename); + + // Guzzle might? close the file for us... $file = fopen($fullPath, 'r'); if (!$file) { return [ @@ -91,7 +92,7 @@ public function uploadVideoByPath(string $videoPath, string $videoFilename) if ($uploadRes->getStatusCode() !== 200) { return [ 'error' => 'Error uploading video', - 'message' => $uploadRes->getBody(), + 'message' => $uploadRes->getBody()->getContents(), ]; } @@ -100,6 +101,53 @@ public function uploadVideoByPath(string $videoPath, string $videoFilename) return $data['result']; } + public function uploadVideoByTus(string $videoPath, string $videoFilename) + { + $client = new GuzzleHttp\Client(); + $fullPath = $this->createSafeFullPath($videoPath, $videoFilename); + + $file = \file_exists($fullPath); + if (!$file) { + return [ + 'error' => 'Video file does not exist', + 'message' => "File '{$fullPath}' not found", + ]; + } + $uploadRes = $client->request('POST', $this->createCfUrl('/stream?direct_user=true'), [ + 'headers' => \array_merge($this->createHttpHeaders(), [ + 'Tus-Resumable' => '1.0.0', + 'Upload-Length' => (string) \filesize($fullPath), + 'Upload-Metadata' => 'name ' . \base64_encode($videoFilename), + ]), + 'http_errors' => false, + ]); + + if ($uploadRes->getStatusCode() !== 201) { + return [ + 'error' => 'Error creating TUS request', + 'message' => $uploadRes->getBody()->getContents(), + ]; + } + + $headers = $uploadRes->getHeaders(); + $location = $headers['Location'][0]; + $uid = $headers['stream-media-id'][0]; + + if (!$location) { + return [ + 'error' => 'Error getting TUS location', + 'message' => $uploadRes->getBody()->getContents(), + ]; + } + + return [ + 'readyToStream' => false, + 'uid' => $uid, + 'location' => $location, + 'fullPath' => $fullPath, + ]; + } + public function getVideo(string $videoUid) { $client = new GuzzleHttp\Client(); @@ -134,7 +182,7 @@ public function deleteVideo(string $videoUid) if ($res->getStatusCode() !== 200) { return [ 'error' => 'Error deleting video', - 'message' => $res->getBody(), + 'message' => $res->getBody()->getContents(), ]; } @@ -156,7 +204,7 @@ public function updateThumbnail(string $videoUid, float $time, float $duration) if ($res->getStatusCode() !== 200) { return [ 'error' => 'Error updating thumbnail', - 'message' => $res->getBody(), + 'message' => $res->getBody()->getContents(), ]; } @@ -164,4 +212,9 @@ public function updateThumbnail(string $videoUid, float $time, float $duration) 'success' => true, ]; } + + private function createSafeFullPath(string $videoPath, string $videoFilename) + { + return rtrim($videoPath, '/') . '/' . $videoFilename; + } } diff --git a/src/jobs/PollVideoJob.php b/src/jobs/PollVideoJob.php index bac4a8a..c0d877b 100644 --- a/src/jobs/PollVideoJob.php +++ b/src/jobs/PollVideoJob.php @@ -19,7 +19,7 @@ class PollVideoJob extends BaseJob public function getTtr() { - return 10 + $this->delay(); + return 2 + $this->delay(); } public function execute($queue): void @@ -104,9 +104,9 @@ public function execute($queue): void // We need to if the process is not completed or if we don't still have a mp4 url if (!$this->completed || !$hasMp4Url) { $this->setProgress($queue, 0, 'Delayed retry'); - // Retry the job after x * 2 seconds + // Retry the job after x * 1.2 seconds $this->lastResult = $result; - $queue->delay($this->delay())->push($this); + \Craft::$app->getQueue()->delay($this->delay())->push($this); } else { // We are done !!! $this->setProgress($queue, 1, 'Done'); @@ -120,7 +120,7 @@ protected function defaultDescription(): ?string private function delay() { - return $this->attempts * 2; + return (int) ($this->attempts * 1.5); } private function setFieldValue($element, array $result) diff --git a/src/jobs/TusUploadVideoJob.php b/src/jobs/TusUploadVideoJob.php new file mode 100644 index 0000000..b3bcb1d --- /dev/null +++ b/src/jobs/TusUploadVideoJob.php @@ -0,0 +1,157 @@ +chunkSize = $envChunkSize; + } + + // Validate the chunk size + $this->chunkSize = max(self::MINIMUM_CHUNK_SIZE, min(self::MAXIMUM_CHUNK_SIZE, $this->chunkSize)); + } + + public function getTtr() + { + // 1Mb per second + return (int) max(5, $this->chunkSize / 1024 / 1024); + } + + public function canRetry($attempt, $error) + { + return $attempt < 1000; + } + + protected function defaultDescription(): ?string + { + return "TUS upload video {$this->videoName}, offset {$this->offset}."; + } + + public function execute($queue): void + { + $this->setProgress($queue, 0, 'Validating job data'); + + // Get the entry or element where the field is located + $element = \Craft::$app->getElements()->getElementById($this->elementId); + if (!$element) { + // Ignore deleted entries + $this->setProgress($queue, 1, 'Element not found'); + + return; + } else if (!$element instanceof Asset) { + throw new Exception('Element not an asset.'); + } + + // Get the CloudflareVideoStreamField by its handle + $field = \Craft::$app->getFields()->getFieldByHandle($this->fieldHandle); + if (!$field) { + // Ignore deleted fields + $this->setProgress($queue, 1, 'Field not found'); + + return; + } + + $this->setProgress($queue, 0.1, 'Validating Cloudflare Video Stream field'); + + // Check if the field is a CloudflareVideoStreamField + if (!$field instanceof CloudflareVideoStreamField) { + $this->setProgress($queue, 0.1, 'ERROR: Field is not a Cloudflare Video Stream field'); + + throw new \Error('Field is not a Cloudflare Video Stream field'); + } + + $client = new GuzzleHttp\Client(); + + // Sync the current offset + $this->setProgress($queue, 0.3, 'Syncing current offset'); + $headRes = $client->request('HEAD', $this->videoLocation, [ + 'headers' => [ + 'Tus-Resumable' => '1.0.0', + ], + 'http_errors' => false, + ]); + $headHeaders = $headRes->getHeaders(); + if ($headRes->getStatusCode() === 200) { + if (!isset($headHeaders['upload-offset'][0])) { + throw new Exception('Missing upload-offset header'); + } + $this->offset = (int) $headHeaders['upload-offset'][0]; + } + + // Upload a chunk of the video + $this->setProgress($queue, 0.3, 'Uploading video to Cloudflare Stream via TUS'); + $file = \fopen($this->videoPath, 'r'); + if (!$file) { + throw new Exception('Failed to open file for reading'); + } + + $fileSize = \filesize($this->videoPath); + if (!$fileSize) { + throw new Exception('Failed to get file size'); + } else if ($fileSize < $this->offset) { + throw new Exception('File size is smaller than the current offset'); + } else if ($fileSize === $this->offset) { + $this->setProgress($queue, 1, 'Upload complete, starting polling job'); + \Craft::$app->getQueue()->push(new PollVideoJob([ + 'elementId' => $this->elementId, + 'fieldHandle' => $this->fieldHandle, + 'videoUid' => $this->videoUid, + ])); + return; + } + + if ($this->offset > 0) { + \fseek($file, $this->offset); + } + $uploadRes = $client->request('PATCH', $this->videoLocation, [ + 'headers' => [ + 'Tus-Resumable' => '1.0.0', + 'Upload-Offset' => $this->offset, + 'Content-Type' => 'application/offset+octet-stream', + 'Expect' => '', + ], + 'body' => \fread($file, $this->chunkSize), + 'http_errors' => false, + ]); + + if (\is_resource($file)) { + \fclose($file); + } + + if ($uploadRes->getStatusCode() === 204) { + $this->offset = min($fileSize, $this->offset + $this->chunkSize); + \Craft::$app->getQueue()->push($this); + $this->setProgress($queue, 1, 'Chunk upload completed, pushed next chunk'); + } else { + throw new Exception("Chunk at offset {$this->offset} failed, retrying"); + } + } +} diff --git a/src/jobs/UploadVideoJob.php b/src/jobs/UploadVideoJob.php index 4256a3a..bcfaa26 100644 --- a/src/jobs/UploadVideoJob.php +++ b/src/jobs/UploadVideoJob.php @@ -2,11 +2,13 @@ namespace deuxhuithuit\cfstream\jobs; +use craft\elements\Asset; use craft\queue\BaseJob; use deuxhuithuit\cfstream\client\CloudflareVideoStreamClient; use deuxhuithuit\cfstream\fields\CloudflareVideoStreamField; use deuxhuithuit\cfstream\models\Settings; use deuxhuithuit\cfstream\Plugin; +use Exception; use yii\queue\RetryableJobInterface; // TODO: Make cancellable, to cancel the upload if the asset is deleted @@ -40,6 +42,8 @@ public function execute($queue): void $this->setProgress($queue, 1, 'Element not found'); return; + } else if (!$element instanceof Asset) { + throw new Exception('Element not an asset.'); } // Get the CloudflareVideoStreamField by its handle @@ -66,9 +70,16 @@ public function execute($queue): void $settings = Plugin::getInstance()->getSettings(); $client = new CloudflareVideoStreamClient($settings); $result = null; + $jobType = 'poll'; if ($settings->isUsingFormUpload()) { - \Craft::info('Uploading video by path', __METHOD__); - $result = $client->uploadVideoByPath($this->videoPath, $this->videoName); + if ($element->size > TusUploadVideoJob::DEFAULT_CHUNK_SIZE) { + \Craft::info('Uploading video by TUS', __METHOD__); + $jobType = 'tus'; + $result = $client->uploadVideoByTus($this->videoPath, $this->videoName); + } else { + \Craft::info('Uploading video by path', __METHOD__); + $result = $client->uploadVideoByPath($this->videoPath, $this->videoName); + } } else { \Craft::info('Uploading video by url', __METHOD__); $result = $client->uploadVideoByUrl($this->videoUrl, $this->videoName, $this->videoTitle); @@ -83,7 +94,7 @@ public function execute($queue): void throw new \Error('Upload request failed'); } if (!empty($result['error'])) { - $this->setProgress($queue, 0.3, 'ERROR: ' . $result['error']); + $this->setProgress($queue, 0.3, 'ERROR: ' . $result['error'] . ': ' . $result['message']); \Craft::error('Upload request failed.' . $result['error'] . ' ' . $result['message'], __METHOD__); throw new \Error($result['error'] . ' ' . $result['message']); @@ -101,14 +112,29 @@ public function execute($queue): void } $this->setProgress($queue, 0.5, 'Craft element saved'); - $this->setProgress($queue, 0.6, 'Pushing polling job'); - $pollingJob = new PollVideoJob([ - 'elementId' => $this->elementId, - 'fieldHandle' => $this->fieldHandle, - 'videoUid' => $result['uid'], - ]); - \Craft::$app->getQueue()->push($pollingJob); - $this->setProgress($queue, 0.7, 'Polling job pushed'); + // Push next job + if ($jobType == 'poll') { + $this->setProgress($queue, 0.6, 'Pushing polling job'); + $pollingJob = new PollVideoJob([ + 'elementId' => $this->elementId, + 'fieldHandle' => $this->fieldHandle, + 'videoUid' => $result['uid'], + ]); + \Craft::$app->getQueue()->push($pollingJob); + $this->setProgress($queue, 0.7, 'Polling job pushed'); + } else if ($jobType == 'tus') { + $this->setProgress($queue, 0.6, 'Pushing TUS job'); + $tusJob = new TusUploadVideoJob([ + 'elementId' => $this->elementId, + 'fieldHandle' => $this->fieldHandle, + 'videoUid' => $result['uid'], + 'videoLocation' => $result['location'], + 'videoPath' => $result['fullPath'], + 'videoName' => $this->videoName, + ]); + \Craft::$app->getQueue()->push($tusJob); + $this->setProgress($queue, 0.7, 'TUS job pushed'); + } // Log the success \Craft::info('Video uploaded to Cloudflare Stream.', __METHOD__);