(feat) Implement proper file index/listing

This commit makes the index asset command work. Beware as it can and will import ALL exisiting images from your CF account. This change should also fix problems with images that contain 3 dots in their filename, which would be parsed has account name and blew everything up.
DeuxHuitHuit · Oct 31, 2024 · f87919c · f87919c
1 parent 18b4ce2
commit f87919c
Show file tree

Hide file tree

Showing 3 changed files with 58 additions and 46 deletions.
diff --git a/src/Filename.php b/src/Filename.php
@@ -2,6 +2,9 @@
 
 namespace deuxhuithuit\cfimages;
 
+/**
+ * @internal
+ */
 class Filename
 {
     public const SEPARATOR = '.';
@@ -25,14 +28,10 @@ public static function toId(string $filename): string
         return $parts['id'];
     }
 
-    public static function tryToFilename(string $filename): string
+    public static function cleanParts(string $filename): string
     {
-        try {
-            $parts = static::toParts($filename);
-            return $parts['filename'];
-        } catch (\Throwable $e) {
-            return $filename;
-        }
+        $sep = static::SEPARATOR;
+        return preg_replace("/(^|\/)[A-z0-9\\+]+\\{$sep}[a-f0-9-]+\\{$sep}/", '', $filename, 1);
     }
 
     public static function fromParts(string $account, string $id, string $filename): string

diff --git a/src/client/CloudflareImagesClient.php b/src/client/CloudflareImagesClient.php
@@ -7,6 +7,8 @@
 
 class CloudflareImagesClient
 {
+    const MAX_PER_PAGE = 10000;
+
     public string $baseUrl = 'https://api.cloudflare.com/client/v4/accounts/';
     /** @var \deuxhuithuit\cfimages\models\Settings? */
     public $config;
@@ -86,7 +88,6 @@ public function uploadImageStream($imageStream, string $imageFilename, array $me
 
     public function moveImage(string $imageFilename, string $imageUid, array $meta = []): array
     {
-        $meta['path'] = $imageFilename;
         $meta['folder'] = \dirname($imageFilename);
         $meta['updated'] = time();
         $client = new GuzzleHttp\Client();
@@ -105,10 +106,15 @@ public function moveImage(string $imageFilename, string $imageUid, array $meta =
         return $data['result'];
     }
 
-    public function listImages($perPage = 10000, $continueToken = null): array
+    public function listImages($perPage = null, $continueToken = null): array
     {
         $client = new GuzzleHttp\Client();
-        $res = $client->request('GET', $this->createCfUrl("/images/v2?per_page=$perPage"), [
+        $perPage = $perPage ? min($perPage, self::MAX_PER_PAGE) : self::MAX_PER_PAGE;
+        $endpoint = "/images/v2?per_page=$perPage";
+        if ($continueToken) {
+            $endpoint .= "&continuation_token=$continueToken";
+        }
+        $res = $client->request('GET', $this->createCfUrl($endpoint), [
             'headers' => $this->createHttpHeaders(),
             'http_errors' => false,
         ]);
@@ -119,13 +125,17 @@ public function listImages($perPage = 10000, $continueToken = null): array
 
         $data = json_decode($res->getBody(), true);
 
-        if (!isset($data['result']['images'])) {
-            return [];
+        if (!isset($data['success']) || !$data['success'] || !isset($data['result']['images'])) {
+            return [
+                'images' => [],
+                'continuation_token' => null,
+            ];
         }
 
-        // TODO: Recursive call for the next page
-
-        return $data['result']['images'];
+        return [
+            'images' => $data['result']['images'],
+            'continuation_token' => $data['result']['continuation_token'],
+        ];
     }
 
     public function getImage(string $imageUid): array

diff --git a/src/fs/CloudflareImagesFs.php b/src/fs/CloudflareImagesFs.php
@@ -64,33 +64,41 @@ public function getRootUrl(): ?string
      */
     public function getFileList(string $directory = '', bool $recursive = true): \Generator
     {
+        if ($directory === '') {
+            $directory = '.';
+        }
         try {
-            $images = $this->client->listImages();
-            foreach ($images as $image) {
-                $dirname = isset($image['meta']['folder']) ? $image['meta']['folder'] : '';
-                $dirname = $dirname === '.' ? '' : $dirname;
-                $filename = isset($image['meta']['path']) ? basename($image['meta']['path']) : $image['filename'];
-
-                if ($recursive) {
-                    if (strlen($directory) > 0 && !\str_starts_with("$dirname/", $directory)) {
-                        continue;
-                    }
-                } else {
-                    if ($dirname !== $directory && "$dirname/" !== $directory) {
-                        continue;
+            $continueToken = null;
+            do {
+                $list = $this->client->listImages(100, $continueToken);
+                $images = $list['images'];
+                $continueToken = $list['continuation_token'];
+
+                foreach ($images as $image) {
+                    $dirname = isset($image['meta']['folder']) ? $image['meta']['folder'] : '';
+                    $filename = isset($image['meta']['path']) ? basename($image['meta']['path']) : $image['filename'];
+
+                    if ($recursive) {
+                        if ($directory != '.' && !\str_starts_with("$dirname/", $directory)) {
+                            continue;
+                        }
+                    } else {
+                        if ($dirname !== $directory && "$dirname/" !== $directory) {
+                            continue;
+                        }
                     }
-                }
 
-                yield new FsListing([
-                    'basename' => Filename::fromParts($this->settings->getAccountHash(), $image['id'], $filename),
-                    'dirname' => $dirname,
-                    'type' => 'file',
-                    'fileSize' => isset($image['meta']['size']) ? $image['meta']['size'] : 0,
-                    'dateModified' => isset($image['meta']['updated'])
-                        ? $image['meta']['updated']
-                        : (isset($image['meta']['created']) ? $image['meta']['created'] : 0),
-                ]);
-            }
+                    yield new FsListing([
+                        'basename' => Filename::fromParts($this->settings->getAccountHash(), $image['id'], $filename),
+                        'dirname' => $dirname,
+                        'type' => 'file',
+                        'fileSize' => isset($image['meta']['size']) ? $image['meta']['size'] : 0,
+                        'dateModified' => isset($image['meta']['updated'])
+                            ? $image['meta']['updated']
+                            : (isset($image['meta']['created']) ? $image['meta']['created'] : 0),
+                    ]);
+                }
+            } while ($continueToken);
         } catch (\Exception $e) {
             throw new FsException($e->getMessage(), $e->getCode(), $e);
         }
@@ -173,7 +181,7 @@ public function saveAsset(Asset $asset): void
         $properFilename = Filename::fromParts(
             $this->settings->getAccountHash(),
             $recentId,
-            Filename::tryToFilename($asset->filename)
+            Filename::cleanParts($asset->filename)
         );
         // If the filename is already correct, we are done.
         // Somehow, we need to break the loop here, otherwise the asset will be saved again and again.
@@ -209,11 +217,6 @@ public function fileExists(string $path): bool
             return false;
         }
 
-        // Check if the file exists in the recent files
-        if (isset($this->recentFiles[$path])) {
-            return true;
-        }
-
         // Check if the file exists in the cloudflare images
         try {
             $imageId = Filename::toId(\basename($path));
@@ -268,7 +271,7 @@ public function renameFile(string $path, string $newPath, array $config = []): v
             $image = $this->client->getImage($imageId);
             // Make sure we get rid of any parts in the new paths.
             // This happens when the file is moved to another folder in the asset manager.
-            $newCleanedPath = \dirname($newPath) . '/' . Filename::tryToFilename(\basename($newPath));
+            $newCleanedPath = \dirname($newPath) . '/' . Filename::cleanParts(\basename($newPath));
             $this->client->moveImage($newCleanedPath, $imageId, $image['meta']);
             $this->recentFiles[$newCleanedPath] = $imageId;
         } catch (\Exception $e) {