utopia-php · christyjacob4 · Sep 14, 2023 · Sep 12, 2023 · Sep 12, 2023 · Sep 13, 2023
diff --git a/playground.php b/playground.php
@@ -8,6 +8,7 @@
 require_once __DIR__.'/vendor/autoload.php';
 
 use Dotenv\Dotenv;
+use Utopia\CLI\Console;
 use Utopia\Migration\Destinations\Appwrite as AppwriteDestination;
 use Utopia\Migration\Destinations\Local;
 use Utopia\Migration\Resource;
@@ -29,30 +30,30 @@
     $_ENV['SOURCE_APPWRITE_TEST_KEY']
 );
 
-$firebase = json_decode($_ENV['FIREBASE_TEST_ACCOUNT'], true);
-
-$sourceFirebase = new Firebase(
-    $firebase,
-    $firebase['project_id'] ?? '',
-);
-
-$sourceNHost = new NHost(
-    $_ENV['NHOST_TEST_SUBDOMAIN'] ?? '',
-    $_ENV['NHOST_TEST_REGION'] ?? '',
-    $_ENV['NHOST_TEST_SECRET'] ?? '',
-    $_ENV['NHOST_TEST_DATABASE'] ?? '',
-    $_ENV['NHOST_TEST_USERNAME'] ?? '',
-    $_ENV['NHOST_TEST_PASSWORD'] ?? '',
-);
-
-$sourceSupabase = new Supabase(
-    $_ENV['SUPABASE_TEST_ENDPOINT'] ?? '',
-    $_ENV['SUPABASE_TEST_KEY'] ?? '',
-    $_ENV['SUPABASE_TEST_HOST'] ?? '',
-    $_ENV['SUPABASE_TEST_DATABASE'] ?? '',
-    $_ENV['SUPABASE_TEST_USERNAME'] ?? '',
-    $_ENV['SUPABASE_TEST_PASSWORD'] ?? '',
-);
+// $firebase = json_decode($_ENV['FIREBASE_TEST_ACCOUNT'], true);
+
+// $sourceFirebase = new Firebase(
+//     $firebase,
+//     $firebase['project_id'] ?? '',
+// );
+
+// $sourceNHost = new NHost(
+//     $_ENV['NHOST_TEST_SUBDOMAIN'] ?? '',
+//     $_ENV['NHOST_TEST_REGION'] ?? '',
+//     $_ENV['NHOST_TEST_SECRET'] ?? '',
+//     $_ENV['NHOST_TEST_DATABASE'] ?? '',
+//     $_ENV['NHOST_TEST_USERNAME'] ?? '',
+//     $_ENV['NHOST_TEST_PASSWORD'] ?? '',
+// );
+
+// $sourceSupabase = new Supabase(
+//     $_ENV['SUPABASE_TEST_ENDPOINT'] ?? '',
+//     $_ENV['SUPABASE_TEST_KEY'] ?? '',
+//     $_ENV['SUPABASE_TEST_HOST'] ?? '',
+//     $_ENV['SUPABASE_TEST_DATABASE'] ?? '',
+//     $_ENV['SUPABASE_TEST_USERNAME'] ?? '',
+//     $_ENV['SUPABASE_TEST_PASSWORD'] ?? '',
+// );
 
 /**
  * Initialise All Destination Adapters
@@ -69,16 +70,14 @@
  * Initialise Transfer Class
  */
 $transfer = new Transfer(
-    $sourceFirebase,
-    $destinationLocal
+    $sourceAppwrite,
+    $destinationAppwrite
 );
 
-$sourceFirebase->report();
-
 // /**
 //  * Run Transfer
 //  */
-$transfer->run($sourceFirebase->getSupportedResources(),
+$transfer->run($sourceAppwrite->getSupportedResources(),
     function (array $resources) {
     }
 );
@@ -87,10 +86,13 @@ function (array $resources) {
 
 $cache = $transfer->getCache()->getAll();
 
-// foreach ($cache as $type => $resources) {
-//     foreach ($resources as $resource) {
-//         if ($resource->getStatus() !== Resource::STATUS_ERROR) {
-//             continue;
-//         }
-//     }
-// }
+foreach ($cache as $type => $resources) {
+    foreach ($resources as $resource) {
+        /** @var resource $resource */
+        if ($resource->getStatus() !== Resource::STATUS_ERROR) {
+            continue;
+        }
+
+        Console::error($resource->getName().' '.$resource->getInternalId().' '.$resource->getMessage());
+    }
+}
diff --git a/src/Migration/Cache.php b/src/Migration/Cache.php
@@ -2,7 +2,6 @@
 
 namespace Utopia\Migration;
 
-use Utopia\Migration\Resources\Functions\Func;
 use Utopia\Migration\Resources\Storage\File;
 
 /**
@@ -37,8 +36,8 @@ public function add($resource)
             $resource->setInternalId(uniqid());
         }
 
-        if ($resource->getName() == Resource::TYPE_FILE || $resource->getName() == Resource::TYPE_FUNCTION) {
-            /** @var File|Func $resource */
+        if ($resource->getName() == Resource::TYPE_FILE || $resource->getName() == Resource::TYPE_DEPLOYMENT) {
+            /** @var File|Deployment $resource */
             $resource->setData(''); // Prevent Memory Leak
         }
 

diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php
@@ -529,8 +529,8 @@ public function importAuthResource(Resource $resource): Resource
                     /** @var User $resource */
                     if (in_array(User::TYPE_EMAIL, $resource->getTypes())) {
                         $this->importPasswordUser($resource);
-                    } elseif (in_array(User::TYPE_ANONYMOUS, $resource->getTypes()) || in_array(User::TYPE_OAUTH, $resource->getTypes())) {
-                        $resource->setStatus(Resource::STATUS_WARNING, 'Anonymous and OAuth users cannot be imported.');
+                    } elseif (in_array(User::TYPE_OAUTH, $resource->getTypes())) {
+                        $resource->setStatus(Resource::STATUS_WARNING, 'OAuth users cannot be imported.');
 
                         return $resource;
                     } else {
@@ -563,6 +563,14 @@ public function importAuthResource(Resource $resource): Resource
                         $userService->updateStatus($resource->getId(), ! $resource->getDisabled());
                     }
 
+                    if ($resource->getPreferences()) {
+                        $userService->updatePrefs($resource->getId(), $resource->getPreferences());
+                    }
+
+                    if ($resource->getLabels()) {
+                        $userService->updateLabels($resource->getId(), $resource->getLabels());
+                    }
+
                     break;
                 case Resource::TYPE_TEAM:
                     /** @var Team $resource */

diff --git a/src/Migration/Resources/Auth/User.php b/src/Migration/Resources/Auth/User.php
@@ -27,6 +27,8 @@ class User extends Resource
 
     protected array $types = [self::TYPE_ANONYMOUS];
 
+    protected array $labels = [];
+
     protected string $oauthProvider = '';
 
     protected bool $emailVerified = false;
@@ -44,6 +46,7 @@ public function __construct(
         Hash $passwordHash = null,
         string $phone = '',
         array $types = [self::TYPE_ANONYMOUS],
+        array $labels = [],
         string $oauthProvider = '',
         bool $emailVerified = false,
         bool $phoneVerified = false,
@@ -56,6 +59,7 @@ public function __construct(
         $this->passwordHash = $passwordHash;
         $this->phone = $phone;
         $this->types = $types;
+        $this->labels = $labels;
         $this->oauthProvider = $oauthProvider;
         $this->emailVerified = $emailVerified;
         $this->phoneVerified = $phoneVerified;
@@ -161,6 +165,24 @@ public function setTypes(string $types): self
         return $this;
     }
 
+    /**
+     * Get Labels
+     */
+    public function getLabels(): array
+    {
+        return $this->labels;
+    }
+
+    /**
+     * Set Labels
+     */
+    public function setLabels(array $labels): self
+    {
+        $this->labels = $labels;
+
+        return $this;
+    }
+
     /**
      * Get OAuth Provider
      */

diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php
@@ -203,15 +203,39 @@ public function report(array $resources = []): array
             if (in_array(Resource::TYPE_FILE, $resources)) {
                 $report[Resource::TYPE_FILE] = 0;
                 $report['size'] = 0;
-                $buckets = $storageClient->listBuckets()['buckets'];
+                $buckets = [];
+                $lastBucket = null;
+
+                while (true) {
+                    $currentBuckets = $storageClient->listBuckets($lastBucket ? [Query::cursorAfter($lastBucket)] : [Query::limit(20)])['buckets'];
+                    $buckets = array_merge($buckets, $currentBuckets);
+                    $lastBucket = $buckets[count($buckets) - 1]['$id'];
+
+                    if (count($currentBuckets) < 20) {
+                        break;
+                    }
+                }
+
                 foreach ($buckets as $bucket) {
-                    $files = $storageClient->listFiles($bucket['$id']);
-                    $report[Resource::TYPE_FILE] += $files['total'];
-                    foreach ($files['files'] as $file) {
+                    $files = [];
+                    $lastFile = null;
+
+                    while (true) {
+                        $currentFiles = $storageClient->listFiles($bucket['$id'], $lastFile ? [Query::cursorAfter($lastFile)] : [Query::limit(20)])['files'];
+                        $files = array_merge($files, $currentFiles);
+                        $lastFile = $files[count($files) - 1]['$id'];
+
+                        if (count($currentFiles) < 20) {
+                            break;
+                        }
+                    }
+
+                    $report[Resource::TYPE_FILE] += count($files);
+                    foreach ($files as $file) {
                         $report['size'] += $storageClient->getFile($bucket['$id'], $file['$id'])['sizeOriginal'];
                     }
                 }
-                $report['size'] = $report['size'] / 1024 / 1024; // MB
+                $report['size'] = $report['size'] / 1000 / 1000; // MB
             }
 
             // Functions
@@ -300,6 +324,7 @@ private function exportUsers(int $batchSize)
                     $user['password'] ? new Hash($user['password'], $user['hash']) : null,
                     $user['phone'],
                     $this->calculateTypes($user),
+                    $user['labels'] ?? [],
                     '',
                     $user['emailVerification'],
                     $user['phoneVerification'],
@@ -650,6 +675,14 @@ private function convertAttribute(array $value, Collection $collection): Attribu
                     $value['onDelete'],
                     $value['side']
                 );
+            case 'datetime':
+                return new DateTime(
+                    $value['key'],
+                    $collection,
+                    $value['required'],
+                    $value['array'],
+                    $value['default']
+                );
         }
 
         throw new \Exception('Unknown attribute type: '.$value['type']);
@@ -1077,12 +1110,43 @@ private function exportDeploymentData(Func $func, array $deployment)
         $end = Transfer::STORAGE_MAX_CHUNK_SIZE - 1;
 
         // Get the file size
-        $fileSize = $deployment['size'];
+        $responseHeaders = [];
+
+        $this->call(
+            'HEAD',
+            "/functions/{$func->getId()}/deployments/{$deployment['$id']}/download",
+            [],
+            [],
+            $responseHeaders
+        );
 
-        if ($end > $fileSize) {
-            $end = $fileSize - 1;
+        // Content-Length header was missing, file is less than max buffer size.
+        if (! array_key_exists('Content-Length', $responseHeaders)) {
+            $file = $this->call(
+                'GET',
+                "/functions/{$func->getId()}/deployments/{$deployment['$id']}/download",
+                [],
+                [],
+                $responseHeaders
+            );
+
+            $deployment = new Deployment(
+                $deployment['$id'],
+                $func,
+                strlen($file),
+                $deployment['entrypoint'],
+                $start,
+                $end,
+                $file,
+                $deployment['activate']
+            );
+            $deployment->setInternalId($deployment->getId());
+
+            return $this->callback([$deployment]);
         }
 
+        $fileSize = $responseHeaders['Content-Length'];
+
         $deployment = new Deployment(
             $deployment['$id'],
             $func,

diff --git a/src/Migration/Sources/Firebase.php b/src/Migration/Sources/Firebase.php
@@ -94,11 +94,11 @@ private function authenticate()
         }
     }
 
-    protected function call(string $method, string $path = '', array $headers = [], array $params = []): array|string
+    protected function call(string $method, string $path = '', array $headers = [], array $params = [], &$responseHeaders = []): array|string
     {
         $this->authenticate();
 
-        return parent::call($method, $path, $headers, $params);
+        return parent::call($method, $path, $headers, $params, $responseHeaders);
     }
 
     /**
@@ -210,6 +210,7 @@ private function exportUsers(int $batchSize)
                     new Hash($user['passwordHash'] ?? '', $user['salt'] ?? '', Hash::ALGORITHM_SCRYPT_MODIFIED, $hashConfig['saltSeparator'] ?? '', $hashConfig['signerKey'] ?? ''),
                     $user['phoneNumber'] ?? '',
                     $this->calculateUserType($user['providerUserInfo'] ?? []),
+                    [],
                     '',
                     $user['emailVerified'],
                     false, // Can't get phone number status on firebase :/

diff --git a/src/Migration/Sources/NHost.php b/src/Migration/Sources/NHost.php
@@ -247,6 +247,7 @@ private function exportUsers(int $batchSize)
                     new Hash($user['password_hash'], '', Hash::ALGORITHM_BCRYPT),
                     $user['phone_number'] ?? '',
                     $this->calculateUserTypes($user),
+                    [],
                     '',
                     $user['email_verified'],
                     $user['phone_number_verified'],

diff --git a/src/Migration/Sources/Supabase.php b/src/Migration/Sources/Supabase.php
@@ -383,6 +383,7 @@ private function exportUsers(int $batchSize)
                     new Hash($user['encrypted_password'], '', Hash::ALGORITHM_BCRYPT),
                     $user['phone'] ?? '',
                     $this->calculateAuthTypes($user),
+                    [],
                     '',
                     ! empty($user['email_confirmed_at']),
                     ! empty($user['phone_confirmed_at']),

diff --git a/src/Migration/Target.php b/src/Migration/Target.php
@@ -68,11 +68,10 @@ abstract public function report(array $resources = []): array;
      *
      * @throws \Exception
      */
-    protected function call(string $method, string $path = '', array $headers = [], array $params = []): array|string
+    protected function call(string $method, string $path = '', array $headers = [], array $params = [], &$responseHeaders = []): array|string
     {
         $headers = array_merge($this->headers, $headers);
         $ch = curl_init((str_contains($path, 'http') ? $path.(($method == 'GET' && ! empty($params)) ? '?'.http_build_query($params) : '') : $this->endpoint.$path.(($method == 'GET' && ! empty($params)) ? '?'.http_build_query($params) : '')));
-        $responseHeaders = [];
         $responseStatus = -1;
         $responseType = '';
         $responseBody = '';
@@ -96,7 +95,11 @@ protected function call(string $method, string $path = '', array $headers = [],
             unset($headers[$i]);
         }
 
-        curl_setopt($ch, CURLOPT_CUSTOMREQUEST, $method);
+        if ($method === 'HEAD') {
+            curl_setopt($ch, CURLOPT_NOBODY, true);
+        } else {
+            curl_setopt($ch, CURLOPT_CUSTOMREQUEST, $method);
+        }
         curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
         curl_setopt($ch, CURLOPT_USERAGENT, php_uname('s').'-'.php_uname('r').':php-'.phpversion());
         curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);