Improve Migrations Stablity

.github/workflows/tests.yml

@@ -0,0 +1,19 @@
+name: Tests
+on:
+  pull_request:
+  push: { branches: [main] }
+
+jobs:
+  tests:
+    name: Run Test Suite
+    runs-on: ubuntu-latest
+    env:
+      COMPOSE_FILE: docker-compose.yml
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Run Tests
+        run: |
+          docker-compose up -d --build && sleep 5 && docker compose exec tests php ./vendor/bin/phpunit

Dockerfile

@@ -1,33 +1,37 @@
-FROM postgres:alpine3.18 as supabase-db
-COPY ./tests/Transfer/resources/supabase/backup.tar /docker-entrypoint-initdb.d/backup.tar
-COPY ./tests/Transfer/resources/restore.sh /docker-entrypoint-initdb.d/restore.sh
+FROM supabase/postgres:15.1.0.96 as supabase-db
+COPY ./tests/Transfer/resources/supabase/1_globals.sql /docker-entrypoint-initdb.d/1_globals.sql
+COPY ./tests/Transfer/resources/supabase/2_main.sql /docker-entrypoint-initdb.d/2_main.sql
+RUN rm -rf /docker-entrypoint-initdb.d/migrate.sh
 
 FROM postgres:alpine3.18 as nhost-db
-COPY ./tests/Transfer/resources/nhost/backup.tar /docker-entrypoint-initdb.d/backup.tar
-COPY ./tests/Transfer/resources/restore.sh /docker-entrypoint-initdb.d/restore.sh
+COPY ./tests/Transfer/resources/nhost/1_globals.sql /docker-entrypoint-initdb.d/1_globals.sql
+COPY ./tests/Transfer/resources/nhost/2_main.sql /docker-entrypoint-initdb.d/2_main.sql
 
 # Use my fork of mockoon while waiting for range headers to be merged
-FROM node:14-alpine3.14 as mock-api
+FROM node:20.4-alpine3.17 as mock-api
 WORKDIR /app
-RUN git clone https://github.com/PineappleIOnic/mockoon.git . 
+RUN apk add --no-cache git
+RUN git clone https://github.com/PineappleIOnic/mockoon.git
+WORKDIR /app/mockoon
+RUN git checkout origin/feat-implement-range
+RUN apk add python3 make gcc g++
 RUN npm run bootstrap
 RUN npm run build:libs
 RUN npm run build:cli
-RUN mv ./packages/cli/dist/run /usr/local/bin/mockoon
+RUN cd packages/cli && npm install -g .
+RUN adduser --shell /bin/sh --disabled-password --gecos "" mockoon
+USER mockoon
+CMD mockoon-cli start --data /mockoon/api.json --port 80 --disable-log-to-file && tail -f /dev/null
 
 FROM composer:2.0 as composer
-
 WORKDIR /usr/local/src/
-
-COPY composer.lock /usr/local/src/
 COPY composer.json /usr/local/src/
-
 RUN composer install --ignore-platform-reqs
 
-FROM php:8.0-fpm-alpine3.14 as tests
+FROM php:8.1.21-fpm-alpine3.18 as tests
 RUN set -ex && apk --no-cache add postgresql-dev
 RUN docker-php-ext-install pdo pdo_pgsql
-COPY ./src /usr/local/src
-COPY ./tests /usr/local/src/tests
-COPY --from=composer /usr/local/src/vendor /usr/local/src/vendor
-CMD php ./vendor/bin/phpunit
+COPY ./src /app/src
+COPY ./tests /app/src/tests
+COPY --from=composer /usr/local/src/vendor /app/vendor
+CMD tail -f /dev/null

composer.json

@@ -28,7 +28,7 @@
     },
     "require": {
         "php": ">=8.0",
-        "utopia-php/cli": "^0.13.0",
+        "utopia-php/cli": "^0.15.0",
         "appwrite/appwrite": "^8.0"
     },
     "require-dev": {

docker-compose.yml

@@ -26,6 +26,24 @@ services:
       POSTGRES_USER: postgres
       POSTGRES_PASSWORD: postgres
       POSTGRES_DB: postgres
+
+  nhost-storage:
+    build:
+      context: .
+      target: mock-api
+    networks:
+      - tests
+    volumes:
+      - ./tests/Transfer/resources/nhost:/mockoon
+
+  supabase-api:
+    build:
+      context: .
+      target: mock-api
+    networks:
+      - tests
+    volumes:
+      - ./tests/Transfer/resources/supabase:/mockoon
 
   tests:
     build:
@@ -34,11 +52,14 @@ services:
     networks:
       - tests
     volumes:
-      - ./:/app
+      - ./tests:/app/tests
+      - ./src:/app/src
+      - ./phpunit.xml:/app/phpunit.xml
     working_dir: /app
     depends_on:
       - supabase-db
       - nhost-db
+      - nhost-storage
     environment:
       - NHOST_DB_URL=postgres://postgres:postgres@nhost-db:5432/postgres
       - SUPABASE_DB_URL=postgres://postgres:postgres@supabase-db:5432/postgres

playground.php

@@ -7,10 +7,10 @@
  */
 require_once __DIR__.'/vendor/autoload.php';
 
-use Appwrite\Query;
 use Dotenv\Dotenv;
 use Utopia\Transfer\Destinations\Appwrite as AppwriteDestination;
 use Utopia\Transfer\Destinations\Local;
+use Utopia\Transfer\Resource;
 use Utopia\Transfer\Sources\Appwrite;
 use Utopia\Transfer\Sources\Firebase;
 use Utopia\Transfer\Sources\NHost;
@@ -20,16 +20,14 @@
 $dotenv = Dotenv::createImmutable(__DIR__);
 $dotenv->load();
 
-cleanupAppwrite();
-
 /**
  * Initialise All Source Adapters
  */
-// $sourceAppwrite = new Appwrite(
-//     $_ENV['SOURCE_APPWRITE_TEST_PROJECT'],
-//     $_ENV['SOURCE_APPWRITE_TEST_ENDPOINT'],
-//     $_ENV['SOURCE_APPWRITE_TEST_KEY']
-// );
+$sourceAppwrite = new Appwrite(
+    $_ENV['SOURCE_APPWRITE_TEST_PROJECT'],
+    $_ENV['SOURCE_APPWRITE_TEST_ENDPOINT'],
+    $_ENV['SOURCE_APPWRITE_TEST_KEY']
+);
 
 $firebase = json_decode($_ENV['FIREBASE_TEST_ACCOUNT'], true);
 
@@ -38,14 +36,14 @@
     $firebase['project_id'] ?? '',
 );
 
-// $sourceNHost = new NHost(
-//     $_ENV['NHOST_TEST_SUBDOMAIN'] ?? '',
-//     $_ENV['NHOST_TEST_REGION'] ?? '',
-//     $_ENV['NHOST_TEST_SECRET'] ?? '',
-//     $_ENV['NHOST_TEST_DATABASE'] ?? '',
-//     $_ENV['NHOST_TEST_USERNAME'] ?? '',
-//     $_ENV['NHOST_TEST_PASSWORD'] ?? '',
-// );
+$sourceNHost = new NHost(
+    $_ENV['NHOST_TEST_SUBDOMAIN'] ?? '',
+    $_ENV['NHOST_TEST_REGION'] ?? '',
+    $_ENV['NHOST_TEST_SECRET'] ?? '',
+    $_ENV['NHOST_TEST_DATABASE'] ?? '',
+    $_ENV['NHOST_TEST_USERNAME'] ?? '',
+    $_ENV['NHOST_TEST_PASSWORD'] ?? '',
+);
 
 $sourceSupabase = new Supabase(
     $_ENV['SUPABASE_TEST_ENDPOINT'] ?? '',
@@ -59,91 +57,42 @@
 /**
  * Initialise All Destination Adapters
  */
-// $destinationAppwrite = new AppwriteDestination(
-//     $_ENV['DESTINATION_APPWRITE_TEST_PROJECT'],
-//     $_ENV['DESTINATION_APPWRITE_TEST_ENDPOINT'],
-//     $_ENV['DESTINATION_APPWRITE_TEST_KEY']
-// );
+$destinationAppwrite = new AppwriteDestination(
+    $_ENV['DESTINATION_APPWRITE_TEST_PROJECT'],
+    $_ENV['DESTINATION_APPWRITE_TEST_ENDPOINT'],
+    $_ENV['DESTINATION_APPWRITE_TEST_KEY']
+);
 
 $destinationLocal = new Local(__DIR__.'/localBackup/');
 
 /**
  * Initialise Transfer Class
  */
 $transfer = new Transfer(
-    $sourceFirebase,
+    $sourceSupabase,
     $destinationLocal
 );
 
-/**
- * Run Transfer
- */
-$transfer->run(
-    Supabase::getSupportedResources(),
-    function (array $resources) {
-    }
-);
+$sourceFirebase->report();
+
+// $sourceSupabase->report();
+
+// /**
+//  * Run Transfer
+//  */
+// $transfer->run($sourceAppwrite->getSupportedResources(),
+//     function (array $resources) {
+//     }
+// );
+
+// $report = [];
+
+// $cache = $transfer->getCache()->getAll();
 
-function cleanupAppwrite()
-{
-    $client = new \Appwrite\Client();
-
-    $client
-        ->setEndpoint($_ENV['DESTINATION_APPWRITE_TEST_ENDPOINT'])
-        ->setProject($_ENV['DESTINATION_APPWRITE_TEST_PROJECT'])
-        ->setKey($_ENV['DESTINATION_APPWRITE_TEST_KEY']);
-
-    $databaseService = new \Appwrite\Services\Databases($client);
-    $listDatabases = $databaseService->list();
-    foreach ($listDatabases['databases'] as $database) {
-        $databaseId = $database['$id'];
-        $listCollections = $databaseService->listCollections($databaseId);
-        foreach ($listCollections['collections'] as $collection) {
-            $collectionId = $collection['$id'];
-            $listDocuments = $databaseService->listDocuments($databaseId, $collectionId);
-            foreach ($listDocuments['documents'] as $document) {
-                $documentId = $document['$id'];
-                $databaseService->deleteDocument($databaseId, $collectionId, $documentId);
-            }
-        }
-
-        $databaseService->delete($databaseId);
-    }
-
-    $usersService = new \Appwrite\Services\Users($client);
-    $listUsers = $usersService->list();
-    if ($listUsers['total'] > count($listUsers['users'])) {
-        while ($listUsers['total'] > count($listUsers['users'])) {
-            $listUsers['users'] = array_merge($listUsers['users'], $usersService->list(
-                [Query::cursorAfter(
-                    $listUsers['users'][count($listUsers['users']) - 1]['$id']
-                )]
-            )['users']);
-        }
-    }
-
-    foreach ($listUsers['users'] as $user) {
-        $userId = $user['$id'];
-        $usersService->delete($userId);
-    }
-
-    $teamsService = new \Appwrite\Services\Teams($client);
-    $listTeams = $teamsService->list();
-    foreach ($listTeams['teams'] as $team) {
-        $teamId = $team['$id'];
-        $teamsService->delete($teamId);
-    }
-
-    $storageService = new \Appwrite\Services\Storage($client);
-    $listBuckets = $storageService->listBuckets();
-    foreach ($listBuckets['buckets'] as $bucket) {
-        $bucketId = $bucket['$id'];
-        $listFiles = $storageService->listFiles($bucketId);
-        foreach ($listFiles['files'] as $file) {
-            $fileId = $file['$id'];
-            $storageService->deleteFile($bucketId, $fileId);
-        }
-    }
-}
-
-var_dump($transfer->getStatusCounters());
+// foreach ($cache as $type => $resources) {
+//     foreach ($resources as $resource) {
+//         if ($resource->getStatus() !== Resource::STATUS_ERROR) {
+//             continue;
+//         }
+//     }
+// }

src/Transfer/Cache.php

@@ -2,6 +2,9 @@
 
 namespace Utopia\Transfer;
 
+use Utopia\Transfer\Resources\Functions\Func;
+use Utopia\Transfer\Resources\Storage\File;
+
 /**
  * Cache stores a local version of all data copied over from the source, This can be used as reference point for
  * previous transfers and also help the destination to determine what needs to be updated, modified,
@@ -33,6 +36,12 @@ public function add($resource)
             }
             $resource->setInternalId(uniqid());
         }
+
+        if ($resource->getName() == Resource::TYPE_FILE || $resource->getName() == Resource::TYPE_FUNCTION) {
+            /** @var File|Func $resource */
+            $resource->setData(''); // Prevent Memory Leak
+        }
+
         $this->cache[$resource->getName()][$resource->getInternalId()] = $resource;
     }
 
@@ -54,8 +63,8 @@ public function addAll(array $resources)
      */
     public function update($resource)
     {
-        if (! in_array($resource, $this->cache[$resource->getName()])) {
-            throw new \Exception('Resource does not exist in cache');
+        if (! in_array($resource->getName(), $this->cache)) {
+            $this->add($resource);
         }
 
         $this->cache[$resource->getName()][$resource->getInternalId()] = $resource;