diff --git a/bin/MigrationCLI.php b/bin/MigrationCLI.php index 4529225..f6024f0 100644 --- a/bin/MigrationCLI.php +++ b/bin/MigrationCLI.php @@ -33,7 +33,7 @@ public function drawFrame() $statusCounters = $this->transfer->getStatusCounters(); - $mask = "| %15.15s | %-7.7s | %10.10s | %7.7s | %7.7s | %8.8s |\n"; + $mask = "| %15.15s | %-7.7s | %10.10s | %7.7s | %7.7s | %8.8s | %8.8s |\n"; printf($mask, 'Resource', 'Pending', 'Processing', 'Skipped', 'Warning', 'Error', 'Success'); printf($mask, '-------------', '-------------', '-------------', '-------------', '-------------', '-------------', '-------------'); foreach ($statusCounters as $resource => $data) { @@ -58,6 +58,25 @@ public function drawFrame() echo $error->getResourceGroup().'['.$error->getResourceId().'] - '.$error->getMessage()."\n"; } } + + // Render Warnings + $sourceWarnings = $this->source->getWarnings(); + if (! empty($sourceWarnings)) { + echo "\n\nSource Warnings:\n"; + foreach ($sourceWarnings as $warning) { + /** @var Utopia\Migration\Warning $warning */ + echo $warning->getResourceName().'['.$warning->getResourceId().'] - '.$warning->getMessage()."\n"; + } + } + + $destWarnings = $this->destination->getWarnings(); + if (! empty($destWarnings)) { + echo "\n\nDestination Warnings:\n"; + foreach ($destWarnings as $warning) { + /** @var Utopia\Migration\Warning $warning */ + echo $warning->getResourceName().'['.$warning->getResourceId().'] - '.$warning->getMessage()."\n"; + } + } } public function getSource(): Source diff --git a/composer.json b/composer.json index 2ea999e..ccc4648 100644 --- a/composer.json +++ b/composer.json @@ -24,6 +24,7 @@ "require-dev": { "phpunit/phpunit": "9.*", "vlucas/phpdotenv": "5.*", - "laravel/pint": "1.*" + "laravel/pint": "1.*", + "utopia-php/cli": "^0.18.0" } } diff --git a/playground.php b/playground.php deleted file mode 100644 index 571a768..0000000 --- a/playground.php +++ /dev/null @@ -1,102 +0,0 @@ -load(); - -/** - * Initialise All Source Adapters - */ -$sourceAppwrite = new Appwrite( - $_ENV['SOURCE_APPWRITE_TEST_PROJECT'], - $_ENV['SOURCE_APPWRITE_TEST_ENDPOINT'], - $_ENV['SOURCE_APPWRITE_TEST_KEY'] -); - -// $firebase = json_decode($_ENV['FIREBASE_TEST_ACCOUNT'], true); - -// $sourceFirebase = new Firebase( -// $firebase, -// $firebase['project_id'] ?? '', -// ); - -// $sourceNHost = new NHost( -// $_ENV['NHOST_TEST_SUBDOMAIN'] ?? '', -// $_ENV['NHOST_TEST_REGION'] ?? '', -// $_ENV['NHOST_TEST_SECRET'] ?? '', -// $_ENV['NHOST_TEST_DATABASE'] ?? '', -// $_ENV['NHOST_TEST_USERNAME'] ?? '', -// $_ENV['NHOST_TEST_PASSWORD'] ?? '', -// ); - -// $sourceSupabase = new Supabase( -// $_ENV['SUPABASE_TEST_ENDPOINT'] ?? '', -// $_ENV['SUPABASE_TEST_KEY'] ?? '', -// $_ENV['SUPABASE_TEST_HOST'] ?? '', -// $_ENV['SUPABASE_TEST_DATABASE'] ?? '', -// $_ENV['SUPABASE_TEST_USERNAME'] ?? '', -// $_ENV['SUPABASE_TEST_PASSWORD'] ?? '', -// ); - -/** - * Initialise All Destination Adapters - */ -$destinationAppwrite = new AppwriteDestination( - $_ENV['DESTINATION_APPWRITE_TEST_PROJECT'], - $_ENV['DESTINATION_APPWRITE_TEST_ENDPOINT'], - $_ENV['DESTINATION_APPWRITE_TEST_KEY'] -); - -$destinationLocal = new Local(__DIR__.'/localBackup/'); - -/** - * Initialise Transfer Class - */ -$transfer = new Transfer( - $sourceAppwrite, - $destinationAppwrite -); - -/** - * Run Transfer - */ -$transfer->run( - $sourceAppwrite->getSupportedResources(), - function (array $resources) use ($transfer) { - var_dump($transfer->getStatusCounters()); - } -); - -$report = []; - -$cache = $transfer->getCache()->getAll(); - -if (count($sourceAppwrite->getErrors()) > 0) { - foreach ($sourceAppwrite->getErrors() as $error) { - /* @var \Utopia\Migration\Exception $error */ - Console::error('[Source] ['.$error->getResourceType().'] '.$error->getMessage()); - } -} - -if (count($destinationAppwrite->getErrors()) > 0) { - foreach ($destinationAppwrite->getErrors() as $error) { - /* @var \Utopia\Migration\Exception $error */ - Console::error('[Destination] ['.$error->getResourceType().'] '.$error->getMessage()); - } -} diff --git a/src/Migration/Sources/NHost.php b/src/Migration/Sources/NHost.php index e4ce5a7..b8306df 100644 --- a/src/Migration/Sources/NHost.php +++ b/src/Migration/Sources/NHost.php @@ -21,6 +21,7 @@ use Utopia\Migration\Resources\Storage\File; use Utopia\Migration\Source; use Utopia\Migration\Transfer; +use Utopia\Migration\Warning; class NHost extends Source { @@ -511,13 +512,51 @@ private function convertAttribute(array $column, Collection $collection): Attrib return new Boolean($column['column_name'], $collection, $column['is_nullable'] === 'NO', $isArray, $column['column_default']); case 'smallint': case 'int2': + if (! is_numeric($column['column_default']) && ! is_null($column['column_default'])) { + $this->addWarning(new Warning( + Resource::TYPE_COLLECTION, + Transfer::GROUP_DATABASES, + 'Functional default values are not supported. Default value for attribute '.$column['column_name'].' will be set to null.', + $collection->getId() + )); + + $collection->setStatus(Resource::STATUS_WARNING); + + $column['column_default'] = null; + } + return new Integer($column['column_name'], $collection, $column['is_nullable'] === 'NO', $isArray, $column['column_default'], -32768, 32767); case 'integer': case 'int4': + if (! is_numeric($column['column_default']) && ! is_null($column['column_default'])) { + $this->addWarning(new Warning( + Resource::TYPE_COLLECTION, + Transfer::GROUP_DATABASES, + 'Functional default values are not supported. Default value for attribute '.$column['column_name'].' will be set to null.', + $collection->getId() + )); + + $collection->setStatus(Resource::STATUS_WARNING); + + $column['column_default'] = null; + } + return new Integer($column['column_name'], $collection, $column['is_nullable'] === 'NO', $isArray, $column['column_default'], -2147483648, 2147483647); case 'bigint': case 'int8': case 'numeric': + if (! is_numeric($column['column_default']) && ! is_null($column['column_default'])) { + $this->addWarning(new Warning( + Resource::TYPE_COLLECTION, + Transfer::GROUP_DATABASES, + 'Functional default values are not supported. Default value for attribute '.$column['column_name'].' will be set to null.', + $collection->getId() + )); + $collection->setStatus(Resource::STATUS_WARNING); + + $column['column_default'] = null; + } + return new Integer($column['column_name'], $collection, $column['is_nullable'] === 'NO', $isArray, $column['column_default']); case 'decimal': case 'real': @@ -525,6 +564,19 @@ private function convertAttribute(array $column, Collection $collection): Attrib case 'float4': case 'float8': case 'money': + if (! is_numeric($column['column_default']) && ! is_null($column['column_default'])) { + $this->addWarning(new Warning( + Resource::TYPE_COLLECTION, + Transfer::GROUP_DATABASES, + 'Functional default values are not supported. Default value for attribute '.$column['column_name'].' will be set to null.', + $collection->getId() + )); + + $collection->setStatus(Resource::STATUS_WARNING); + + $column['column_default'] = null; + } + return new Decimal($column['column_name'], $collection, $column['is_nullable'] === 'NO', $isArray, $column['column_default']); // Time (Conversion happens with documents) case 'timestamp with time zone': diff --git a/src/Migration/Target.php b/src/Migration/Target.php index b8a9574..e07e5e3 100644 --- a/src/Migration/Target.php +++ b/src/Migration/Target.php @@ -15,8 +15,20 @@ abstract class Target public $cache; + /** + * Errors + * + * @var array + */ public $errors = []; + /** + * Warnings + * + * @var array + */ + public $warnings = []; + protected $endpoint = ''; abstract public static function getName(): string; @@ -160,7 +172,7 @@ protected function flatten(array $data, string $prefix = ''): array /** * Get Errors * - * @returns Error[] + * @returns array */ public function getErrors(): array { @@ -168,17 +180,28 @@ public function getErrors(): array } /** - * Set Errors + * Add Error + */ + public function addError(Exception $error): void + { + $this->errors[] = $error; + } + + /** + * Get Warnings * - * @param Error[] $errors + * @returns array */ - public function setErrors(array $errors): void + public function getWarnings(): array { - $this->errors = $errors; + return $this->warnings; } - public function addError(Exception $error): void + /** + * Add Warning + */ + public function addWarning(Warning $warning): void { - $this->errors[] = $error; + $this->warnings[] = $warning; } } diff --git a/src/Migration/Transfer.php b/src/Migration/Transfer.php index c0e45c8..7bce6e8 100644 --- a/src/Migration/Transfer.php +++ b/src/Migration/Transfer.php @@ -62,12 +62,6 @@ public function __construct(Source $source, Destination $destination) */ protected Cache $cache; - protected array $options = []; - - protected array $callbacks = []; - - protected array $events = []; - protected array $resources = []; public function getStatusCounters() @@ -115,8 +109,9 @@ public function getStatusCounters() // Process Source Errprs foreach ($this->source->getErrors() as $error) { - if (isset($status[$error->getResourceType()])) { - $status[$error->getResourceType()][Resource::STATUS_ERROR]++; + /** @var Exception $error */ + if (isset($status[$error->getResourceGroup()])) { + $status[$error->getResourceGroup()][Resource::STATUS_ERROR]++; } } diff --git a/src/Migration/Warning.php b/src/Migration/Warning.php new file mode 100644 index 0000000..c354971 --- /dev/null +++ b/src/Migration/Warning.php @@ -0,0 +1,42 @@ +resourceName = $resourceName; + $this->resourceId = $resourceId; + $this->resourceGroup = $resourceGroup; + $this->message = $message; + } + + public function getResourceName(): string + { + return $this->resourceName; + } + + public function getResourceGroup(): string + { + return $this->resourceGroup; + } + + public function getResourceId(): string + { + return $this->resourceId; + } + + public function getMessage(): string + { + return $this->message; + } +} diff --git a/tests/Migration/E2E/Sources/NHostTest.php b/tests/Migration/E2E/Sources/NHostTest.php index 0551247..cf75c90 100644 --- a/tests/Migration/E2E/Sources/NHostTest.php +++ b/tests/Migration/E2E/Sources/NHostTest.php @@ -161,7 +161,7 @@ public function testValidateUserTransfer($state): void /** * @depends testValidateTransfer */ - public function testValidateDatabaseTransfer($state): void + public function testValidateDatabaseTransfer($state) { // Find known database $databases = $state['source']->cache->get(Resource::TYPE_DATABASE); @@ -194,9 +194,9 @@ public function testValidateDatabaseTransfer($state): void /** @var \Utopia\Migration\Resources\Database\Collection $collection */ if ($collection->getCollectionName() === 'TestTable') { $foundCollection = $collection; - } - break; + break; + } } if (! $foundCollection) { @@ -209,6 +209,38 @@ public function testValidateDatabaseTransfer($state): void $this->assertEquals('TestTable', $foundCollection->getCollectionName()); $this->assertEquals('TestTable', $foundCollection->getId()); $this->assertEquals('public', $foundCollection->getDatabase()->getId()); + + return $state; + } + + /** + * @depends testValidateDatabaseTransfer + */ + public function testDatabaseFunctionalDefaultsWarn($state): void + { + // Find known collection + $collections = $state['source']->cache->get(Resource::TYPE_COLLECTION); + $foundCollection = null; + + foreach ($collections as $collection) { + /** @var \Utopia\Migration\Resources\Database\Collection $collection */ + if ($collection->getCollectionName() === 'FunctionalDefaultTestTable') { + $foundCollection = $collection; + } + + break; + } + + if (! $foundCollection) { + $this->fail('Collection "FunctionalDefaultTestTable" not found'); + + return; + } + + $this->assertEquals('warning', $foundCollection->getStatus()); + $this->assertEquals('FunctionalDefaultTestTable', $foundCollection->getCollectionName()); + $this->assertEquals('FunctionalDefaultTestTable', $foundCollection->getId()); + $this->assertEquals('public', $foundCollection->getDatabase()->getId()); } /** diff --git a/tests/Migration/E2E/Sources/SupabaseTest.php b/tests/Migration/E2E/Sources/SupabaseTest.php index 231efc0..ba09804 100644 --- a/tests/Migration/E2E/Sources/SupabaseTest.php +++ b/tests/Migration/E2E/Sources/SupabaseTest.php @@ -140,7 +140,7 @@ public function testValidateUserTransfer($state): void /** * @depends testValidateTransfer */ - public function testValidateDatabaseTransfer($state): void + public function testValidateDatabaseTransfer($state) { // Find known database $databases = $state['source']->cache->get(Resource::TYPE_DATABASE); @@ -151,9 +151,9 @@ public function testValidateDatabaseTransfer($state): void /** @var \Utopia\Migration\Resources\Database $database */ if ($database->getDBName() === 'public') { $foundDatabase = $database; - } - break; + break; + } } if (! $foundDatabase) { @@ -214,6 +214,38 @@ public function testValidateDatabaseTransfer($state): void } $this->assertEquals('success', $foundDocument->getStatus()); + + return $state; + } + + /** + * @depends testValidateDatabaseTransfer + */ + public function testDatabaseFunctionalDefaultsWarn($state): void + { + // Find known collection + $collections = $state['source']->cache->get(Resource::TYPE_COLLECTION); + $foundCollection = null; + + foreach ($collections as $collection) { + /** @var \Utopia\Migration\Resources\Database\Collection $collection */ + if ($collection->getCollectionName() === 'FunctionalDefaultTestTable') { + $foundCollection = $collection; + } + + break; + } + + if (! $foundCollection) { + $this->fail('Collection "FunctionalDefaultTestTable" not found'); + + return; + } + + $this->assertEquals('warning', $foundCollection->getStatus()); + $this->assertEquals('FunctionalDefaultTestTable', $foundCollection->getCollectionName()); + $this->assertEquals('FunctionalDefaultTestTable', $foundCollection->getId()); + $this->assertEquals('public', $foundCollection->getDatabase()->getId()); } /** diff --git a/tests/Migration/resources/nhost/2_main.sql b/tests/Migration/resources/nhost/2_main.sql index 1546377..2c29e39 100644 --- a/tests/Migration/resources/nhost/2_main.sql +++ b/tests/Migration/resources/nhost/2_main.sql @@ -618,6 +618,23 @@ CREATE TABLE storage.files ( ALTER TABLE storage.files OWNER TO nhost_storage_admin; +-- +-- Name: FunctionalDefaultTestTable; Type: TABLE; Schema: public; Owner: nhost_hasura +-- +CREATE SEQUENCE IF NOT EXISTS public.test_data_id_seq; + +-- Table Definition +CREATE TABLE public."FunctionalDefaultTestTable" ( + "id" int4 NOT NULL DEFAULT nextval('public.test_data_id_seq'::regclass), + PRIMARY KEY ("id") +); + +-- Indices +CREATE UNIQUE INDEX test_data_pkey ON public."FunctionalDefaultTestTable" USING btree (id); + +-- Change table owner +ALTER TABLE public."FunctionalDefaultTestTable" OWNER TO nhost_hasura; + -- -- Name: schema_migrations; Type: TABLE; Schema: storage; Owner: nhost_storage_admin -- diff --git a/tests/Migration/resources/supabase/2_main.sql b/tests/Migration/resources/supabase/2_main.sql index 7e25991..55ca839 100644 --- a/tests/Migration/resources/supabase/2_main.sql +++ b/tests/Migration/resources/supabase/2_main.sql @@ -1305,6 +1305,17 @@ ALTER TABLE public.test OWNER TO postgres; COMMENT ON TABLE public.test IS 'test'; +CREATE SEQUENCE IF NOT EXISTS public.test_data_id_seq; + +-- Table Definition +CREATE TABLE public."FunctionalDefaultTestTable" ( + "id" int4 NOT NULL DEFAULT nextval('public.test_data_id_seq'::regclass), + PRIMARY KEY ("id") +); + +-- Indices +CREATE UNIQUE INDEX test_data_pkey ON public."FunctionalDefaultTestTable" USING btree (id); + -- -- Name: test2; Type: TABLE; Schema: public; Owner: postgres