[PLA-2023] Improve metadata caching

config/enjin-platform.php

@@ -291,4 +291,16 @@
         'attempts' => env('RATE_LIMIT_ATTEMPTS', 500),
         'time' => env('RATE_LIMIT_TIME', 1), // minutes
     ],
+
+    /*
+    |--------------------------------------------------------------------------
+    | Attribute metadata syncing
+    |--------------------------------------------------------------------------
+    |
+    | Here you may configure how the attribute metadata is synced
+    |
+    */
+    'sync_metadata' => [
+        'data_chunk_size' => env('SYNC_METADATA_CHUNK_SIZE', 10000),
+    ],
 ];

lang/en/mutation.php

@@ -116,4 +116,5 @@
     'update_wallet_external_id.cannot_update_id_on_managed_wallet' => 'Cannot update the external id on a managed wallet.',
     'verify_account.description' => 'The wallet calls this mutation to prove the ownership of the user account.',
     'operator_transfer_token.args.keepAlive' => '(DEPRECATED) If true, the transaction will fail if the balance drops below the minimum requirement. Defaults to False.',
+    'refresh_metadata.description' => "Refresh the collection's or token's metadata.",
 ];

src/Commands/SyncMetadata.php

@@ -0,0 +1,57 @@
+<?php
+
+namespace Enjin\Platform\Commands;
+
+use Enjin\Platform\Jobs\SyncMetadata as SyncMetadataJob;
+use Enjin\Platform\Models\Attribute;
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\Log;
+
+class SyncMetadata extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'platform:sync-metadata';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Sync attributes metadata to cache.';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle(): void
+    {
+        $query = Attribute::query()
+            ->select('key', 'value', 'token_id', 'collection_id')
+            ->where('key', '0x757269'); // uri hex
+
+        if (($total = $query->count()) == 0) {
+            $this->info('No attributes found to sync.');
+
+            return;
+        }
+
+        $progress = $this->output->createProgressBar($total);
+        $progress->start();
+        Log::info('Syncing metadata for ' . $total . ' attributes.');
+
+        $withs = [
+            'token:id,token_chain_id',
+            'collection:id,collection_chain_id',
+        ];
+        foreach ($query->with($withs)->lazy(config('enjin-platform.sync_metadata.data_chunk_size')) as $attribute) {
+            SyncMetadataJob::dispatch($attribute->value_string);
+            $progress->advance();
+        }
+
+        $progress->finish();
+        Log::info('Finished syncing metadata.');
+    }
+}

src/CoreServiceProvider.php

@@ -6,6 +6,7 @@
 use Enjin\Platform\Commands\Ingest;
 use Enjin\Platform\Commands\RelayWatcher;
 use Enjin\Platform\Commands\Sync;
+use Enjin\Platform\Commands\SyncMetadata;
 use Enjin\Platform\Commands\TransactionChecker;
 use Enjin\Platform\Commands\Transactions;
 use Enjin\Platform\Enums\Global\PlatformCache;
@@ -78,6 +79,7 @@ public function configurePackage(Package $package): void
             ->hasCommand(ClearCache::class)
             ->hasCommand(TransactionChecker::class)
             ->hasCommand(RelayWatcher::class)
+            ->hasCommand(SyncMetadata::class)
             ->hasTranslations();
     }
 

src/GraphQL/Schemas/Primary/Substrate/Mutations/RefreshMetadataMutation.php

@@ -0,0 +1,111 @@
+<?php
+
+namespace Enjin\Platform\GraphQL\Schemas\Primary\Substrate\Mutations;
+
+use Closure;
+use Enjin\Platform\GraphQL\Base\Mutation;
+use Enjin\Platform\GraphQL\Schemas\Primary\Substrate\Traits\HasEncodableTokenId;
+use Enjin\Platform\GraphQL\Schemas\Primary\Substrate\Traits\InPrimarySubstrateSchema;
+use Enjin\Platform\GraphQL\Schemas\Primary\Traits\HasTokenIdFieldRules;
+use Enjin\Platform\GraphQL\Types\Input\Substrate\Traits\HasTokenIdFields;
+use Enjin\Platform\Interfaces\PlatformGraphQlMutation;
+use Enjin\Platform\Models\Attribute;
+use Enjin\Platform\Rules\MaxBigInt;
+use Enjin\Platform\Rules\MinBigInt;
+use Enjin\Platform\Services\Database\MetadataService;
+use Enjin\Platform\Services\Serialization\Interfaces\SerializationServiceInterface;
+use Enjin\Platform\Support\Hex;
+use GraphQL\Type\Definition\ResolveInfo;
+use GraphQL\Type\Definition\Type;
+use Illuminate\Support\Arr;
+use Illuminate\Validation\Rule;
+use Rebing\GraphQL\Support\Facades\GraphQL;
+
+class RefreshMetadataMutation extends Mutation implements PlatformGraphQlMutation
+{
+    use HasEncodableTokenId;
+    use HasTokenIdFieldRules;
+    use HasTokenIdFields;
+    use InPrimarySubstrateSchema;
+
+    /**
+     * Get the mutation's attributes.
+     */
+    public function attributes(): array
+    {
+        return [
+            'name' => 'RefreshMetadata',
+            'description' => __('enjin-platform::mutation.refresh_metadata.description'),
+        ];
+    }
+
+    /**
+     * Get the mutation's return type.
+     */
+    public function type(): Type
+    {
+        return GraphQL::type('Boolean!');
+    }
+
+    /**
+     * Get the mutation's arguments definition.
+     */
+    public function args(): array
+    {
+        return [
+            'collectionId' => [
+                'type' => GraphQL::type('BigInt!'),
+                'description' => __('enjin-platform::mutation.approve_collection.args.collectionId'),
+            ],
+            ...$this->getTokenFields(__('enjin-platform::args.common.tokenId'), true),
+        ];
+    }
+
+    /**
+     * Resolve the mutation's request.
+     */
+    public function resolve(
+        $root,
+        array $args,
+        $context,
+        ResolveInfo $resolveInfo,
+        Closure $getSelectFields,
+        SerializationServiceInterface $serializationService,
+        MetadataService $metadataService,
+    ): mixed {
+        Attribute::query()
+            ->select('key', 'value', 'token_id', 'collection_id')
+            ->with([
+                'token:id,collection_id,token_chain_id',
+                'collection:id,collection_chain_id',
+            ])->whereHas(
+                'collection',
+                fn ($query) => $query->where('collection_chain_id', Arr::get($args, 'collectionId'))
+            )->when(
+                $tokenId = $this->encodeTokenId($args),
+                fn ($query) => $query->whereHas(
+                    'token',
+                    fn ($query) => $query->where('token_chain_id', $tokenId)
+                )
+            )
+            ->get()
+            ->each(fn ($attribute) => $metadataService->fetchAndCache($attribute->value_string));
+
+        return true;
+    }
+
+    /**
+     * Get the mutation's validation rules.
+     */
+    protected function rulesWithValidation(array $args): array
+    {
+        return [
+            'collectionId' => [
+                new MinBigInt(0),
+                new MaxBigInt(Hex::MAX_UINT128),
+                Rule::exists('collections', 'collection_chain_id'),
+            ],
+            ...$this->getOptionalTokenFieldRulesExist(),
+        ];
+    }
+}

src/Jobs/SyncMetadata.php

@@ -0,0 +1,40 @@
+<?php
+
+namespace Enjin\Platform\Jobs;
+
+use Enjin\Platform\Services\Database\MetadataService;
+use Enjin\Platform\Support\Hex;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Log;
+use Throwable;
+
+class SyncMetadata implements ShouldQueue
+{
+    use Dispatchable;
+    use InteractsWithQueue;
+    use Queueable;
+    use SerializesModels;
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct(protected string $url) {}
+
+    /**
+     * Execute the job.
+     */
+    public function handle(MetadataService $service): void
+    {
+        try {
+            $service->fetchAndCache(
+                Hex::isHexEncoded($this->url) ? Hex::safeConvertToString($this->url) : $this->url
+            );
+        } catch (Throwable $e) {
+            Log::error("Unable to sync metadata for url {$this->url}", $e->getMessage());
+        }
+    }
+}

src/Models/Laravel/Attribute.php

@@ -58,8 +58,16 @@ protected function valueString(): AttributeCasts
                 $key = Hex::safeConvertToString($attributes['key']);
                 $value = Hex::safeConvertToString($attributes['value']);
 
-                if ($key == 'uri' && str_contains($value, '{id}') && $this->token_id) {
-                    return Str::replace('{id}', "{$this->token->collection->collection_chain_id}-{$this->token->token_chain_id}", $value);
+                if ($key == 'uri' && str_contains($value, '{id}')) {
+                    if (!$this->relationLoaded('collection')) {
+                        $this->load('collection:id,collection_chain_id');
+                    }
+
+                    if (!$this->relationLoaded('token')) {
+                        $this->load('token:id,token_chain_id');
+                    }
+
+                    return Str::replace('{id}', "{$this->collection->collection_chain_id}-{$this->token->token_chain_id}", $value);
                 }
 
                 return $value;

src/Models/Laravel/Token.php

@@ -9,11 +9,9 @@
 use Enjin\Platform\Models\BaseModel;
 use Enjin\Platform\Models\Laravel\Traits\EagerLoadSelectFields;
 use Enjin\Platform\Models\Laravel\Traits\Token as TokenMethods;
-use Enjin\Platform\Support\Hex;
 use Facades\Enjin\Platform\Services\Database\MetadataService;
 use Illuminate\Database\Eloquent\Casts\Attribute;
 use Illuminate\Database\Eloquent\Factories\HasFactory;
-use Illuminate\Support\Str;
 
 class Token extends BaseModel
 {
@@ -136,7 +134,7 @@ protected function fetchMetadata(): Attribute
             get: fn () => $this->attributes['fetch_metadata'] ?? false,
             set: function ($value): void {
                 if ($value === true) {
-                    $this->attributes['metadata'] = MetadataService::fetch($this->getRelation('attributes')->first());
+                    $this->attributes['metadata'] = MetadataService::getCache($this->getRelation('attributes')->first());
                 }
                 $this->attributes['fetch_metadata'] = $value;
             }
@@ -149,28 +147,9 @@ protected function fetchMetadata(): Attribute
     protected function metadata(): Attribute
     {
         return new Attribute(
-            get: function () {
-                $tokenUriAttribute = $this->fetchUriAttribute($this);
-                if ($tokenUriAttribute) {
-                    $tokenUriAttribute->value = Hex::safeConvertToString($tokenUriAttribute->value);
-                }
-                $fetchedMetadata = $this->attributes['metadata'] ?? MetadataService::fetch($tokenUriAttribute);
-
-                if (!$fetchedMetadata) {
-                    $collectionUriAttribute = $this->fetchUriAttribute($this->collection);
-                    if ($collectionUriAttribute) {
-                        $collectionUriAttribute->value = Hex::safeConvertToString($collectionUriAttribute->value);
-                    }
-
-                    if ($collectionUriAttribute?->value && Str::contains($collectionUriAttribute->value, '{id}')) {
-                        $collectionUriAttribute->value = Str::replace('{id}', "{$this->collection->collection_chain_id}-{$this->token_chain_id}", $collectionUriAttribute->value);
-                    }
-
-                    $fetchedMetadata = MetadataService::fetch($collectionUriAttribute);
-                }
-
-                return $fetchedMetadata;
-            },
+            get: fn () => $this->attributes['metadata']
+                ?? MetadataService::getCache($this->fetchUriAttribute($this)?->value_string ?? '')
+                ?? MetadataService::getCache($this->fetchUriAttribute($this->collection)->value_string ?? ''),
         );
     }
 

src/Services/Database/MetadataService.php

@@ -4,9 +4,13 @@
 
 use Enjin\Platform\Clients\Implementations\MetadataHttpClient;
 use Enjin\Platform\Models\Laravel\Attribute;
+use Enjin\Platform\Support\Hex;
+use Illuminate\Support\Facades\Cache;
 
 class MetadataService
 {
+    public static $cacheKey = 'platform:attributeMetadata';
+
     /**
      * Create a new instance.
      */
@@ -25,4 +29,54 @@ public function fetch(?Attribute $attribute): mixed
 
         return $response ?: null;
     }
+
+    public function fetchUrl(string $url): mixed
+    {
+        $url = $this->convertHexToString($url);
+
+        if (!filter_var($url, FILTER_VALIDATE_URL)) {
+            return null;
+        }
+
+        return $this->client->fetch($url) ?: null;
+    }
+
+    public function fetchAndCache(string $url, bool $forget = true): mixed
+    {
+        $url = $this->convertHexToString($url);
+
+        if (!filter_var($url, FILTER_VALIDATE_URL)) {
+            return null;
+        }
+
+        if ($forget) {
+            Cache::forget($this->cacheKey($url));
+        }
+
+        return Cache::rememberForever(
+            $this->cacheKey($url),
+            fn () => $this->fetchUrl($url)
+        );
+    }
+
+    public function getCache(string $url): mixed
+    {
+        $url = $this->convertHexToString($url);
+
+        if (!filter_var($url, FILTER_VALIDATE_URL)) {
+            return null;
+        }
+
+        return Cache::get($this->cacheKey($url), $this->fetchAndCache($url, false));
+    }
+
+    protected function convertHexToString(string $url): string
+    {
+        return Hex::isHexEncoded($url) ? Hex::safeConvertToString($url) : $url;
+    }
+
+    protected function cacheKey(string $suffix): string
+    {
+        return self::$cacheKey . ':' . $suffix;
+    }
 }