[PLA-2023] Improve metadata caching

config/enjin-platform.php

@@ -291,4 +291,18 @@
         'attempts' => env('RATE_LIMIT_ATTEMPTS', 500),
         'time' => env('RATE_LIMIT_TIME', 1), // minutes
     ],
+
+    /*
+    |--------------------------------------------------------------------------
+    | Attribute metadata syncing
+    |--------------------------------------------------------------------------
+    |
+    | Here you may configure how the attribute metadata is synced
+    |
+    */
+    'sync_metadata' => [
+        'data_chunk_size' => env('SYNC_METADATA_CHUNK_SIZE', 10000),
+        'refresh_max_attempts' => env('REFRESH_METADATA_MAX_ATTEMPTS', 10),
+        'refresh_decay_seconds' => env('REFRESH_METADATA_DECAY_SECONDS', 60),
+    ],
 ];

lang/en/error.php

@@ -59,4 +59,5 @@
     'verification.unable_to_generate_verification_id' => 'Unable to generate a verification id.',
     'verification.verification_not_found' => 'Verification not found.',
     'wallet_is_immutable' => 'The wallet account is immutable once set.',
+    'too_many_requests' =>  'Too many requests. Retry in :num seconds',
 ];

lang/en/mutation.php

@@ -119,4 +119,5 @@
     'update_wallet_external_id.cannot_update_id_on_managed_wallet' => 'Cannot update the external id on a managed wallet.',
     'verify_account.description' => 'The wallet calls this mutation to prove the ownership of the user account.',
     'operator_transfer_token.args.keepAlive' => '(DEPRECATED) If true, the transaction will fail if the balance drops below the minimum requirement. Defaults to False.',
+    'refresh_metadata.description' => "Refresh the collection's or token's metadata.",
 ];

src/Commands/SyncMetadata.php

@@ -0,0 +1,52 @@
+<?php
+
+namespace Enjin\Platform\Commands;
+
+use Enjin\Platform\Jobs\SyncMetadata as SyncMetadataJob;
+use Enjin\Platform\Models\Attribute;
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\Log;
+
+class SyncMetadata extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'platform:sync-metadata';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Sync attributes metadata to cache.';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle(): void
+    {
+        $query = Attribute::query()
+            ->select('id')
+            ->where('key', '0x757269'); // uri hex
+        if (($total = $query->count()) == 0) {
+            $this->info('No attributes found to sync.');
+
+            return;
+        }
+
+        $progress = $this->output->createProgressBar($total);
+        $progress->start();
+        Log::debug('Syncing metadata for ' . $total . ' attributes.');
+
+        foreach ($query->lazy(config('enjin-platform.sync_metadata.data_chunk_size')) as $attribute) {
+            SyncMetadataJob::dispatch($attribute->id);
+            $progress->advance();
+        }
+
+        $progress->finish();
+        Log::debug('Finished syncing metadata.');
+    }
+}

src/CoreServiceProvider.php

@@ -6,6 +6,7 @@
 use Enjin\Platform\Commands\Ingest;
 use Enjin\Platform\Commands\RelayWatcher;
 use Enjin\Platform\Commands\Sync;
+use Enjin\Platform\Commands\SyncMetadata;
 use Enjin\Platform\Commands\TransactionChecker;
 use Enjin\Platform\Commands\Transactions;
 use Enjin\Platform\Enums\Global\PlatformCache;
@@ -80,6 +81,7 @@ public function configurePackage(Package $package): void
             ->hasCommand(ClearCache::class)
             ->hasCommand(TransactionChecker::class)
             ->hasCommand(RelayWatcher::class)
+            ->hasCommand(SyncMetadata::class)
             ->hasTranslations();
     }
 

src/Enums/Global/PlatformCache.php

@@ -24,10 +24,9 @@ enum PlatformCache: string implements PlatformCacheable
     case FEE = 'fee';
     case DEPOSIT = 'deposit';
     case RELEASE_DIFF = 'releaseDiff';
-
     case BLOCK_EVENT_COUNT = 'blockEventCount';
-
     case BLOCK_TRANSACTION = 'blockTransaction';
+    case REFRESH_METADATA = 'refreshMetadata';
 
     public function key(?string $suffix = null, ?string $network = null): string
     {

src/Events/Substrate/MultiTokens/MetadataUpdated.php

@@ -0,0 +1,29 @@
+<?php
+
+namespace Enjin\Platform\Events\Substrate\MultiTokens;
+
+use Enjin\Platform\Channels\PlatformAppChannel;
+use Enjin\Platform\Events\PlatformBroadcastEvent;
+use Illuminate\Broadcasting\Channel;
+
+class MetadataUpdated extends PlatformBroadcastEvent
+{
+    /**
+     * Create a new event instance.
+     */
+    public function __construct(string $collectionId, ?string $tokenId = null)
+    {
+        parent::__construct();
+
+        $this->broadcastData = [
+            'collectionId' => $collectionId,
+            'tokenId'      => $tokenId,
+        ];
+
+        $this->broadcastChannels = [
+            new Channel("collection;{$collectionId}"),
+            new Channel("token;{$collectionId}-{$tokenId}"),
+            new PlatformAppChannel(),
+        ];
+    }
+}

src/GraphQL/Schemas/Primary/Substrate/Mutations/RefreshMetadataMutation.php

@@ -0,0 +1,125 @@
+<?php
+
+namespace Enjin\Platform\GraphQL\Schemas\Primary\Substrate\Mutations;
+
+use Closure;
+use Enjin\Platform\Enums\Global\PlatformCache;
+use Enjin\Platform\Exceptions\PlatformException;
+use Enjin\Platform\GraphQL\Base\Mutation;
+use Enjin\Platform\GraphQL\Schemas\Primary\Substrate\Traits\HasEncodableTokenId;
+use Enjin\Platform\GraphQL\Schemas\Primary\Substrate\Traits\InPrimarySubstrateSchema;
+use Enjin\Platform\GraphQL\Schemas\Primary\Traits\HasTokenIdFieldRules;
+use Enjin\Platform\GraphQL\Types\Input\Substrate\Traits\HasTokenIdFields;
+use Enjin\Platform\Interfaces\PlatformGraphQlMutation;
+use Enjin\Platform\Interfaces\PlatformPublicGraphQlOperation;
+use Enjin\Platform\Models\Attribute;
+use Enjin\Platform\Rules\MaxBigInt;
+use Enjin\Platform\Rules\MinBigInt;
+use Enjin\Platform\Services\Database\MetadataService;
+use Enjin\Platform\Services\Serialization\Interfaces\SerializationServiceInterface;
+use Enjin\Platform\Support\Hex;
+use GraphQL\Type\Definition\ResolveInfo;
+use GraphQL\Type\Definition\Type;
+use Illuminate\Support\Arr;
+use Illuminate\Support\Facades\RateLimiter;
+use Illuminate\Validation\Rule;
+use Rebing\GraphQL\Support\Facades\GraphQL;
+
+class RefreshMetadataMutation extends Mutation implements PlatformGraphQlMutation, PlatformPublicGraphQlOperation
+{
+    use HasEncodableTokenId;
+    use HasTokenIdFieldRules;
+    use HasTokenIdFields;
+    use InPrimarySubstrateSchema;
+
+    /**
+     * Get the mutation's attributes.
+     */
+    public function attributes(): array
+    {
+        return [
+            'name' => 'RefreshMetadata',
+            'description' => __('enjin-platform::mutation.refresh_metadata.description'),
+        ];
+    }
+
+    /**
+     * Get the mutation's return type.
+     */
+    public function type(): Type
+    {
+        return GraphQL::type('Boolean!');
+    }
+
+    /**
+     * Get the mutation's arguments definition.
+     */
+    public function args(): array
+    {
+        return [
+            'collectionId' => [
+                'type' => GraphQL::type('BigInt!'),
+                'description' => __('enjin-platform::mutation.approve_collection.args.collectionId'),
+            ],
+            ...$this->getTokenFields(__('enjin-platform::args.common.tokenId'), true),
+        ];
+    }
+
+    /**
+     * Resolve the mutation's request.
+     */
+    public function resolve(
+        $root,
+        array $args,
+        $context,
+        ResolveInfo $resolveInfo,
+        Closure $getSelectFields,
+        SerializationServiceInterface $serializationService,
+        MetadataService $metadataService,
+    ): mixed {
+        $key = PlatformCache::REFRESH_METADATA->key(
+            ($collectionId = Arr::get($args, 'collectionId')) . ':' . ($tokenId = $this->encodeTokenId($args))
+        );
+        if (RateLimiter::tooManyAttempts($key, config('enjin-platform.sync_metadata.refresh_max_attempts'))) {
+            throw new PlatformException(
+                __('enjin-platform::error.too_many_requests', ['num' => RateLimiter::availableIn($key)])
+            );
+        }
+        RateLimiter::hit($key, config('enjin-platform.sync_metadata.refresh_decay_seconds'));
+
+        Attribute::query()
+            ->select('key', 'value', 'token_id', 'collection_id')
+            ->with([
+                'token:id,collection_id,token_chain_id',
+                'collection:id,collection_chain_id',
+            ])->whereHas(
+                'collection',
+                fn ($query) => $query->where('collection_chain_id', $collectionId)
+            )->when(
+                $tokenId,
+                fn ($query) => $query->whereHas(
+                    'token',
+                    fn ($query) => $query->where('token_chain_id', $tokenId)
+                )
+            )
+            ->get()
+            ->each(fn ($attribute) => $metadataService->fetchAttributeWithEvent($attribute));
+
+        return true;
+    }
+
+    /**
+     * Get the mutation's validation rules.
+     */
+    protected function rulesWithValidation(array $args): array
+    {
+        return [
+            'collectionId' => [
+                new MinBigInt(0),
+                new MaxBigInt(Hex::MAX_UINT128),
+                Rule::exists('collections', 'collection_chain_id'),
+            ],
+            ...$this->getOptionalTokenFieldRulesExist(),
+        ];
+    }
+}

src/Jobs/SyncMetadata.php

@@ -0,0 +1,43 @@
+<?php
+
+namespace Enjin\Platform\Jobs;
+
+use Enjin\Platform\Models\Laravel\Attribute;
+use Enjin\Platform\Services\Database\MetadataService;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Log;
+use Throwable;
+
+class SyncMetadata implements ShouldQueue
+{
+    use Dispatchable;
+    use InteractsWithQueue;
+    use Queueable;
+    use SerializesModels;
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct(protected int $attributeId) {}
+
+    /**
+     * Execute the job.
+     */
+    public function handle(MetadataService $service): void
+    {
+        try {
+            $service->fetchAttributeWithEvent(
+                Attribute::with([
+                    'token:id,token_chain_id',
+                    'collection:id,collection_chain_id',
+                ])->find($this->attributeId)
+            );
+        } catch (Throwable $e) {
+            Log::error("Unable to sync metadata for attribute ID {$this->attributeId}", $e->getMessage());
+        }
+    }
+}

src/Models/Laravel/Attribute.php

@@ -58,8 +58,16 @@ protected function valueString(): AttributeCasts
                 $key = Hex::safeConvertToString($attributes['key']);
                 $value = Hex::safeConvertToString($attributes['value']);
 
-                if ($key == 'uri' && str_contains($value, '{id}') && $this->token_id) {
-                    return Str::replace('{id}', "{$this->token->collection->collection_chain_id}-{$this->token->token_chain_id}", $value);
+                if ($key == 'uri' && str_contains($value, '{id}')) {
+                    if (!$this->relationLoaded('collection')) {
+                        $this->load('collection:id,collection_chain_id');
+                    }
+
+                    if (!$this->relationLoaded('token')) {
+                        $this->load('token:id,token_chain_id');
+                    }
+
+                    return Str::replace('{id}', "{$this->collection->collection_chain_id}-{$this->token->token_chain_id}", $value);
                 }
 
                 return $value;

src/Models/Laravel/Token.php

@@ -9,11 +9,9 @@
 use Enjin\Platform\Models\BaseModel;
 use Enjin\Platform\Models\Laravel\Traits\EagerLoadSelectFields;
 use Enjin\Platform\Models\Laravel\Traits\Token as TokenMethods;
-use Enjin\Platform\Support\Hex;
 use Facades\Enjin\Platform\Services\Database\MetadataService;
 use Illuminate\Database\Eloquent\Casts\Attribute;
 use Illuminate\Database\Eloquent\Factories\HasFactory;
-use Illuminate\Support\Str;
 
 class Token extends BaseModel
 {
@@ -136,7 +134,7 @@ protected function fetchMetadata(): Attribute
             get: fn () => $this->attributes['fetch_metadata'] ?? false,
             set: function ($value): void {
                 if ($value === true) {
-                    $this->attributes['metadata'] = MetadataService::fetch($this->getRelation('attributes')->first());
+                    $this->attributes['metadata'] = MetadataService::getCache($this->getRelation('attributes')->first());
                 }
                 $this->attributes['fetch_metadata'] = $value;
             }
@@ -149,28 +147,9 @@ protected function fetchMetadata(): Attribute
     protected function metadata(): Attribute
     {
         return new Attribute(
-            get: function () {
-                $tokenUriAttribute = $this->fetchUriAttribute($this);
-                if ($tokenUriAttribute) {
-                    $tokenUriAttribute->value = Hex::safeConvertToString($tokenUriAttribute->value);
-                }
-                $fetchedMetadata = $this->attributes['metadata'] ?? MetadataService::fetch($tokenUriAttribute);
-
-                if (!$fetchedMetadata) {
-                    $collectionUriAttribute = $this->fetchUriAttribute($this->collection);
-                    if ($collectionUriAttribute) {
-                        $collectionUriAttribute->value = Hex::safeConvertToString($collectionUriAttribute->value);
-                    }
-
-                    if ($collectionUriAttribute?->value && Str::contains($collectionUriAttribute->value, '{id}')) {
-                        $collectionUriAttribute->value = Str::replace('{id}', "{$this->collection->collection_chain_id}-{$this->token_chain_id}", $collectionUriAttribute->value);
-                    }
-
-                    $fetchedMetadata = MetadataService::fetch($collectionUriAttribute);
-                }
-
-                return $fetchedMetadata;
-            },
+            get: fn () => $this->attributes['metadata']
+                ?? MetadataService::getCache($this->fetchUriAttribute($this)?->value_string ?? '')
+                ?? MetadataService::getCache($this->fetchUriAttribute($this->collection)->value_string ?? ''),
         );
     }
 
@@ -184,6 +163,10 @@ protected static function newFactory(): TokenFactory
 
     protected function pivotIdentifier(): Attribute
     {
+        if (!$this->relationLoaded('collection')) {
+            $this->load('collection:id,collection_chain_id');
+        }
+
         if (!$collection = $this->collection) {
             throw new PlatformException(__('enjin-platform::error.no_collection', ['tokenId' => $this->token_chain_id]));
         }