Update dependencies, fix lints and set updated packages (#107)

## Details - [x] Retain current version of polkadotjs, multiformats - [x] Remove unused dependencies - [x] Update linter and fix all the linter related errors, most of them were `namespace` errors - [x] Update remaning depdencies to new version ## Tests - [x] Run e2e tests to ensure everything is working --------- Co-authored-by: Wil Wade <[email protected]>
ProjectLibertyLabs · May 9, 2024 · e07f423 · e07f423
1 parent b0fea94
commit e07f423
Show file tree

Hide file tree

Showing 46 changed files with 7,779 additions and 7,518 deletions.
diff --git a/apps/api/src/api.controller.ts b/apps/api/src/api.controller.ts
@@ -54,7 +54,7 @@ export class ApiController {
         }),
     )
     files: // eslint-disable-next-line no-undef
-    Array<Express.Multer.File>,
+    Express.Multer.File[],
   ): Promise<UploadResponseDto> {
     return this.apiService.addAssets(files);
   }

diff --git a/apps/api/src/api.module.ts b/apps/api/src/api.module.ts
@@ -2,14 +2,14 @@ import { Module } from '@nestjs/common';
 import { EventEmitterModule } from '@nestjs/event-emitter';
 import { BullModule } from '@nestjs/bullmq';
 import { ScheduleModule } from '@nestjs/schedule';
-import { RedisModule } from '@liaoliaots/nestjs-redis';
+import { RedisModule } from '@songkeys/nestjs-redis';
 import { BullBoardModule } from '@bull-board/nestjs';
 import { BullMQAdapter } from '@bull-board/api/bullMQAdapter';
 import { ExpressAdapter } from '@bull-board/express';
 import { MulterModule } from '@nestjs/platform-express';
 import { ApiController } from './api.controller';
 import { DevelopmentController } from './development.controller';
-import { QueueConstants } from '../../../libs/common/src';
+import * as QueueConstants from '../../../libs/common/src';
 import { ApiService } from './api.service';
 import { IpfsService } from '../../../libs/common/src/utils/ipfs.client';
 import { ConfigModule } from '../../../libs/common/src/config/config.module';

diff --git a/apps/api/src/api.service.ts b/apps/api/src/api.service.ts
@@ -3,33 +3,33 @@ import { Injectable, Logger } from '@nestjs/common';
 import { Queue } from 'bullmq';
 import { createHash } from 'crypto';
 import { BulkJobOptions } from 'bullmq/dist/esm/interfaces';
-import { InjectRedis } from '@liaoliaots/nestjs-redis';
+import { InjectRedis } from '@songkeys/nestjs-redis';
 import Redis from 'ioredis';
 import { HttpErrorByCode } from '@nestjs/common/utils/http-error-by-code.util';
 import {
   AnnouncementResponseDto,
   AnnouncementTypeDto,
+  ASSET_QUEUE_NAME,
   AssetIncludedRequestDto,
   IRequestJob,
   isImage,
-  QueueConstants,
+  REQUEST_QUEUE_NAME,
   RequestTypeDto,
   UploadResponseDto,
 } from '../../../libs/common/src';
 import { calculateIpfsCID } from '../../../libs/common/src/utils/ipfs';
 import { IAssetJob, IAssetMetadata } from '../../../libs/common/src/interfaces/asset-job.interface';
-import { RedisUtils } from '../../../libs/common/src/utils/redis';
-import getAssetDataKey = RedisUtils.getAssetDataKey;
-import getAssetMetadataKey = RedisUtils.getAssetMetadataKey;
+import { getAssetDataKey, getAssetMetadataKey , STORAGE_EXPIRE_UPPER_LIMIT_SECONDS, } from '../../../libs/common/src/utils/redis';
+
 
 @Injectable()
 export class ApiService {
   private readonly logger: Logger;
 
   constructor(
     @InjectRedis() private redis: Redis,
-    @InjectQueue(QueueConstants.REQUEST_QUEUE_NAME) private requestQueue: Queue,
-    @InjectQueue(QueueConstants.ASSET_QUEUE_NAME) private assetQueue: Queue,
+    @InjectQueue(REQUEST_QUEUE_NAME) private requestQueue: Queue,
+    @InjectQueue(ASSET_QUEUE_NAME) private assetQueue: Queue,
   ) {
     this.logger = new Logger(this.constructor.name);
   }
@@ -60,7 +60,7 @@ export class ApiService {
   }
 
   async validateAssetsAndFetchMetadata(content: AssetIncludedRequestDto): Promise<Map<string, string> | undefined> {
-    const checkingList: Array<{ onlyImage: boolean; referenceId: string }> = [];
+    const checkingList: { onlyImage: boolean; referenceId: string }[] = [];
     if (content.profile) {
       content.profile.icon?.forEach((reference) => checkingList.push({ onlyImage: true, referenceId: reference.referenceId }));
     } else if (content.content) {
@@ -98,7 +98,7 @@ export class ApiService {
   }
 
   // eslint-disable-next-line no-undef
-  async addAssets(files: Array<Express.Multer.File>): Promise<UploadResponseDto> {
+  async addAssets(files: Express.Multer.File[]): Promise<UploadResponseDto> {
     // calculate ipfs cid references
     const referencePromises: Promise<string>[] = files.map((file) => calculateIpfsCID(file.buffer));
     const references = await Promise.all(referencePromises);
@@ -108,10 +108,10 @@ export class ApiService {
     const jobs: any[] = [];
     files.forEach((f, index) => {
       // adding data and metadata to the transaction
-      dataTransaction = dataTransaction.setex(getAssetDataKey(references[index]), RedisUtils.STORAGE_EXPIRE_UPPER_LIMIT_SECONDS, f.buffer);
+      dataTransaction = dataTransaction.setex(getAssetDataKey(references[index]), STORAGE_EXPIRE_UPPER_LIMIT_SECONDS, f.buffer);
       metadataTransaction = metadataTransaction.setex(
         getAssetMetadataKey(references[index]),
-        RedisUtils.STORAGE_EXPIRE_UPPER_LIMIT_SECONDS,
+        STORAGE_EXPIRE_UPPER_LIMIT_SECONDS,
         JSON.stringify({
           ipfsCid: references[index],
           mimeType: f.mimetype,

diff --git a/apps/api/src/development.controller.ts b/apps/api/src/development.controller.ts
@@ -7,7 +7,7 @@ import { Controller, Get, Logger, NotFoundException, Param, Post } from '@nestjs
 import { InjectQueue } from '@nestjs/bullmq';
 import { Queue } from 'bullmq';
 import { Job } from 'bullmq/dist/esm/classes/job';
-import { AnnouncementTypeDto, QueueConstants } from '../../../libs/common/src';
+import * as QueueConstants from '../../../libs/common/src';
 import { IpfsService } from '../../../libs/common/src/utils/ipfs.client';
 import { AnnouncementType, createBroadcast, createProfile, createReaction, createReply, createTombstone, createUpdate } from '../../../libs/common/src/interfaces/dsnp';
 import { calculateDsnpHash } from '../../../libs/common/src/utils/ipfs';
@@ -16,7 +16,7 @@ import { calculateDsnpHash } from '../../../libs/common/src/utils/ipfs';
 export class DevelopmentController {
   private readonly logger: Logger;
 
-  private readonly queueMapper: Map<AnnouncementTypeDto, Queue>;
+  private readonly queueMapper: Map<QueueConstants.AnnouncementTypeDto, Queue>;
 
   constructor(
     @InjectQueue(QueueConstants.REQUEST_QUEUE_NAME) private requestQueue: Queue,
@@ -30,12 +30,12 @@ export class DevelopmentController {
   ) {
     this.logger = new Logger(this.constructor.name);
     this.queueMapper = new Map([
-      [AnnouncementTypeDto.BROADCAST, broadcastQueue],
-      [AnnouncementTypeDto.REPLY, replyQueue],
-      [AnnouncementTypeDto.REACTION, reactionQueue],
-      [AnnouncementTypeDto.UPDATE, updateQueue],
-      [AnnouncementTypeDto.PROFILE, profileQueue],
-      [AnnouncementTypeDto.TOMBSTONE, tombstoneQueue],
+      [QueueConstants.AnnouncementTypeDto.BROADCAST, broadcastQueue],
+      [QueueConstants.AnnouncementTypeDto.REPLY, replyQueue],
+      [QueueConstants.AnnouncementTypeDto.REACTION, reactionQueue],
+      [QueueConstants.AnnouncementTypeDto.UPDATE, updateQueue],
+      [QueueConstants.AnnouncementTypeDto.PROFILE, profileQueue],
+      [QueueConstants.AnnouncementTypeDto.TOMBSTONE, tombstoneQueue],
     ]);
   }
 
@@ -57,7 +57,7 @@ export class DevelopmentController {
   }
 
   @Post('/dummy/announcement/:queueType/:count')
-  async populate(@Param('queueType') queueType: AnnouncementTypeDto, @Param('count') count: number) {
+  async populate(@Param('queueType') queueType: QueueConstants.AnnouncementTypeDto, @Param('count') count: number) {
     const promises: Promise<Job>[] = [];
     // eslint-disable-next-line no-plusplus
     for (let i = 0; i < count; i++) {
@@ -66,27 +66,27 @@ export class DevelopmentController {
       const fromId = `${Math.floor(Math.random() * 100000000)}`;
       const hash = `${Math.floor(Math.random() * 100000000)}`;
       switch (queueType) {
-        case AnnouncementTypeDto.BROADCAST:
+        case QueueConstants.AnnouncementTypeDto.BROADCAST:
           data = createBroadcast(fromId, `https://example.com/${Math.floor(Math.random() * 100000000)}`, hash);
           break;
-        case AnnouncementTypeDto.PROFILE:
+        case QueueConstants.AnnouncementTypeDto.PROFILE:
           data = createProfile(fromId, `https://example.com/${Math.floor(Math.random() * 100000000)}`, hash);
           break;
-        case AnnouncementTypeDto.UPDATE:
+        case QueueConstants.AnnouncementTypeDto.UPDATE:
           data = createUpdate(fromId, `https://example.com/${Math.floor(Math.random() * 100000000)}`, hash, AnnouncementType.Broadcast, `${Math.floor(Math.random() * 100000000)}`);
           break;
-        case AnnouncementTypeDto.REPLY:
+        case QueueConstants.AnnouncementTypeDto.REPLY:
           data = createReply(
             fromId,
             `https://example.com/${Math.floor(Math.random() * 100000000)}`,
             hash,
             `dsnp://0x${Math.floor(Math.random() * 100000000)}/0x${Math.floor(Math.random() * 100000000)}`,
           );
           break;
-        case AnnouncementTypeDto.REACTION:
+        case QueueConstants.AnnouncementTypeDto.REACTION:
           data = createReaction(fromId, '🤌🏼', `dsnp://0x${Math.floor(Math.random() * 100000000)}/0x${Math.floor(Math.random() * 100000000)}`, 1);
           break;
-        case AnnouncementTypeDto.TOMBSTONE:
+        case QueueConstants.AnnouncementTypeDto.TOMBSTONE:
           data = createTombstone(fromId, AnnouncementType.Reply, hash);
           break;
         default:

diff --git a/apps/api/test/app.e2e-spec.ts b/apps/api/test/app.e2e-spec.ts
@@ -1,4 +1,3 @@
-/* eslint-disable import/no-extraneous-dependencies */
 /* eslint-disable no-undef */
 import { INestApplication, ValidationPipe } from '@nestjs/common';
 import { Test, TestingModule } from '@nestjs/testing';

diff --git a/apps/worker/src/BaseConsumer.ts b/apps/worker/src/BaseConsumer.ts
@@ -1,7 +1,7 @@
 import { OnWorkerEvent, WorkerHost } from '@nestjs/bullmq';
 import { Logger, OnModuleDestroy } from '@nestjs/common';
-import { Job, Worker } from 'bullmq';
-import { ProcessingUtils } from '../../../libs/common/src/utils/processing';
+import { Job, Worker, delay } from 'bullmq';
+import { DELAY_TO_CHECK_FOR_SHUTDOWN_MS, MAX_WAIT_FOR_GRACE_FULL_SHUTDOWN_MS } from '../../../libs/common/src/utils/processing';
 
 export abstract class BaseConsumer<T extends Worker = Worker> extends WorkerHost<T> implements OnModuleDestroy {
   protected logger: Logger;
@@ -24,11 +24,11 @@ export abstract class BaseConsumer<T extends Worker = Worker> extends WorkerHost
 
   async onModuleDestroy(): Promise<any> {
     await this.worker?.close(false);
-    let maxWaitMs = ProcessingUtils.MAX_WAIT_FOR_GRACE_FULL_SHUTDOWN_MS;
+    let maxWaitMs = MAX_WAIT_FOR_GRACE_FULL_SHUTDOWN_MS;
     while (this.actives.size > 0 && maxWaitMs > 0) {
       // eslint-disable-next-line no-await-in-loop
-      await ProcessingUtils.delay(ProcessingUtils.DELAY_TO_CHECK_FOR_SHUTDOWN_MS);
-      maxWaitMs -= ProcessingUtils.DELAY_TO_CHECK_FOR_SHUTDOWN_MS;
+      await delay(DELAY_TO_CHECK_FOR_SHUTDOWN_MS);
+      maxWaitMs -= DELAY_TO_CHECK_FOR_SHUTDOWN_MS;
     }
   }
 

diff --git a/apps/worker/src/asset_processor/asset.processor.module.ts b/apps/worker/src/asset_processor/asset.processor.module.ts
@@ -4,8 +4,8 @@ https://docs.nestjs.com/modules
 
 import { BullModule } from '@nestjs/bullmq';
 import { Module } from '@nestjs/common';
-import { RedisModule } from '@liaoliaots/nestjs-redis';
-import { QueueConstants } from '../../../../libs/common/src';
+import { RedisModule } from '@songkeys/nestjs-redis';
+import { ASSET_QUEUE_NAME } from '../../../../libs/common/src';
 import { AssetProcessorService } from './asset.processor.service';
 import { IpfsService } from '../../../../libs/common/src/utils/ipfs.client';
 import { ConfigModule } from '../../../../libs/common/src/config/config.module';
@@ -48,7 +48,7 @@ import { ConfigService } from '../../../../libs/common/src/config/config.service
       inject: [ConfigService],
     }),
     BullModule.registerQueue({
-      name: QueueConstants.ASSET_QUEUE_NAME,
+      name: ASSET_QUEUE_NAME,
     }),
   ],
   providers: [AssetProcessorService, IpfsService],

diff --git a/apps/worker/src/asset_processor/asset.processor.service.ts b/apps/worker/src/asset_processor/asset.processor.service.ts
@@ -1,16 +1,16 @@
-import { InjectRedis } from '@liaoliaots/nestjs-redis';
+import { InjectRedis } from '@songkeys/nestjs-redis';
 import { Processor, WorkerHost, OnWorkerEvent } from '@nestjs/bullmq';
 import { Injectable, Logger } from '@nestjs/common';
 import { Job } from 'bullmq';
 import Redis from 'ioredis';
 import { ConfigService } from '../../../../libs/common/src/config/config.service';
-import { QueueConstants } from '../../../../libs/common/src';
+import { ASSET_QUEUE_NAME } from '../../../../libs/common/src';
 import { IAssetJob } from '../../../../libs/common/src/interfaces/asset-job.interface';
 import { IpfsService } from '../../../../libs/common/src/utils/ipfs.client';
 import { BaseConsumer } from '../BaseConsumer';
 
 @Injectable()
-@Processor(QueueConstants.ASSET_QUEUE_NAME)
+@Processor(ASSET_QUEUE_NAME)
 export class AssetProcessorService extends BaseConsumer {
   constructor(
     @InjectRedis() private redis: Redis,

diff --git a/apps/worker/src/batch_announcer/batch.announcer.module.ts b/apps/worker/src/batch_announcer/batch.announcer.module.ts
@@ -5,12 +5,12 @@ https://docs.nestjs.com/modules
 import { BullModule } from '@nestjs/bullmq';
 import { Module } from '@nestjs/common';
 import { EventEmitterModule } from '@nestjs/event-emitter';
-import { RedisModule } from '@liaoliaots/nestjs-redis';
+import { RedisModule } from '@songkeys/nestjs-redis';
 import { BatchAnnouncementService } from './batch.announcer.service';
 import { ConfigModule } from '../../../../libs/common/src/config/config.module';
 import { ConfigService } from '../../../../libs/common/src/config/config.service';
 import { BatchAnnouncer } from './batch.announcer';
-import { QueueConstants } from '../../../../libs/common/src';
+import { BATCH_QUEUE_NAME, PUBLISH_QUEUE_NAME } from '../../../../libs/common/src';
 import { BlockchainModule } from '../../../../libs/common/src/blockchain/blockchain.module';
 import { IpfsService } from '../../../../libs/common/src/utils/ipfs.client';
 
@@ -54,7 +54,7 @@ import { IpfsService } from '../../../../libs/common/src/utils/ipfs.client';
     }),
     BullModule.registerQueue(
       {
-        name: QueueConstants.PUBLISH_QUEUE_NAME,
+        name: PUBLISH_QUEUE_NAME,
         defaultJobOptions: {
           attempts: 1,
           backoff: {
@@ -65,7 +65,7 @@ import { IpfsService } from '../../../../libs/common/src/utils/ipfs.client';
         },
       },
       {
-        name: QueueConstants.BATCH_QUEUE_NAME,
+        name: BATCH_QUEUE_NAME,
         defaultJobOptions: {
           attempts: 1,
           backoff: {

diff --git a/apps/worker/src/batch_announcer/batch.announcer.service.ts b/apps/worker/src/batch_announcer/batch.announcer.service.ts
@@ -1,4 +1,4 @@
-import { InjectRedis } from '@liaoliaots/nestjs-redis';
+import { InjectRedis } from '@songkeys/nestjs-redis';
 import { Processor, WorkerHost, OnWorkerEvent, InjectQueue } from '@nestjs/bullmq';
 import { Injectable, Logger, OnApplicationBootstrap, OnModuleDestroy } from '@nestjs/common';
 import { Job, Queue } from 'bullmq';
@@ -9,17 +9,17 @@ import { ConfigService } from '../../../../libs/common/src/config/config.service
 import { BatchAnnouncer } from './batch.announcer';
 import { CAPACITY_EPOCH_TIMEOUT_NAME } from '../../../../libs/common/src/constants';
 import { IBatchAnnouncerJobData } from '../interfaces/batch-announcer.job.interface';
-import { QueueConstants } from '../../../../libs/common/src';
+import { BATCH_QUEUE_NAME, PUBLISH_QUEUE_NAME } from '../../../../libs/common/src';
 import { BaseConsumer } from '../BaseConsumer';
 
 @Injectable()
-@Processor(QueueConstants.BATCH_QUEUE_NAME, {
+@Processor(BATCH_QUEUE_NAME, {
   concurrency: 2,
 })
 export class BatchAnnouncementService extends BaseConsumer implements OnModuleDestroy {
   constructor(
     @InjectRedis() private cacheManager: Redis,
-    @InjectQueue(QueueConstants.PUBLISH_QUEUE_NAME) private publishQueue: Queue,
+    @InjectQueue(PUBLISH_QUEUE_NAME) private publishQueue: Queue,
     private configService: ConfigService,
     private ipfsPublisher: BatchAnnouncer,
     private schedulerRegistry: SchedulerRegistry,

diff --git a/apps/worker/src/batch_announcer/batch.announcer.spec.ts b/apps/worker/src/batch_announcer/batch.announcer.spec.ts
@@ -1,8 +1,6 @@
 import { expect, describe, jest, it, beforeEach } from '@jest/globals';
 import assert from 'assert';
-// eslint-disable-next-line import/no-extraneous-dependencies
 import { FrequencyParquetSchema } from '@dsnp/frequency-schemas/types/frequency';
-// eslint-disable-next-line import/no-extraneous-dependencies
 import Redis from 'ioredis-mock';
 import { stringToHex } from '@polkadot/util';
 import { Bytes } from '@polkadot/types';

diff --git a/apps/worker/src/batch_announcer/batch.announcer.ts b/apps/worker/src/batch_announcer/batch.announcer.ts
@@ -2,10 +2,10 @@ import { Injectable, Logger } from '@nestjs/common';
 import { PassThrough } from 'node:stream';
 import { ParquetWriter } from '@dsnp/parquetjs';
 import { fromFrequencySchema } from '@dsnp/frequency-schemas/parquet';
-import { InjectRedis } from '@liaoliaots/nestjs-redis';
+import { InjectRedis } from '@songkeys/nestjs-redis';
 import Redis from 'ioredis';
 import { hexToString } from '@polkadot/util';
-import { RedisUtils } from '../../../../libs/common/src/utils/redis';
+import {  STORAGE_EXPIRE_UPPER_LIMIT_SECONDS } from '../../../../libs/common/src/utils/redis';
 import { BlockchainService } from '../../../../libs/common/src/blockchain/blockchain.service';
 import { ConfigService } from '../../../../libs/common/src/config/config.service';
 import { IBatchAnnouncerJobData } from '../interfaces/batch-announcer.job.interface';
@@ -34,7 +34,7 @@ export class BatchAnnouncer {
     if (!cachedSchema) {
       const schemaResponse = await this.blockchainService.getSchemaPayload(schemaId);
       cachedSchema = JSON.stringify(schemaResponse);
-      await this.cacheManager.setex(schemaCacheKey, RedisUtils.STORAGE_EXPIRE_UPPER_LIMIT_SECONDS, cachedSchema);
+      await this.cacheManager.setex(schemaCacheKey, STORAGE_EXPIRE_UPPER_LIMIT_SECONDS, cachedSchema);
     }
 
     const frequencySchemaPayload = JSON.parse(cachedSchema);
@@ -50,7 +50,7 @@ export class BatchAnnouncer {
     const writer = await ParquetWriter.openStream(parquetSchema, publishStream as any, writerOptions);
     // eslint-disable-next-line no-restricted-syntax
     for await (const announcement of announcements) {
-      await writer.appendRow(announcement);
+      await writer.appendRow(announcement as Record<string, any>);
     }
     await writer.close();
 

diff --git a/apps/worker/src/batching_processor/batching.processor.module.ts b/apps/worker/src/batching_processor/batching.processor.module.ts
@@ -4,11 +4,11 @@ https://docs.nestjs.com/modules
 
 import { BullModule } from '@nestjs/bullmq';
 import { Module } from '@nestjs/common';
-import { RedisModule } from '@liaoliaots/nestjs-redis';
+import { RedisModule } from '@songkeys/nestjs-redis';
 import { ScheduleModule } from '@nestjs/schedule';
 import { ConfigModule } from '../../../../libs/common/src/config/config.module';
 import { ConfigService } from '../../../../libs/common/src/config/config.service';
-import { QueueConstants } from '../../../../libs/common/src';
+import * as QueueConstants from '../../../../libs/common/src';
 import { BatchingProcessorService } from './batching.processor.service';
 import { BroadcastWorker } from './workers/broadcast.worker';
 import { ReplyWorker } from './workers/reply.worker';
@@ -55,7 +55,7 @@ import { ProfileWorker } from './workers/profile.worker';
     }),
     ScheduleModule.forRoot(),
     BullModule.registerQueue({
-      name: QueueConstants.BATCH_QUEUE_NAME,
+      name:QueueConstants.BATCH_QUEUE_NAME,
     }),
     BullModule.registerQueue({
       name: QueueConstants.BROADCAST_QUEUE_NAME,