diff --git a/apps/api/src/config/config.service.spec.ts b/apps/api/src/config/config.service.spec.ts index 52bdd19..549c54c 100644 --- a/apps/api/src/config/config.service.spec.ts +++ b/apps/api/src/config/config.service.spec.ts @@ -38,6 +38,10 @@ describe('ContentPublishingConfigService', () => { const ALL_ENV: { [key: string]: string | undefined } = { REDIS_URL: undefined, FREQUENCY_URL: undefined, + IPFS_ENDPOINT: undefined, + IPFS_GATEWAY_URL: undefined, + IPFS_BASIC_AUTH_USER: undefined, + IPFS_BASIC_AUTH_SECRET: undefined, PROVIDER_ID: undefined, BLOCKCHAIN_SCAN_INTERVAL_MINUTES: undefined, QUEUE_HIGH_WATER: undefined, diff --git a/apps/api/src/config/env.config.ts b/apps/api/src/config/env.config.ts index 8e83ddf..a6bd641 100644 --- a/apps/api/src/config/env.config.ts +++ b/apps/api/src/config/env.config.ts @@ -5,6 +5,10 @@ import { mnemonicValidate } from '@polkadot/util-crypto'; export const configModuleOptions: ConfigModuleOptions = { isGlobal: true, validationSchema: Joi.object({ + IPFS_ENDPOINT: Joi.string().uri().required(), + IPFS_GATEWAY_URL: Joi.string().required(), // This is parse as string as the required format of this not a valid uri, check .env.template + IPFS_BASIC_AUTH_USER: Joi.string().allow('').default(''), + IPFS_BASIC_AUTH_SECRET: Joi.string().allow('').default(''), REDIS_URL: Joi.string().uri().required(), FREQUENCY_URL: Joi.string().uri().required(), PROVIDER_ID: Joi.required().custom((value: string, helpers) => { diff --git a/apps/worker/src/batch_announcer/batch.announcer.module.ts b/apps/worker/src/batch_announcer/batch.announcer.module.ts index 206e128..6c2c1f8 100644 --- a/apps/worker/src/batch_announcer/batch.announcer.module.ts +++ b/apps/worker/src/batch_announcer/batch.announcer.module.ts @@ -9,12 +9,15 @@ import { RedisModule } from '@liaoliaots/nestjs-redis'; import { BatchAnnouncementService } from './batch.announcer.service'; import { ConfigModule } from '../../../api/src/config/config.module'; import { ConfigService } from '../../../api/src/config/config.service'; -import { IPFSAnnouncer } from './ipfs.announcer'; +import { BatchAnnouncer } from './batch.announcer'; import { QueueConstants } from '../../../../libs/common/src'; +import { BlockchainModule } from '../blockchain/blockchain.module'; +import { IpfsService } from '../../../../libs/common/src/utils/ipfs.client'; @Module({ imports: [ ConfigModule, + BlockchainModule, EventEmitterModule, RedisModule.forRootAsync( { @@ -75,7 +78,7 @@ import { QueueConstants } from '../../../../libs/common/src'; ), ], controllers: [], - providers: [BatchAnnouncementService, IPFSAnnouncer], - exports: [BullModule, BatchAnnouncementService, IPFSAnnouncer], + providers: [BatchAnnouncementService, BatchAnnouncer, IpfsService], + exports: [BullModule, BatchAnnouncementService, BatchAnnouncer, IpfsService], }) export class BatchAnnouncerModule {} diff --git a/apps/worker/src/batch_announcer/batch.announcer.service.ts b/apps/worker/src/batch_announcer/batch.announcer.service.ts index 9a9ce51..0bb623d 100644 --- a/apps/worker/src/batch_announcer/batch.announcer.service.ts +++ b/apps/worker/src/batch_announcer/batch.announcer.service.ts @@ -6,7 +6,7 @@ import Redis from 'ioredis'; import { SchedulerRegistry } from '@nestjs/schedule'; import { EventEmitter2 } from '@nestjs/event-emitter'; import { ConfigService } from '../../../api/src/config/config.service'; -import { IPFSAnnouncer } from './ipfs.announcer'; +import { BatchAnnouncer } from './batch.announcer'; import { CAPACITY_EPOCH_TIMEOUT_NAME } from '../../../../libs/common/src/constants'; import { IBatchAnnouncerJobData } from '../interfaces/batch-announcer.job.interface'; import { QueueConstants } from '../../../../libs/common/src'; @@ -18,13 +18,11 @@ import { QueueConstants } from '../../../../libs/common/src'; export class BatchAnnouncementService extends WorkerHost implements OnApplicationBootstrap, OnModuleDestroy { private logger: Logger; - private capacityExhausted = false; - constructor( @InjectRedis() private cacheManager: Redis, @InjectQueue(QueueConstants.PUBLISH_QUEUE_NAME) private publishQueue: Queue, private configService: ConfigService, - private ipfsPublisher: IPFSAnnouncer, + private ipfsPublisher: BatchAnnouncer, private schedulerRegistry: SchedulerRegistry, private eventEmitter: EventEmitter2, ) { @@ -47,7 +45,9 @@ export class BatchAnnouncementService extends WorkerHost implements OnApplicatio async process(job: Job): Promise { this.logger.log(`Processing job ${job.id} of type ${job.name}`); try { - await this.ipfsPublisher.announce(job.data); + const publisherJob = await this.ipfsPublisher.announce(job.data); + + await this.publishQueue.add(publisherJob.id, publisherJob); this.logger.log(`Completed job ${job.id} of type ${job.name}`); return job.data; } catch (e) { diff --git a/apps/worker/src/batch_announcer/batch.announcer.spec.ts b/apps/worker/src/batch_announcer/batch.announcer.spec.ts new file mode 100644 index 0000000..52ffefd --- /dev/null +++ b/apps/worker/src/batch_announcer/batch.announcer.spec.ts @@ -0,0 +1,88 @@ +import { expect, describe, jest, it, beforeEach } from '@jest/globals'; +import assert from 'assert'; +import { FrequencyParquetSchema } from '@dsnp/frequency-schemas/types/frequency'; +import Redis from 'ioredis-mock'; +import { BatchAnnouncer } from './batch.announcer'; + +// Create a mock for the dependencies +const mockConfigService = { + getIpfsCidPlaceholder: jest.fn(), +}; + +const mockBlockchainService = { + getSchema: jest.fn(), +}; + +const mockIpfsService = { + getPinned: jest.fn(), + ipfsPin: jest.fn(), +}; + +describe('BatchAnnouncer', () => { + let ipfsAnnouncer: BatchAnnouncer; + + const broadcast: FrequencyParquetSchema = [ + { + name: 'announcementType', + column_type: { + INTEGER: { + bit_width: 32, + sign: true, + }, + }, + compression: 'GZIP', + bloom_filter: false, + }, + { + name: 'contentHash', + column_type: 'BYTE_ARRAY', + compression: 'GZIP', + bloom_filter: true, + }, + { + name: 'fromId', + column_type: { + INTEGER: { + bit_width: 64, + sign: false, + }, + }, + compression: 'GZIP', + bloom_filter: true, + }, + { + name: 'url', + column_type: 'STRING', + compression: 'GZIP', + bloom_filter: false, + }, + ]; + const mockClient = new Redis(); + + beforeEach(async () => { + ipfsAnnouncer = new BatchAnnouncer(mockClient, mockConfigService as any, mockBlockchainService as any, mockIpfsService as any); + }); + it('should be defined', () => { + expect(ipfsAnnouncer).toBeDefined(); + }); + + // Write your test cases here + it('should announce a batch to IPFS', async () => { + // Mock the necessary dependencies' behavior + mockConfigService.getIpfsCidPlaceholder.mockReturnValue('mockIpfsUrl'); + mockBlockchainService.getSchema.mockReturnValue({ model: JSON.stringify(broadcast) }); + mockIpfsService.getPinned.mockReturnValue(Buffer.from('mockContentBuffer')); + mockIpfsService.ipfsPin.mockReturnValue({ cid: 'mockCid', size: 'mockSize' }); + + const batchJob = { + batchId: 'mockBatchId', + schemaId: 123, + announcements: [], + }; + + const result = await ipfsAnnouncer.announce(batchJob); + assert(result); + expect(mockConfigService.getIpfsCidPlaceholder).toHaveBeenCalledWith('mockCid'); + expect(mockBlockchainService.getSchema).toHaveBeenCalledWith(123); + }); +}); diff --git a/apps/worker/src/batch_announcer/batch.announcer.ts b/apps/worker/src/batch_announcer/batch.announcer.ts new file mode 100644 index 0000000..1f39ced --- /dev/null +++ b/apps/worker/src/batch_announcer/batch.announcer.ts @@ -0,0 +1,89 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { PassThrough } from 'node:stream'; +import { ParquetWriter } from '@dsnp/parquetjs'; +import { fromFrequencySchema } from '@dsnp/frequency-schemas/parquet'; +import { InjectRedis } from '@liaoliaots/nestjs-redis'; +import Redis from 'ioredis'; +import { PalletSchemasSchema } from '@polkadot/types/lookup'; +import { BlockchainService } from '../blockchain/blockchain.service'; +import { ConfigService } from '../../../api/src/config/config.service'; +import { IBatchAnnouncerJobData } from '../interfaces/batch-announcer.job.interface'; +import { IPublisherJob } from '../interfaces/publisher-job.interface'; +import { IpfsService } from '../../../../libs/common/src/utils/ipfs.client'; + +@Injectable() +export class BatchAnnouncer { + private logger: Logger; + + constructor( + @InjectRedis() private cacheManager: Redis, + private configService: ConfigService, + private blockchainService: BlockchainService, + private ipfsService: IpfsService, + ) { + this.logger = new Logger(BatchAnnouncer.name); + } + + public async announce(batchJob: IBatchAnnouncerJobData): Promise { + this.logger.debug(`Announcing batch ${batchJob.batchId} on IPFS`); + const { batchId, schemaId, announcements } = batchJob; + + let frequencySchema: PalletSchemasSchema; + + const schemaCacheKey = `schema:${schemaId}`; + const cachedSchema = await this.cacheManager.get(schemaCacheKey); + if (cachedSchema) { + frequencySchema = JSON.parse(cachedSchema); + } else { + frequencySchema = await this.blockchainService.getSchema(schemaId); + await this.cacheManager.set(schemaCacheKey, JSON.stringify(frequencySchema)); + } + + const schema = JSON.parse(frequencySchema.model.toString()); + if (!schema) { + throw new Error(`Unable to parse schema for schemaId ${schemaId}`); + } + + const [parquetSchema, writerOptions] = fromFrequencySchema(schema); + const publishStream = new PassThrough(); + + const writer = await ParquetWriter.openStream(parquetSchema, publishStream as any, writerOptions); + + announcements.forEach(async (announcement) => { + writer.appendRow(announcement); + }); + + await writer.close(); + const buffer = await this.bufferPublishStream(publishStream); + const [cid, hash] = await this.pinStringToIPFS(buffer); + const ipfsUrl = await this.formIpfsUrl(cid); + this.logger.debug(`Batch ${batchId} published to IPFS at ${ipfsUrl}`); + this.logger.debug(`Batch ${batchId} hash: ${hash}`); + return { id: batchId, schemaId, data: { cid, payloadLength: buffer.length } }; + } + + private async bufferPublishStream(publishStream: PassThrough): Promise { + this.logger.debug('Buffering publish stream'); + return new Promise((resolve, reject) => { + const buffers: Buffer[] = []; + publishStream.on('data', (data) => { + buffers.push(data); + }); + publishStream.on('end', () => { + resolve(Buffer.concat(buffers)); + }); + publishStream.on('error', (err) => { + reject(err); + }); + }); + } + + private async pinStringToIPFS(buf: Buffer): Promise<[string, string]> { + const { cid, size } = await this.ipfsService.ipfsPin('application/octet-stream', buf); + return [cid.toString(), size.toString()]; + } + + private async formIpfsUrl(cid: string): Promise { + return this.configService.getIpfsCidPlaceholder(cid); + } +} diff --git a/apps/worker/src/batch_announcer/ipfs.announcer.spec.ts b/apps/worker/src/batch_announcer/ipfs.announcer.spec.ts deleted file mode 100644 index fafa3c6..0000000 --- a/apps/worker/src/batch_announcer/ipfs.announcer.spec.ts +++ /dev/null @@ -1,13 +0,0 @@ -// test file for ipfs announcer -import { describe, it, beforeEach } from '@jest/globals'; -import { IPFSAnnouncer } from './ipfs.announcer'; - -describe('IPFSAnnouncer', () => { - let ipfsAnnouncer: IPFSAnnouncer; - - beforeEach(async () => {}); - - describe('announce', () => { - it('should announce a batch on ipfs', async () => {}); - }); -}); diff --git a/apps/worker/src/batch_announcer/ipfs.announcer.ts b/apps/worker/src/batch_announcer/ipfs.announcer.ts deleted file mode 100644 index f548271..0000000 --- a/apps/worker/src/batch_announcer/ipfs.announcer.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { ConfigService } from '../../../api/src/config/config.service'; -import { IBatchAnnouncerJobData } from '../interfaces/batch-announcer.job.interface'; - -@Injectable() -export class IPFSAnnouncer { - private logger: Logger; - - constructor(private configService: ConfigService) { - this.logger = new Logger(IPFSAnnouncer.name); - } - - public async announce(batchJob: IBatchAnnouncerJobData): Promise { - this.logger.log(`Announcing batch ${batchJob.batchId} on IPFS`); - } -} diff --git a/apps/worker/src/blockchain/blockchain.service.ts b/apps/worker/src/blockchain/blockchain.service.ts index 3cdd832..00b303d 100644 --- a/apps/worker/src/blockchain/blockchain.service.ts +++ b/apps/worker/src/blockchain/blockchain.service.ts @@ -7,8 +7,8 @@ import { KeyringPair } from '@polkadot/keyring/types'; import { BlockHash, BlockNumber } from '@polkadot/types/interfaces'; import { SubmittableExtrinsic } from '@polkadot/api/types'; import { AnyNumber, ISubmittableResult } from '@polkadot/types/types'; -import { u32, Option, u128 } from '@polkadot/types'; -import { PalletCapacityCapacityDetails, PalletCapacityEpochInfo } from '@polkadot/types/lookup'; +import { u32, Option } from '@polkadot/types'; +import { PalletCapacityCapacityDetails, PalletCapacityEpochInfo, PalletSchemasSchema } from '@polkadot/types/lookup'; import { ConfigService } from '../../../api/src/config/config.service'; import { Extrinsic } from './extrinsic'; @@ -139,4 +139,9 @@ export class BlockchainService implements OnApplicationBootstrap, OnApplicationS public async capacityBatchLimit(): Promise { return this.api.consts.frequencyTxPayment.maximumCapacityBatchLength.toNumber(); } + + public async getSchema(schemaId: number): Promise { + const schema: PalletSchemasSchema = await this.query('schemas', 'schemas', schemaId); + return schema; + } } diff --git a/apps/worker/src/interfaces/batch-announcer.job.interface.ts b/apps/worker/src/interfaces/batch-announcer.job.interface.ts index 25d5bbf..10cb3b0 100644 --- a/apps/worker/src/interfaces/batch-announcer.job.interface.ts +++ b/apps/worker/src/interfaces/batch-announcer.job.interface.ts @@ -1,4 +1,7 @@ +import { Announcement } from '../../../../libs/common/src/interfaces/dsnp'; + export interface IBatchAnnouncerJobData { batchId: string; schemaId: number; + announcements: Announcement[]; } diff --git a/env.template b/env.template index 8afa058..bc6fd8a 100644 --- a/env.template +++ b/env.template @@ -1,4 +1,8 @@ # Copy this file to ".env.dev" and ".env.docker.dev", and then tweak values for local development +IPFS_ENDPOINT="https://ipfs.infura.io:5001" +IPFS_BASIC_AUTH_USER="Infura Project ID Here or Blank for Kubo RPC" +IPFS_BASIC_AUTH_SECRET="Infura Secret Here or Blank for Kubo RPC" +IPFS_GATEWAY_URL="https://ipfs.io/ipfs/[CID]" FREQUENCY_URL=ws://0.0.0.0:9944 PROVIDER_ID=1 REDIS_URL=redis://0.0.0.0:6379 diff --git a/libs/common/src/interfaces/dsnp.ts b/libs/common/src/interfaces/dsnp.ts new file mode 100644 index 0000000..5b0170d --- /dev/null +++ b/libs/common/src/interfaces/dsnp.ts @@ -0,0 +1,183 @@ +/** + * AnnouncementType: an enum representing different types of DSNP announcements + */ + +import { ActivityContentNote } from '@dsnp/activity-content/types'; + +// eslint-disable-next-line no-shadow +export enum AnnouncementType { + Tombstone = 0, + Broadcast = 2, + Reply = 3, + Reaction = 4, + Profile = 5, + Update = 6, + PublicFollows = 113, +} + +type TombstoneFields = { + announcementType: AnnouncementType.Tombstone; + targetAnnouncementType: AnnouncementType; + targetSignature: string; +}; + +type BroadcastFields = { + announcementType: AnnouncementType.Broadcast; + contentHash: string; + url: string; +}; + +type ReplyFields = { + announcementType: AnnouncementType.Reply; + contentHash: string; + inReplyTo: string; + url: string; +}; + +type ReactionFields = { + announcementType: AnnouncementType.Reaction; + emoji: string; + inReplyTo: string; +}; + +type ProfileFields = { + announcementType: AnnouncementType.Profile; + contentHash: string; + url: string; +}; + +/** + * TypedAnnouncement: an Announcement with a particular AnnouncementType + */ +export type TypedAnnouncement = { + announcementType: T; + fromId: string; +} & (TombstoneFields | BroadcastFields | ReplyFields | ReactionFields | ProfileFields); + +/** + * Announcement: an Announcement intended for inclusion in a batch file + */ +export type Announcement = TypedAnnouncement; + +/** + * ProfileAnnouncement: an Announcement of type Profile + */ +export type ProfileAnnouncement = TypedAnnouncement; + +/** + * TombstoneAnnouncement: an Announcement of type Tombstone + */ +export type TombstoneAnnouncement = TypedAnnouncement; + +/** + * BroadcastAnnouncement: an Announcement of type Broadcast + */ +export type BroadcastAnnouncement = TypedAnnouncement; + +/** + * ReplyAnnouncement: am announcement of type Reply + */ +export type ReplyAnnouncement = TypedAnnouncement; + +/** + * ReactionAnnouncement: an Announcement of type Reaction + */ +export type ReactionAnnouncement = TypedAnnouncement; + +/** + * createTombstone() generates a tombstone announcement from a given URL and + * hash. + * + * @param fromId - The id of the user from whom the announcement is posted + * @param targetType - The DSNP announcement type of the target announcement + * @param targetSignature - The signature of the target announcement + * @returns A TombstoneAnnouncement + */ +export const createTombstone = (fromId: string, targetType: AnnouncementType, targetSignature: string): TombstoneAnnouncement => ({ + announcementType: AnnouncementType.Tombstone, + targetAnnouncementType: targetType, + targetSignature, + fromId, +}); + +/** + * createBroadcast() generates a broadcast announcement from a given URL and + * hash. + * + * @param fromId - The id of the user from whom the announcement is posted + * @param url - The URL of the activity content to reference + * @param hash - The hash of the content at the URL + * @returns A BroadcastAnnouncement + */ +export const createBroadcast = (fromId: string, url: string, hash: string): BroadcastAnnouncement => ({ + announcementType: AnnouncementType.Broadcast, + contentHash: hash, + fromId, + url, +}); + +/** + * createReply() generates a reply announcement from a given URL, hash and + * content uri. + * + * @param fromId - The id of the user from whom the announcement is posted + * @param url - The URL of the activity content to reference + * @param hash - The hash of the content at the URL + * @param inReplyTo - The DSNP Content Uri of the parent announcement + * @returns A ReplyAnnouncement + */ +export const createReply = (fromId: string, url: string, hash: string, inReplyTo: string): ReplyAnnouncement => ({ + announcementType: AnnouncementType.Reply, + contentHash: hash, + fromId, + inReplyTo, + url, +}); + +/** + * createReaction() generates a reaction announcement from a given URL, hash and + * content uri. + * + * @param fromId - The id of the user from whom the announcement is posted + * @param emoji - The emoji to respond with + * @param inReplyTo - The DSNP Content Uri of the parent announcement + * @returns A ReactionAnnouncement + */ +export const createReaction = (fromId: string, emoji: string, inReplyTo: string): ReactionAnnouncement => ({ + announcementType: AnnouncementType.Reaction, + emoji, + fromId, + inReplyTo, +}); + +/** + * createProfile() generates a profile announcement from a given URL and hash. + * + * @param fromId - The id of the user from whom the announcement is posted + * @param url - The URL of the activity content to reference + * @param hash - The hash of the content at the URL + * @returns A ProfileAnnouncement + */ +export const createProfile = (fromId: string, url: string, hash: string): ProfileAnnouncement => ({ + announcementType: AnnouncementType.Profile, + contentHash: hash, + fromId, + url, +}); + +/** + * createNote() provides a simple factory for generating an ActivityContentNote + * object. + * @param content - The text content to include in the note + * @param published - the Date that the note was claimed to be published + * @param options - Overrides default fields for the ActivityContentNote + * @returns An ActivityContentNote object + */ +export const createNote = (content: string, published: Date, options?: Partial): ActivityContentNote => ({ + '@context': 'https://www.w3.org/ns/activitystreams', + type: 'Note', + mediaType: 'text/plain', + published: published.toISOString(), + content, + ...options, +}); diff --git a/libs/common/src/utils/dsnpTypeConverter.ts b/libs/common/src/utils/dsnpTypeConverter.ts new file mode 100644 index 0000000..014c70d --- /dev/null +++ b/libs/common/src/utils/dsnpTypeConverter.ts @@ -0,0 +1,176 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { + ActivityContentTag, + ActivityContentAttachment, + ActivityContentLink, + ActivityContentImageLink, + ActivityContentImage, + ActivityContentVideoLink, + ActivityContentVideo, + ActivityContentAudioLink, + ActivityContentAudio, +} from '@dsnp/activity-content/types'; +import { TagTypeDto, AssetDto, AttachmentTypeDto } from '../dtos/activity.dto'; +import { createNote } from '../interfaces/dsnp'; +import { calculateDsnpHash } from './ipfs'; +import { IpfsService } from './ipfs.client'; +import { ConfigService } from '../../../../apps/api/src/config/config.service'; + +@Injectable() +export class BatchAnnouncer { + private logger: Logger; + + constructor( + private configService: ConfigService, + private ipfsService: IpfsService, + ) { + this.logger = new Logger(BatchAnnouncer.name); + } + + public async prepareNote(noteContent?: any): Promise<[string, string, string]> { + this.logger.debug(`Preparing note`); + const tags: ActivityContentTag[] = []; + if (noteContent?.content.tag) { + noteContent.content.tag.forEach((tag) => { + switch (tag.type) { + case TagTypeDto.Hashtag: + tags.push({ name: tag.name || '' }); + break; + case TagTypeDto.Mention: + tags.push({ + name: tag.name || '', + type: 'Mention', + id: tag.mentionedId || '', + }); + break; + default: + throw new Error(`Unsupported tag type ${typeof tag.type}`); + } + }); + } + + const attachments: ActivityContentAttachment[] = []; + if (noteContent?.content.assets) { + noteContent.content.assets.forEach(async (asset: AssetDto) => { + switch (asset.type) { + case AttachmentTypeDto.LINK: { + const link: ActivityContentLink = { + type: 'Link', + href: asset.href || '', + name: asset.name || '', + }; + + attachments.push(link); + break; + } + case AttachmentTypeDto.IMAGE: { + const imageLinks: ActivityContentImageLink[] = []; + asset.references?.forEach(async (reference) => { + const contentBuffer = await this.ipfsService.getPinned(reference.referenceId); + const hashedContent = await calculateDsnpHash(contentBuffer); + const image: ActivityContentImageLink = { + mediaType: 'image', // TODO + hash: [hashedContent], + height: reference.height, + width: reference.width, + type: 'Link', + href: await this.formIpfsUrl(reference.referenceId), + }; + imageLinks.push(image); + }); + const imageActivity: ActivityContentImage = { + type: 'Image', + name: asset.name || '', + url: imageLinks, + }; + + attachments.push(imageActivity); + break; + } + case AttachmentTypeDto.VIDEO: { + const videoLinks: ActivityContentVideoLink[] = []; + let duration = ''; + asset.references?.forEach(async (reference) => { + const contentBuffer = await this.ipfsService.getPinned(reference.referenceId); + const hashedContent = await calculateDsnpHash(contentBuffer); + const video: ActivityContentVideoLink = { + mediaType: 'video', // TODO + hash: [hashedContent], + height: reference.height, + width: reference.width, + type: 'Link', + href: await this.formIpfsUrl(reference.referenceId), + }; + duration = reference.duration ?? ''; + videoLinks.push(video); + }); + const videoActivity: ActivityContentVideo = { + type: 'Video', + name: asset.name || '', + url: videoLinks, + duration, + }; + + attachments.push(videoActivity); + break; + } + case AttachmentTypeDto.AUDIO: { + const audioLinks: ActivityContentAudioLink[] = []; + let duration = ''; + asset.references?.forEach(async (reference) => { + const contentBuffer = await this.ipfsService.getPinned(reference.referenceId); + const hashedContent = await calculateDsnpHash(contentBuffer); + duration = reference.duration ?? ''; + const audio: ActivityContentAudioLink = { + mediaType: 'audio', // TODO + hash: [hashedContent], + type: 'Link', + href: await this.formIpfsUrl(reference.referenceId), + }; + audioLinks.push(audio); + }); + const audioActivity: ActivityContentAudio = { + type: 'Audio', + name: asset.name || '', + url: audioLinks, + duration, + }; + + attachments.push(audioActivity); + break; + } + default: + throw new Error(`Unsupported attachment type ${typeof asset.type}`); + } + }); + } + + const note = createNote(noteContent?.content.content ?? '', new Date(noteContent?.content.published ?? ''), { + name: noteContent?.content.name, + location: { + latitude: noteContent?.content.location?.latitude, + longitude: noteContent?.content.location?.longitude, + radius: noteContent?.content.location?.radius, + altitude: noteContent?.content.location?.altitude, + accuracy: noteContent?.content.location?.accuracy, + name: noteContent?.content.location?.name || '', + type: 'Place', + }, + tag: tags, + attachment: attachments, + }); + const noteString = JSON.stringify(note); + const [cid, hash] = await this.pinStringToIPFS(Buffer.from(noteString)); + const ipfsUrl = await this.formIpfsUrl(cid); + return [cid, hash, ipfsUrl]; + } + + private async pinStringToIPFS(buf: Buffer): Promise<[string, string]> { + const { cid, size } = await this.ipfsService.ipfsPin('application/octet-stream', buf); + return [cid.toString(), size.toString()]; + } + + private async formIpfsUrl(cid: string): Promise { + return this.configService.getIpfsCidPlaceholder(cid); + } +} diff --git a/libs/common/src/utils/ipfs.client.ts b/libs/common/src/utils/ipfs.client.ts index 0729d64..7cc9c32 100644 --- a/libs/common/src/utils/ipfs.client.ts +++ b/libs/common/src/utils/ipfs.client.ts @@ -6,9 +6,9 @@ import FormData from 'form-data'; import { extension as getExtension } from 'mime-types'; import { CID } from 'multiformats/cid'; import { blake2b256 as hasher } from '@multiformats/blake2/blake2b'; -import { base58btc } from 'multiformats/bases/base58'; import { create } from 'multiformats/hashes/digest'; import { randomUUID } from 'crypto'; +import { base58btc } from 'multiformats/bases/base58'; import { ConfigService } from '../../../../apps/api/src/config/config.service'; export interface FilePin { diff --git a/package-lock.json b/package-lock.json index db2c709..67a5f2f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,8 @@ "version": "0.1.0", "license": "Apache-2.0", "dependencies": { + "@dsnp/activity-content": "^1.1.0", + "@dsnp/frequency-schemas": "^1.0.2", "@dsnp/parquetjs": "^1.3.4", "@frequency-chain/api-augment": "1.7.0", "@jest/globals": "^29.5.0", @@ -43,7 +45,7 @@ "ipfs-only-hash": "^4.0.0", "joi": "^17.9.1", "mime-types": "^2.1.35", - "multiformats": "^9.9.0", + "multiformats": "9.9.0", "rxjs": "^7.8.1", "time-constants": "^1.0.3" }, @@ -67,6 +69,7 @@ "eslint-plugin-nestjs": "^1.2.3", "eslint-plugin-prettier": "^5.0.0", "eslint-plugin-promise": "^6.1.1", + "ioredis-mock": "^8.8.3", "jest": "^29.5.0", "license-report": "^6.4.0", "prettier": "^3.0.2", @@ -764,7 +767,6 @@ }, "node_modules/@cspotcode/source-map-support": { "version": "0.8.1", - "devOptional": true, "license": "MIT", "dependencies": { "@jridgewell/trace-mapping": "0.3.9" @@ -773,6 +775,49 @@ "node": ">=12" } }, + "node_modules/@dsnp/activity-content": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@dsnp/activity-content/-/activity-content-1.1.0.tgz", + "integrity": "sha512-T83Bi3Nn4uUOySAECaUuNZ+1XkL1c3VENAxKv1m9n76J0GApeLcRfujtWZnPNJGZJp+wgLbmjoTBxulwjbDW4w==", + "dependencies": { + "@multiformats/blake2": "^1.0.13", + "multiformats": "^11.0.2" + } + }, + "node_modules/@dsnp/activity-content/node_modules/multiformats": { + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@dsnp/frequency-schemas": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@dsnp/frequency-schemas/-/frequency-schemas-1.0.2.tgz", + "integrity": "sha512-+u2Fwv9aYbMn7MI5LbiDn92dWK+YxcJJEwdy6r/wdQwFVz/jZtE5lR56KqPYS2piun/vINMJU+HNZUVYL4zkOg==", + "dependencies": { + "@frequency-chain/api-augment": "0.0.0-45e306", + "@polkadot/api": "^10.7.3", + "json-stringify-pretty-compact": "^4.0.0", + "ts-node": "^10.9.1", + "typescript": "^5.0.4" + }, + "optionalDependencies": { + "@dsnp/parquetjs": "^1.3.0" + } + }, + "node_modules/@dsnp/frequency-schemas/node_modules/@frequency-chain/api-augment": { + "version": "0.0.0-45e306", + "resolved": "https://registry.npmjs.org/@frequency-chain/api-augment/-/api-augment-0.0.0-45e306.tgz", + "integrity": "sha512-VzZIFwMX8LaY6dJfzsJ6t9WKZ28DzGA4+lEKys0OR6O+CIAV/fVWNb8FaHG9c6It0GmSz2Os0ehvn5p/rgWfug==", + "dependencies": { + "@polkadot/api": "^10.7.3", + "@polkadot/rpc-provider": "^10.7.3", + "@polkadot/types": "^10.7.3" + } + }, "node_modules/@dsnp/parquetjs": { "version": "1.3.4", "resolved": "https://registry.npmjs.org/@dsnp/parquetjs/-/parquetjs-1.3.4.tgz", @@ -898,6 +943,12 @@ "dev": true, "license": "BSD-3-Clause" }, + "node_modules/@ioredis/as-callback": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@ioredis/as-callback/-/as-callback-3.0.0.tgz", + "integrity": "sha512-Kqv1rZ3WbgOrS+hgzJ5xG5WQuhvzzSTRYvNeyPMLOAM78MHSnuKI20JeJGbpuAt//LCuP0vsexZcorqW7kWhJg==", + "dev": true + }, "node_modules/@ioredis/commands": { "version": "1.2.0", "license": "MIT" @@ -2684,22 +2735,18 @@ }, "node_modules/@tsconfig/node10": { "version": "1.0.9", - "devOptional": true, "license": "MIT" }, "node_modules/@tsconfig/node12": { "version": "1.0.11", - "devOptional": true, "license": "MIT" }, "node_modules/@tsconfig/node14": { "version": "1.0.3", - "devOptional": true, "license": "MIT" }, "node_modules/@tsconfig/node16": { "version": "1.0.4", - "devOptional": true, "license": "MIT" }, "node_modules/@types/babel__core": { @@ -2830,6 +2877,16 @@ "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.1.tgz", "integrity": "sha512-/K3ds8TRAfBvi5vfjuz8y6+GiAYBZ0x4tXv1Av6CWBWn0IlADc+ZX9pMq7oU0fNQPnBwIZl3rmeLp6SBApbxSQ==" }, + "node_modules/@types/ioredis-mock": { + "version": "8.2.2", + "resolved": "https://registry.npmjs.org/@types/ioredis-mock/-/ioredis-mock-8.2.2.tgz", + "integrity": "sha512-bnbPHOjxy4TUDjRh61MMoK2QvDNZqrMDXJYrEDZP/HPFvBubR24CQ0DBi5lgWhLxG4lvVsXPRDXtZ03+JgonoQ==", + "dev": true, + "peer": true, + "dependencies": { + "ioredis": ">=5" + } + }, "node_modules/@types/istanbul-lib-coverage": { "version": "2.0.4", "license": "MIT" @@ -3619,7 +3676,6 @@ }, "node_modules/acorn-walk": { "version": "8.2.0", - "devOptional": true, "license": "MIT", "engines": { "node": ">=0.4.0" @@ -3764,7 +3820,6 @@ }, "node_modules/arg": { "version": "4.1.3", - "devOptional": true, "license": "MIT" }, "node_modules/argparse": { @@ -4877,7 +4932,6 @@ }, "node_modules/create-require": { "version": "1.1.1", - "devOptional": true, "license": "MIT" }, "node_modules/cron": { @@ -5180,7 +5234,6 @@ }, "node_modules/diff": { "version": "4.0.2", - "devOptional": true, "license": "BSD-3-Clause", "engines": { "node": ">=0.3.1" @@ -6197,6 +6250,32 @@ "bser": "2.1.1" } }, + "node_modules/fengari": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/fengari/-/fengari-0.1.4.tgz", + "integrity": "sha512-6ujqUuiIYmcgkGz8MGAdERU57EIluGGPSUgGPTsco657EHa+srq0S3/YUl/r9kx1+D+d4rGfYObd+m8K22gB1g==", + "dev": true, + "dependencies": { + "readline-sync": "^1.4.9", + "sprintf-js": "^1.1.1", + "tmp": "^0.0.33" + } + }, + "node_modules/fengari-interop": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/fengari-interop/-/fengari-interop-0.1.3.tgz", + "integrity": "sha512-EtZ+oTu3kEwVJnoymFPBVLIbQcCoy9uWCVnMA6h3M/RqHkUBsLYp29+RRHf9rKr6GwjubWREU1O7RretFIXjHw==", + "dev": true, + "peerDependencies": { + "fengari": "^0.1.0" + } + }, + "node_modules/fengari/node_modules/sprintf-js": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.2.tgz", + "integrity": "sha512-VE0SOVEHCk7Qc8ulkWw3ntAzXuqf7S2lvwQaDLRnUeIEaKNQJzV6BwmLKhOqT61aGhfUMrXeaBk+oDGCzvhcug==", + "dev": true + }, "node_modules/fetch-blob": { "version": "3.2.0", "funding": [ @@ -7103,6 +7182,26 @@ "url": "https://opencollective.com/ioredis" } }, + "node_modules/ioredis-mock": { + "version": "8.8.3", + "resolved": "https://registry.npmjs.org/ioredis-mock/-/ioredis-mock-8.8.3.tgz", + "integrity": "sha512-LkF17WIyYkPfUhvp0fjIZ+HKhILEoq1J2b71vv+9EOW054UlkySVEvgQ2RolXM+eI759MteHtXQvv0oRn0lkUg==", + "dev": true, + "dependencies": { + "@ioredis/as-callback": "^3.0.0", + "@ioredis/commands": "^1.2.0", + "fengari": "^0.1.4", + "fengari-interop": "^0.1.3", + "semver": "^7.5.4" + }, + "engines": { + "node": ">=12.22" + }, + "peerDependencies": { + "@types/ioredis-mock": "^8", + "ioredis": "^5" + } + }, "node_modules/ipaddr.js": { "version": "1.9.1", "license": "MIT", @@ -8423,6 +8522,11 @@ "dev": true, "license": "MIT" }, + "node_modules/json-stringify-pretty-compact": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/json-stringify-pretty-compact/-/json-stringify-pretty-compact-4.0.0.tgz", + "integrity": "sha512-3CNZ2DnrpByG9Nqj6Xo8vqbjT4F6N+tb4Gb28ESAZjYZ5yqvmc56J+/kuIwkaAMOyblTQhUW7PxMkUb8Q36N3Q==" + }, "node_modules/json-stringify-safe": { "version": "5.0.1", "license": "ISC" @@ -8679,7 +8783,6 @@ }, "node_modules/make-error": { "version": "1.3.6", - "devOptional": true, "license": "ISC" }, "node_modules/makeerror": { @@ -10114,6 +10217,15 @@ "node": ">=8.10.0" } }, + "node_modules/readline-sync": { + "version": "1.4.10", + "resolved": "https://registry.npmjs.org/readline-sync/-/readline-sync-1.4.10.tgz", + "integrity": "sha512-gNva8/6UAe8QYepIQH/jQ2qn91Qj0B9sYjMBBs3QOB8F2CXcKgLxQaJRP76sWVRQt+QU+8fAkCbCvjjMFu7Ycw==", + "dev": true, + "engines": { + "node": ">= 0.8.0" + } + }, "node_modules/rechoir": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz", @@ -10551,8 +10663,9 @@ } }, "node_modules/semver": { - "version": "7.5.3", - "license": "ISC", + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", + "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", "dependencies": { "lru-cache": "^6.0.0" }, @@ -11389,7 +11502,6 @@ }, "node_modules/ts-node": { "version": "10.9.1", - "devOptional": true, "license": "MIT", "dependencies": { "@cspotcode/source-map-support": "^0.8.0", @@ -11899,7 +12011,6 @@ }, "node_modules/v8-compile-cache-lib": { "version": "3.0.1", - "devOptional": true, "license": "MIT" }, "node_modules/v8-to-istanbul": { @@ -12355,7 +12466,6 @@ }, "node_modules/yn": { "version": "3.1.1", - "devOptional": true, "license": "MIT", "engines": { "node": ">=6" diff --git a/package.json b/package.json index 9855c77..8380475 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,8 @@ }, "homepage": "https://github.com/AmplicaLabs/content-publishing-service#readme", "dependencies": { + "@dsnp/activity-content": "^1.1.0", + "@dsnp/frequency-schemas": "^1.0.2", "@dsnp/parquetjs": "^1.3.4", "@frequency-chain/api-augment": "1.7.0", "@jest/globals": "^29.5.0", @@ -71,7 +73,7 @@ "ipfs-only-hash": "^4.0.0", "joi": "^17.9.1", "mime-types": "^2.1.35", - "multiformats": "^9.9.0", + "multiformats": "9.9.0", "rxjs": "^7.8.1", "time-constants": "^1.0.3" }, @@ -95,6 +97,7 @@ "eslint-plugin-nestjs": "^1.2.3", "eslint-plugin-prettier": "^5.0.0", "eslint-plugin-promise": "^6.1.1", + "ioredis-mock": "^8.8.3", "jest": "^29.5.0", "license-report": "^6.4.0", "prettier": "^3.0.2",