From 0059ce31648e030051c757c4c6c2d941292b9ff3 Mon Sep 17 00:00:00 2001 From: tsa96 Date: Fri, 18 Oct 2024 18:58:40 +0100 Subject: [PATCH] refactor(back): add header to static map list files --- apps/backend-e2e/src/admin.e2e-spec.ts | 40 +++--- apps/backend-e2e/src/maps.e2e-spec.ts | 11 +- .../app/modules/maps/map-list.service.spec.ts | 127 ++++++++++++------ .../src/app/modules/maps/map-list.service.ts | 50 ++++++- libs/test-utils/src/utils/s3.util.ts | 7 +- scripts/src/seed.script.ts | 26 +++- 6 files changed, 191 insertions(+), 70 deletions(-) diff --git a/apps/backend-e2e/src/admin.e2e-spec.ts b/apps/backend-e2e/src/admin.e2e-spec.ts index a6e63a8ca..51653cc7d 100644 --- a/apps/backend-e2e/src/admin.e2e-spec.ts +++ b/apps/backend-e2e/src/admin.e2e-spec.ts @@ -1769,23 +1769,27 @@ describe('Admin', () => { oldVersion.body.submissions + 1 ); - const approvedMapList = await fileStore.getMapListVersion( + const approved = await fileStore.getMapListVersion( FlatMapList.APPROVED, - newVersion.body.approved + newVersion.body.submissions ); - expect(approvedMapList).toHaveLength(1); - expect(approvedMapList[0]).toMatchObject({ + expect(approved.ident).toBe('MSML'); + expect(approved.numMaps).toBe(1); + expect(approved.data).toHaveLength(1); + expect(approved.data[0]).toMatchObject({ id: map.id, leaderboards: expect.anything(), info: expect.anything() }); - expect(approvedMapList[0]).not.toHaveProperty('zones'); + expect(approved.data[0]).not.toHaveProperty('zones'); - const submissionMapList = await fileStore.getMapListVersion( + const submission = await fileStore.getMapListVersion( FlatMapList.SUBMISSION, newVersion.body.submissions ); - expect(submissionMapList).toHaveLength(0); + expect(submission.ident).toBe('MSML'); + expect(submission.numMaps).toBe(0); + expect(submission.data).toHaveLength(0); }); it('should 400 when moving from FA to approved if leaderboards are not provided', async () => { @@ -1832,12 +1836,14 @@ describe('Admin', () => { expect(newVersion.body.submissions).toBe(oldVersion.body.submissions); expect(newVersion.body.approved).toBe(oldVersion.body.approved + 1); - const approvedMapList = await fileStore.getMapListVersion( - FlatMapList.APPROVED, - newVersion.body.approved + const { ident, numMaps, data } = await fileStore.getMapListVersion( + FlatMapList.SUBMISSION, + newVersion.body.submissions ); - expect(approvedMapList).toHaveLength(1); - expect(approvedMapList[0]).toMatchObject({ id: map2.id }); + expect(ident).toBe('MSML'); + expect(numMaps).toBe(1); + expect(data).toHaveLength(1); + expect(data[0].id).toBe(map2.id); }); it('should return 404 if map not found', () => @@ -1973,11 +1979,13 @@ describe('Admin', () => { expect(newVersion.body.submissions).toBe(oldVersion.body.submissions); expect(newVersion.body.approved).toBe(oldVersion.body.approved + 1); - const approvedMapList = await fileStore.getMapListVersion( - FlatMapList.APPROVED, - newVersion.body.approved + const { ident, numMaps, data } = await fileStore.getMapListVersion( + FlatMapList.SUBMISSION, + newVersion.body.submissions ); - expect(approvedMapList).toHaveLength(0); + expect(ident).toBe('MSML'); + expect(numMaps).toBe(0); + expect(data).toHaveLength(0); }); it('should return 404 if map not found', () => diff --git a/apps/backend-e2e/src/maps.e2e-spec.ts b/apps/backend-e2e/src/maps.e2e-spec.ts index 623c0beda..56cd42c17 100644 --- a/apps/backend-e2e/src/maps.e2e-spec.ts +++ b/apps/backend-e2e/src/maps.e2e-spec.ts @@ -2116,13 +2116,15 @@ describe('Maps', () => { oldListVersion.body.submissions + 1 ); - const submissionMapList = await fileStore.getMapListVersion( + const { ident, numMaps, data } = await fileStore.getMapListVersion( FlatMapList.SUBMISSION, newListVersion.body.submissions ); - expect(submissionMapList).toHaveLength(1); - expect(submissionMapList[0].id).toBe(map.id); - expect(submissionMapList[0]).not.toHaveProperty('zones'); + expect(ident).toBe('MSML'); + expect(numMaps).toBe(1); + expect(data).toHaveLength(1); + expect(data[0].id).toBe(map.id); + expect(data[0]).not.toHaveProperty('zones'); }); it('should 400 for bad zones', async () => { @@ -3012,7 +3014,6 @@ describe('Maps', () => { const newListVersion = await req.get({ url: 'maps/maplistversion', status: 200, - token }); diff --git a/apps/backend/src/app/modules/maps/map-list.service.spec.ts b/apps/backend/src/app/modules/maps/map-list.service.spec.ts index b7cbc9a26..cafad7e21 100644 --- a/apps/backend/src/app/modules/maps/map-list.service.spec.ts +++ b/apps/backend/src/app/modules/maps/map-list.service.spec.ts @@ -1,16 +1,24 @@ import { FlatMapList } from '@momentum/constants'; import { MapListService } from './map-list.service'; import { Test, TestingModule } from '@nestjs/testing'; -import { PRISMA_MOCK_PROVIDER } from '../../../../test/prisma-mock.const'; +import { + PRISMA_MOCK_PROVIDER, + PrismaMock +} from '../../../../test/prisma-mock.const'; import { mockDeep } from 'jest-mock-extended'; import { FileStoreService } from '../filestore/file-store.service'; +import { EXTENDED_PRISMA_SERVICE } from '../database/db.constants'; +import { promisify } from 'node:util'; +import * as zlib from 'node:zlib'; describe('MapListService', () => { describe('onModuleInit', () => { - let service: MapListService; + let service: MapListService, db: PrismaMock; const fileStoreMock = { listFileKeys: jest.fn(() => Promise.resolve([])), - deleteFiles: jest.fn() + storeFile: jest.fn(), + deleteFiles: jest.fn(), + deleteFile: jest.fn() }; beforeEach(async () => { @@ -25,57 +33,98 @@ describe('MapListService', () => { .compile(); service = module.get(MapListService); + db = module.get(EXTENDED_PRISMA_SERVICE); }); - it('should set version values based on files in storage', async () => { - fileStoreMock.listFileKeys.mockResolvedValueOnce([ - 'maplist/approved/1.dat' - ]); - fileStoreMock.listFileKeys.mockResolvedValueOnce([ - 'maplist/submissions/15012024.dat' - ]); + describe('onModuleInit', () => { + it('should set version values based on files in storage', async () => { + fileStoreMock.listFileKeys.mockResolvedValueOnce([ + 'maplist/approved/1.dat' + ]); + fileStoreMock.listFileKeys.mockResolvedValueOnce([ + 'maplist/submissions/15012024.dat' + ]); - await service.onModuleInit(); + await service.onModuleInit(); - expect(service['version']).toMatchObject({ - [FlatMapList.APPROVED]: 1, - [FlatMapList.SUBMISSION]: 15012024 + expect(service['version']).toMatchObject({ + [FlatMapList.APPROVED]: 1, + [FlatMapList.SUBMISSION]: 15012024 + }); + + expect(fileStoreMock.deleteFiles).not.toHaveBeenCalled(); }); - expect(fileStoreMock.deleteFiles).not.toHaveBeenCalled(); - }); + it('should set version to 0 when no versions exist in storage', async () => { + await service.onModuleInit(); - it('should set version to 0 when no versions exist in storage', async () => { - await service.onModuleInit(); + expect(service['version']).toMatchObject({ + [FlatMapList.APPROVED]: 0, + [FlatMapList.SUBMISSION]: 0 + }); - expect(service['version']).toMatchObject({ - [FlatMapList.APPROVED]: 0, - [FlatMapList.SUBMISSION]: 0 + expect(fileStoreMock.deleteFiles).not.toHaveBeenCalled(); }); - expect(fileStoreMock.deleteFiles).not.toHaveBeenCalled(); - }); + it('should pick most recent when multiple versions exist in storage, and wipe old versions', async () => { + fileStoreMock.listFileKeys.mockResolvedValueOnce([ + 'maplist/approved/4.dat', + 'maplist/approved/5.dat', + 'maplist/approved/3.dat', + 'maplist/approved/1.dat' + ]); - it('should pick most recent when multiple versions exist in storage, and wipe old versions', async () => { - fileStoreMock.listFileKeys.mockResolvedValueOnce([ - 'maplist/approved/4.dat', - 'maplist/approved/5.dat', - 'maplist/approved/3.dat', - 'maplist/approved/1.dat' - ]); + await service.onModuleInit(); - await service.onModuleInit(); + expect(service['version']).toMatchObject({ + [FlatMapList.APPROVED]: 5, + [FlatMapList.SUBMISSION]: 0 + }); - expect(service['version']).toMatchObject({ - [FlatMapList.APPROVED]: 5, - [FlatMapList.SUBMISSION]: 0 + expect(fileStoreMock.deleteFiles).toHaveBeenCalledWith([ + 'maplist/approved/4.dat', + 'maplist/approved/3.dat', + 'maplist/approved/1.dat' + ]); }); + }); + + describe('updateMapList', () => { + // prettier-ignore + const storedMap = { + id: 1, + name: 'The Map', + status: 0, + images: [ 'f2fecc26-34a0-448b-a3c7-007f43b9ec7e', 'a797e52e-3efc-4174-9f66-36e2c57ff55c', 'dee8bbd5-cec2-4341-9ddf-bdadd8337cdd' ], + info: { description: 'A map that makes me think I am becoming a better person', youtubeID: null, creationDate: '2024-09-27T10:18:42.318Z', mapID: 1 }, + leaderboards: [ { mapID: 12345, gamemode: 8, trackType: 0, trackNum: 1, style: 0, tier: 3, linear: false, type: 1, tags: [] } ], + credits: [ { type: 1, description: 'who am i', user: { id: 674, alias: 'John God', avatar: '0227a240393e6d62f539ee7b306dd048b0830eeb', steamID: '43576820710' } } ], + createdAt: '2024-09-27T22:31:12.846Z', + currentVersion: { id: 'fc89afc9-7ad2-4590-853c-a9ff4f41ddd5', versionNum: 3, bspHash: 'ddd39cbfc070e98e1e68131bab0f40df1d06645f', zoneHash: '608437d3bb461dd6e4abfff881f6b16827629d0b', hasVmf: false, submitterID: null, createdAt: '2024-09-27T18:12:52.465Z' } + }; + + it('should generate a Momentum Static Map List file and send to filestore', async () => { + db.mMap.findMany.mockResolvedValueOnce([storedMap as any]); - expect(fileStoreMock.deleteFiles).toHaveBeenCalledWith([ - 'maplist/approved/4.dat', - 'maplist/approved/3.dat', - 'maplist/approved/1.dat' - ]); + await service.updateMapList(FlatMapList.APPROVED); + + const buffer: Buffer = fileStoreMock.storeFile.mock.calls[0][0]; + expect(buffer.subarray(0, 4)).toMatchObject( + Buffer.from('MSML', 'utf8') + ); + expect(buffer.readUInt32LE(8)).toBe(1); + + const decompressed = await promisify(zlib.inflate)(buffer.subarray(12)); + expect(buffer.readUInt32LE(4)).toBe(decompressed.length); + + const parsed = JSON.parse(decompressed.toString('utf8')); + + // Can't do full toMatchObject, we've run through class-transformer. + expect(parsed[0]).toMatchObject({ + id: storedMap.id, + name: storedMap.name + }); + }); }); }); }); diff --git a/apps/backend/src/app/modules/maps/map-list.service.ts b/apps/backend/src/app/modules/maps/map-list.service.ts index 7dcc5668f..adce1957e 100644 --- a/apps/backend/src/app/modules/maps/map-list.service.ts +++ b/apps/backend/src/app/modules/maps/map-list.service.ts @@ -1,4 +1,4 @@ -import { Inject, Injectable, OnModuleInit } from '@nestjs/common'; +import { Inject, Injectable, Logger, OnModuleInit } from '@nestjs/common'; import { FileStoreService } from '../filestore/file-store.service'; import { EXTENDED_PRISMA_SERVICE } from '../database/db.constants'; import { ExtendedPrismaService } from '../database/prisma.extension'; @@ -27,6 +27,8 @@ export class MapListService implements OnModuleInit { [FlatMapList.SUBMISSION]: 0 }; + private readonly logger = new Logger('Map List Service'); + async onModuleInit(): Promise { for (const type of [FlatMapList.APPROVED, FlatMapList.SUBMISSION]) { const keys = await this.fileStoreService.listFileKeys(mapListDir(type)); @@ -83,27 +85,65 @@ export class MapListService implements OnModuleInit { } }, createdAt: true, - currentVersion: { omit: { zones: true, changelog: true } }, + currentVersion: { omit: { zones: true, changelog: true, mapID: true } }, ...(type === FlatMapList.SUBMISSION - ? { submission: true, versions: { omit: { zones: true } } } + ? { + submission: true, + versions: { omit: { zones: true, mapID: true } } + } : {}) } }); + const t1 = Date.now(); + // Convert to DTO then serialize back to JSON so any class-transformer // transformations are applied. const mapListJson = JSON.stringify( maps.map((map) => instanceToPlain(plainToInstance(MapDto, map))) ); - const compressed = await promisify(zlib.deflate)(mapListJson); + + // Momentum Static Map List + // + // -- Header [12 bytes] -- + // Ident [4 bytes - "MSML" 4D 53 4D 4C] + // Length of uncompressed data [4 bytes - uint32 LE] + // Total number of maps [4 bytes - uint32 LE] + // + // -- Contents [Variable] -- + // Deflate compressed map list + + // Not very memory-efficent, could be improved using streams, but if doing + // that we should hook up streaming uploads to API and I can't be fucked + // with the S3 API. (see https://stackoverflow.com/a/73332454). + // This takes me ~100ms for ~3000 maps. + // + // Hilariously, class-transformer serialization takes about 10 TIMES + // (~1000ms) the time to JSON.stringify, compress, and concat the buffers. + // So this isn't worth optimising whilst we're still using that piece of + // crap library. + const uncompressed = Buffer.from(mapListJson); + const header = Buffer.alloc(12); + + header.write('MSML', 0, 'utf8'); + header.writeUInt32LE(uncompressed.length, 4); + header.writeUInt32LE(maps.length, 8); + + const compressed = await promisify(zlib.deflate)(uncompressed); + + const outBuf = Buffer.concat([header, compressed]); const oldVersion = this.version[type]; const newVersion = this.updateMapListVersion(type); const oldKey = mapListPath(type, oldVersion); const newKey = mapListPath(type, newVersion); + this.logger.log( + `Updating ${type} map list from v${oldVersion} to v${newVersion}, ${maps.length} maps, encoding took ${Date.now() - t1}ms` + ); + await this.fileStoreService.deleteFile(oldKey); - await this.fileStoreService.storeFile(compressed, newKey); + await this.fileStoreService.storeFile(outBuf, newKey); } private updateMapListVersion(type: FlatMapList): number { diff --git a/libs/test-utils/src/utils/s3.util.ts b/libs/test-utils/src/utils/s3.util.ts index 27648e8c9..45ff46e96 100644 --- a/libs/test-utils/src/utils/s3.util.ts +++ b/libs/test-utils/src/utils/s3.util.ts @@ -156,6 +156,11 @@ export class FileStoreUtil { type === FlatMapList.APPROVED ? 'approved' : 'submissions' }/${version}.dat` ); - return JSON.parse(zlib.inflateSync(file).toString()); + return { + ident: file.subarray(0, 4).toString(), + uncompressedLength: file.readUInt32LE(4), + numMaps: file.readUInt32LE(8), + data: JSON.parse(zlib.inflateSync(file.subarray(12)).toString()) + }; } } diff --git a/scripts/src/seed.script.ts b/scripts/src/seed.script.ts index d86c97e0e..9678bf0d3 100644 --- a/scripts/src/seed.script.ts +++ b/scripts/src/seed.script.ts @@ -1042,9 +1042,12 @@ prismaWrapper(async (prisma: PrismaClient) => { } }, createdAt: true, - currentVersion: { omit: { zones: true, changelog: true } }, + currentVersion: { omit: { zones: true, changelog: true, mapID: true } }, ...(type === FlatMapList.SUBMISSION - ? { submission: true, versions: { omit: { zones: true } } } + ? { + submission: true, + versions: { omit: { zones: true, mapID: true } } + } : {}) } }); @@ -1087,12 +1090,27 @@ prismaWrapper(async (prisma: PrismaClient) => { writeFileSync(`./map-list-${type}.json`, mapListJson); } - const compressed = await promisify(zlib.deflate)(mapListJson); + // This is copied directly from map-list-service.ts, see there + const t1 = Date.now(); + + const uncompressed = Buffer.from(mapListJson); + const header = Buffer.alloc(12); + + header.write('MSML', 0, 'utf8'); + header.writeUInt32LE(uncompressed.length, 4); + header.writeUInt32LE(maps.length, 8); + + const compressed = await promisify(zlib.deflate)(uncompressed); + + const outBuf = Buffer.concat([header, compressed]); + + console.log(`Generated map list, encoding took ${Date.now() - t1}ms`); + await s3.send( new PutObjectCommand({ Bucket: s3BucketName, Key: mapListPath(type, 1), - Body: compressed + Body: outBuf }) ); }