Skip to content

Commit

Permalink
refactor(back): add header to static map list files
Browse files Browse the repository at this point in the history
  • Loading branch information
@tsa96
tsa96 committed Oct 23, 2024
1 parent f24be95 commit 0059ce3
Show file tree
Hide file tree
Showing 6 changed files with 191 additions and 70 deletions.
40 changes: 24 additions & 16 deletions apps/backend-e2e/src/admin.e2e-spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1769,23 +1769,27 @@ describe('Admin', () => {
oldVersion.body.submissions + 1
);

const approvedMapList = await fileStore.getMapListVersion(
const approved = await fileStore.getMapListVersion(
FlatMapList.APPROVED,
newVersion.body.approved
newVersion.body.submissions
);
expect(approvedMapList).toHaveLength(1);
expect(approvedMapList[0]).toMatchObject({
expect(approved.ident).toBe('MSML');
expect(approved.numMaps).toBe(1);
expect(approved.data).toHaveLength(1);
expect(approved.data[0]).toMatchObject({
id: map.id,
leaderboards: expect.anything(),
info: expect.anything()
});
expect(approvedMapList[0]).not.toHaveProperty('zones');
expect(approved.data[0]).not.toHaveProperty('zones');

const submissionMapList = await fileStore.getMapListVersion(
const submission = await fileStore.getMapListVersion(
FlatMapList.SUBMISSION,
newVersion.body.submissions
);
expect(submissionMapList).toHaveLength(0);
expect(submission.ident).toBe('MSML');
expect(submission.numMaps).toBe(0);
expect(submission.data).toHaveLength(0);
});

it('should 400 when moving from FA to approved if leaderboards are not provided', async () => {
Expand Down Expand Up @@ -1832,12 +1836,14 @@ describe('Admin', () => {
expect(newVersion.body.submissions).toBe(oldVersion.body.submissions);
expect(newVersion.body.approved).toBe(oldVersion.body.approved + 1);

const approvedMapList = await fileStore.getMapListVersion(
FlatMapList.APPROVED,
newVersion.body.approved
const { ident, numMaps, data } = await fileStore.getMapListVersion(
FlatMapList.SUBMISSION,
newVersion.body.submissions
);
expect(approvedMapList).toHaveLength(1);
expect(approvedMapList[0]).toMatchObject({ id: map2.id });
expect(ident).toBe('MSML');
expect(numMaps).toBe(1);
expect(data).toHaveLength(1);
expect(data[0].id).toBe(map2.id);
});

it('should return 404 if map not found', () =>
Expand Down Expand Up @@ -1973,11 +1979,13 @@ describe('Admin', () => {
expect(newVersion.body.submissions).toBe(oldVersion.body.submissions);
expect(newVersion.body.approved).toBe(oldVersion.body.approved + 1);

const approvedMapList = await fileStore.getMapListVersion(
FlatMapList.APPROVED,
newVersion.body.approved
const { ident, numMaps, data } = await fileStore.getMapListVersion(
FlatMapList.SUBMISSION,
newVersion.body.submissions
);
expect(approvedMapList).toHaveLength(0);
expect(ident).toBe('MSML');
expect(numMaps).toBe(0);
expect(data).toHaveLength(0);
});

it('should return 404 if map not found', () =>
Expand Down
11 changes: 6 additions & 5 deletions apps/backend-e2e/src/maps.e2e-spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2116,13 +2116,15 @@ describe('Maps', () => {
oldListVersion.body.submissions + 1
);

const submissionMapList = await fileStore.getMapListVersion(
const { ident, numMaps, data } = await fileStore.getMapListVersion(
FlatMapList.SUBMISSION,
newListVersion.body.submissions
);
expect(submissionMapList).toHaveLength(1);
expect(submissionMapList[0].id).toBe(map.id);
expect(submissionMapList[0]).not.toHaveProperty('zones');
expect(ident).toBe('MSML');
expect(numMaps).toBe(1);
expect(data).toHaveLength(1);
expect(data[0].id).toBe(map.id);
expect(data[0]).not.toHaveProperty('zones');
});

it('should 400 for bad zones', async () => {
Expand Down Expand Up @@ -3012,7 +3014,6 @@ describe('Maps', () => {
const newListVersion = await req.get({
url: 'maps/maplistversion',
status: 200,

token
});

Expand Down
127 changes: 88 additions & 39 deletions apps/backend/src/app/modules/maps/map-list.service.spec.ts
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,24 @@
import { FlatMapList } from '@momentum/constants';
import { MapListService } from './map-list.service';
import { Test, TestingModule } from '@nestjs/testing';
import { PRISMA_MOCK_PROVIDER } from '../../../../test/prisma-mock.const';
import {
PRISMA_MOCK_PROVIDER,
PrismaMock
} from '../../../../test/prisma-mock.const';
import { mockDeep } from 'jest-mock-extended';
import { FileStoreService } from '../filestore/file-store.service';
import { EXTENDED_PRISMA_SERVICE } from '../database/db.constants';
import { promisify } from 'node:util';
import * as zlib from 'node:zlib';

describe('MapListService', () => {
describe('onModuleInit', () => {
let service: MapListService;
let service: MapListService, db: PrismaMock;
const fileStoreMock = {
listFileKeys: jest.fn(() => Promise.resolve([])),
deleteFiles: jest.fn()
storeFile: jest.fn(),
deleteFiles: jest.fn(),
deleteFile: jest.fn()
};

beforeEach(async () => {
Expand All @@ -25,57 +33,98 @@ describe('MapListService', () => {
.compile();

service = module.get(MapListService);
db = module.get(EXTENDED_PRISMA_SERVICE);
});

it('should set version values based on files in storage', async () => {
fileStoreMock.listFileKeys.mockResolvedValueOnce([
'maplist/approved/1.dat'
]);
fileStoreMock.listFileKeys.mockResolvedValueOnce([
'maplist/submissions/15012024.dat'
]);
describe('onModuleInit', () => {
it('should set version values based on files in storage', async () => {
fileStoreMock.listFileKeys.mockResolvedValueOnce([
'maplist/approved/1.dat'
]);
fileStoreMock.listFileKeys.mockResolvedValueOnce([
'maplist/submissions/15012024.dat'
]);

await service.onModuleInit();
await service.onModuleInit();

expect(service['version']).toMatchObject({
[FlatMapList.APPROVED]: 1,
[FlatMapList.SUBMISSION]: 15012024
expect(service['version']).toMatchObject({
[FlatMapList.APPROVED]: 1,
[FlatMapList.SUBMISSION]: 15012024
});

expect(fileStoreMock.deleteFiles).not.toHaveBeenCalled();
});

expect(fileStoreMock.deleteFiles).not.toHaveBeenCalled();
});
it('should set version to 0 when no versions exist in storage', async () => {
await service.onModuleInit();

it('should set version to 0 when no versions exist in storage', async () => {
await service.onModuleInit();
expect(service['version']).toMatchObject({
[FlatMapList.APPROVED]: 0,
[FlatMapList.SUBMISSION]: 0
});

expect(service['version']).toMatchObject({
[FlatMapList.APPROVED]: 0,
[FlatMapList.SUBMISSION]: 0
expect(fileStoreMock.deleteFiles).not.toHaveBeenCalled();
});

expect(fileStoreMock.deleteFiles).not.toHaveBeenCalled();
});
it('should pick most recent when multiple versions exist in storage, and wipe old versions', async () => {
fileStoreMock.listFileKeys.mockResolvedValueOnce([
'maplist/approved/4.dat',
'maplist/approved/5.dat',
'maplist/approved/3.dat',
'maplist/approved/1.dat'
]);

it('should pick most recent when multiple versions exist in storage, and wipe old versions', async () => {
fileStoreMock.listFileKeys.mockResolvedValueOnce([
'maplist/approved/4.dat',
'maplist/approved/5.dat',
'maplist/approved/3.dat',
'maplist/approved/1.dat'
]);
await service.onModuleInit();

await service.onModuleInit();
expect(service['version']).toMatchObject({
[FlatMapList.APPROVED]: 5,
[FlatMapList.SUBMISSION]: 0
});

expect(service['version']).toMatchObject({
[FlatMapList.APPROVED]: 5,
[FlatMapList.SUBMISSION]: 0
expect(fileStoreMock.deleteFiles).toHaveBeenCalledWith([
'maplist/approved/4.dat',
'maplist/approved/3.dat',
'maplist/approved/1.dat'
]);
});
});

describe('updateMapList', () => {
// prettier-ignore
const storedMap = {
id: 1,
name: 'The Map',
status: 0,
images: [ 'f2fecc26-34a0-448b-a3c7-007f43b9ec7e', 'a797e52e-3efc-4174-9f66-36e2c57ff55c', 'dee8bbd5-cec2-4341-9ddf-bdadd8337cdd' ],
info: { description: 'A map that makes me think I am becoming a better person', youtubeID: null, creationDate: '2024-09-27T10:18:42.318Z', mapID: 1 },
leaderboards: [ { mapID: 12345, gamemode: 8, trackType: 0, trackNum: 1, style: 0, tier: 3, linear: false, type: 1, tags: [] } ],
credits: [ { type: 1, description: 'who am i', user: { id: 674, alias: 'John God', avatar: '0227a240393e6d62f539ee7b306dd048b0830eeb', steamID: '43576820710' } } ],
createdAt: '2024-09-27T22:31:12.846Z',
currentVersion: { id: 'fc89afc9-7ad2-4590-853c-a9ff4f41ddd5', versionNum: 3, bspHash: 'ddd39cbfc070e98e1e68131bab0f40df1d06645f', zoneHash: '608437d3bb461dd6e4abfff881f6b16827629d0b', hasVmf: false, submitterID: null, createdAt: '2024-09-27T18:12:52.465Z' }
};

it('should generate a Momentum Static Map List file and send to filestore', async () => {
db.mMap.findMany.mockResolvedValueOnce([storedMap as any]);

expect(fileStoreMock.deleteFiles).toHaveBeenCalledWith([
'maplist/approved/4.dat',
'maplist/approved/3.dat',
'maplist/approved/1.dat'
]);
await service.updateMapList(FlatMapList.APPROVED);

const buffer: Buffer = fileStoreMock.storeFile.mock.calls[0][0];
expect(buffer.subarray(0, 4)).toMatchObject(
Buffer.from('MSML', 'utf8')
);
expect(buffer.readUInt32LE(8)).toBe(1);

const decompressed = await promisify(zlib.inflate)(buffer.subarray(12));
expect(buffer.readUInt32LE(4)).toBe(decompressed.length);

const parsed = JSON.parse(decompressed.toString('utf8'));

// Can't do full toMatchObject, we've run through class-transformer.
expect(parsed[0]).toMatchObject({
id: storedMap.id,
name: storedMap.name
});
});
});
});
});
50 changes: 45 additions & 5 deletions apps/backend/src/app/modules/maps/map-list.service.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { Inject, Injectable, OnModuleInit } from '@nestjs/common';
import { Inject, Injectable, Logger, OnModuleInit } from '@nestjs/common';
import { FileStoreService } from '../filestore/file-store.service';
import { EXTENDED_PRISMA_SERVICE } from '../database/db.constants';
import { ExtendedPrismaService } from '../database/prisma.extension';
Expand Down Expand Up @@ -27,6 +27,8 @@ export class MapListService implements OnModuleInit {
[FlatMapList.SUBMISSION]: 0
};

private readonly logger = new Logger('Map List Service');

async onModuleInit(): Promise<void> {
for (const type of [FlatMapList.APPROVED, FlatMapList.SUBMISSION]) {
const keys = await this.fileStoreService.listFileKeys(mapListDir(type));
Expand Down Expand Up @@ -83,27 +85,65 @@ export class MapListService implements OnModuleInit {
}
},
createdAt: true,
currentVersion: { omit: { zones: true, changelog: true } },
currentVersion: { omit: { zones: true, changelog: true, mapID: true } },
...(type === FlatMapList.SUBMISSION
? { submission: true, versions: { omit: { zones: true } } }
? {
submission: true,
versions: { omit: { zones: true, mapID: true } }
}
: {})
}
});

const t1 = Date.now();

// Convert to DTO then serialize back to JSON so any class-transformer
// transformations are applied.
const mapListJson = JSON.stringify(
maps.map((map) => instanceToPlain(plainToInstance(MapDto, map)))
);
const compressed = await promisify(zlib.deflate)(mapListJson);

// Momentum Static Map List
//
// -- Header [12 bytes] --
// Ident [4 bytes - "MSML" 4D 53 4D 4C]
// Length of uncompressed data [4 bytes - uint32 LE]
// Total number of maps [4 bytes - uint32 LE]
//
// -- Contents [Variable] --
// Deflate compressed map list

// Not very memory-efficent, could be improved using streams, but if doing
// that we should hook up streaming uploads to API and I can't be fucked
// with the S3 API. (see https://stackoverflow.com/a/73332454).
// This takes me ~100ms for ~3000 maps.
//
// Hilariously, class-transformer serialization takes about 10 TIMES
// (~1000ms) the time to JSON.stringify, compress, and concat the buffers.
// So this isn't worth optimising whilst we're still using that piece of
// crap library.
const uncompressed = Buffer.from(mapListJson);
const header = Buffer.alloc(12);

header.write('MSML', 0, 'utf8');
header.writeUInt32LE(uncompressed.length, 4);
header.writeUInt32LE(maps.length, 8);

const compressed = await promisify(zlib.deflate)(uncompressed);

const outBuf = Buffer.concat([header, compressed]);

const oldVersion = this.version[type];
const newVersion = this.updateMapListVersion(type);
const oldKey = mapListPath(type, oldVersion);
const newKey = mapListPath(type, newVersion);

this.logger.log(
`Updating ${type} map list from v${oldVersion} to v${newVersion}, ${maps.length} maps, encoding took ${Date.now() - t1}ms`
);

await this.fileStoreService.deleteFile(oldKey);
await this.fileStoreService.storeFile(compressed, newKey);
await this.fileStoreService.storeFile(outBuf, newKey);
}

private updateMapListVersion(type: FlatMapList): number {
Expand Down
7 changes: 6 additions & 1 deletion libs/test-utils/src/utils/s3.util.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,11 @@ export class FileStoreUtil {
type === FlatMapList.APPROVED ? 'approved' : 'submissions'
}/${version}.dat`
);
return JSON.parse(zlib.inflateSync(file).toString());
return {
ident: file.subarray(0, 4).toString(),
uncompressedLength: file.readUInt32LE(4),
numMaps: file.readUInt32LE(8),
data: JSON.parse(zlib.inflateSync(file.subarray(12)).toString())
};
}
}
Loading

0 comments on commit 0059ce3

Please sign in to comment.