From 60269ebea37394acca7ebe24113acddd7d71bea4 Mon Sep 17 00:00:00 2001 From: Andrew Chou Date: Thu, 18 Jan 2024 18:40:18 -0500 Subject: [PATCH 01/11] refactor MediaServer and MapeoManager --- src/index.js | 1 + src/mapeo-manager.js | 23 +++++++++++++++---- src/media-server.js | 44 ++++++++++++++++-------------------- test-e2e/core-ownership.js | 3 +++ test-e2e/device-info.js | 9 ++++++++ test-e2e/manager-basic.js | 13 +++++++++++ test-e2e/media-server.js | 8 +++++++ test-e2e/project-settings.js | 3 +++ test-e2e/utils.js | 5 ++++ 9 files changed, 80 insertions(+), 29 deletions(-) diff --git a/src/index.js b/src/index.js index 51286ba9..e46c7bae 100644 --- a/src/index.js +++ b/src/index.js @@ -1 +1,2 @@ +export { MediaServer } from './media-server.js' export { MapeoManager } from './mapeo-manager.js' diff --git a/src/mapeo-manager.js b/src/mapeo-manager.js index 6b01af91..9c7d638b 100644 --- a/src/mapeo-manager.js +++ b/src/mapeo-manager.js @@ -11,6 +11,7 @@ import { TypedEmitter } from 'tiny-typed-emitter' import { IndexWriter } from './index-writer/index.js' import { MapeoProject, + kBlobStore, kProjectLeave, kSetOwnDeviceInfo, } from './mapeo-project.js' @@ -30,9 +31,11 @@ import { projectKeyToPublicId, } from './utils.js' import { RandomAccessFilePool } from './core-manager/random-access-file-pool.js' +import BlobServerPlugin from './fastify-plugins/blobs.js' +import IconServerPlugin from './fastify-plugins/icons.js' import { LocalPeers } from './local-peers.js' import { InviteApi } from './invite-api.js' -import { MediaServer } from './media-server.js' +import { BLOBS_PREFIX, ICONS_PREFIX } from './media-server.js' import { LocalDiscovery } from './discovery/local-discovery.js' import { Capabilities } from './capabilities.js' import NoiseSecretStream from '@hyperswarm/secret-stream' @@ -92,7 +95,7 @@ export class MapeoManager extends TypedEmitter { * @param {string} opts.projectMigrationsFolder path for drizzle migrations folder for project database * @param {string} opts.clientMigrationsFolder path for drizzle migrations folder for client database * @param {string | import('./types.js').CoreStorage} opts.coreStorage Folder for hypercore storage or a function that returns a RandomAccessStorage instance - * @param {{ port?: number, logger: import('fastify').FastifyServerOptions['logger'] }} [opts.mediaServerOpts] + * @param {import('./media-server.js').MediaServer} opts.mediaServer Media server instance */ constructor({ rootKey, @@ -100,7 +103,7 @@ export class MapeoManager extends TypedEmitter { projectMigrationsFolder, clientMigrationsFolder, coreStorage, - mediaServerOpts, + mediaServer, }) { super() this.#keyManager = new KeyManager(rootKey) @@ -160,8 +163,16 @@ export class MapeoManager extends TypedEmitter { this.#coreStorage = coreStorage } - this.#mediaServer = new MediaServer({ - logger: mediaServerOpts?.logger, + this.#mediaServer = mediaServer + this.#mediaServer.registerPlugin(BlobServerPlugin, { + prefix: BLOBS_PREFIX, + getBlobStore: async (projectPublicId) => { + const project = await this.getProject(projectPublicId) + return project[kBlobStore] + }, + }) + this.#mediaServer.registerPlugin(IconServerPlugin, { + prefix: ICONS_PREFIX, getProject: this.getProject.bind(this), }) @@ -654,6 +665,7 @@ export class MapeoManager extends TypedEmitter { return this.#invite } + // TODO: Can we remove this? /** * @param {import('./media-server.js').StartOpts} [opts] */ @@ -661,6 +673,7 @@ export class MapeoManager extends TypedEmitter { await this.#mediaServer.start(opts) } + // TODO: Can we remove this? async stopMediaServer() { await this.#mediaServer.stop() } diff --git a/src/media-server.js b/src/media-server.js index cbfebcde..d32b19ca 100644 --- a/src/media-server.js +++ b/src/media-server.js @@ -1,17 +1,14 @@ import { once } from 'events' import { promisify } from 'util' -import fastify from 'fastify' +import Fastify from 'fastify' import pTimeout from 'p-timeout' import StateMachine from 'start-stop-state-machine' -import BlobServerPlugin from './fastify-plugins/blobs.js' -import IconServerPlugin from './fastify-plugins/icons.js' - -import { kBlobStore } from './mapeo-project.js' - export const BLOBS_PREFIX = 'blobs' export const ICONS_PREFIX = 'icons' +export const kFastify = Symbol('fastify') + /** * @typedef {Object} StartOpts * @@ -27,29 +24,15 @@ export class MediaServer { #serverState /** - * @param {object} params - * @param {(projectPublicId: string) => Promise} params.getProject + * @param {Object} params * @param {import('fastify').FastifyServerOptions['logger']} [params.logger] */ - constructor({ getProject, logger }) { + constructor({ logger } = {}) { this.#fastifyStarted = false this.#host = '127.0.0.1' this.#port = 0 - this.#fastify = fastify({ logger }) - - this.#fastify.register(BlobServerPlugin, { - prefix: BLOBS_PREFIX, - getBlobStore: async (projectPublicId) => { - const project = await getProject(projectPublicId) - return project[kBlobStore] - }, - }) - - this.#fastify.register(IconServerPlugin, { - prefix: ICONS_PREFIX, - getProject, - }) + this.#fastify = Fastify({ logger }) this.#serverState = new StateMachine({ start: this.#startServer.bind(this), @@ -57,6 +40,19 @@ export class MediaServer { }) } + get [kFastify]() { + return this.#fastify + } + + /** + * @template {import('fastify').FastifyPluginOptions} Options + * @param {import('fastify').FastifyPluginAsync} plugin + * @param {Options} opts + */ + registerPlugin(plugin, opts) { + return this.#fastify.register(plugin, opts) + } + /** * @param {StartOpts} [opts] */ @@ -123,7 +119,7 @@ export class MediaServer { } /** - * @param {'blobs' | 'icons'} mediaType + * @param {'blobs' | 'icons' | 'maps'} mediaType * @returns {Promise} */ async getMediaAddress(mediaType) { diff --git a/test-e2e/core-ownership.js b/test-e2e/core-ownership.js index 910d4adb..57926183 100644 --- a/test-e2e/core-ownership.js +++ b/test-e2e/core-ownership.js @@ -5,10 +5,12 @@ import { kCoreOwnership } from '../src/mapeo-project.js' import { parseVersionId } from '@mapeo/schema' import RAM from 'random-access-memory' import { discoveryKey } from 'hypercore-crypto' +import { MediaServer } from '../src/media-server.js' test('CoreOwnership', async (t) => { const rootKey = KeyManager.generateRootKey() const km = new KeyManager(rootKey) + const mediaServer = new MediaServer() const manager = new MapeoManager({ rootKey, projectMigrationsFolder: new URL('../drizzle/project', import.meta.url) @@ -17,6 +19,7 @@ test('CoreOwnership', async (t) => { .pathname, dbFolder: ':memory:', coreStorage: () => new RAM(), + mediaServer, }) const projectId = await manager.createProject() diff --git a/test-e2e/device-info.js b/test-e2e/device-info.js index 25ebcb46..b2938d6b 100644 --- a/test-e2e/device-info.js +++ b/test-e2e/device-info.js @@ -4,6 +4,7 @@ import { KeyManager } from '@mapeo/crypto' import RAM from 'random-access-memory' import { MapeoManager } from '../src/mapeo-manager.js' +import { MediaServer } from '../src/media-server.js' const projectMigrationsFolder = new URL('../drizzle/project', import.meta.url) .pathname @@ -12,12 +13,14 @@ const clientMigrationsFolder = new URL('../drizzle/client', import.meta.url) test('write and read deviceInfo', async (t) => { const rootKey = KeyManager.generateRootKey() + const mediaServer = new MediaServer() const manager = new MapeoManager({ rootKey, projectMigrationsFolder, clientMigrationsFolder, dbFolder: ':memory:', coreStorage: () => new RAM(), + mediaServer, }) const info1 = { name: 'my device' } @@ -32,12 +35,14 @@ test('write and read deviceInfo', async (t) => { test('device info written to projects', (t) => { t.test('when creating project', async (st) => { + const mediaServer = new MediaServer() const manager = new MapeoManager({ rootKey: KeyManager.generateRootKey(), projectMigrationsFolder, clientMigrationsFolder, dbFolder: ':memory:', coreStorage: () => new RAM(), + mediaServer, }) await manager.setDeviceInfo({ name: 'mapeo' }) @@ -52,12 +57,14 @@ test('device info written to projects', (t) => { }) t.test('when adding project', async (st) => { + const mediaServer = new MediaServer() const manager = new MapeoManager({ rootKey: KeyManager.generateRootKey(), projectMigrationsFolder, clientMigrationsFolder, dbFolder: ':memory:', coreStorage: () => new RAM(), + mediaServer, }) await manager.setDeviceInfo({ name: 'mapeo' }) @@ -78,12 +85,14 @@ test('device info written to projects', (t) => { }) t.test('after updating global device info', async (st) => { + const mediaServer = new MediaServer() const manager = new MapeoManager({ rootKey: KeyManager.generateRootKey(), projectMigrationsFolder, clientMigrationsFolder, dbFolder: ':memory:', coreStorage: () => new RAM(), + mediaServer, }) await manager.setDeviceInfo({ name: 'before' }) diff --git a/test-e2e/manager-basic.js b/test-e2e/manager-basic.js index e171702e..f979e11d 100644 --- a/test-e2e/manager-basic.js +++ b/test-e2e/manager-basic.js @@ -4,6 +4,7 @@ import { randomBytes, createHash } from 'crypto' import { KeyManager } from '@mapeo/crypto' import RAM from 'random-access-memory' import { MapeoManager } from '../src/mapeo-manager.js' +import { MediaServer } from '../src/media-server.js' const projectMigrationsFolder = new URL('../drizzle/project', import.meta.url) .pathname @@ -11,12 +12,15 @@ const clientMigrationsFolder = new URL('../drizzle/client', import.meta.url) .pathname test('Managing created projects', async (t) => { + const mediaServer = new MediaServer() + const manager = new MapeoManager({ rootKey: KeyManager.generateRootKey(), projectMigrationsFolder, clientMigrationsFolder, dbFolder: ':memory:', coreStorage: () => new RAM(), + mediaServer, }) const project1Id = await manager.createProject() @@ -115,12 +119,14 @@ test('Managing created projects', async (t) => { }) test('Managing added projects', async (t) => { + const mediaServer = new MediaServer() const manager = new MapeoManager({ rootKey: KeyManager.generateRootKey(), projectMigrationsFolder, clientMigrationsFolder, dbFolder: ':memory:', coreStorage: () => new RAM(), + mediaServer, }) const project1Id = await manager.addProject( @@ -189,12 +195,14 @@ test('Managing added projects', async (t) => { }) test('Managing both created and added projects', async (t) => { + const mediaServer = new MediaServer() const manager = new MapeoManager({ rootKey: KeyManager.generateRootKey(), projectMigrationsFolder, clientMigrationsFolder, dbFolder: ':memory:', coreStorage: () => new RAM(), + mediaServer, }) const createdProjectId = await manager.createProject({ @@ -232,12 +240,14 @@ test('Managing both created and added projects', async (t) => { }) test('Manager cannot add project that already exists', async (t) => { + const mediaServer = new MediaServer() const manager = new MapeoManager({ rootKey: KeyManager.generateRootKey(), projectMigrationsFolder, clientMigrationsFolder, dbFolder: ':memory:', coreStorage: () => new RAM(), + mediaServer, }) const existingProjectId = await manager.createProject() @@ -259,6 +269,8 @@ test('Manager cannot add project that already exists', async (t) => { }) test('Consistent storage folders', async (t) => { + const mediaServer = new MediaServer() + /** @type {string[]} */ const storageNames = [] const manager = new MapeoManager({ @@ -266,6 +278,7 @@ test('Consistent storage folders', async (t) => { projectMigrationsFolder, clientMigrationsFolder, dbFolder: ':memory:', + mediaServer, coreStorage: (name) => { storageNames.push(name) return new RAM() diff --git a/test-e2e/media-server.js b/test-e2e/media-server.js index f210fb08..4a0a4ccf 100644 --- a/test-e2e/media-server.js +++ b/test-e2e/media-server.js @@ -10,6 +10,7 @@ import fs from 'fs/promises' import RAM from 'random-access-memory' import { MapeoManager } from '../src/mapeo-manager.js' +import { MediaServer } from '../src/media-server.js' const BLOB_FIXTURES_DIR = fileURLToPath( new URL('../tests/fixtures/blob-api/', import.meta.url) @@ -21,12 +22,15 @@ const clientMigrationsFolder = new URL('../drizzle/client', import.meta.url) .pathname test('start/stop lifecycle', async (t) => { + const mediaServer = new MediaServer() + const manager = new MapeoManager({ rootKey: KeyManager.generateRootKey(), projectMigrationsFolder, clientMigrationsFolder, dbFolder: ':memory:', coreStorage: () => new RAM(), + mediaServer, }) const project = await manager.getProject(await manager.createProject()) @@ -84,12 +88,14 @@ test('retrieving blobs using url', async (t) => { const clock = FakeTimers.install({ shouldAdvanceTime: true }) t.teardown(() => clock.uninstall()) + const mediaServer = new MediaServer() const manager = new MapeoManager({ rootKey: KeyManager.generateRootKey(), projectMigrationsFolder, clientMigrationsFolder, dbFolder: ':memory:', coreStorage: () => new RAM(), + mediaServer, }) const project = await manager.getProject(await manager.createProject()) @@ -175,12 +181,14 @@ test('retrieving icons using url', async (t) => { const clock = FakeTimers.install({ shouldAdvanceTime: true }) t.teardown(() => clock.uninstall()) + const mediaServer = new MediaServer() const manager = new MapeoManager({ rootKey: KeyManager.generateRootKey(), projectMigrationsFolder, clientMigrationsFolder, dbFolder: ':memory:', coreStorage: () => new RAM(), + mediaServer, }) const project = await manager.getProject(await manager.createProject()) diff --git a/test-e2e/project-settings.js b/test-e2e/project-settings.js index 709eae72..2dc52cbb 100644 --- a/test-e2e/project-settings.js +++ b/test-e2e/project-settings.js @@ -4,8 +4,10 @@ import { MapeoManager } from '../src/mapeo-manager.js' import { MapeoProject } from '../src/mapeo-project.js' import { removeUndefinedFields } from './utils.js' import RAM from 'random-access-memory' +import { MediaServer } from '../src/media-server.js' test('Project settings create, read, and update operations', async (t) => { + const mediaServer = new MediaServer() const manager = new MapeoManager({ rootKey: KeyManager.generateRootKey(), projectMigrationsFolder: new URL('../drizzle/project', import.meta.url) @@ -14,6 +16,7 @@ test('Project settings create, read, and update operations', async (t) => { .pathname, dbFolder: ':memory:', coreStorage: () => new RAM(), + mediaServer, }) const projectId = await manager.createProject() diff --git a/test-e2e/utils.js b/test-e2e/utils.js index 8c8b10ca..438f2c58 100644 --- a/test-e2e/utils.js +++ b/test-e2e/utils.js @@ -12,6 +12,7 @@ import { temporaryDirectory } from 'tempy' import fsPromises from 'node:fs/promises' import { MEMBER_ROLE_ID } from '../src/capabilities.js' import { kSyncState } from '../src/sync/sync-api.js' +import { MediaServer } from '../src/media-server.js' const FAST_TESTS = !!process.env.FAST_TESTS const projectMigrationsFolder = new URL('../drizzle/project', import.meta.url) @@ -174,6 +175,9 @@ export async function createManagers(count, t) { export function createManager(seed, t) { const dbFolder = FAST_TESTS ? ':memory:' : temporaryDirectory() const coreStorage = FAST_TESTS ? () => new RAM() : temporaryDirectory() + + const mediaServer = new MediaServer() + t.teardown(async () => { if (FAST_TESTS) return await Promise.all([ @@ -192,6 +196,7 @@ export function createManager(seed, t) { clientMigrationsFolder, dbFolder, coreStorage, + mediaServer, }) } From 0a39dd8896d2317eec7b9e9c30e34337eb80976c Mon Sep 17 00:00:00 2001 From: Andrew Chou Date: Fri, 19 Jan 2024 14:17:50 -0500 Subject: [PATCH 02/11] further refactor based on feedback --- src/fastify-controller.js | 91 +++++++++ src/fastify-plugins/utils.js | 29 +++ src/index.js | 2 +- src/mapeo-manager.js | 65 ++++--- src/media-server.js | 176 ------------------ test-e2e/core-ownership.js | 11 +- test-e2e/device-info.js | 18 +- test-e2e/manager-basic.js | 19 +- ...ia-server.js => manager-fastify-server.js} | 41 ++-- test-e2e/project-settings.js | 10 +- test-e2e/utils.js | 6 +- tests/fastify-controller.js | 31 +++ tests/media-server.js | 126 ------------- 13 files changed, 246 insertions(+), 379 deletions(-) create mode 100644 src/fastify-controller.js create mode 100644 src/fastify-plugins/utils.js delete mode 100644 src/media-server.js rename test-e2e/{media-server.js => manager-fastify-server.js} (89%) create mode 100644 tests/fastify-controller.js delete mode 100644 tests/media-server.js diff --git a/src/fastify-controller.js b/src/fastify-controller.js new file mode 100644 index 00000000..ffb82e35 --- /dev/null +++ b/src/fastify-controller.js @@ -0,0 +1,91 @@ +import { promisify } from 'util' +import StateMachine from 'start-stop-state-machine' + +/** + * @typedef {Object} StartOpts + * + * @property {string} [host] + * @property {number} [port] + */ + +// Class to properly manage the server lifecycle of a Fastify instance +export class FastifyController { + #fastify + #fastifyStarted + #host + #port + #serverState + + /** + * @param {Object} opts + * @param {import('fastify').FastifyInstance} opts.fastify + */ + constructor({ fastify }) { + this.#fastifyStarted = false + this.#host = '127.0.0.1' + this.#port = 0 + + this.#fastify = fastify + + this.#serverState = new StateMachine({ + start: this.#startServer.bind(this), + stop: this.#stopServer.bind(this), + }) + } + + /** + * @param {StartOpts} [opts] + */ + async #startServer({ host = '127.0.0.1', port = 0 } = {}) { + this.#host = host + this.#port = port + + if (!this.#fastifyStarted) { + await this.#fastify.listen({ host: this.#host, port: this.#port }) + this.#fastifyStarted = true + return + } + + const { server } = this.#fastify + + await new Promise((res, rej) => { + server.listen.call(server, { port: this.#port, host: this.#host }) + + server.once('listening', onListening) + server.once('error', onError) + + function onListening() { + server.removeListener('error', onError) + res(null) + } + + /** + * @param {Error} err + */ + function onError(err) { + server.removeListener('listening', onListening) + rej(err) + } + }) + } + + async #stopServer() { + const { server } = this.#fastify + await promisify(server.close.bind(server))() + } + + /** + * @param {StartOpts} [opts] + */ + async start(opts) { + await this.#serverState.start(opts) + } + + async started() { + return this.#serverState.started() + } + + async stop() { + await this.#serverState.stop() + } +} diff --git a/src/fastify-plugins/utils.js b/src/fastify-plugins/utils.js new file mode 100644 index 00000000..4fa56c9a --- /dev/null +++ b/src/fastify-plugins/utils.js @@ -0,0 +1,29 @@ +import { once } from 'node:events' + +/** + * @param {import('node:http').Server} server + * @returns {Promise} + */ +export async function getFastifyServerAddress(server) { + const address = server.address() + + if (!address) { + await once(server, 'listening') + return getFastifyServerAddress(server) + } + + if (typeof address === 'string') { + return address + } + + // Full address construction for non unix-socket address + // https://github.com/fastify/fastify/blob/7aa802ed224b91ca559edec469a6b903e89a7f88/lib/server.js#L413 + let addr = '' + if (address.address.indexOf(':') === -1) { + addr += address.address + ':' + address.port + } else { + addr += '[' + address.address + ']:' + address.port + } + + return 'http://' + addr +} diff --git a/src/index.js b/src/index.js index e46c7bae..2ca428eb 100644 --- a/src/index.js +++ b/src/index.js @@ -1,2 +1,2 @@ -export { MediaServer } from './media-server.js' +export { FastifyController as MediaServer } from './fastify-controller.js' export { MapeoManager } from './mapeo-manager.js' diff --git a/src/mapeo-manager.js b/src/mapeo-manager.js index 9c7d638b..952e2162 100644 --- a/src/mapeo-manager.js +++ b/src/mapeo-manager.js @@ -6,6 +6,7 @@ import { eq } from 'drizzle-orm' import { drizzle } from 'drizzle-orm/better-sqlite3' import { migrate } from 'drizzle-orm/better-sqlite3/migrator' import Hypercore from 'hypercore' +import pTimeout from 'p-timeout' import { TypedEmitter } from 'tiny-typed-emitter' import { IndexWriter } from './index-writer/index.js' @@ -33,9 +34,9 @@ import { import { RandomAccessFilePool } from './core-manager/random-access-file-pool.js' import BlobServerPlugin from './fastify-plugins/blobs.js' import IconServerPlugin from './fastify-plugins/icons.js' +import { getFastifyServerAddress } from './fastify-plugins/utils.js' import { LocalPeers } from './local-peers.js' import { InviteApi } from './invite-api.js' -import { BLOBS_PREFIX, ICONS_PREFIX } from './media-server.js' import { LocalDiscovery } from './discovery/local-discovery.js' import { Capabilities } from './capabilities.js' import NoiseSecretStream from '@hyperswarm/secret-stream' @@ -53,6 +54,10 @@ const CLIENT_SQLITE_FILE_NAME = 'client.db' // other things e.g. SQLite and other parts of the app. const MAX_FILE_DESCRIPTORS = 768 +// Prefix names for routes registered with http server +const BLOBS_PREFIX = 'blobs' +const ICONS_PREFIX = 'icons' + export const kRPC = Symbol('rpc') export const kManagerReplicate = Symbol('replicate manager') @@ -83,7 +88,7 @@ export class MapeoManager extends TypedEmitter { #deviceId #localPeers #invite - #mediaServer + #fastify #localDiscovery #loggerBase #l @@ -95,7 +100,7 @@ export class MapeoManager extends TypedEmitter { * @param {string} opts.projectMigrationsFolder path for drizzle migrations folder for project database * @param {string} opts.clientMigrationsFolder path for drizzle migrations folder for client database * @param {string | import('./types.js').CoreStorage} opts.coreStorage Folder for hypercore storage or a function that returns a RandomAccessStorage instance - * @param {import('./media-server.js').MediaServer} opts.mediaServer Media server instance + * @param {import('fastify').FastifyInstance} opts.fastify Fastify server instance */ constructor({ rootKey, @@ -103,7 +108,7 @@ export class MapeoManager extends TypedEmitter { projectMigrationsFolder, clientMigrationsFolder, coreStorage, - mediaServer, + fastify, }) { super() this.#keyManager = new KeyManager(rootKey) @@ -163,15 +168,15 @@ export class MapeoManager extends TypedEmitter { this.#coreStorage = coreStorage } - this.#mediaServer = mediaServer - this.#mediaServer.registerPlugin(BlobServerPlugin, { + this.#fastify = fastify + this.#fastify.register(BlobServerPlugin, { prefix: BLOBS_PREFIX, getBlobStore: async (projectPublicId) => { const project = await this.getProject(projectPublicId) return project[kBlobStore] }, }) - this.#mediaServer.registerPlugin(IconServerPlugin, { + this.#fastify.register(IconServerPlugin, { prefix: ICONS_PREFIX, getProject: this.getProject.bind(this), }) @@ -210,6 +215,35 @@ export class MapeoManager extends TypedEmitter { return this.#replicate(noiseStream) } + /** + * @param {'blobs' | 'icons' | 'maps'} mediaType + * @returns {Promise} + */ + async #getMediaAddress(mediaType) { + /** @type {string | null} */ + let prefix = null + + switch (mediaType) { + case 'blobs': { + prefix = BLOBS_PREFIX + break + } + case 'icons': { + prefix = ICONS_PREFIX + break + } + default: { + throw new Error(`Unsupported media type ${mediaType}`) + } + } + + const base = await pTimeout(getFastifyServerAddress(this.#fastify.server), { + milliseconds: 1000, + }) + + return base + '/' + prefix + } + /** * @param {NoiseSecretStream} noiseStream */ @@ -414,9 +448,7 @@ export class MapeoManager extends TypedEmitter { sharedIndexWriter: this.#projectSettingsIndexWriter, localPeers: this.#localPeers, logger: this.#loggerBase, - getMediaBaseUrl: this.#mediaServer.getMediaAddress.bind( - this.#mediaServer - ), + getMediaBaseUrl: this.#getMediaAddress.bind(this), }) } @@ -665,19 +697,6 @@ export class MapeoManager extends TypedEmitter { return this.#invite } - // TODO: Can we remove this? - /** - * @param {import('./media-server.js').StartOpts} [opts] - */ - async startMediaServer(opts) { - await this.#mediaServer.start(opts) - } - - // TODO: Can we remove this? - async stopMediaServer() { - await this.#mediaServer.stop() - } - async startLocalPeerDiscovery() { return this.#localDiscovery.start() } diff --git a/src/media-server.js b/src/media-server.js deleted file mode 100644 index d32b19ca..00000000 --- a/src/media-server.js +++ /dev/null @@ -1,176 +0,0 @@ -import { once } from 'events' -import { promisify } from 'util' -import Fastify from 'fastify' -import pTimeout from 'p-timeout' -import StateMachine from 'start-stop-state-machine' - -export const BLOBS_PREFIX = 'blobs' -export const ICONS_PREFIX = 'icons' - -export const kFastify = Symbol('fastify') - -/** - * @typedef {Object} StartOpts - * - * @property {string} [host] - * @property {number} [port] - */ - -export class MediaServer { - #fastify - #fastifyStarted - #host - #port - #serverState - - /** - * @param {Object} params - * @param {import('fastify').FastifyServerOptions['logger']} [params.logger] - */ - constructor({ logger } = {}) { - this.#fastifyStarted = false - this.#host = '127.0.0.1' - this.#port = 0 - - this.#fastify = Fastify({ logger }) - - this.#serverState = new StateMachine({ - start: this.#startServer.bind(this), - stop: this.#stopServer.bind(this), - }) - } - - get [kFastify]() { - return this.#fastify - } - - /** - * @template {import('fastify').FastifyPluginOptions} Options - * @param {import('fastify').FastifyPluginAsync} plugin - * @param {Options} opts - */ - registerPlugin(plugin, opts) { - return this.#fastify.register(plugin, opts) - } - - /** - * @param {StartOpts} [opts] - */ - async #startServer({ host = '127.0.0.1', port = 0 } = {}) { - this.#host = host - this.#port = port - - if (!this.#fastifyStarted) { - await this.#fastify.listen({ host: this.#host, port: this.#port }) - this.#fastifyStarted = true - return - } - - const { server } = this.#fastify - - await new Promise((res, rej) => { - server.listen.call(server, { port: this.#port, host: this.#host }) - - server.once('listening', onListening) - server.once('error', onError) - - function onListening() { - server.removeListener('error', onError) - res(null) - } - - /** - * @param {Error} err - */ - function onError(err) { - server.removeListener('listening', onListening) - rej(err) - } - }) - } - - async #stopServer() { - const { server } = this.#fastify - await promisify(server.close.bind(server))() - } - - /** - * @returns {Promise} - */ - async #getAddress() { - return pTimeout(getServerAddress(this.#fastify.server), { - milliseconds: 1000, - }) - } - - /** - * @param {StartOpts} [opts] - */ - async start(opts) { - await this.#serverState.start(opts) - } - - async started() { - return this.#serverState.started() - } - - async stop() { - await this.#serverState.stop() - } - - /** - * @param {'blobs' | 'icons' | 'maps'} mediaType - * @returns {Promise} - */ - async getMediaAddress(mediaType) { - /** @type {string | null} */ - let prefix = null - - switch (mediaType) { - case 'blobs': { - prefix = BLOBS_PREFIX - break - } - case 'icons': { - prefix = ICONS_PREFIX - break - } - default: { - throw new Error(`Unsupported media type ${mediaType}`) - } - } - - const base = await this.#getAddress() - - return base + '/' + prefix - } -} - -/** - * @param {import('node:http').Server} server - * - * @returns {Promise} - */ -async function getServerAddress(server) { - const address = server.address() - - if (!address) { - await once(server, 'listening') - return getServerAddress(server) - } - - if (typeof address === 'string') { - return address - } - - // Full address construction for non unix-socket address - // https://github.com/fastify/fastify/blob/7aa802ed224b91ca559edec469a6b903e89a7f88/lib/server.js#L413 - let addr = '' - if (address.address.indexOf(':') === -1) { - addr += address.address + ':' + address.port - } else { - addr += '[' + address.address + ']:' + address.port - } - - return 'http://' + addr -} diff --git a/test-e2e/core-ownership.js b/test-e2e/core-ownership.js index 57926183..caaecaaf 100644 --- a/test-e2e/core-ownership.js +++ b/test-e2e/core-ownership.js @@ -1,16 +1,17 @@ import { test } from 'brittle' import { KeyManager } from '@mapeo/crypto' -import { MapeoManager } from '../src/mapeo-manager.js' -import { kCoreOwnership } from '../src/mapeo-project.js' import { parseVersionId } from '@mapeo/schema' import RAM from 'random-access-memory' import { discoveryKey } from 'hypercore-crypto' -import { MediaServer } from '../src/media-server.js' +import Fastify from 'fastify' + +import { kCoreOwnership } from '../src/mapeo-project.js' +import { MapeoManager } from '../src/mapeo-manager.js' test('CoreOwnership', async (t) => { const rootKey = KeyManager.generateRootKey() const km = new KeyManager(rootKey) - const mediaServer = new MediaServer() + const fastify = Fastify() const manager = new MapeoManager({ rootKey, projectMigrationsFolder: new URL('../drizzle/project', import.meta.url) @@ -19,7 +20,7 @@ test('CoreOwnership', async (t) => { .pathname, dbFolder: ':memory:', coreStorage: () => new RAM(), - mediaServer, + fastify, }) const projectId = await manager.createProject() diff --git a/test-e2e/device-info.js b/test-e2e/device-info.js index b2938d6b..ec0781cc 100644 --- a/test-e2e/device-info.js +++ b/test-e2e/device-info.js @@ -2,9 +2,9 @@ import { test } from 'brittle' import { randomBytes } from 'crypto' import { KeyManager } from '@mapeo/crypto' import RAM from 'random-access-memory' +import Fastify from 'fastify' import { MapeoManager } from '../src/mapeo-manager.js' -import { MediaServer } from '../src/media-server.js' const projectMigrationsFolder = new URL('../drizzle/project', import.meta.url) .pathname @@ -12,15 +12,15 @@ const clientMigrationsFolder = new URL('../drizzle/client', import.meta.url) .pathname test('write and read deviceInfo', async (t) => { + const fastify = Fastify() const rootKey = KeyManager.generateRootKey() - const mediaServer = new MediaServer() const manager = new MapeoManager({ rootKey, projectMigrationsFolder, clientMigrationsFolder, dbFolder: ':memory:', coreStorage: () => new RAM(), - mediaServer, + fastify, }) const info1 = { name: 'my device' } @@ -35,14 +35,14 @@ test('write and read deviceInfo', async (t) => { test('device info written to projects', (t) => { t.test('when creating project', async (st) => { - const mediaServer = new MediaServer() + const fastify = Fastify() const manager = new MapeoManager({ rootKey: KeyManager.generateRootKey(), projectMigrationsFolder, clientMigrationsFolder, dbFolder: ':memory:', coreStorage: () => new RAM(), - mediaServer, + fastify, }) await manager.setDeviceInfo({ name: 'mapeo' }) @@ -57,14 +57,14 @@ test('device info written to projects', (t) => { }) t.test('when adding project', async (st) => { - const mediaServer = new MediaServer() + const fastify = Fastify() const manager = new MapeoManager({ rootKey: KeyManager.generateRootKey(), projectMigrationsFolder, clientMigrationsFolder, dbFolder: ':memory:', coreStorage: () => new RAM(), - mediaServer, + fastify, }) await manager.setDeviceInfo({ name: 'mapeo' }) @@ -85,14 +85,14 @@ test('device info written to projects', (t) => { }) t.test('after updating global device info', async (st) => { - const mediaServer = new MediaServer() + const fastify = Fastify() const manager = new MapeoManager({ rootKey: KeyManager.generateRootKey(), projectMigrationsFolder, clientMigrationsFolder, dbFolder: ':memory:', coreStorage: () => new RAM(), - mediaServer, + fastify, }) await manager.setDeviceInfo({ name: 'before' }) diff --git a/test-e2e/manager-basic.js b/test-e2e/manager-basic.js index f979e11d..14f46ac1 100644 --- a/test-e2e/manager-basic.js +++ b/test-e2e/manager-basic.js @@ -4,7 +4,7 @@ import { randomBytes, createHash } from 'crypto' import { KeyManager } from '@mapeo/crypto' import RAM from 'random-access-memory' import { MapeoManager } from '../src/mapeo-manager.js' -import { MediaServer } from '../src/media-server.js' +import Fastify from 'fastify' const projectMigrationsFolder = new URL('../drizzle/project', import.meta.url) .pathname @@ -12,15 +12,13 @@ const clientMigrationsFolder = new URL('../drizzle/client', import.meta.url) .pathname test('Managing created projects', async (t) => { - const mediaServer = new MediaServer() - const manager = new MapeoManager({ rootKey: KeyManager.generateRootKey(), projectMigrationsFolder, clientMigrationsFolder, dbFolder: ':memory:', coreStorage: () => new RAM(), - mediaServer, + fastify: Fastify(), }) const project1Id = await manager.createProject() @@ -119,14 +117,13 @@ test('Managing created projects', async (t) => { }) test('Managing added projects', async (t) => { - const mediaServer = new MediaServer() const manager = new MapeoManager({ rootKey: KeyManager.generateRootKey(), projectMigrationsFolder, clientMigrationsFolder, dbFolder: ':memory:', coreStorage: () => new RAM(), - mediaServer, + fastify: Fastify(), }) const project1Id = await manager.addProject( @@ -195,14 +192,13 @@ test('Managing added projects', async (t) => { }) test('Managing both created and added projects', async (t) => { - const mediaServer = new MediaServer() const manager = new MapeoManager({ rootKey: KeyManager.generateRootKey(), projectMigrationsFolder, clientMigrationsFolder, dbFolder: ':memory:', coreStorage: () => new RAM(), - mediaServer, + fastify: Fastify(), }) const createdProjectId = await manager.createProject({ @@ -240,14 +236,13 @@ test('Managing both created and added projects', async (t) => { }) test('Manager cannot add project that already exists', async (t) => { - const mediaServer = new MediaServer() const manager = new MapeoManager({ rootKey: KeyManager.generateRootKey(), projectMigrationsFolder, clientMigrationsFolder, dbFolder: ':memory:', coreStorage: () => new RAM(), - mediaServer, + fastify: Fastify(), }) const existingProjectId = await manager.createProject() @@ -269,8 +264,6 @@ test('Manager cannot add project that already exists', async (t) => { }) test('Consistent storage folders', async (t) => { - const mediaServer = new MediaServer() - /** @type {string[]} */ const storageNames = [] const manager = new MapeoManager({ @@ -278,7 +271,7 @@ test('Consistent storage folders', async (t) => { projectMigrationsFolder, clientMigrationsFolder, dbFolder: ':memory:', - mediaServer, + fastify: Fastify(), coreStorage: (name) => { storageNames.push(name) return new RAM() diff --git a/test-e2e/media-server.js b/test-e2e/manager-fastify-server.js similarity index 89% rename from test-e2e/media-server.js rename to test-e2e/manager-fastify-server.js index 4a0a4ccf..cb24902b 100644 --- a/test-e2e/media-server.js +++ b/test-e2e/manager-fastify-server.js @@ -8,9 +8,10 @@ import FakeTimers from '@sinonjs/fake-timers' import { Agent, fetch as uFetch } from 'undici' import fs from 'fs/promises' import RAM from 'random-access-memory' +import Fastify from 'fastify' import { MapeoManager } from '../src/mapeo-manager.js' -import { MediaServer } from '../src/media-server.js' +import { FastifyController } from '../src/fastify-controller.js' const BLOB_FIXTURES_DIR = fileURLToPath( new URL('../tests/fixtures/blob-api/', import.meta.url) @@ -22,7 +23,8 @@ const clientMigrationsFolder = new URL('../drizzle/client', import.meta.url) .pathname test('start/stop lifecycle', async (t) => { - const mediaServer = new MediaServer() + const fastify = Fastify() + const fastifyController = new FastifyController({ fastify }) const manager = new MapeoManager({ rootKey: KeyManager.generateRootKey(), @@ -30,12 +32,13 @@ test('start/stop lifecycle', async (t) => { clientMigrationsFolder, dbFolder: ':memory:', coreStorage: () => new RAM(), - mediaServer, + fastify, }) - const project = await manager.getProject(await manager.createProject()) + // Manager should await for the server to start internally + fastifyController.start() - await manager.startMediaServer() + const project = await manager.getProject(await manager.createProject()) const blobUrl1 = await project.$blobs.getUrl({ driveId: randomBytes(32).toString('hex'), @@ -46,8 +49,6 @@ test('start/stop lifecycle', async (t) => { const response1 = await fetch(blobUrl1) t.is(response1.status, 404, 'server started and listening') - await manager.startMediaServer() - const blobUrl2 = await project.$blobs.getUrl({ driveId: randomBytes(32).toString('hex'), name: randomBytes(8).toString('hex'), @@ -60,13 +61,14 @@ test('start/stop lifecycle', async (t) => { 'server port is the same' ) - await manager.stopMediaServer() + await fastifyController.stop() await t.exception.all(async () => { await fetch(blobUrl2) }, 'failed to fetch due to connection error') - await manager.startMediaServer() + // Manager should await for the server to start internally + fastifyController.start() const blobUrl3 = await project.$blobs.getUrl({ driveId: randomBytes(32).toString('hex'), @@ -77,7 +79,7 @@ test('start/stop lifecycle', async (t) => { const response3 = await fetch(blobUrl3) t.is(response3.status, 404, 'server started and listening') - await manager.stopMediaServer() + await fastifyController.stop() await t.exception.all(async () => { await fetch(blobUrl3) @@ -88,14 +90,15 @@ test('retrieving blobs using url', async (t) => { const clock = FakeTimers.install({ shouldAdvanceTime: true }) t.teardown(() => clock.uninstall()) - const mediaServer = new MediaServer() + const fastify = Fastify() + const fastifyController = new FastifyController({ fastify }) const manager = new MapeoManager({ rootKey: KeyManager.generateRootKey(), projectMigrationsFolder, clientMigrationsFolder, dbFolder: ':memory:', coreStorage: () => new RAM(), - mediaServer, + fastify, }) const project = await manager.getProject(await manager.createProject()) @@ -112,7 +115,8 @@ test('retrieving blobs using url', async (t) => { clock.tick(100_000) await exceptionPromise1 - await manager.startMediaServer() + // Manager should await for the server to start internally + fastifyController.start() await t.test('blob does not exist', async (st) => { const blobUrl = await project.$blobs.getUrl({ @@ -163,7 +167,7 @@ test('retrieving blobs using url', async (t) => { st.alike(body, expected, 'matching reponse body') }) - await manager.stopMediaServer() + await fastifyController.stop() const exceptionPromise2 = t.exception(async () => { await project.$blobs.getUrl({ @@ -181,14 +185,15 @@ test('retrieving icons using url', async (t) => { const clock = FakeTimers.install({ shouldAdvanceTime: true }) t.teardown(() => clock.uninstall()) - const mediaServer = new MediaServer() + const fastify = Fastify() + const fastifyController = new FastifyController({ fastify }) const manager = new MapeoManager({ rootKey: KeyManager.generateRootKey(), projectMigrationsFolder, clientMigrationsFolder, dbFolder: ':memory:', coreStorage: () => new RAM(), - mediaServer, + fastify, }) const project = await manager.getProject(await manager.createProject()) @@ -204,7 +209,7 @@ test('retrieving icons using url', async (t) => { clock.tick(100_000) await exceptionPromise1 - await manager.startMediaServer() + await fastifyController.start() await t.test('icon does not exist', async (st) => { const nonExistentIconId = randomBytes(32).toString('hex') @@ -263,7 +268,7 @@ test('retrieving icons using url', async (t) => { st.alike(body, iconBuffer, 'matching response body') }) - await manager.stopMediaServer() + await fastifyController.stop() const exceptionPromise2 = t.exception(async () => { await project.$icons.getIconUrl(randomBytes(32).toString('hex'), { diff --git a/test-e2e/project-settings.js b/test-e2e/project-settings.js index 2dc52cbb..9a8c0b27 100644 --- a/test-e2e/project-settings.js +++ b/test-e2e/project-settings.js @@ -1,13 +1,15 @@ import { test } from 'brittle' import { KeyManager } from '@mapeo/crypto' +import RAM from 'random-access-memory' +import Fastify from 'fastify' + import { MapeoManager } from '../src/mapeo-manager.js' import { MapeoProject } from '../src/mapeo-project.js' import { removeUndefinedFields } from './utils.js' -import RAM from 'random-access-memory' -import { MediaServer } from '../src/media-server.js' test('Project settings create, read, and update operations', async (t) => { - const mediaServer = new MediaServer() + const fastify = Fastify() + const manager = new MapeoManager({ rootKey: KeyManager.generateRootKey(), projectMigrationsFolder: new URL('../drizzle/project', import.meta.url) @@ -16,7 +18,7 @@ test('Project settings create, read, and update operations', async (t) => { .pathname, dbFolder: ':memory:', coreStorage: () => new RAM(), - mediaServer, + fastify, }) const projectId = await manager.createProject() diff --git a/test-e2e/utils.js b/test-e2e/utils.js index 438f2c58..44453658 100644 --- a/test-e2e/utils.js +++ b/test-e2e/utils.js @@ -1,6 +1,7 @@ // @ts-check import sodium from 'sodium-universal' import RAM from 'random-access-memory' +import Fastify from 'fastify' import { MapeoManager } from '../src/index.js' import { kManagerReplicate, kRPC } from '../src/mapeo-manager.js' @@ -12,7 +13,6 @@ import { temporaryDirectory } from 'tempy' import fsPromises from 'node:fs/promises' import { MEMBER_ROLE_ID } from '../src/capabilities.js' import { kSyncState } from '../src/sync/sync-api.js' -import { MediaServer } from '../src/media-server.js' const FAST_TESTS = !!process.env.FAST_TESTS const projectMigrationsFolder = new URL('../drizzle/project', import.meta.url) @@ -176,8 +176,6 @@ export function createManager(seed, t) { const dbFolder = FAST_TESTS ? ':memory:' : temporaryDirectory() const coreStorage = FAST_TESTS ? () => new RAM() : temporaryDirectory() - const mediaServer = new MediaServer() - t.teardown(async () => { if (FAST_TESTS) return await Promise.all([ @@ -196,7 +194,7 @@ export function createManager(seed, t) { clientMigrationsFolder, dbFolder, coreStorage, - mediaServer, + fastify: Fastify(), }) } diff --git a/tests/fastify-controller.js b/tests/fastify-controller.js new file mode 100644 index 00000000..46ee1cff --- /dev/null +++ b/tests/fastify-controller.js @@ -0,0 +1,31 @@ +// @ts-check +import { test } from 'brittle' +import Fastify from 'fastify' + +import { FastifyController } from '../src/fastify-controller.js' + +test('lifecycle', async (t) => { + const fastify = Fastify() + const fastifyController = new FastifyController({ fastify }) + + const startOptsFixtures = [ + {}, + { port: 1234 }, + { port: 4321, host: '0.0.0.0' }, + { host: '0.0.0.0' }, + ] + + for (const opts of startOptsFixtures) { + await fastifyController.start(opts) + await fastifyController.start(opts) + await fastifyController.stop() + await fastifyController.stop() + + fastifyController.start(opts) + await fastifyController.started() + await fastifyController.started() + await fastifyController.stop() + + t.pass('server lifecycle works with valid opts') + } +}) diff --git a/tests/media-server.js b/tests/media-server.js deleted file mode 100644 index 32cfbac2..00000000 --- a/tests/media-server.js +++ /dev/null @@ -1,126 +0,0 @@ -// @ts-check -import { test } from 'brittle' -import FakeTimers from '@sinonjs/fake-timers' -import { BLOBS_PREFIX, ICONS_PREFIX, MediaServer } from '../src/media-server.js' - -const MEDIA_TYPES = /** @type {const} */ ([BLOBS_PREFIX, ICONS_PREFIX]) - -test('lifecycle', async (t) => { - const server = new MediaServer({ - getProject: async () => { - throw new Error("Shouldn't be calling") - }, - }) - - const startOptsFixtures = [ - {}, - { port: 1234 }, - { port: 4321, host: '0.0.0.0' }, - { host: '0.0.0.0' }, - ] - - for (const opts of startOptsFixtures) { - await server.start(opts) - await server.start(opts) - await server.stop() - await server.stop() - - server.start(opts) - await server.started() - await server.started() - await server.stop() - - t.pass('server lifecycle works with valid opts') - } -}) - -test('getMediaAddress()', async (t) => { - const clock = FakeTimers.install({ shouldAdvanceTime: true }) - - t.teardown(() => clock.uninstall()) - - const server = new MediaServer({ - getProject: async () => { - throw new Error("Shouldn't be calling") - }, - }) - - const exceptionPromise = t.exception(async () => { - await server.getMediaAddress('blobs') - }, 'getMediaAddress() throws before start() is called') - - clock.tick(10_000) - - await exceptionPromise - - const startOptsFixtures = [ - {}, - { port: 1234 }, - { port: 4321, host: '0.0.0.0' }, - { host: '0.0.0.0' }, - ] - - for (const startOpts of startOptsFixtures) { - const exceptionPromiseBlobs = t.exception(async () => { - await server.getMediaAddress('blobs') - }, 'getting media address fails if start() has not been called yet') - - clock.tick(10_000) - - await exceptionPromiseBlobs - - const exceptionPromiseIcons = t.exception(async () => { - await server.getMediaAddress('icons') - }, 'getting media address fails if start() has not been called yet') - - clock.tick(10_000) - - await exceptionPromiseIcons - - await server.start(startOpts) - - for (const mediaType of MEDIA_TYPES) { - const address = await server.getMediaAddress(mediaType) - - t.ok(address, 'address is retrievable after starting server') - - const parsedUrl = new URL(address) - - t.ok( - parsedUrl.pathname.startsWith('/' + mediaType), - `${mediaType} url starts with '${mediaType}' prefix` - ) - - t.is(parsedUrl.protocol, 'http:', 'url uses http protocol') - - const expectedHostname = startOpts.host || '127.0.0.1' - - t.is(parsedUrl.hostname, expectedHostname, 'expected hostname') - - if (typeof startOpts.port === 'number') { - t.is( - parsedUrl.port, - startOpts.port.toString(), - 'port matches value specified when calling start()' - ) - } else { - t.ok( - !isNaN(parseInt(parsedUrl.port, 10)), - 'port automatically assigned when not specified in start()' - ) - } - } - - await server.stop() - - for (const mediaType of MEDIA_TYPES) { - const exceptionPromise = t.exception(async () => { - await server.getMediaAddress(mediaType) - }, `getting ${mediaType} media address fails if stop() has been called`) - - clock.tick(10_000) - - await exceptionPromise - } - } -}) From dee0a86164443c55f5cb0b0186b44cb1e259631e Mon Sep 17 00:00:00 2001 From: Andrew Chou Date: Fri, 19 Jan 2024 14:26:49 -0500 Subject: [PATCH 03/11] update docs --- .../{media-server.md => mapeo-http-server.md} | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) rename docs/guides/{media-server.md => mapeo-http-server.md} (89%) diff --git a/docs/guides/media-server.md b/docs/guides/mapeo-http-server.md similarity index 89% rename from docs/guides/media-server.md rename to docs/guides/mapeo-http-server.md index 9b371775..496bb609 100644 --- a/docs/guides/media-server.md +++ b/docs/guides/mapeo-http-server.md @@ -1,21 +1,30 @@ -# Mapeo's Media Server +# Mapeo's HTTP Server -Each Mapeo manager instance includes an embedded HTTP server that is responsible for serving media assets over HTTP. Each server is responsible for handling requests for assets that can live in any Mapeo project (the URL structure reflects this, as we will show later on). +Each Mapeo manager instance requires an adjacent Fastify server instance that is responsible for serving assets over HTTP. Each Fastify instance is responsible for handling requests for assets that can live in any Mapeo project (the URL structure reflects this, as we will show later on). Some boilerplate for getting started with a Mapeo project: ```js +// Create Fastify instance +const fastify = Fastify() + +// Create FastifyController instance for managing the starting and stopping the Fastify server (handles it more gracefully and allows pausing and restarting) +const fastifyController = new FastifyController({ fastify }) + // Create the manager instance (truncated for brevity) -const manager = new MapeoManager({...}) +const manager = new MapeoManager({ fastify, ... }) -// Start the media server (no need to await in most cases, unless you need to immediately access the HTTP endpoints) -manager.startMediaServer() +// Start the HTTP server using the controller (awaitable but no need to await in most cases, unless you need to immediately access the HTTP endpoints) +fastifyController.start() // Create a project const projectPublicId = await manager.createProject() // Get the project instance const project = await manager.getProject(projectPublicId) + +// Whenever you need to stop the HTTP server, use the controller +await fastifyController.stop() ``` The example code in the following sections assume that some variation of the above has been done already. From 0a9fbe578d720d0cce2810a1ee5a86222e950c10 Mon Sep 17 00:00:00 2001 From: Andrew Chou Date: Tue, 23 Jan 2024 11:10:22 -0500 Subject: [PATCH 04/11] update wording in docs --- docs/guides/mapeo-http-server.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/guides/mapeo-http-server.md b/docs/guides/mapeo-http-server.md index 496bb609..95685c8e 100644 --- a/docs/guides/mapeo-http-server.md +++ b/docs/guides/mapeo-http-server.md @@ -31,7 +31,7 @@ The example code in the following sections assume that some variation of the abo ## Working with blobs -Blobs represent any binary objects. In the case of Mapeo, that will most likely be media assets such as photos, videos, and audio files. Mapeo provides a project-scoped API that is used for creating and retrieving blobs. Combined with the media server, applications can access them using HTTP requests. +Blobs represent any binary objects. In the case of Mapeo, that will most likely be media assets such as photos, videos, and audio files. Mapeo provides a project-scoped API that is used for creating and retrieving blobs. Combined with the HTTP server, applications can access them using HTTP requests. In the case of an observation record, there can be any number of references to "attachments" (in most cases, an image). In order to create these attachments we need to work with a project's blob API, which can be accessed using `project.$blobs`. @@ -63,7 +63,7 @@ const observation = await project.observation.create({ }) ``` -The attachment provides the information that is needed to create a HTTP URL that can be used to access the asset from the media server: +The attachment provides the information that is needed to create a HTTP URL that can be used to access the asset from the HTTP server: ```js // If you don't already have the observation record, you may need to retrieve it by doing the following @@ -90,7 +90,7 @@ http://{HOST_NAME}:{PORT}/blobs/{PROJECT_PUBLIC_ID}/{DRIVE_DISCOVERY_ID}/{TYPE}/ Explanation of the different parts of this URL: - `HOST_NAME`: Hostname of the server. Defaults to `127.0.0.1` (localhost) -- `PORT`: Port that's being listened on. A random available port is used when the media server is started. +- `PORT`: Port that's being listened on. A random available port is used when the HTTP server is started. - `PROJECT_PUBLIC_ID`: The public ID used to identify the project of interest. - `DRIVE_DISCOVERY_ID`: The discovery ID of the Hyperdrive instance where the blob of interest is located. - `TYPE`: The asset type. Can be `'photo'`, `'video'`, or `'audio'`. @@ -113,7 +113,7 @@ You can then use this URL with anything that uses HTTP to fetch media. Some exam ## Working with icons -Icons are primarily used in the context of project presets, where they are displayed as visual representations of a particular category when recording observations. Mapeo provides a project-scoped API for creating and retrieving icons. Combined with the media server, applications can access them using HTTP requests. +Icons are primarily used in the context of project presets, where they are displayed as visual representations of a particular category when recording observations. Mapeo provides a project-scoped API for creating and retrieving icons. Combined with the HTTP server, applications can access them using HTTP requests. In order to create an icon we need to work with a project's icon API, which can be accessed using `project.$icons`: @@ -192,7 +192,7 @@ http://{HOST_NAME}:{PORT}/icons/{PROJECT_PUBLIC_ID}/{ICON_ID}/{SIZE}{PIXEL_DENSI Explanation of the different parts of this URL: - `HOST_NAME`: Hostname of the server. Defaults to `127.0.0.1` (localhost) -- `PORT`: Port that's being listened on. A random available port is used when the media server is started. +- `PORT`: Port that's being listened on. A random available port is used when the HTTP server is started. - `PROJECT_PUBLIC_ID`: The public ID used to identify the project of interest. - `ICON_ID`: The ID of the icon record associated with the asset. - `SIZE`: The denoted size of the asset. Can be `'small'`, `'medium'`, or `'large'`. From d3af758105bd839c6a3f0dcf88e6c6cefa6a025b Mon Sep 17 00:00:00 2001 From: Andrew Chou Date: Tue, 23 Jan 2024 11:10:31 -0500 Subject: [PATCH 05/11] fix export name --- src/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index.js b/src/index.js index 2ca428eb..34b8c776 100644 --- a/src/index.js +++ b/src/index.js @@ -1,2 +1,2 @@ -export { FastifyController as MediaServer } from './fastify-controller.js' +export { FastifyController } from './fastify-controller.js' export { MapeoManager } from './mapeo-manager.js' From 7f5cab7793c68d6ec4e076e0f3a8a47dc8c7b594 Mon Sep 17 00:00:00 2001 From: Andrew Chou Date: Tue, 23 Jan 2024 11:12:12 -0500 Subject: [PATCH 06/11] update test assertion descriptions --- test-e2e/manager-fastify-server.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test-e2e/manager-fastify-server.js b/test-e2e/manager-fastify-server.js index cb24902b..42fff166 100644 --- a/test-e2e/manager-fastify-server.js +++ b/test-e2e/manager-fastify-server.js @@ -110,7 +110,7 @@ test('retrieving blobs using url', async (t) => { type: 'photo', variant: 'original', }) - }, 'getting blob url fails if manager.startMediaServer() has not been called yet') + }, 'getting blob url fails if fastifyController.start() has not been called yet') clock.tick(100_000) await exceptionPromise1 @@ -128,7 +128,7 @@ test('retrieving blobs using url', async (t) => { st.ok( new URL(blobUrl), - 'retrieving url based on media server resolves after starting it' + 'retrieving url based on HTTP server resolves after starting it' ) const response = await fetch(blobUrl) @@ -149,7 +149,7 @@ test('retrieving blobs using url', async (t) => { st.ok( new URL(blobUrl), - 'retrieving url based on media server resolves after starting it' + 'retrieving url based on HTTP server resolves after starting it' ) const response = await fetch(blobUrl) @@ -176,7 +176,7 @@ test('retrieving blobs using url', async (t) => { type: 'photo', variant: 'original', }) - }, 'getting url after manager.stop() has been called fails') + }, 'getting url after fastifyController.stop() has been called fails') clock.tick(100_000) await exceptionPromise2 }) @@ -204,7 +204,7 @@ test('retrieving icons using url', async (t) => { pixelDensity: 1, size: 'small', }) - }, 'getting icon url fails if manager.startMediaServer() has not been called yet') + }, 'getting icon url fails if fastifyController.start() has not been called yet') clock.tick(100_000) await exceptionPromise1 @@ -222,7 +222,7 @@ test('retrieving icons using url', async (t) => { st.ok( new URL(iconUrl), - 'retrieving url based on media server resolves after starting it' + 'retrieving url based on HTTP server resolves after starting it' ) const response = await fetch(iconUrl) @@ -253,7 +253,7 @@ test('retrieving icons using url', async (t) => { st.ok( new URL(iconUrl), - 'retrieving url based on media server resolves after starting it' + 'retrieving url based on HTTP server resolves after starting it' ) const response = await fetch(iconUrl) @@ -276,7 +276,7 @@ test('retrieving icons using url', async (t) => { pixelDensity: 1, size: 'small', }) - }, 'getting url after manager.stop() has been called fails') + }, 'getting url after fastifyController.stop() has been called fails') clock.tick(100_000) await exceptionPromise2 }) From 3fb05ddf439efde39531c72407e0ea6c523a3af8 Mon Sep 17 00:00:00 2001 From: Andrew Chou Date: Wed, 24 Jan 2024 16:42:08 -0500 Subject: [PATCH 07/11] update docs --- docs/guides/mapeo-http-server.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/guides/mapeo-http-server.md b/docs/guides/mapeo-http-server.md index 95685c8e..1f3a0665 100644 --- a/docs/guides/mapeo-http-server.md +++ b/docs/guides/mapeo-http-server.md @@ -8,14 +8,17 @@ Some boilerplate for getting started with a Mapeo project: // Create Fastify instance const fastify = Fastify() -// Create FastifyController instance for managing the starting and stopping the Fastify server (handles it more gracefully and allows pausing and restarting) -const fastifyController = new FastifyController({ fastify }) - // Create the manager instance (truncated for brevity) const manager = new MapeoManager({ fastify, ... }) -// Start the HTTP server using the controller (awaitable but no need to await in most cases, unless you need to immediately access the HTTP endpoints) -fastifyController.start() +// Start the HTTP server (awaitable but no need to await in most cases, unless you need to immediately access the HTTP endpoints) +fastify.listen() + +// (optional) Create FastifyController instance for managing the starting and stopping the Fastify server (handles it more gracefully and allows pausing and restarting) +// This is useful if you are working in a context that needs to pause or restart the server frequently. +// e.g. +// const fastifyController = new FastifyController({ fastify }) +// fastifyController.start() // Create a project const projectPublicId = await manager.createProject() From f80ac9a6490da9e3a2f23510d911e88f369ebf5a Mon Sep 17 00:00:00 2001 From: Andrew Chou Date: Wed, 24 Jan 2024 16:46:15 -0500 Subject: [PATCH 08/11] remove host and port instance properties in FastifyController --- src/fastify-controller.js | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/fastify-controller.js b/src/fastify-controller.js index ffb82e35..c09e5bb1 100644 --- a/src/fastify-controller.js +++ b/src/fastify-controller.js @@ -12,8 +12,6 @@ import StateMachine from 'start-stop-state-machine' export class FastifyController { #fastify #fastifyStarted - #host - #port #serverState /** @@ -22,8 +20,6 @@ export class FastifyController { */ constructor({ fastify }) { this.#fastifyStarted = false - this.#host = '127.0.0.1' - this.#port = 0 this.#fastify = fastify @@ -37,11 +33,8 @@ export class FastifyController { * @param {StartOpts} [opts] */ async #startServer({ host = '127.0.0.1', port = 0 } = {}) { - this.#host = host - this.#port = port - if (!this.#fastifyStarted) { - await this.#fastify.listen({ host: this.#host, port: this.#port }) + await this.#fastify.listen({ host, port }) this.#fastifyStarted = true return } @@ -49,7 +42,7 @@ export class FastifyController { const { server } = this.#fastify await new Promise((res, rej) => { - server.listen.call(server, { port: this.#port, host: this.#host }) + server.listen.call(server, { host, port }) server.once('listening', onListening) server.once('error', onError) From a772a425585260f9ffe436707ea95232946dece3 Mon Sep 17 00:00:00 2001 From: Andrew Chou Date: Wed, 24 Jan 2024 16:47:20 -0500 Subject: [PATCH 09/11] small fix in FastifyController.#startServer() --- src/fastify-controller.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fastify-controller.js b/src/fastify-controller.js index c09e5bb1..a84c600a 100644 --- a/src/fastify-controller.js +++ b/src/fastify-controller.js @@ -34,8 +34,8 @@ export class FastifyController { */ async #startServer({ host = '127.0.0.1', port = 0 } = {}) { if (!this.#fastifyStarted) { - await this.#fastify.listen({ host, port }) this.#fastifyStarted = true + await this.#fastify.listen({ host, port }) return } From 3aafbc01327688488971d330e4d98482ac2d93b1 Mon Sep 17 00:00:00 2001 From: Andrew Chou Date: Wed, 24 Jan 2024 16:51:14 -0500 Subject: [PATCH 10/11] add timeout option to getFastifyServerAddress() --- src/fastify-plugins/utils.js | 7 +++++-- src/mapeo-manager.js | 5 ++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/fastify-plugins/utils.js b/src/fastify-plugins/utils.js index 4fa56c9a..4a5c832e 100644 --- a/src/fastify-plugins/utils.js +++ b/src/fastify-plugins/utils.js @@ -2,13 +2,16 @@ import { once } from 'node:events' /** * @param {import('node:http').Server} server + * @param {{ timeout?: number }} [options] * @returns {Promise} */ -export async function getFastifyServerAddress(server) { +export async function getFastifyServerAddress(server, { timeout } = {}) { const address = server.address() if (!address) { - await once(server, 'listening') + await once(server, 'listening', { + signal: timeout ? AbortSignal.timeout(timeout) : undefined, + }) return getFastifyServerAddress(server) } diff --git a/src/mapeo-manager.js b/src/mapeo-manager.js index 952e2162..93919f4b 100644 --- a/src/mapeo-manager.js +++ b/src/mapeo-manager.js @@ -6,7 +6,6 @@ import { eq } from 'drizzle-orm' import { drizzle } from 'drizzle-orm/better-sqlite3' import { migrate } from 'drizzle-orm/better-sqlite3/migrator' import Hypercore from 'hypercore' -import pTimeout from 'p-timeout' import { TypedEmitter } from 'tiny-typed-emitter' import { IndexWriter } from './index-writer/index.js' @@ -237,8 +236,8 @@ export class MapeoManager extends TypedEmitter { } } - const base = await pTimeout(getFastifyServerAddress(this.#fastify.server), { - milliseconds: 1000, + const base = await getFastifyServerAddress(this.#fastify.server, { + timeout: 5000, }) return base + '/' + prefix From 29727df112b333a93d57d99834d0f0f178c42a75 Mon Sep 17 00:00:00 2001 From: Andrew Chou Date: Wed, 24 Jan 2024 16:52:18 -0500 Subject: [PATCH 11/11] rename private method in MapeoManager --- src/mapeo-manager.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mapeo-manager.js b/src/mapeo-manager.js index 93919f4b..544deffb 100644 --- a/src/mapeo-manager.js +++ b/src/mapeo-manager.js @@ -218,7 +218,7 @@ export class MapeoManager extends TypedEmitter { * @param {'blobs' | 'icons' | 'maps'} mediaType * @returns {Promise} */ - async #getMediaAddress(mediaType) { + async #getMediaBaseUrl(mediaType) { /** @type {string | null} */ let prefix = null @@ -447,7 +447,7 @@ export class MapeoManager extends TypedEmitter { sharedIndexWriter: this.#projectSettingsIndexWriter, localPeers: this.#localPeers, logger: this.#loggerBase, - getMediaBaseUrl: this.#getMediaAddress.bind(this), + getMediaBaseUrl: this.#getMediaBaseUrl.bind(this), }) }