diff --git a/docs/env.md b/docs/env.md index 834719a1333..0fe0e1c7363 100644 --- a/docs/env.md +++ b/docs/env.md @@ -97,6 +97,20 @@ Description: When nonempty, create pretend prepopulated tokens like "moola" and Lifetime: until chain is mature enough not to need any pretend tokens +## LMDB_MAP_SIZE + +Affects: cosmic-swingset + +Purpose: set the minimum size limit for swing-store's LMDB key-value store + +Description: default is `2147483648` (2GB), and you need to set higher if you +receive `Error: MDB_MAP_FULL: Environment mapsize limit reached` + +Can always be increased, and does not decrease once a transaction has been +written with the new mapSize. + +Lifetime: until we no longer use LMDB in swing-store + ## OTEL_EXPORTER_PROMETHEUS_HOST ## OTEL_EXPORTER_PROMETHEUS_PORT @@ -109,15 +123,21 @@ for the Prometheus scrape endpoint to export telemetry. Lifetime: until we decide not to support Prometheus for metrics export +## SOLO_LMDB_MAP_SIZE + +Affects: solo + +Same as `LMDB_MAP_SIZE`, but for solo instead of chain. + ## SOLO_SLOGFILE -Same as SLOGFILE, but for solo instead of chain. +Same as `SLOGFILE`, but for solo instead of chain. Lifetime: ? ## SOLO_SLOGSENDER -Same as SLOGSENDER, but for solo instead of chain. +Same as `SLOGSENDER`, but for solo instead of chain. Lifetime: ? diff --git a/packages/cosmic-swingset/src/chain-main.js b/packages/cosmic-swingset/src/chain-main.js index e0b41526d81..a8e6a522edd 100644 --- a/packages/cosmic-swingset/src/chain-main.js +++ b/packages/cosmic-swingset/src/chain-main.js @@ -251,27 +251,29 @@ export default async function main(progname, args, { env, homedir, agcc }) { ).pathname; const { metricsProvider } = getTelemetryProviders({ console, env }); - const { SLOGFILE, SLOGSENDER } = env; + const { SLOGFILE, SLOGSENDER, LMDB_MAP_SIZE } = env; const slogSender = await makeSlogSenderFromModule(SLOGSENDER, { stateDir: stateDBDir, }); + const mapSize = (LMDB_MAP_SIZE && parseInt(LMDB_MAP_SIZE, 10)) || undefined; + // We want to make it hard for a validator to accidentally disable // consensusMode. const consensusMode = true; - const s = await launch( - stateDBDir, + const s = await launch({ + kernelStateDBDir: stateDBDir, mailboxStorage, setActivityhash, - doOutboundBridge, + bridgeOutbound: doOutboundBridge, vatconfig, argv, - undefined, metricsProvider, - SLOGFILE, + slogFile: SLOGFILE, slogSender, consensusMode, - ); + mapSize, + }); return s; } diff --git a/packages/cosmic-swingset/src/launch-chain.js b/packages/cosmic-swingset/src/launch-chain.js index 12499ce15c2..5dc73b4a475 100644 --- a/packages/cosmic-swingset/src/launch-chain.js +++ b/packages/cosmic-swingset/src/launch-chain.js @@ -12,7 +12,7 @@ import { loadSwingsetConfigFile, } from '@agoric/swingset-vat'; import { assert, details as X } from '@agoric/assert'; -import { openSwingStore } from '@agoric/swing-store'; +import { openSwingStore, DEFAULT_LMDB_MAP_SIZE } from '@agoric/swing-store'; import { DEFAULT_METER_PROVIDER, exportKernelStats, @@ -135,7 +135,7 @@ function neverStop() { }); } -export async function launch( +export async function launch({ kernelStateDBDir, mailboxStorage, setActivityhash, @@ -143,15 +143,18 @@ export async function launch( vatconfig, argv, debugName = undefined, - meterProvider = DEFAULT_METER_PROVIDER, + metricsProvider = DEFAULT_METER_PROVIDER, slogFile = undefined, slogSender, consensusMode = true, -) { + mapSize = DEFAULT_LMDB_MAP_SIZE, +}) { console.info('Launching SwingSet kernel'); - const { kvStore, streamStore, snapStore, commit } = - openSwingStore(kernelStateDBDir); + const { kvStore, streamStore, snapStore, commit } = openSwingStore( + kernelStateDBDir, + { mapSize }, + ); const hostStorage = { kvStore, streamStore, @@ -159,7 +162,7 @@ export async function launch( }; // Not to be confused with the gas model, this meter is for OpenTelemetry. - const metricMeter = meterProvider.getMeter('ag-chain-cosmos'); + const metricMeter = metricsProvider.getMeter('ag-chain-cosmos'); const METRIC_LABELS = { app: 'ag-chain-cosmos' }; const slogCallbacks = makeSlogCallbacks({ diff --git a/packages/cosmic-swingset/src/sim-chain.js b/packages/cosmic-swingset/src/sim-chain.js index 430f57f8f96..fed9ccc00ce 100644 --- a/packages/cosmic-swingset/src/sim-chain.js +++ b/packages/cosmic-swingset/src/sim-chain.js @@ -86,26 +86,26 @@ export async function connectToFakeChain(basedir, GCI, delay, inbound) { env: process.env, }); - const { SLOGFILE, SLOGSENDER } = process.env; + const { SLOGFILE, SLOGSENDER, LMDB_MAP_SIZE } = process.env; + const mapSize = (LMDB_MAP_SIZE && parseInt(LMDB_MAP_SIZE, 10)) || undefined; const slogSender = await makeSlogSenderFromModule(SLOGSENDER, { stateDir: stateDBdir, }); // We don't want to force a sim chain to use consensus mode. const consensusMode = false; - const s = await launch( - stateDBdir, + const s = await launch({ + kernelStateDBDir: stateDBdir, mailboxStorage, - undefined, - undefined, vatconfig, argv, - GCI, // debugName + debugName: GCI, metricsProvider, - SLOGFILE, + slogFile: SLOGFILE, slogSender, consensusMode, - ); + mapSize, + }); const { savedHeight, savedActions, savedChainSends } = s; const blockManager = makeBlockManager({ ...s, flushChainSends }); diff --git a/packages/solo/src/start.js b/packages/solo/src/start.js index 25a559f9d8e..ad806ced7ad 100644 --- a/packages/solo/src/start.js +++ b/packages/solo/src/start.js @@ -144,8 +144,17 @@ const buildSwingset = async ( plugin: { ...plugin.endowments }, }; - const { kvStore, streamStore, snapStore, commit } = - openSwingStore(kernelStateDBDir); + const { + SOLO_SLOGFILE: slogFile, + SOLO_SLOGSENDER, + SOLO_LMDB_MAP_SIZE, + } = process.env; + const mapSize = + (SOLO_LMDB_MAP_SIZE && parseInt(SOLO_LMDB_MAP_SIZE, 10)) || undefined; + const { kvStore, streamStore, snapStore, commit } = openSwingStore( + kernelStateDBDir, + { mapSize }, + ); const hostStorage = { kvStore, streamStore, @@ -158,7 +167,6 @@ const buildSwingset = async ( } await initializeSwingset(config, argv, hostStorage); } - const { SOLO_SLOGFILE: slogFile, SOLO_SLOGSENDER } = process.env; const slogSender = await makeSlogSenderFromModule(SOLO_SLOGSENDER, { stateDir: kernelStateDBDir, }); diff --git a/packages/swing-store/src/swingStore.js b/packages/swing-store/src/swingStore.js index 8a2b02d2d37..2fd8469751d 100644 --- a/packages/swing-store/src/swingStore.js +++ b/packages/swing-store/src/swingStore.js @@ -13,6 +13,8 @@ import { sqlStreamStore } from './sqlStreamStore.js'; import { makeSnapStore } from './snapStore.js'; import { initEphemeralSwingStore } from './ephemeralSwingStore.js'; +export const DEFAULT_LMDB_MAP_SIZE = 2 * 1024 * 1024 * 1024; + export { makeSnapStore }; export function makeSnapStoreIO() { @@ -126,12 +128,18 @@ function makeSwingStore(dirPath, forceReset, options) { } fs.mkdirSync(dirPath, { recursive: true }); - const { mapSize = 2 * 1024 * 1024 * 1024 } = options; + const { mapSize = DEFAULT_LMDB_MAP_SIZE } = options; let lmdbEnv = new lmdb.Env(); + + // Use the new mapSize, persisting if it's bigger than before and there's a + // write txn commit. + // http://www.lmdb.tech/doc/group__mdb.html#gaa2506ec8dab3d969b0e609cd82e619e5 + lmdbEnv.resize(mapSize); + lmdbEnv.open({ path: dirPath, mapSize, - // Turn off useWritemap on the Mac. The userWritemap option is currently + // Turn off useWritemap on the Mac. The useWritemap option is currently // required for LMDB to function correctly on Linux running under WSL, but // we don't yet have a convenient recipe to probe our environment at // runtime to distinguish that species of Linux from the others. For now