diff --git a/.github/spot-runner-action/dist/index.js b/.github/spot-runner-action/dist/index.js index 422a423df1d..a612434a95a 100644 --- a/.github/spot-runner-action/dist/index.js +++ b/.github/spot-runner-action/dist/index.js @@ -281,8 +281,8 @@ class Ec2Instance { Ebs: { VolumeSize: 64, VolumeType: 'gp3', - Throughput: 1000, - Iops: 5000 + Throughput: 125, + Iops: 3000 }, }, ], @@ -699,10 +699,11 @@ function pollSpotStatus(config, ec2Client, ghClient) { } try { core.info("Found ec2 instance, looking for runners."); - if (process.env.WAIT_FOR_RUNNERS === "false" || (yield ghClient.hasRunner([config.githubJobId]))) { - // we have runners - return instances[0].InstanceId; - } + // TODO find out whatever happened here but we seem to not be able to wait for runners + //if (process.env.WAIT_FOR_RUNNERS === "false" || await ghClient.hasRunner([config.githubJobId])) { + // we have runners + return instances[0].InstanceId; + //} } catch (err) { } // wait 10 seconds @@ -835,12 +836,11 @@ function startWithGithubRunners(config) { return false; } yield setupGithubRunners(ip, config); - if (instanceId) - yield ghClient.pollForRunnerCreation([config.githubJobId]); - else { - core.error("Instance failed to register with Github Actions"); - throw Error("Instance failed to register with Github Actions"); - } + // if (instanceId) await ghClient.pollForRunnerCreation([config.githubJobId]); + // else { + // core.error("Instance failed to register with Github Actions"); + // throw Error("Instance failed to register with Github Actions"); + // } core.info("Done setting up runner."); } // Export to github environment @@ -1578,7 +1578,7 @@ exports.prepareKeyValueMessage = exports.issueFileCommand = void 0; /* eslint-disable @typescript-eslint/no-explicit-any */ const fs = __importStar(__nccwpck_require__(57147)); const os = __importStar(__nccwpck_require__(22037)); -const uuid_1 = __nccwpck_require__(78974); +const uuid_1 = __nccwpck_require__(75840); const utils_1 = __nccwpck_require__(5278); function issueFileCommand(command, message) { const filePath = process.env[`GITHUB_${command}`]; @@ -2098,777 +2098,363 @@ exports.toCommandProperties = toCommandProperties; /***/ }), -/***/ 78974: -/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { - -"use strict"; - - -Object.defineProperty(exports, "__esModule", ({ - value: true -})); -Object.defineProperty(exports, "v1", ({ - enumerable: true, - get: function () { - return _v.default; - } -})); -Object.defineProperty(exports, "v3", ({ - enumerable: true, - get: function () { - return _v2.default; - } -})); -Object.defineProperty(exports, "v4", ({ - enumerable: true, - get: function () { - return _v3.default; - } -})); -Object.defineProperty(exports, "v5", ({ - enumerable: true, - get: function () { - return _v4.default; - } -})); -Object.defineProperty(exports, "NIL", ({ - enumerable: true, - get: function () { - return _nil.default; - } -})); -Object.defineProperty(exports, "version", ({ - enumerable: true, - get: function () { - return _version.default; - } -})); -Object.defineProperty(exports, "validate", ({ - enumerable: true, - get: function () { - return _validate.default; - } -})); -Object.defineProperty(exports, "stringify", ({ - enumerable: true, - get: function () { - return _stringify.default; - } -})); -Object.defineProperty(exports, "parse", ({ - enumerable: true, - get: function () { - return _parse.default; - } -})); - -var _v = _interopRequireDefault(__nccwpck_require__(81595)); - -var _v2 = _interopRequireDefault(__nccwpck_require__(26993)); - -var _v3 = _interopRequireDefault(__nccwpck_require__(51472)); - -var _v4 = _interopRequireDefault(__nccwpck_require__(16217)); - -var _nil = _interopRequireDefault(__nccwpck_require__(32381)); - -var _version = _interopRequireDefault(__nccwpck_require__(40427)); - -var _validate = _interopRequireDefault(__nccwpck_require__(92609)); - -var _stringify = _interopRequireDefault(__nccwpck_require__(61458)); - -var _parse = _interopRequireDefault(__nccwpck_require__(26385)); - -function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } - -/***/ }), - -/***/ 5842: +/***/ 74087: /***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { "use strict"; - -Object.defineProperty(exports, "__esModule", ({ - value: true -})); -exports["default"] = void 0; - -var _crypto = _interopRequireDefault(__nccwpck_require__(6113)); - -function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } - -function md5(bytes) { - if (Array.isArray(bytes)) { - bytes = Buffer.from(bytes); - } else if (typeof bytes === 'string') { - bytes = Buffer.from(bytes, 'utf8'); - } - - return _crypto.default.createHash('md5').update(bytes).digest(); +Object.defineProperty(exports, "__esModule", ({ value: true })); +exports.Context = void 0; +const fs_1 = __nccwpck_require__(57147); +const os_1 = __nccwpck_require__(22037); +class Context { + /** + * Hydrate the context from the environment + */ + constructor() { + var _a, _b, _c; + this.payload = {}; + if (process.env.GITHUB_EVENT_PATH) { + if ((0, fs_1.existsSync)(process.env.GITHUB_EVENT_PATH)) { + this.payload = JSON.parse((0, fs_1.readFileSync)(process.env.GITHUB_EVENT_PATH, { encoding: 'utf8' })); + } + else { + const path = process.env.GITHUB_EVENT_PATH; + process.stdout.write(`GITHUB_EVENT_PATH ${path} does not exist${os_1.EOL}`); + } + } + this.eventName = process.env.GITHUB_EVENT_NAME; + this.sha = process.env.GITHUB_SHA; + this.ref = process.env.GITHUB_REF; + this.workflow = process.env.GITHUB_WORKFLOW; + this.action = process.env.GITHUB_ACTION; + this.actor = process.env.GITHUB_ACTOR; + this.job = process.env.GITHUB_JOB; + this.runNumber = parseInt(process.env.GITHUB_RUN_NUMBER, 10); + this.runId = parseInt(process.env.GITHUB_RUN_ID, 10); + this.apiUrl = (_a = process.env.GITHUB_API_URL) !== null && _a !== void 0 ? _a : `https://api.github.com`; + this.serverUrl = (_b = process.env.GITHUB_SERVER_URL) !== null && _b !== void 0 ? _b : `https://github.com`; + this.graphqlUrl = + (_c = process.env.GITHUB_GRAPHQL_URL) !== null && _c !== void 0 ? _c : `https://api.github.com/graphql`; + } + get issue() { + const payload = this.payload; + return Object.assign(Object.assign({}, this.repo), { number: (payload.issue || payload.pull_request || payload).number }); + } + get repo() { + if (process.env.GITHUB_REPOSITORY) { + const [owner, repo] = process.env.GITHUB_REPOSITORY.split('/'); + return { owner, repo }; + } + if (this.payload.repository) { + return { + owner: this.payload.repository.owner.login, + repo: this.payload.repository.name + }; + } + throw new Error("context.repo requires a GITHUB_REPOSITORY environment variable like 'owner/repo'"); + } } - -var _default = md5; -exports["default"] = _default; +exports.Context = Context; +//# sourceMappingURL=context.js.map /***/ }), -/***/ 32381: -/***/ ((__unused_webpack_module, exports) => { +/***/ 95438: +/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) { "use strict"; - -Object.defineProperty(exports, "__esModule", ({ - value: true +var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + var desc = Object.getOwnPropertyDescriptor(m, k); + if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) { + desc = { enumerable: true, get: function() { return m[k]; } }; + } + Object.defineProperty(o, k2, desc); +}) : (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + o[k2] = m[k]; })); -exports["default"] = void 0; -var _default = '00000000-0000-0000-0000-000000000000'; -exports["default"] = _default; +var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { + Object.defineProperty(o, "default", { enumerable: true, value: v }); +}) : function(o, v) { + o["default"] = v; +}); +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); + __setModuleDefault(result, mod); + return result; +}; +Object.defineProperty(exports, "__esModule", ({ value: true })); +exports.getOctokit = exports.context = void 0; +const Context = __importStar(__nccwpck_require__(74087)); +const utils_1 = __nccwpck_require__(73030); +exports.context = new Context.Context(); +/** + * Returns a hydrated octokit ready to use for GitHub Actions + * + * @param token the repo PAT or GITHUB_TOKEN + * @param options other options to set + */ +function getOctokit(token, options, ...additionalPlugins) { + const GitHubWithPlugins = utils_1.GitHub.plugin(...additionalPlugins); + return new GitHubWithPlugins((0, utils_1.getOctokitOptions)(token, options)); +} +exports.getOctokit = getOctokit; +//# sourceMappingURL=github.js.map /***/ }), -/***/ 26385: -/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { +/***/ 47914: +/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) { "use strict"; - -Object.defineProperty(exports, "__esModule", ({ - value: true +var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + var desc = Object.getOwnPropertyDescriptor(m, k); + if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) { + desc = { enumerable: true, get: function() { return m[k]; } }; + } + Object.defineProperty(o, k2, desc); +}) : (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + o[k2] = m[k]; })); -exports["default"] = void 0; - -var _validate = _interopRequireDefault(__nccwpck_require__(92609)); - -function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } - -function parse(uuid) { - if (!(0, _validate.default)(uuid)) { - throw TypeError('Invalid UUID'); - } - - let v; - const arr = new Uint8Array(16); // Parse ########-....-....-....-............ - - arr[0] = (v = parseInt(uuid.slice(0, 8), 16)) >>> 24; - arr[1] = v >>> 16 & 0xff; - arr[2] = v >>> 8 & 0xff; - arr[3] = v & 0xff; // Parse ........-####-....-....-............ - - arr[4] = (v = parseInt(uuid.slice(9, 13), 16)) >>> 8; - arr[5] = v & 0xff; // Parse ........-....-####-....-............ - - arr[6] = (v = parseInt(uuid.slice(14, 18), 16)) >>> 8; - arr[7] = v & 0xff; // Parse ........-....-....-####-............ - - arr[8] = (v = parseInt(uuid.slice(19, 23), 16)) >>> 8; - arr[9] = v & 0xff; // Parse ........-....-....-....-############ - // (Use "/" to avoid 32-bit truncation when bit-shifting high-order bytes) - - arr[10] = (v = parseInt(uuid.slice(24, 36), 16)) / 0x10000000000 & 0xff; - arr[11] = v / 0x100000000 & 0xff; - arr[12] = v >>> 24 & 0xff; - arr[13] = v >>> 16 & 0xff; - arr[14] = v >>> 8 & 0xff; - arr[15] = v & 0xff; - return arr; +var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { + Object.defineProperty(o, "default", { enumerable: true, value: v }); +}) : function(o, v) { + o["default"] = v; +}); +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); + __setModuleDefault(result, mod); + return result; +}; +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +Object.defineProperty(exports, "__esModule", ({ value: true })); +exports.getApiBaseUrl = exports.getProxyFetch = exports.getProxyAgentDispatcher = exports.getProxyAgent = exports.getAuthString = void 0; +const httpClient = __importStar(__nccwpck_require__(96255)); +const undici_1 = __nccwpck_require__(41773); +function getAuthString(token, options) { + if (!token && !options.auth) { + throw new Error('Parameter token or opts.auth is required'); + } + else if (token && options.auth) { + throw new Error('Parameters token and opts.auth may not both be specified'); + } + return typeof options.auth === 'string' ? options.auth : `token ${token}`; } - -var _default = parse; -exports["default"] = _default; +exports.getAuthString = getAuthString; +function getProxyAgent(destinationUrl) { + const hc = new httpClient.HttpClient(); + return hc.getAgent(destinationUrl); +} +exports.getProxyAgent = getProxyAgent; +function getProxyAgentDispatcher(destinationUrl) { + const hc = new httpClient.HttpClient(); + return hc.getAgentDispatcher(destinationUrl); +} +exports.getProxyAgentDispatcher = getProxyAgentDispatcher; +function getProxyFetch(destinationUrl) { + const httpDispatcher = getProxyAgentDispatcher(destinationUrl); + const proxyFetch = (url, opts) => __awaiter(this, void 0, void 0, function* () { + return (0, undici_1.fetch)(url, Object.assign(Object.assign({}, opts), { dispatcher: httpDispatcher })); + }); + return proxyFetch; +} +exports.getProxyFetch = getProxyFetch; +function getApiBaseUrl() { + return process.env['GITHUB_API_URL'] || 'https://api.github.com'; +} +exports.getApiBaseUrl = getApiBaseUrl; +//# sourceMappingURL=utils.js.map /***/ }), -/***/ 86230: -/***/ ((__unused_webpack_module, exports) => { +/***/ 73030: +/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) { "use strict"; - -Object.defineProperty(exports, "__esModule", ({ - value: true +var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + var desc = Object.getOwnPropertyDescriptor(m, k); + if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) { + desc = { enumerable: true, get: function() { return m[k]; } }; + } + Object.defineProperty(o, k2, desc); +}) : (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + o[k2] = m[k]; })); -exports["default"] = void 0; -var _default = /^(?:[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}|00000000-0000-0000-0000-000000000000)$/i; -exports["default"] = _default; +var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { + Object.defineProperty(o, "default", { enumerable: true, value: v }); +}) : function(o, v) { + o["default"] = v; +}); +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); + __setModuleDefault(result, mod); + return result; +}; +Object.defineProperty(exports, "__esModule", ({ value: true })); +exports.getOctokitOptions = exports.GitHub = exports.defaults = exports.context = void 0; +const Context = __importStar(__nccwpck_require__(74087)); +const Utils = __importStar(__nccwpck_require__(47914)); +// octokit + plugins +const core_1 = __nccwpck_require__(76762); +const plugin_rest_endpoint_methods_1 = __nccwpck_require__(83044); +const plugin_paginate_rest_1 = __nccwpck_require__(64193); +exports.context = new Context.Context(); +const baseUrl = Utils.getApiBaseUrl(); +exports.defaults = { + baseUrl, + request: { + agent: Utils.getProxyAgent(baseUrl), + fetch: Utils.getProxyFetch(baseUrl) + } +}; +exports.GitHub = core_1.Octokit.plugin(plugin_rest_endpoint_methods_1.restEndpointMethods, plugin_paginate_rest_1.paginateRest).defaults(exports.defaults); +/** + * Convience function to correctly format Octokit Options to pass into the constructor. + * + * @param token the repo PAT or GITHUB_TOKEN + * @param options other options to set + */ +function getOctokitOptions(token, options) { + const opts = Object.assign({}, options || {}); // Shallow clone - don't mutate the object provided by the caller + // Auth + const auth = Utils.getAuthString(token, opts); + if (auth) { + opts.auth = auth; + } + return opts; +} +exports.getOctokitOptions = getOctokitOptions; +//# sourceMappingURL=utils.js.map /***/ }), -/***/ 9784: -/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { +/***/ 35526: +/***/ (function(__unused_webpack_module, exports) { "use strict"; - -Object.defineProperty(exports, "__esModule", ({ - value: true -})); -exports["default"] = rng; - -var _crypto = _interopRequireDefault(__nccwpck_require__(6113)); - -function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } - -const rnds8Pool = new Uint8Array(256); // # of random values to pre-allocate - -let poolPtr = rnds8Pool.length; - -function rng() { - if (poolPtr > rnds8Pool.length - 16) { - _crypto.default.randomFillSync(rnds8Pool); - - poolPtr = 0; - } - - return rnds8Pool.slice(poolPtr, poolPtr += 16); +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +Object.defineProperty(exports, "__esModule", ({ value: true })); +exports.PersonalAccessTokenCredentialHandler = exports.BearerCredentialHandler = exports.BasicCredentialHandler = void 0; +class BasicCredentialHandler { + constructor(username, password) { + this.username = username; + this.password = password; + } + prepareRequest(options) { + if (!options.headers) { + throw Error('The request has no headers'); + } + options.headers['Authorization'] = `Basic ${Buffer.from(`${this.username}:${this.password}`).toString('base64')}`; + } + // This handler cannot handle 401 + canHandleAuthentication() { + return false; + } + handleAuthentication() { + return __awaiter(this, void 0, void 0, function* () { + throw new Error('not implemented'); + }); + } +} +exports.BasicCredentialHandler = BasicCredentialHandler; +class BearerCredentialHandler { + constructor(token) { + this.token = token; + } + // currently implements pre-authorization + // TODO: support preAuth = false where it hooks on 401 + prepareRequest(options) { + if (!options.headers) { + throw Error('The request has no headers'); + } + options.headers['Authorization'] = `Bearer ${this.token}`; + } + // This handler cannot handle 401 + canHandleAuthentication() { + return false; + } + handleAuthentication() { + return __awaiter(this, void 0, void 0, function* () { + throw new Error('not implemented'); + }); + } +} +exports.BearerCredentialHandler = BearerCredentialHandler; +class PersonalAccessTokenCredentialHandler { + constructor(token) { + this.token = token; + } + // currently implements pre-authorization + // TODO: support preAuth = false where it hooks on 401 + prepareRequest(options) { + if (!options.headers) { + throw Error('The request has no headers'); + } + options.headers['Authorization'] = `Basic ${Buffer.from(`PAT:${this.token}`).toString('base64')}`; + } + // This handler cannot handle 401 + canHandleAuthentication() { + return false; + } + handleAuthentication() { + return __awaiter(this, void 0, void 0, function* () { + throw new Error('not implemented'); + }); + } } +exports.PersonalAccessTokenCredentialHandler = PersonalAccessTokenCredentialHandler; +//# sourceMappingURL=auth.js.map /***/ }), -/***/ 38844: -/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { +/***/ 96255: +/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) { "use strict"; - -Object.defineProperty(exports, "__esModule", ({ - value: true -})); -exports["default"] = void 0; - -var _crypto = _interopRequireDefault(__nccwpck_require__(6113)); - -function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } - -function sha1(bytes) { - if (Array.isArray(bytes)) { - bytes = Buffer.from(bytes); - } else if (typeof bytes === 'string') { - bytes = Buffer.from(bytes, 'utf8'); - } - - return _crypto.default.createHash('sha1').update(bytes).digest(); -} - -var _default = sha1; -exports["default"] = _default; - -/***/ }), - -/***/ 61458: -/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { - -"use strict"; - - -Object.defineProperty(exports, "__esModule", ({ - value: true -})); -exports["default"] = void 0; - -var _validate = _interopRequireDefault(__nccwpck_require__(92609)); - -function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } - -/** - * Convert array of 16 byte values to UUID string format of the form: - * XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX - */ -const byteToHex = []; - -for (let i = 0; i < 256; ++i) { - byteToHex.push((i + 0x100).toString(16).substr(1)); -} - -function stringify(arr, offset = 0) { - // Note: Be careful editing this code! It's been tuned for performance - // and works in ways you may not expect. See https://github.com/uuidjs/uuid/pull/434 - const uuid = (byteToHex[arr[offset + 0]] + byteToHex[arr[offset + 1]] + byteToHex[arr[offset + 2]] + byteToHex[arr[offset + 3]] + '-' + byteToHex[arr[offset + 4]] + byteToHex[arr[offset + 5]] + '-' + byteToHex[arr[offset + 6]] + byteToHex[arr[offset + 7]] + '-' + byteToHex[arr[offset + 8]] + byteToHex[arr[offset + 9]] + '-' + byteToHex[arr[offset + 10]] + byteToHex[arr[offset + 11]] + byteToHex[arr[offset + 12]] + byteToHex[arr[offset + 13]] + byteToHex[arr[offset + 14]] + byteToHex[arr[offset + 15]]).toLowerCase(); // Consistency check for valid UUID. If this throws, it's likely due to one - // of the following: - // - One or more input array values don't map to a hex octet (leading to - // "undefined" in the uuid) - // - Invalid input values for the RFC `version` or `variant` fields - - if (!(0, _validate.default)(uuid)) { - throw TypeError('Stringified UUID is invalid'); - } - - return uuid; -} - -var _default = stringify; -exports["default"] = _default; - -/***/ }), - -/***/ 81595: -/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { - -"use strict"; - - -Object.defineProperty(exports, "__esModule", ({ - value: true -})); -exports["default"] = void 0; - -var _rng = _interopRequireDefault(__nccwpck_require__(9784)); - -var _stringify = _interopRequireDefault(__nccwpck_require__(61458)); - -function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } - -// **`v1()` - Generate time-based UUID** -// -// Inspired by https://github.com/LiosK/UUID.js -// and http://docs.python.org/library/uuid.html -let _nodeId; - -let _clockseq; // Previous uuid creation time - - -let _lastMSecs = 0; -let _lastNSecs = 0; // See https://github.com/uuidjs/uuid for API details - -function v1(options, buf, offset) { - let i = buf && offset || 0; - const b = buf || new Array(16); - options = options || {}; - let node = options.node || _nodeId; - let clockseq = options.clockseq !== undefined ? options.clockseq : _clockseq; // node and clockseq need to be initialized to random values if they're not - // specified. We do this lazily to minimize issues related to insufficient - // system entropy. See #189 - - if (node == null || clockseq == null) { - const seedBytes = options.random || (options.rng || _rng.default)(); - - if (node == null) { - // Per 4.5, create and 48-bit node id, (47 random bits + multicast bit = 1) - node = _nodeId = [seedBytes[0] | 0x01, seedBytes[1], seedBytes[2], seedBytes[3], seedBytes[4], seedBytes[5]]; - } - - if (clockseq == null) { - // Per 4.2.2, randomize (14 bit) clockseq - clockseq = _clockseq = (seedBytes[6] << 8 | seedBytes[7]) & 0x3fff; - } - } // UUID timestamps are 100 nano-second units since the Gregorian epoch, - // (1582-10-15 00:00). JSNumbers aren't precise enough for this, so - // time is handled internally as 'msecs' (integer milliseconds) and 'nsecs' - // (100-nanoseconds offset from msecs) since unix epoch, 1970-01-01 00:00. - - - let msecs = options.msecs !== undefined ? options.msecs : Date.now(); // Per 4.2.1.2, use count of uuid's generated during the current clock - // cycle to simulate higher resolution clock - - let nsecs = options.nsecs !== undefined ? options.nsecs : _lastNSecs + 1; // Time since last uuid creation (in msecs) - - const dt = msecs - _lastMSecs + (nsecs - _lastNSecs) / 10000; // Per 4.2.1.2, Bump clockseq on clock regression - - if (dt < 0 && options.clockseq === undefined) { - clockseq = clockseq + 1 & 0x3fff; - } // Reset nsecs if clock regresses (new clockseq) or we've moved onto a new - // time interval - - - if ((dt < 0 || msecs > _lastMSecs) && options.nsecs === undefined) { - nsecs = 0; - } // Per 4.2.1.2 Throw error if too many uuids are requested - - - if (nsecs >= 10000) { - throw new Error("uuid.v1(): Can't create more than 10M uuids/sec"); - } - - _lastMSecs = msecs; - _lastNSecs = nsecs; - _clockseq = clockseq; // Per 4.1.4 - Convert from unix epoch to Gregorian epoch - - msecs += 12219292800000; // `time_low` - - const tl = ((msecs & 0xfffffff) * 10000 + nsecs) % 0x100000000; - b[i++] = tl >>> 24 & 0xff; - b[i++] = tl >>> 16 & 0xff; - b[i++] = tl >>> 8 & 0xff; - b[i++] = tl & 0xff; // `time_mid` - - const tmh = msecs / 0x100000000 * 10000 & 0xfffffff; - b[i++] = tmh >>> 8 & 0xff; - b[i++] = tmh & 0xff; // `time_high_and_version` - - b[i++] = tmh >>> 24 & 0xf | 0x10; // include version - - b[i++] = tmh >>> 16 & 0xff; // `clock_seq_hi_and_reserved` (Per 4.2.2 - include variant) - - b[i++] = clockseq >>> 8 | 0x80; // `clock_seq_low` - - b[i++] = clockseq & 0xff; // `node` - - for (let n = 0; n < 6; ++n) { - b[i + n] = node[n]; - } - - return buf || (0, _stringify.default)(b); -} - -var _default = v1; -exports["default"] = _default; - -/***/ }), - -/***/ 26993: -/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { - -"use strict"; - - -Object.defineProperty(exports, "__esModule", ({ - value: true -})); -exports["default"] = void 0; - -var _v = _interopRequireDefault(__nccwpck_require__(65920)); - -var _md = _interopRequireDefault(__nccwpck_require__(5842)); - -function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } - -const v3 = (0, _v.default)('v3', 0x30, _md.default); -var _default = v3; -exports["default"] = _default; - -/***/ }), - -/***/ 65920: -/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { - -"use strict"; - - -Object.defineProperty(exports, "__esModule", ({ - value: true -})); -exports["default"] = _default; -exports.URL = exports.DNS = void 0; - -var _stringify = _interopRequireDefault(__nccwpck_require__(61458)); - -var _parse = _interopRequireDefault(__nccwpck_require__(26385)); - -function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } - -function stringToBytes(str) { - str = unescape(encodeURIComponent(str)); // UTF8 escape - - const bytes = []; - - for (let i = 0; i < str.length; ++i) { - bytes.push(str.charCodeAt(i)); - } - - return bytes; -} - -const DNS = '6ba7b810-9dad-11d1-80b4-00c04fd430c8'; -exports.DNS = DNS; -const URL = '6ba7b811-9dad-11d1-80b4-00c04fd430c8'; -exports.URL = URL; - -function _default(name, version, hashfunc) { - function generateUUID(value, namespace, buf, offset) { - if (typeof value === 'string') { - value = stringToBytes(value); - } - - if (typeof namespace === 'string') { - namespace = (0, _parse.default)(namespace); - } - - if (namespace.length !== 16) { - throw TypeError('Namespace must be array-like (16 iterable integer values, 0-255)'); - } // Compute hash of namespace and value, Per 4.3 - // Future: Use spread syntax when supported on all platforms, e.g. `bytes = - // hashfunc([...namespace, ... value])` - - - let bytes = new Uint8Array(16 + value.length); - bytes.set(namespace); - bytes.set(value, namespace.length); - bytes = hashfunc(bytes); - bytes[6] = bytes[6] & 0x0f | version; - bytes[8] = bytes[8] & 0x3f | 0x80; - - if (buf) { - offset = offset || 0; - - for (let i = 0; i < 16; ++i) { - buf[offset + i] = bytes[i]; - } - - return buf; - } - - return (0, _stringify.default)(bytes); - } // Function#name is not settable on some platforms (#270) - - - try { - generateUUID.name = name; // eslint-disable-next-line no-empty - } catch (err) {} // For CommonJS default export support - - - generateUUID.DNS = DNS; - generateUUID.URL = URL; - return generateUUID; -} - -/***/ }), - -/***/ 51472: -/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { - -"use strict"; - - -Object.defineProperty(exports, "__esModule", ({ - value: true -})); -exports["default"] = void 0; - -var _rng = _interopRequireDefault(__nccwpck_require__(9784)); - -var _stringify = _interopRequireDefault(__nccwpck_require__(61458)); - -function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } - -function v4(options, buf, offset) { - options = options || {}; - - const rnds = options.random || (options.rng || _rng.default)(); // Per 4.4, set bits for version and `clock_seq_hi_and_reserved` - - - rnds[6] = rnds[6] & 0x0f | 0x40; - rnds[8] = rnds[8] & 0x3f | 0x80; // Copy bytes to buffer, if provided - - if (buf) { - offset = offset || 0; - - for (let i = 0; i < 16; ++i) { - buf[offset + i] = rnds[i]; - } - - return buf; - } - - return (0, _stringify.default)(rnds); -} - -var _default = v4; -exports["default"] = _default; - -/***/ }), - -/***/ 16217: -/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { - -"use strict"; - - -Object.defineProperty(exports, "__esModule", ({ - value: true -})); -exports["default"] = void 0; - -var _v = _interopRequireDefault(__nccwpck_require__(65920)); - -var _sha = _interopRequireDefault(__nccwpck_require__(38844)); - -function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } - -const v5 = (0, _v.default)('v5', 0x50, _sha.default); -var _default = v5; -exports["default"] = _default; - -/***/ }), - -/***/ 92609: -/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { - -"use strict"; - - -Object.defineProperty(exports, "__esModule", ({ - value: true -})); -exports["default"] = void 0; - -var _regex = _interopRequireDefault(__nccwpck_require__(86230)); - -function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } - -function validate(uuid) { - return typeof uuid === 'string' && _regex.default.test(uuid); -} - -var _default = validate; -exports["default"] = _default; - -/***/ }), - -/***/ 40427: -/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { - -"use strict"; - - -Object.defineProperty(exports, "__esModule", ({ - value: true -})); -exports["default"] = void 0; - -var _validate = _interopRequireDefault(__nccwpck_require__(92609)); - -function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } - -function version(uuid) { - if (!(0, _validate.default)(uuid)) { - throw TypeError('Invalid UUID'); - } - - return parseInt(uuid.substr(14, 1), 16); -} - -var _default = version; -exports["default"] = _default; - -/***/ }), - -/***/ 74087: -/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { - -"use strict"; - -Object.defineProperty(exports, "__esModule", ({ value: true })); -exports.Context = void 0; -const fs_1 = __nccwpck_require__(57147); -const os_1 = __nccwpck_require__(22037); -class Context { - /** - * Hydrate the context from the environment - */ - constructor() { - var _a, _b, _c; - this.payload = {}; - if (process.env.GITHUB_EVENT_PATH) { - if ((0, fs_1.existsSync)(process.env.GITHUB_EVENT_PATH)) { - this.payload = JSON.parse((0, fs_1.readFileSync)(process.env.GITHUB_EVENT_PATH, { encoding: 'utf8' })); - } - else { - const path = process.env.GITHUB_EVENT_PATH; - process.stdout.write(`GITHUB_EVENT_PATH ${path} does not exist${os_1.EOL}`); - } - } - this.eventName = process.env.GITHUB_EVENT_NAME; - this.sha = process.env.GITHUB_SHA; - this.ref = process.env.GITHUB_REF; - this.workflow = process.env.GITHUB_WORKFLOW; - this.action = process.env.GITHUB_ACTION; - this.actor = process.env.GITHUB_ACTOR; - this.job = process.env.GITHUB_JOB; - this.runNumber = parseInt(process.env.GITHUB_RUN_NUMBER, 10); - this.runId = parseInt(process.env.GITHUB_RUN_ID, 10); - this.apiUrl = (_a = process.env.GITHUB_API_URL) !== null && _a !== void 0 ? _a : `https://api.github.com`; - this.serverUrl = (_b = process.env.GITHUB_SERVER_URL) !== null && _b !== void 0 ? _b : `https://github.com`; - this.graphqlUrl = - (_c = process.env.GITHUB_GRAPHQL_URL) !== null && _c !== void 0 ? _c : `https://api.github.com/graphql`; - } - get issue() { - const payload = this.payload; - return Object.assign(Object.assign({}, this.repo), { number: (payload.issue || payload.pull_request || payload).number }); - } - get repo() { - if (process.env.GITHUB_REPOSITORY) { - const [owner, repo] = process.env.GITHUB_REPOSITORY.split('/'); - return { owner, repo }; - } - if (this.payload.repository) { - return { - owner: this.payload.repository.owner.login, - repo: this.payload.repository.name - }; - } - throw new Error("context.repo requires a GITHUB_REPOSITORY environment variable like 'owner/repo'"); - } -} -exports.Context = Context; -//# sourceMappingURL=context.js.map - -/***/ }), - -/***/ 95438: -/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) { - -"use strict"; - -var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { - if (k2 === undefined) k2 = k; - var desc = Object.getOwnPropertyDescriptor(m, k); - if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) { - desc = { enumerable: true, get: function() { return m[k]; } }; - } - Object.defineProperty(o, k2, desc); -}) : (function(o, m, k, k2) { - if (k2 === undefined) k2 = k; - o[k2] = m[k]; -})); -var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { - Object.defineProperty(o, "default", { enumerable: true, value: v }); -}) : function(o, v) { - o["default"] = v; -}); -var __importStar = (this && this.__importStar) || function (mod) { - if (mod && mod.__esModule) return mod; - var result = {}; - if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); - __setModuleDefault(result, mod); - return result; -}; -Object.defineProperty(exports, "__esModule", ({ value: true })); -exports.getOctokit = exports.context = void 0; -const Context = __importStar(__nccwpck_require__(74087)); -const utils_1 = __nccwpck_require__(73030); -exports.context = new Context.Context(); -/** - * Returns a hydrated octokit ready to use for GitHub Actions - * - * @param token the repo PAT or GITHUB_TOKEN - * @param options other options to set - */ -function getOctokit(token, options, ...additionalPlugins) { - const GitHubWithPlugins = utils_1.GitHub.plugin(...additionalPlugins); - return new GitHubWithPlugins((0, utils_1.getOctokitOptions)(token, options)); -} -exports.getOctokit = getOctokit; -//# sourceMappingURL=github.js.map - -/***/ }), - -/***/ 47914: -/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) { - -"use strict"; - -var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { - if (k2 === undefined) k2 = k; - var desc = Object.getOwnPropertyDescriptor(m, k); - if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) { - desc = { enumerable: true, get: function() { return m[k]; } }; - } - Object.defineProperty(o, k2, desc); -}) : (function(o, m, k, k2) { - if (k2 === undefined) k2 = k; - o[k2] = m[k]; +/* eslint-disable @typescript-eslint/no-explicit-any */ +var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + var desc = Object.getOwnPropertyDescriptor(m, k); + if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) { + desc = { enumerable: true, get: function() { return m[k]; } }; + } + Object.defineProperty(o, k2, desc); +}) : (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + o[k2] = m[k]; })); var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { Object.defineProperty(o, "default", { enumerable: true, value: v }); @@ -2892,243 +2478,11 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge }); }; Object.defineProperty(exports, "__esModule", ({ value: true })); -exports.getApiBaseUrl = exports.getProxyFetch = exports.getProxyAgentDispatcher = exports.getProxyAgent = exports.getAuthString = void 0; -const httpClient = __importStar(__nccwpck_require__(96255)); -const undici_1 = __nccwpck_require__(41773); -function getAuthString(token, options) { - if (!token && !options.auth) { - throw new Error('Parameter token or opts.auth is required'); - } - else if (token && options.auth) { - throw new Error('Parameters token and opts.auth may not both be specified'); - } - return typeof options.auth === 'string' ? options.auth : `token ${token}`; -} -exports.getAuthString = getAuthString; -function getProxyAgent(destinationUrl) { - const hc = new httpClient.HttpClient(); - return hc.getAgent(destinationUrl); -} -exports.getProxyAgent = getProxyAgent; -function getProxyAgentDispatcher(destinationUrl) { - const hc = new httpClient.HttpClient(); - return hc.getAgentDispatcher(destinationUrl); -} -exports.getProxyAgentDispatcher = getProxyAgentDispatcher; -function getProxyFetch(destinationUrl) { - const httpDispatcher = getProxyAgentDispatcher(destinationUrl); - const proxyFetch = (url, opts) => __awaiter(this, void 0, void 0, function* () { - return (0, undici_1.fetch)(url, Object.assign(Object.assign({}, opts), { dispatcher: httpDispatcher })); - }); - return proxyFetch; -} -exports.getProxyFetch = getProxyFetch; -function getApiBaseUrl() { - return process.env['GITHUB_API_URL'] || 'https://api.github.com'; -} -exports.getApiBaseUrl = getApiBaseUrl; -//# sourceMappingURL=utils.js.map - -/***/ }), - -/***/ 73030: -/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) { - -"use strict"; - -var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { - if (k2 === undefined) k2 = k; - var desc = Object.getOwnPropertyDescriptor(m, k); - if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) { - desc = { enumerable: true, get: function() { return m[k]; } }; - } - Object.defineProperty(o, k2, desc); -}) : (function(o, m, k, k2) { - if (k2 === undefined) k2 = k; - o[k2] = m[k]; -})); -var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { - Object.defineProperty(o, "default", { enumerable: true, value: v }); -}) : function(o, v) { - o["default"] = v; -}); -var __importStar = (this && this.__importStar) || function (mod) { - if (mod && mod.__esModule) return mod; - var result = {}; - if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); - __setModuleDefault(result, mod); - return result; -}; -Object.defineProperty(exports, "__esModule", ({ value: true })); -exports.getOctokitOptions = exports.GitHub = exports.defaults = exports.context = void 0; -const Context = __importStar(__nccwpck_require__(74087)); -const Utils = __importStar(__nccwpck_require__(47914)); -// octokit + plugins -const core_1 = __nccwpck_require__(76762); -const plugin_rest_endpoint_methods_1 = __nccwpck_require__(83044); -const plugin_paginate_rest_1 = __nccwpck_require__(64193); -exports.context = new Context.Context(); -const baseUrl = Utils.getApiBaseUrl(); -exports.defaults = { - baseUrl, - request: { - agent: Utils.getProxyAgent(baseUrl), - fetch: Utils.getProxyFetch(baseUrl) - } -}; -exports.GitHub = core_1.Octokit.plugin(plugin_rest_endpoint_methods_1.restEndpointMethods, plugin_paginate_rest_1.paginateRest).defaults(exports.defaults); -/** - * Convience function to correctly format Octokit Options to pass into the constructor. - * - * @param token the repo PAT or GITHUB_TOKEN - * @param options other options to set - */ -function getOctokitOptions(token, options) { - const opts = Object.assign({}, options || {}); // Shallow clone - don't mutate the object provided by the caller - // Auth - const auth = Utils.getAuthString(token, opts); - if (auth) { - opts.auth = auth; - } - return opts; -} -exports.getOctokitOptions = getOctokitOptions; -//# sourceMappingURL=utils.js.map - -/***/ }), - -/***/ 35526: -/***/ (function(__unused_webpack_module, exports) { - -"use strict"; - -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; -Object.defineProperty(exports, "__esModule", ({ value: true })); -exports.PersonalAccessTokenCredentialHandler = exports.BearerCredentialHandler = exports.BasicCredentialHandler = void 0; -class BasicCredentialHandler { - constructor(username, password) { - this.username = username; - this.password = password; - } - prepareRequest(options) { - if (!options.headers) { - throw Error('The request has no headers'); - } - options.headers['Authorization'] = `Basic ${Buffer.from(`${this.username}:${this.password}`).toString('base64')}`; - } - // This handler cannot handle 401 - canHandleAuthentication() { - return false; - } - handleAuthentication() { - return __awaiter(this, void 0, void 0, function* () { - throw new Error('not implemented'); - }); - } -} -exports.BasicCredentialHandler = BasicCredentialHandler; -class BearerCredentialHandler { - constructor(token) { - this.token = token; - } - // currently implements pre-authorization - // TODO: support preAuth = false where it hooks on 401 - prepareRequest(options) { - if (!options.headers) { - throw Error('The request has no headers'); - } - options.headers['Authorization'] = `Bearer ${this.token}`; - } - // This handler cannot handle 401 - canHandleAuthentication() { - return false; - } - handleAuthentication() { - return __awaiter(this, void 0, void 0, function* () { - throw new Error('not implemented'); - }); - } -} -exports.BearerCredentialHandler = BearerCredentialHandler; -class PersonalAccessTokenCredentialHandler { - constructor(token) { - this.token = token; - } - // currently implements pre-authorization - // TODO: support preAuth = false where it hooks on 401 - prepareRequest(options) { - if (!options.headers) { - throw Error('The request has no headers'); - } - options.headers['Authorization'] = `Basic ${Buffer.from(`PAT:${this.token}`).toString('base64')}`; - } - // This handler cannot handle 401 - canHandleAuthentication() { - return false; - } - handleAuthentication() { - return __awaiter(this, void 0, void 0, function* () { - throw new Error('not implemented'); - }); - } -} -exports.PersonalAccessTokenCredentialHandler = PersonalAccessTokenCredentialHandler; -//# sourceMappingURL=auth.js.map - -/***/ }), - -/***/ 96255: -/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) { - -"use strict"; - -/* eslint-disable @typescript-eslint/no-explicit-any */ -var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { - if (k2 === undefined) k2 = k; - var desc = Object.getOwnPropertyDescriptor(m, k); - if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) { - desc = { enumerable: true, get: function() { return m[k]; } }; - } - Object.defineProperty(o, k2, desc); -}) : (function(o, m, k, k2) { - if (k2 === undefined) k2 = k; - o[k2] = m[k]; -})); -var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { - Object.defineProperty(o, "default", { enumerable: true, value: v }); -}) : function(o, v) { - o["default"] = v; -}); -var __importStar = (this && this.__importStar) || function (mod) { - if (mod && mod.__esModule) return mod; - var result = {}; - if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); - __setModuleDefault(result, mod); - return result; -}; -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; -Object.defineProperty(exports, "__esModule", ({ value: true })); -exports.HttpClient = exports.isHttps = exports.HttpClientResponse = exports.HttpClientError = exports.getProxyUrl = exports.MediaTypes = exports.Headers = exports.HttpCodes = void 0; -const http = __importStar(__nccwpck_require__(13685)); -const https = __importStar(__nccwpck_require__(95687)); -const pm = __importStar(__nccwpck_require__(19835)); -const tunnel = __importStar(__nccwpck_require__(74294)); +exports.HttpClient = exports.isHttps = exports.HttpClientResponse = exports.HttpClientError = exports.getProxyUrl = exports.MediaTypes = exports.Headers = exports.HttpCodes = void 0; +const http = __importStar(__nccwpck_require__(13685)); +const https = __importStar(__nccwpck_require__(95687)); +const pm = __importStar(__nccwpck_require__(19835)); +const tunnel = __importStar(__nccwpck_require__(74294)); const undici_1 = __nccwpck_require__(41773); var HttpCodes; (function (HttpCodes) { @@ -3593,7 +2947,7 @@ class HttpClient { if (this._keepAlive && useProxy) { agent = this._proxyAgent; } - if (!useProxy) { + if (this._keepAlive && !useProxy) { agent = this._agent; } // if agent is already assigned use that agent. @@ -3625,12 +2979,16 @@ class HttpClient { agent = tunnelAgent(agentOptions); this._proxyAgent = agent; } - // if tunneling agent isn't assigned create a new agent - if (!agent) { + // if reusing agent across request and tunneling agent isn't assigned create a new agent + if (this._keepAlive && !agent) { const options = { keepAlive: this._keepAlive, maxSockets }; agent = usingSsl ? new https.Agent(options) : new http.Agent(options); this._agent = agent; } + // if not using private agent and tunnel agent isn't setup then use global agent + if (!agent) { + agent = usingSsl ? https.globalAgent : http.globalAgent; + } if (usingSsl && this._ignoreSslError) { // we don't want to set NODE_TLS_REJECT_UNAUTHORIZED=0 since that will affect request for entire process // http.RequestOptions doesn't expose a way to modify RequestOptions.agent.options @@ -3652,7 +3010,7 @@ class HttpClient { } const usingSsl = parsedUrl.protocol === 'https:'; proxyAgent = new undici_1.ProxyAgent(Object.assign({ uri: proxyUrl.href, pipelining: !this._keepAlive ? 0 : 1 }, ((proxyUrl.username || proxyUrl.password) && { - token: `Basic ${Buffer.from(`${proxyUrl.username}:${proxyUrl.password}`).toString('base64')}` + token: `${proxyUrl.username}:${proxyUrl.password}` }))); this._proxyAgentDispatcher = proxyAgent; if (usingSsl && this._ignoreSslError) { @@ -3766,11 +3124,11 @@ function getProxyUrl(reqUrl) { })(); if (proxyVar) { try { - return new DecodedURL(proxyVar); + return new URL(proxyVar); } catch (_a) { if (!proxyVar.startsWith('http://') && !proxyVar.startsWith('https://')) - return new DecodedURL(`http://${proxyVar}`); + return new URL(`http://${proxyVar}`); } } else { @@ -3829,19 +3187,6 @@ function isLoopbackAddress(host) { hostLower.startsWith('[::1]') || hostLower.startsWith('[0:0:0:0:0:0:0:1]')); } -class DecodedURL extends URL { - constructor(url, base) { - super(url, base); - this._decodedUsername = decodeURIComponent(super.username); - this._decodedPassword = decodeURIComponent(super.password); - } - get username() { - return this._decodedUsername; - } - get password() { - return this._decodedPassword; - } -} //# sourceMappingURL=proxy.js.map /***/ }), @@ -3967,7 +3312,7 @@ var import_graphql = __nccwpck_require__(88467); var import_auth_token = __nccwpck_require__(40334); // pkg/dist-src/version.js -var VERSION = "5.2.0"; +var VERSION = "5.0.2"; // pkg/dist-src/index.js var noop = () => { @@ -4134,7 +3479,7 @@ module.exports = __toCommonJS(dist_src_exports); var import_universal_user_agent = __nccwpck_require__(45030); // pkg/dist-src/version.js -var VERSION = "9.0.5"; +var VERSION = "9.0.4"; // pkg/dist-src/defaults.js var userAgent = `octokit-endpoint.js/${VERSION} ${(0, import_universal_user_agent.getUserAgent)()}`; @@ -4519,7 +3864,7 @@ var import_request3 = __nccwpck_require__(36234); var import_universal_user_agent = __nccwpck_require__(45030); // pkg/dist-src/version.js -var VERSION = "7.1.0"; +var VERSION = "7.0.2"; // pkg/dist-src/with-defaults.js var import_request2 = __nccwpck_require__(36234); @@ -4676,7 +4021,7 @@ __export(dist_src_exports, { module.exports = __toCommonJS(dist_src_exports); // pkg/dist-src/version.js -var VERSION = "9.2.1"; +var VERSION = "9.1.5"; // pkg/dist-src/normalize-paginated-list-response.js function normalizePaginatedListResponse(response) { @@ -4837,8 +4182,6 @@ var paginatingEndpoints = [ "GET /orgs/{org}/members/{username}/codespaces", "GET /orgs/{org}/migrations", "GET /orgs/{org}/migrations/{migration_id}/repositories", - "GET /orgs/{org}/organization-roles/{role_id}/teams", - "GET /orgs/{org}/organization-roles/{role_id}/users", "GET /orgs/{org}/outside_collaborators", "GET /orgs/{org}/packages", "GET /orgs/{org}/packages/{package_type}/{package_name}/versions", @@ -5075,7 +4418,7 @@ __export(dist_src_exports, { module.exports = __toCommonJS(dist_src_exports); // pkg/dist-src/version.js -var VERSION = "10.4.1"; +var VERSION = "10.2.0"; // pkg/dist-src/generated/endpoints.js var Endpoints = { @@ -5202,9 +4545,6 @@ var Endpoints = { "GET /repos/{owner}/{repo}/actions/permissions/selected-actions" ], getArtifact: ["GET /repos/{owner}/{repo}/actions/artifacts/{artifact_id}"], - getCustomOidcSubClaimForRepo: [ - "GET /repos/{owner}/{repo}/actions/oidc/customization/sub" - ], getEnvironmentPublicKey: [ "GET /repositories/{repository_id}/environments/{environment_name}/secrets/public-key" ], @@ -5357,9 +4697,6 @@ var Endpoints = { setCustomLabelsForSelfHostedRunnerForRepo: [ "PUT /repos/{owner}/{repo}/actions/runners/{runner_id}/labels" ], - setCustomOidcSubClaimForRepo: [ - "PUT /repos/{owner}/{repo}/actions/oidc/customization/sub" - ], setGithubActionsDefaultWorkflowPermissionsOrganization: [ "PUT /orgs/{org}/actions/permissions/workflow" ], @@ -5429,7 +4766,6 @@ var Endpoints = { listWatchersForRepo: ["GET /repos/{owner}/{repo}/subscribers"], markNotificationsAsRead: ["PUT /notifications"], markRepoNotificationsAsRead: ["PUT /repos/{owner}/{repo}/notifications"], - markThreadAsDone: ["DELETE /notifications/threads/{thread_id}"], markThreadAsRead: ["PATCH /notifications/threads/{thread_id}"], setRepoSubscription: ["PUT /repos/{owner}/{repo}/subscription"], setThreadSubscription: [ @@ -5706,10 +5042,10 @@ var Endpoints = { updateForAuthenticatedUser: ["PATCH /user/codespaces/{codespace_name}"] }, copilot: { - addCopilotSeatsForTeams: [ + addCopilotForBusinessSeatsForTeams: [ "POST /orgs/{org}/copilot/billing/selected_teams" ], - addCopilotSeatsForUsers: [ + addCopilotForBusinessSeatsForUsers: [ "POST /orgs/{org}/copilot/billing/selected_users" ], cancelCopilotSeatAssignmentForTeams: [ @@ -6022,24 +5358,10 @@ var Endpoints = { } ] }, - oidc: { - getOidcCustomSubTemplateForOrg: [ - "GET /orgs/{org}/actions/oidc/customization/sub" - ], - updateOidcCustomSubTemplateForOrg: [ - "PUT /orgs/{org}/actions/oidc/customization/sub" - ] - }, orgs: { addSecurityManagerTeam: [ "PUT /orgs/{org}/security-managers/teams/{team_slug}" ], - assignTeamToOrgRole: [ - "PUT /orgs/{org}/organization-roles/teams/{team_slug}/{role_id}" - ], - assignUserToOrgRole: [ - "PUT /orgs/{org}/organization-roles/users/{username}/{role_id}" - ], blockUser: ["PUT /orgs/{org}/blocks/{username}"], cancelInvitation: ["DELETE /orgs/{org}/invitations/{invitation_id}"], checkBlockedUser: ["GET /orgs/{org}/blocks/{username}"], @@ -6048,7 +5370,6 @@ var Endpoints = { convertMemberToOutsideCollaborator: [ "PUT /orgs/{org}/outside_collaborators/{username}" ], - createCustomOrganizationRole: ["POST /orgs/{org}/organization-roles"], createInvitation: ["POST /orgs/{org}/invitations"], createOrUpdateCustomProperties: ["PATCH /orgs/{org}/properties/schema"], createOrUpdateCustomPropertiesValuesForRepos: [ @@ -6059,9 +5380,6 @@ var Endpoints = { ], createWebhook: ["POST /orgs/{org}/hooks"], delete: ["DELETE /orgs/{org}"], - deleteCustomOrganizationRole: [ - "DELETE /orgs/{org}/organization-roles/{role_id}" - ], deleteWebhook: ["DELETE /orgs/{org}/hooks/{hook_id}"], enableOrDisableSecurityProductOnAllOrgRepos: [ "POST /orgs/{org}/{security_product}/{enablement}" @@ -6073,7 +5391,6 @@ var Endpoints = { ], getMembershipForAuthenticatedUser: ["GET /user/memberships/orgs/{org}"], getMembershipForUser: ["GET /orgs/{org}/memberships/{username}"], - getOrgRole: ["GET /orgs/{org}/organization-roles/{role_id}"], getWebhook: ["GET /orgs/{org}/hooks/{hook_id}"], getWebhookConfigForOrg: ["GET /orgs/{org}/hooks/{hook_id}/config"], getWebhookDelivery: [ @@ -6089,12 +5406,6 @@ var Endpoints = { listInvitationTeams: ["GET /orgs/{org}/invitations/{invitation_id}/teams"], listMembers: ["GET /orgs/{org}/members"], listMembershipsForAuthenticatedUser: ["GET /user/memberships/orgs"], - listOrgRoleTeams: ["GET /orgs/{org}/organization-roles/{role_id}/teams"], - listOrgRoleUsers: ["GET /orgs/{org}/organization-roles/{role_id}/users"], - listOrgRoles: ["GET /orgs/{org}/organization-roles"], - listOrganizationFineGrainedPermissions: [ - "GET /orgs/{org}/organization-fine-grained-permissions" - ], listOutsideCollaborators: ["GET /orgs/{org}/outside_collaborators"], listPatGrantRepositories: [ "GET /orgs/{org}/personal-access-tokens/{pat_id}/repositories" @@ -6109,9 +5420,6 @@ var Endpoints = { listSecurityManagerTeams: ["GET /orgs/{org}/security-managers"], listWebhookDeliveries: ["GET /orgs/{org}/hooks/{hook_id}/deliveries"], listWebhooks: ["GET /orgs/{org}/hooks"], - patchCustomOrganizationRole: [ - "PATCH /orgs/{org}/organization-roles/{role_id}" - ], pingWebhook: ["POST /orgs/{org}/hooks/{hook_id}/pings"], redeliverWebhookDelivery: [ "POST /orgs/{org}/hooks/{hook_id}/deliveries/{delivery_id}/attempts" @@ -6136,18 +5444,6 @@ var Endpoints = { reviewPatGrantRequestsInBulk: [ "POST /orgs/{org}/personal-access-token-requests" ], - revokeAllOrgRolesTeam: [ - "DELETE /orgs/{org}/organization-roles/teams/{team_slug}" - ], - revokeAllOrgRolesUser: [ - "DELETE /orgs/{org}/organization-roles/users/{username}" - ], - revokeOrgRoleTeam: [ - "DELETE /orgs/{org}/organization-roles/teams/{team_slug}/{role_id}" - ], - revokeOrgRoleUser: [ - "DELETE /orgs/{org}/organization-roles/users/{username}/{role_id}" - ], setMembershipForUser: ["PUT /orgs/{org}/memberships/{username}"], setPublicMembershipForAuthenticatedUser: [ "PUT /orgs/{org}/public_members/{username}" @@ -6438,9 +5734,6 @@ var Endpoints = { {}, { mapToData: "users" } ], - cancelPagesDeployment: [ - "POST /repos/{owner}/{repo}/pages/deployments/{pages_deployment_id}/cancel" - ], checkAutomatedSecurityFixes: [ "GET /repos/{owner}/{repo}/automated-security-fixes" ], @@ -6476,15 +5769,12 @@ var Endpoints = { createForAuthenticatedUser: ["POST /user/repos"], createFork: ["POST /repos/{owner}/{repo}/forks"], createInOrg: ["POST /orgs/{org}/repos"], - createOrUpdateCustomPropertiesValues: [ - "PATCH /repos/{owner}/{repo}/properties/values" - ], createOrUpdateEnvironment: [ "PUT /repos/{owner}/{repo}/environments/{environment_name}" ], createOrUpdateFileContents: ["PUT /repos/{owner}/{repo}/contents/{path}"], createOrgRuleset: ["POST /orgs/{org}/rulesets"], - createPagesDeployment: ["POST /repos/{owner}/{repo}/pages/deployments"], + createPagesDeployment: ["POST /repos/{owner}/{repo}/pages/deployment"], createPagesSite: ["POST /repos/{owner}/{repo}/pages"], createRelease: ["POST /repos/{owner}/{repo}/releases"], createRepoRuleset: ["POST /repos/{owner}/{repo}/rulesets"], @@ -6637,9 +5927,6 @@ var Endpoints = { getOrgRulesets: ["GET /orgs/{org}/rulesets"], getPages: ["GET /repos/{owner}/{repo}/pages"], getPagesBuild: ["GET /repos/{owner}/{repo}/pages/builds/{build_id}"], - getPagesDeployment: [ - "GET /repos/{owner}/{repo}/pages/deployments/{pages_deployment_id}" - ], getPagesHealthCheck: ["GET /repos/{owner}/{repo}/pages/health"], getParticipationStats: ["GET /repos/{owner}/{repo}/stats/participation"], getPullRequestReviewProtection: [ @@ -6850,9 +6137,6 @@ var Endpoints = { ] }, securityAdvisories: { - createFork: [ - "POST /repos/{owner}/{repo}/security-advisories/{ghsa_id}/forks" - ], createPrivateVulnerabilityReport: [ "POST /repos/{owner}/{repo}/security-advisories/reports" ], @@ -7344,7 +6628,7 @@ var import_endpoint = __nccwpck_require__(59440); var import_universal_user_agent = __nccwpck_require__(45030); // pkg/dist-src/version.js -var VERSION = "8.4.0"; +var VERSION = "8.1.6"; // pkg/dist-src/is-plain-object.js function isPlainObject(value) { @@ -7369,7 +6653,7 @@ function getBufferResponse(response) { // pkg/dist-src/fetch-wrapper.js function fetchWrapper(requestOptions) { - var _a, _b, _c, _d; + var _a, _b, _c; const log = requestOptions.request && requestOptions.request.log ? requestOptions.request.log : console; const parseSuccessResponseBody = ((_a = requestOptions.request) == null ? void 0 : _a.parseSuccessResponseBody) !== false; if (isPlainObject(requestOptions.body) || Array.isArray(requestOptions.body)) { @@ -7390,9 +6674,8 @@ function fetchWrapper(requestOptions) { return fetch(requestOptions.url, { method: requestOptions.method, body: requestOptions.body, - redirect: (_c = requestOptions.request) == null ? void 0 : _c.redirect, headers: requestOptions.headers, - signal: (_d = requestOptions.request) == null ? void 0 : _d.signal, + signal: (_c = requestOptions.request) == null ? void 0 : _c.signal, // duplex must be set if request.body is ReadableStream or Async Iterables. // See https://fetch.spec.whatwg.org/#dom-requestinit-duplex. ...requestOptions.body && { duplex: "half" } @@ -7489,17 +6772,11 @@ async function getResponseData(response) { function toErrorMessage(data) { if (typeof data === "string") return data; - let suffix; - if ("documentation_url" in data) { - suffix = ` - ${data.documentation_url}`; - } else { - suffix = ""; - } if ("message" in data) { if (Array.isArray(data.errors)) { - return `${data.message}: ${data.errors.map(JSON.stringify).join(", ")}${suffix}`; + return `${data.message}: ${data.errors.map(JSON.stringify).join(", ")}`; } - return `${data.message}${suffix}`; + return data.message; } return `Unknown error: ${JSON.stringify(data)}`; } @@ -7641,6 +6918,31 @@ Object.defineProperty(apiLoader.services['acmpca'], '2017-08-22', { module.exports = AWS.ACMPCA; +/***/ }), + +/***/ 14578: +/***/ ((module, __unused_webpack_exports, __nccwpck_require__) => { + +__nccwpck_require__(73639); +var AWS = __nccwpck_require__(28437); +var Service = AWS.Service; +var apiLoader = AWS.apiLoader; + +apiLoader.services['alexaforbusiness'] = {}; +AWS.AlexaForBusiness = Service.defineService('alexaforbusiness', ['2017-11-09']); +Object.defineProperty(apiLoader.services['alexaforbusiness'], '2017-11-09', { + get: function get() { + var model = __nccwpck_require__(69786); + model.paginators = (__nccwpck_require__(21009)/* .pagination */ .o); + return model; + }, + enumerable: true, + configurable: true +}); + +module.exports = AWS.AlexaForBusiness; + + /***/ }), /***/ 26296: @@ -7745,6 +7047,7 @@ module.exports = { WAFRegional: __nccwpck_require__(23153), WorkDocs: __nccwpck_require__(38835), WorkSpaces: __nccwpck_require__(25513), + CodeStar: __nccwpck_require__(98336), LexModelBuildingService: __nccwpck_require__(37397), MarketplaceEntitlementService: __nccwpck_require__(53707), Athena: __nccwpck_require__(29434), @@ -7753,6 +7056,7 @@ module.exports = { MigrationHub: __nccwpck_require__(14688), CloudHSMV2: __nccwpck_require__(70889), Glue: __nccwpck_require__(31658), + Mobile: __nccwpck_require__(39782), Pricing: __nccwpck_require__(92765), CostExplorer: __nccwpck_require__(79523), MediaConvert: __nccwpck_require__(57220), @@ -7772,6 +7076,7 @@ module.exports = { SageMaker: __nccwpck_require__(77657), Translate: __nccwpck_require__(72544), ResourceGroups: __nccwpck_require__(58756), + AlexaForBusiness: __nccwpck_require__(14578), Cloud9: __nccwpck_require__(85473), ServerlessApplicationRepository: __nccwpck_require__(62402), ServiceDiscovery: __nccwpck_require__(91569), @@ -7868,6 +7173,7 @@ module.exports = { IoTSiteWise: __nccwpck_require__(89690), Macie2: __nccwpck_require__(57330), CodeArtifact: __nccwpck_require__(91983), + Honeycode: __nccwpck_require__(38889), IVS: __nccwpck_require__(67701), Braket: __nccwpck_require__(35429), IdentityStore: __nccwpck_require__(60222), @@ -7954,6 +7260,7 @@ module.exports = { RedshiftServerless: __nccwpck_require__(29987), RolesAnywhere: __nccwpck_require__(83604), LicenseManagerUserSubscriptions: __nccwpck_require__(37725), + BackupStorage: __nccwpck_require__(82304), PrivateNetworks: __nccwpck_require__(63088), SupportApp: __nccwpck_require__(51288), ControlTower: __nccwpck_require__(77574), @@ -7963,6 +7270,7 @@ module.exports = { ResourceExplorer2: __nccwpck_require__(74071), Scheduler: __nccwpck_require__(94840), ChimeSDKVoice: __nccwpck_require__(349), + IoTRoboRunner: __nccwpck_require__(22163), SsmSap: __nccwpck_require__(44552), OAM: __nccwpck_require__(9319), ARCZonalShift: __nccwpck_require__(54280), @@ -8001,6 +7309,7 @@ module.exports = { DataZone: __nccwpck_require__(31763), LaunchWizard: __nccwpck_require__(71060), TrustedAdvisor: __nccwpck_require__(4992), + CloudFrontKeyValueStore: __nccwpck_require__(47859), InspectorScan: __nccwpck_require__(25467), BCMDataExports: __nccwpck_require__(56703), CostOptimizationHub: __nccwpck_require__(55443), @@ -8016,23 +7325,9 @@ module.exports = { CleanRoomsML: __nccwpck_require__(47594), MarketplaceAgreement: __nccwpck_require__(50379), MarketplaceDeployment: __nccwpck_require__(56811), + NeptuneGraph: __nccwpck_require__(77598), NetworkMonitor: __nccwpck_require__(77614), - SupplyChain: __nccwpck_require__(39674), - Artifact: __nccwpck_require__(63151), - Chatbot: __nccwpck_require__(14373), - TimestreamInfluxDB: __nccwpck_require__(13610), - CodeConnections: __nccwpck_require__(19123), - Deadline: __nccwpck_require__(29242), - ControlCatalog: __nccwpck_require__(87324), - Route53Profiles: __nccwpck_require__(13907), - MailManager: __nccwpck_require__(46253), - TaxSettings: __nccwpck_require__(44688), - ApplicationSignals: __nccwpck_require__(17535), - PcaConnectorScep: __nccwpck_require__(72523), - AppTest: __nccwpck_require__(50505), - QApps: __nccwpck_require__(2725), - SSMQuickSetup: __nccwpck_require__(99330), - PCS: __nccwpck_require__(6301) + SupplyChain: __nccwpck_require__(39674) }; /***/ }), @@ -8414,31 +7709,6 @@ Object.defineProperty(apiLoader.services['applicationinsights'], '2018-11-25', { module.exports = AWS.ApplicationInsights; -/***/ }), - -/***/ 17535: -/***/ ((module, __unused_webpack_exports, __nccwpck_require__) => { - -__nccwpck_require__(73639); -var AWS = __nccwpck_require__(28437); -var Service = AWS.Service; -var apiLoader = AWS.apiLoader; - -apiLoader.services['applicationsignals'] = {}; -AWS.ApplicationSignals = Service.defineService('applicationsignals', ['2024-04-15']); -Object.defineProperty(apiLoader.services['applicationsignals'], '2024-04-15', { - get: function get() { - var model = __nccwpck_require__(75196); - model.paginators = (__nccwpck_require__(15302)/* .pagination */ .o); - return model; - }, - enumerable: true, - configurable: true -}); - -module.exports = AWS.ApplicationSignals; - - /***/ }), /***/ 69226: @@ -8549,32 +7819,6 @@ Object.defineProperty(apiLoader.services['appsync'], '2017-07-25', { module.exports = AWS.AppSync; -/***/ }), - -/***/ 50505: -/***/ ((module, __unused_webpack_exports, __nccwpck_require__) => { - -__nccwpck_require__(73639); -var AWS = __nccwpck_require__(28437); -var Service = AWS.Service; -var apiLoader = AWS.apiLoader; - -apiLoader.services['apptest'] = {}; -AWS.AppTest = Service.defineService('apptest', ['2022-12-06']); -Object.defineProperty(apiLoader.services['apptest'], '2022-12-06', { - get: function get() { - var model = __nccwpck_require__(53250); - model.paginators = (__nccwpck_require__(32351)/* .pagination */ .o); - model.waiters = (__nccwpck_require__(60289)/* .waiters */ .V); - return model; - }, - enumerable: true, - configurable: true -}); - -module.exports = AWS.AppTest; - - /***/ }), /***/ 54280: @@ -8600,32 +7844,6 @@ Object.defineProperty(apiLoader.services['arczonalshift'], '2022-10-30', { module.exports = AWS.ARCZonalShift; -/***/ }), - -/***/ 63151: -/***/ ((module, __unused_webpack_exports, __nccwpck_require__) => { - -__nccwpck_require__(73639); -var AWS = __nccwpck_require__(28437); -var Service = AWS.Service; -var apiLoader = AWS.apiLoader; - -apiLoader.services['artifact'] = {}; -AWS.Artifact = Service.defineService('artifact', ['2018-05-10']); -Object.defineProperty(apiLoader.services['artifact'], '2018-05-10', { - get: function get() { - var model = __nccwpck_require__(76591); - model.paginators = (__nccwpck_require__(2961)/* .pagination */ .o); - model.waiters = (__nccwpck_require__(35293)/* .waiters */ .V); - return model; - }, - enumerable: true, - configurable: true -}); - -module.exports = AWS.Artifact; - - /***/ }), /***/ 29434: @@ -8826,6 +8044,31 @@ Object.defineProperty(apiLoader.services['backupgateway'], '2021-01-01', { module.exports = AWS.BackupGateway; +/***/ }), + +/***/ 82304: +/***/ ((module, __unused_webpack_exports, __nccwpck_require__) => { + +__nccwpck_require__(73639); +var AWS = __nccwpck_require__(28437); +var Service = AWS.Service; +var apiLoader = AWS.apiLoader; + +apiLoader.services['backupstorage'] = {}; +AWS.BackupStorage = Service.defineService('backupstorage', ['2018-04-10']); +Object.defineProperty(apiLoader.services['backupstorage'], '2018-04-10', { + get: function get() { + var model = __nccwpck_require__(97436); + model.paginators = (__nccwpck_require__(73644)/* .pagination */ .o); + return model; + }, + enumerable: true, + configurable: true +}); + +module.exports = AWS.BackupStorage; + + /***/ }), /***/ 10000: @@ -9054,31 +8297,6 @@ Object.defineProperty(apiLoader.services['budgets'], '2016-10-20', { module.exports = AWS.Budgets; -/***/ }), - -/***/ 14373: -/***/ ((module, __unused_webpack_exports, __nccwpck_require__) => { - -__nccwpck_require__(73639); -var AWS = __nccwpck_require__(28437); -var Service = AWS.Service; -var apiLoader = AWS.apiLoader; - -apiLoader.services['chatbot'] = {}; -AWS.Chatbot = Service.defineService('chatbot', ['2017-10-11']); -Object.defineProperty(apiLoader.services['chatbot'], '2017-10-11', { - get: function get() { - var model = __nccwpck_require__(39802); - model.paginators = (__nccwpck_require__(81040)/* .pagination */ .o); - return model; - }, - enumerable: true, - configurable: true -}); - -module.exports = AWS.Chatbot; - - /***/ }), /***/ 84646: @@ -9271,7 +8489,6 @@ Object.defineProperty(apiLoader.services['cleanroomsml'], '2023-09-06', { get: function get() { var model = __nccwpck_require__(1867); model.paginators = (__nccwpck_require__(89767)/* .pagination */ .o); - model.waiters = (__nccwpck_require__(39743)/* .waiters */ .V); return model; }, enumerable: true, @@ -9479,6 +8696,31 @@ Object.defineProperty(apiLoader.services['cloudfront'], '2020-05-31', { module.exports = AWS.CloudFront; +/***/ }), + +/***/ 47859: +/***/ ((module, __unused_webpack_exports, __nccwpck_require__) => { + +__nccwpck_require__(73639); +var AWS = __nccwpck_require__(28437); +var Service = AWS.Service; +var apiLoader = AWS.apiLoader; + +apiLoader.services['cloudfrontkeyvaluestore'] = {}; +AWS.CloudFrontKeyValueStore = Service.defineService('cloudfrontkeyvaluestore', ['2022-07-26']); +Object.defineProperty(apiLoader.services['cloudfrontkeyvaluestore'], '2022-07-26', { + get: function get() { + var model = __nccwpck_require__(49651); + model.paginators = (__nccwpck_require__(41274)/* .pagination */ .o); + return model; + }, + enumerable: true, + configurable: true +}); + +module.exports = AWS.CloudFrontKeyValueStore; + + /***/ }), /***/ 59976: @@ -9815,31 +9057,6 @@ Object.defineProperty(apiLoader.services['codecommit'], '2015-04-13', { module.exports = AWS.CodeCommit; -/***/ }), - -/***/ 19123: -/***/ ((module, __unused_webpack_exports, __nccwpck_require__) => { - -__nccwpck_require__(73639); -var AWS = __nccwpck_require__(28437); -var Service = AWS.Service; -var apiLoader = AWS.apiLoader; - -apiLoader.services['codeconnections'] = {}; -AWS.CodeConnections = Service.defineService('codeconnections', ['2023-12-01']); -Object.defineProperty(apiLoader.services['codeconnections'], '2023-12-01', { - get: function get() { - var model = __nccwpck_require__(21781); - model.paginators = (__nccwpck_require__(96720)/* .pagination */ .o); - return model; - }, - enumerable: true, - configurable: true -}); - -module.exports = AWS.CodeConnections; - - /***/ }), /***/ 54599: @@ -9967,6 +9184,31 @@ Object.defineProperty(apiLoader.services['codepipeline'], '2015-07-09', { module.exports = AWS.CodePipeline; +/***/ }), + +/***/ 98336: +/***/ ((module, __unused_webpack_exports, __nccwpck_require__) => { + +__nccwpck_require__(73639); +var AWS = __nccwpck_require__(28437); +var Service = AWS.Service; +var apiLoader = AWS.apiLoader; + +apiLoader.services['codestar'] = {}; +AWS.CodeStar = Service.defineService('codestar', ['2017-04-19']); +Object.defineProperty(apiLoader.services['codestar'], '2017-04-19', { + get: function get() { + var model = __nccwpck_require__(12425); + model.paginators = (__nccwpck_require__(70046)/* .pagination */ .o); + return model; + }, + enumerable: true, + configurable: true +}); + +module.exports = AWS.CodeStar; + + /***/ }), /***/ 78270: @@ -10317,32 +9559,6 @@ Object.defineProperty(apiLoader.services['connectparticipant'], '2018-09-07', { module.exports = AWS.ConnectParticipant; -/***/ }), - -/***/ 87324: -/***/ ((module, __unused_webpack_exports, __nccwpck_require__) => { - -__nccwpck_require__(73639); -var AWS = __nccwpck_require__(28437); -var Service = AWS.Service; -var apiLoader = AWS.apiLoader; - -apiLoader.services['controlcatalog'] = {}; -AWS.ControlCatalog = Service.defineService('controlcatalog', ['2018-05-10']); -Object.defineProperty(apiLoader.services['controlcatalog'], '2018-05-10', { - get: function get() { - var model = __nccwpck_require__(65015); - model.paginators = (__nccwpck_require__(31095)/* .pagination */ .o); - model.waiters = (__nccwpck_require__(7580)/* .waiters */ .V); - return model; - }, - enumerable: true, - configurable: true -}); - -module.exports = AWS.ControlCatalog; - - /***/ }), /***/ 77574: @@ -10409,7 +9625,6 @@ Object.defineProperty(apiLoader.services['costoptimizationhub'], '2022-07-26', { get: function get() { var model = __nccwpck_require__(56073); model.paginators = (__nccwpck_require__(70563)/* .pagination */ .o); - model.waiters = (__nccwpck_require__(17029)/* .waiters */ .V); return model; }, enumerable: true, @@ -10620,32 +9835,6 @@ Object.defineProperty(apiLoader.services['dax'], '2017-04-19', { module.exports = AWS.DAX; -/***/ }), - -/***/ 29242: -/***/ ((module, __unused_webpack_exports, __nccwpck_require__) => { - -__nccwpck_require__(73639); -var AWS = __nccwpck_require__(28437); -var Service = AWS.Service; -var apiLoader = AWS.apiLoader; - -apiLoader.services['deadline'] = {}; -AWS.Deadline = Service.defineService('deadline', ['2023-10-12']); -Object.defineProperty(apiLoader.services['deadline'], '2023-10-12', { - get: function get() { - var model = __nccwpck_require__(27799); - model.paginators = (__nccwpck_require__(60855)/* .pagination */ .o); - model.waiters = (__nccwpck_require__(54096)/* .waiters */ .V); - return model; - }, - enumerable: true, - configurable: true -}); - -module.exports = AWS.Deadline; - - /***/ }), /***/ 60674: @@ -12076,6 +11265,31 @@ Object.defineProperty(apiLoader.services['healthlake'], '2017-07-01', { module.exports = AWS.HealthLake; +/***/ }), + +/***/ 38889: +/***/ ((module, __unused_webpack_exports, __nccwpck_require__) => { + +__nccwpck_require__(73639); +var AWS = __nccwpck_require__(28437); +var Service = AWS.Service; +var apiLoader = AWS.apiLoader; + +apiLoader.services['honeycode'] = {}; +AWS.Honeycode = Service.defineService('honeycode', ['2020-03-01']); +Object.defineProperty(apiLoader.services['honeycode'], '2020-03-01', { + get: function get() { + var model = __nccwpck_require__(27577); + model.paginators = (__nccwpck_require__(12243)/* .pagination */ .o); + return model; + }, + enumerable: true, + configurable: true +}); + +module.exports = AWS.Honeycode; + + /***/ }), /***/ 50058: @@ -12554,6 +11768,31 @@ Object.defineProperty(apiLoader.services['iotjobsdataplane'], '2017-09-29', { module.exports = AWS.IoTJobsDataPlane; +/***/ }), + +/***/ 22163: +/***/ ((module, __unused_webpack_exports, __nccwpck_require__) => { + +__nccwpck_require__(73639); +var AWS = __nccwpck_require__(28437); +var Service = AWS.Service; +var apiLoader = AWS.apiLoader; + +apiLoader.services['iotroborunner'] = {}; +AWS.IoTRoboRunner = Service.defineService('iotroborunner', ['2018-05-10']); +Object.defineProperty(apiLoader.services['iotroborunner'], '2018-05-10', { + get: function get() { + var model = __nccwpck_require__(11483); + model.paginators = (__nccwpck_require__(82393)/* .pagination */ .o); + return model; + }, + enumerable: true, + configurable: true +}); + +module.exports = AWS.IoTRoboRunner; + + /***/ }), /***/ 98562: @@ -12722,7 +11961,6 @@ Object.defineProperty(apiLoader.services['ivschat'], '2020-07-14', { get: function get() { var model = __nccwpck_require__(77512); model.paginators = (__nccwpck_require__(85556)/* .pagination */ .o); - model.waiters = (__nccwpck_require__(617)/* .waiters */ .V); return model; }, enumerable: true, @@ -12748,7 +11986,6 @@ Object.defineProperty(apiLoader.services['ivsrealtime'], '2020-07-14', { get: function get() { var model = __nccwpck_require__(23084); model.paginators = (__nccwpck_require__(64507)/* .pagination */ .o); - model.waiters = (__nccwpck_require__(40441)/* .waiters */ .V); return model; }, enumerable: true, @@ -13575,31 +12812,6 @@ Object.defineProperty(apiLoader.services['macie2'], '2020-01-01', { module.exports = AWS.Macie2; -/***/ }), - -/***/ 46253: -/***/ ((module, __unused_webpack_exports, __nccwpck_require__) => { - -__nccwpck_require__(73639); -var AWS = __nccwpck_require__(28437); -var Service = AWS.Service; -var apiLoader = AWS.apiLoader; - -apiLoader.services['mailmanager'] = {}; -AWS.MailManager = Service.defineService('mailmanager', ['2023-10-17']); -Object.defineProperty(apiLoader.services['mailmanager'], '2023-10-17', { - get: function get() { - var model = __nccwpck_require__(69303); - model.paginators = (__nccwpck_require__(14795)/* .pagination */ .o); - return model; - }, - enumerable: true, - configurable: true -}); - -module.exports = AWS.MailManager; - - /***/ }), /***/ 85143: @@ -14231,6 +13443,31 @@ Object.defineProperty(apiLoader.services['migrationhubstrategy'], '2020-02-19', module.exports = AWS.MigrationHubStrategy; +/***/ }), + +/***/ 39782: +/***/ ((module, __unused_webpack_exports, __nccwpck_require__) => { + +__nccwpck_require__(73639); +var AWS = __nccwpck_require__(28437); +var Service = AWS.Service; +var apiLoader = AWS.apiLoader; + +apiLoader.services['mobile'] = {}; +AWS.Mobile = Service.defineService('mobile', ['2017-07-01']); +Object.defineProperty(apiLoader.services['mobile'], '2017-07-01', { + get: function get() { + var model = __nccwpck_require__(51691); + model.paginators = (__nccwpck_require__(43522)/* .pagination */ .o); + return model; + }, + enumerable: true, + configurable: true +}); + +module.exports = AWS.Mobile; + + /***/ }), /***/ 66690: @@ -14382,6 +13619,33 @@ Object.defineProperty(apiLoader.services['neptunedata'], '2023-08-01', { module.exports = AWS.Neptunedata; +/***/ }), + +/***/ 77598: +/***/ ((module, __unused_webpack_exports, __nccwpck_require__) => { + +__nccwpck_require__(73639); +var AWS = __nccwpck_require__(28437); +var Service = AWS.Service; +var apiLoader = AWS.apiLoader; + +apiLoader.services['neptunegraph'] = {}; +AWS.NeptuneGraph = Service.defineService('neptunegraph', ['2023-11-29']); +__nccwpck_require__(71963); +Object.defineProperty(apiLoader.services['neptunegraph'], '2023-11-29', { + get: function get() { + var model = __nccwpck_require__(19121); + model.paginators = (__nccwpck_require__(85871)/* .pagination */ .o); + model.waiters = (__nccwpck_require__(91832)/* .waiters */ .V); + return model; + }, + enumerable: true, + configurable: true +}); + +module.exports = AWS.NeptuneGraph; + + /***/ }), /***/ 84626: @@ -14753,7 +14017,6 @@ Object.defineProperty(apiLoader.services['paymentcryptography'], '2021-09-14', { get: function get() { var model = __nccwpck_require__(86072); model.paginators = (__nccwpck_require__(17819)/* .pagination */ .o); - model.waiters = (__nccwpck_require__(60238)/* .waiters */ .V); return model; }, enumerable: true, @@ -14779,7 +14042,6 @@ Object.defineProperty(apiLoader.services['paymentcryptographydata'], '2022-02-03 get: function get() { var model = __nccwpck_require__(68578); model.paginators = (__nccwpck_require__(89757)/* .pagination */ .o); - model.waiters = (__nccwpck_require__(48855)/* .waiters */ .V); return model; }, enumerable: true, @@ -14814,58 +14076,6 @@ Object.defineProperty(apiLoader.services['pcaconnectorad'], '2018-05-10', { module.exports = AWS.PcaConnectorAd; -/***/ }), - -/***/ 72523: -/***/ ((module, __unused_webpack_exports, __nccwpck_require__) => { - -__nccwpck_require__(73639); -var AWS = __nccwpck_require__(28437); -var Service = AWS.Service; -var apiLoader = AWS.apiLoader; - -apiLoader.services['pcaconnectorscep'] = {}; -AWS.PcaConnectorScep = Service.defineService('pcaconnectorscep', ['2018-05-10']); -Object.defineProperty(apiLoader.services['pcaconnectorscep'], '2018-05-10', { - get: function get() { - var model = __nccwpck_require__(99967); - model.paginators = (__nccwpck_require__(82984)/* .pagination */ .o); - model.waiters = (__nccwpck_require__(60372)/* .waiters */ .V); - return model; - }, - enumerable: true, - configurable: true -}); - -module.exports = AWS.PcaConnectorScep; - - -/***/ }), - -/***/ 6301: -/***/ ((module, __unused_webpack_exports, __nccwpck_require__) => { - -__nccwpck_require__(73639); -var AWS = __nccwpck_require__(28437); -var Service = AWS.Service; -var apiLoader = AWS.apiLoader; - -apiLoader.services['pcs'] = {}; -AWS.PCS = Service.defineService('pcs', ['2023-02-10']); -Object.defineProperty(apiLoader.services['pcs'], '2023-02-10', { - get: function get() { - var model = __nccwpck_require__(54950); - model.paginators = (__nccwpck_require__(66315)/* .pagination */ .o); - model.waiters = (__nccwpck_require__(83363)/* .waiters */ .V); - return model; - }, - enumerable: true, - configurable: true -}); - -module.exports = AWS.PCS; - - /***/ }), /***/ 33696: @@ -15081,7 +14291,6 @@ Object.defineProperty(apiLoader.services['pipes'], '2015-10-07', { get: function get() { var model = __nccwpck_require__(40616); model.paginators = (__nccwpck_require__(17710)/* .pagination */ .o); - model.waiters = (__nccwpck_require__(95823)/* .waiters */ .V); return model; }, enumerable: true, @@ -15194,32 +14403,6 @@ Object.defineProperty(apiLoader.services['proton'], '2020-07-20', { module.exports = AWS.Proton; -/***/ }), - -/***/ 2725: -/***/ ((module, __unused_webpack_exports, __nccwpck_require__) => { - -__nccwpck_require__(73639); -var AWS = __nccwpck_require__(28437); -var Service = AWS.Service; -var apiLoader = AWS.apiLoader; - -apiLoader.services['qapps'] = {}; -AWS.QApps = Service.defineService('qapps', ['2023-11-27']); -Object.defineProperty(apiLoader.services['qapps'], '2023-11-27', { - get: function get() { - var model = __nccwpck_require__(41128); - model.paginators = (__nccwpck_require__(17220)/* .pagination */ .o); - model.waiters = (__nccwpck_require__(87485)/* .waiters */ .V); - return model; - }, - enumerable: true, - configurable: true -}); - -module.exports = AWS.QApps; - - /***/ }), /***/ 26842: @@ -15236,7 +14419,6 @@ Object.defineProperty(apiLoader.services['qbusiness'], '2023-11-27', { get: function get() { var model = __nccwpck_require__(12388); model.paginators = (__nccwpck_require__(51051)/* .pagination */ .o); - model.waiters = (__nccwpck_require__(81994)/* .waiters */ .V); return model; }, enumerable: true, @@ -15815,31 +14997,6 @@ Object.defineProperty(apiLoader.services['route53domains'], '2014-05-15', { module.exports = AWS.Route53Domains; -/***/ }), - -/***/ 13907: -/***/ ((module, __unused_webpack_exports, __nccwpck_require__) => { - -__nccwpck_require__(73639); -var AWS = __nccwpck_require__(28437); -var Service = AWS.Service; -var apiLoader = AWS.apiLoader; - -apiLoader.services['route53profiles'] = {}; -AWS.Route53Profiles = Service.defineService('route53profiles', ['2018-05-10']); -Object.defineProperty(apiLoader.services['route53profiles'], '2018-05-10', { - get: function get() { - var model = __nccwpck_require__(38685); - model.paginators = (__nccwpck_require__(18297)/* .pagination */ .o); - return model; - }, - enumerable: true, - configurable: true -}); - -module.exports = AWS.Route53Profiles; - - /***/ }), /***/ 35738: @@ -16826,31 +15983,6 @@ Object.defineProperty(apiLoader.services['ssmincidents'], '2018-05-10', { module.exports = AWS.SSMIncidents; -/***/ }), - -/***/ 99330: -/***/ ((module, __unused_webpack_exports, __nccwpck_require__) => { - -__nccwpck_require__(73639); -var AWS = __nccwpck_require__(28437); -var Service = AWS.Service; -var apiLoader = AWS.apiLoader; - -apiLoader.services['ssmquicksetup'] = {}; -AWS.SSMQuickSetup = Service.defineService('ssmquicksetup', ['2018-05-10']); -Object.defineProperty(apiLoader.services['ssmquicksetup'], '2018-05-10', { - get: function get() { - var model = __nccwpck_require__(49291); - model.paginators = (__nccwpck_require__(73317)/* .pagination */ .o); - return model; - }, - enumerable: true, - configurable: true -}); - -module.exports = AWS.SSMQuickSetup; - - /***/ }), /***/ 44552: @@ -17153,31 +16285,6 @@ Object.defineProperty(apiLoader.services['synthetics'], '2017-10-11', { module.exports = AWS.Synthetics; -/***/ }), - -/***/ 44688: -/***/ ((module, __unused_webpack_exports, __nccwpck_require__) => { - -__nccwpck_require__(73639); -var AWS = __nccwpck_require__(28437); -var Service = AWS.Service; -var apiLoader = AWS.apiLoader; - -apiLoader.services['taxsettings'] = {}; -AWS.TaxSettings = Service.defineService('taxsettings', ['2018-05-10']); -Object.defineProperty(apiLoader.services['taxsettings'], '2018-05-10', { - get: function get() { - var model = __nccwpck_require__(82948); - model.paginators = (__nccwpck_require__(60890)/* .pagination */ .o); - return model; - }, - enumerable: true, - configurable: true -}); - -module.exports = AWS.TaxSettings; - - /***/ }), /***/ 58523: @@ -17203,31 +16310,6 @@ Object.defineProperty(apiLoader.services['textract'], '2018-06-27', { module.exports = AWS.Textract; -/***/ }), - -/***/ 13610: -/***/ ((module, __unused_webpack_exports, __nccwpck_require__) => { - -__nccwpck_require__(73639); -var AWS = __nccwpck_require__(28437); -var Service = AWS.Service; -var apiLoader = AWS.apiLoader; - -apiLoader.services['timestreaminfluxdb'] = {}; -AWS.TimestreamInfluxDB = Service.defineService('timestreaminfluxdb', ['2023-01-27']); -Object.defineProperty(apiLoader.services['timestreaminfluxdb'], '2023-01-27', { - get: function get() { - var model = __nccwpck_require__(63987); - model.paginators = (__nccwpck_require__(25823)/* .pagination */ .o); - return model; - }, - enumerable: true, - configurable: true -}); - -module.exports = AWS.TimestreamInfluxDB; - - /***/ }), /***/ 24529: @@ -18881,7 +17963,7 @@ AWS.util.update(AWS, { /** * @constant */ - VERSION: '2.1687.0', + VERSION: '2.1535.0', /** * @api private @@ -19461,15 +18543,6 @@ var STS = __nccwpck_require__(57513); * identity providers. See {constructor} for an example on creating a credentials * object with proper property values. * - * DISCLAIMER: This convenience method leverages the Enhanced (simplified) Authflow. The underlying - * implementation calls Cognito's `getId()` and `GetCredentialsForIdentity()`. - * In this flow there is no way to explicitly set a session policy, resulting in - * STS attaching the default policy and limiting the permissions of the federated role. - * To be able to explicitly set a session policy, do not use this convenience method. - * Instead, you can use the Cognito client to call `getId()`, `GetOpenIdToken()` and then use - * that token with your desired session policy to call STS's `AssumeRoleWithWebIdentity()` - * For further reading refer to: https://docs.aws.amazon.com/cognito/latest/developerguide/authentication-flow.html - * * ## Refreshing Credentials from Identity Service * * In addition to AWS credentials expiring after a given amount of time, the @@ -20576,16 +19649,13 @@ AWS.ProcessCredentials = AWS.util.inherit(AWS.Credentials, { /***/ 88764: /***/ ((__unused_webpack_module, __unused_webpack_exports, __nccwpck_require__) => { -var fs = __nccwpck_require__(57147); - var AWS = __nccwpck_require__(28437), ENV_RELATIVE_URI = 'AWS_CONTAINER_CREDENTIALS_RELATIVE_URI', ENV_FULL_URI = 'AWS_CONTAINER_CREDENTIALS_FULL_URI', ENV_AUTH_TOKEN = 'AWS_CONTAINER_AUTHORIZATION_TOKEN', - ENV_AUTH_TOKEN_FILE = 'AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE', FULL_URI_UNRESTRICTED_PROTOCOLS = ['https:'], FULL_URI_ALLOWED_PROTOCOLS = ['http:', 'https:'], - FULL_URI_ALLOWED_HOSTNAMES = ['localhost', '127.0.0.1', '169.254.170.23'], + FULL_URI_ALLOWED_HOSTNAMES = ['localhost', '127.0.0.1'], RELATIVE_URI_HOST = '169.254.170.2'; /** @@ -20694,16 +19764,7 @@ AWS.RemoteCredentials = AWS.util.inherit(AWS.Credentials, { * @api private */ getECSAuthToken: function getECSAuthToken() { - if (process && process.env && (process.env[ENV_FULL_URI] || process.env[ENV_AUTH_TOKEN_FILE])) { - if (!process.env[ENV_AUTH_TOKEN] && process.env[ENV_AUTH_TOKEN_FILE]) { - try { - var data = fs.readFileSync(process.env[ENV_AUTH_TOKEN_FILE]).toString(); - return data; - } catch (error) { - console.error('Error reading token file:', error); - throw error; // Re-throw the error to propagate it - } - } + if (process && process.env && process.env[ENV_FULL_URI]) { return process.env[ENV_AUTH_TOKEN]; } }, @@ -21373,7 +20434,7 @@ AWS.SsoCredentials = AWS.util.inherit(AWS.Credentials, { var ssoTokenProvider = new AWS.SSOTokenProvider({ profile: profileName, }); - ssoTokenProvider.get(function (err) { + ssoTokenProvider.load(function (err) { if (err) { return callback(err); } @@ -24708,7 +23769,6 @@ AWS.EventListeners = { this.httpRequest.endpoint = new AWS.Endpoint(resp.httpResponse.headers['location']); this.httpRequest.headers['Host'] = this.httpRequest.endpoint.host; - this.httpRequest.path = this.httpRequest.endpoint.path; resp.error.redirect = true; resp.error.retryable = true; } @@ -25513,10 +24573,9 @@ module.exports = JsonParser; /***/ ((module) => { var warning = [ - 'The AWS SDK for JavaScript (v2) will enter maintenance mode', - 'on September 8, 2024 and reach end-of-support on September 8, 2025.\n', + 'We are formalizing our plans to enter AWS SDK for JavaScript (v2) into maintenance mode in 2023.\n', 'Please migrate your code to use AWS SDK for JavaScript (v3).', - 'For more information, check blog post at https://a.co/cUPnyil' + 'For more information, check the migration guide at https://a.co/7PzMCcy' ].join('\n'); module.exports = { @@ -25689,8 +24748,9 @@ AWS.MetadataService = inherit({ loadCredentialsCallbacks: [], /** - * Fetches metadata token used for authenticating against the instance metadata service. + * Fetches metadata token used for getting credentials * + * @api private * @callback callback function(err, token) * Called when token is loaded from the resource */ @@ -27771,7 +26831,6 @@ module.exports = { /***/ 5883: /***/ ((module, __unused_webpack_exports, __nccwpck_require__) => { -var AWS = __nccwpck_require__(28437); var util = __nccwpck_require__(77985); var Rest = __nccwpck_require__(98200); var Json = __nccwpck_require__(30083); @@ -27849,7 +26908,7 @@ function extractData(resp) { var body = resp.httpResponse.body; if (payloadMember.isEventStream) { parser = new JsonParser(); - resp.data[rules.payload] = util.createEventStream( + resp.data[payload] = util.createEventStream( AWS.HttpClient.streamsApiVersion === 2 ? resp.httpResponse.stream : body, parser, payloadMember @@ -28265,9 +27324,7 @@ function serializeList(name, list, rules, fn) { var memberRules = rules.member || {}; if (list.length === 0) { - if (rules.api.protocol !== 'ec2') { - fn.call(this, name, null); - } + fn.call(this, name, null); return; } @@ -28673,9 +27730,7 @@ function getEndpointSuffix(region) { '^cn\\-\\w+\\-\\d+$': 'amazonaws.com.cn', '^us\\-gov\\-\\w+\\-\\d+$': 'amazonaws.com', '^us\\-iso\\-\\w+\\-\\d+$': 'c2s.ic.gov', - '^us\\-isob\\-\\w+\\-\\d+$': 'sc2s.sgov.gov', - '^eu\\-isoe\\-west\\-1$': 'cloud.adc-e.uk', - '^us\\-isof\\-\\w+\\-\\d+$': 'csp.hci.ic.gov', + '^us\\-isob\\-\\w+\\-\\d+$': 'sc2s.sgov.gov' }; var defaultSuffix = 'amazonaws.com'; var regexes = Object.keys(regionRegexes); @@ -31934,9 +30989,6 @@ AWS.util.update(AWS.CloudSearchDomain.prototype, { */ convertGetToPost: function(request) { var httpRequest = request.httpRequest; - if (httpRequest.method === 'POST') { - return; - } // convert queries to POST to avoid length restrictions var path = httpRequest.path.split('?'); httpRequest.method = 'POST'; @@ -32461,6 +31513,27 @@ AWS.util.update(AWS.Neptune.prototype, { }); +/***/ }), + +/***/ 71963: +/***/ ((__unused_webpack_module, __unused_webpack_exports, __nccwpck_require__) => { + +var AWS = __nccwpck_require__(28437); + +if (AWS.NeptuneGraph) { + AWS.util.update(AWS.NeptuneGraph.prototype, { + /** + * @api private + */ + validateService: function validateService() { + var msg = 'AWS Neptune Graph is not available in the AWS SDK for JavaScript v2, consider using the AWS SDK for JavaScript v3: https://www.npmjs.com/package/@aws-sdk/client-neptune-graph'; + throw AWS.util.error(new Error(), + {name: 'ServiceExcludedFromV2', message: msg}); + }, + }); +}; + + /***/ }), /***/ 53199: @@ -32751,7 +31824,6 @@ AWS.util.update(AWS.S3.prototype, { * @api private */ setupRequestListeners: function setupRequestListeners(request) { - request.addListener('validateResponse', this.setExpiresString); var prependListener = true; request.addListener('validate', this.validateScheme); request.addListener('validate', this.validateBucketName, prependListener); @@ -33163,21 +32235,17 @@ AWS.util.update(AWS.S3.prototype, { * @api private */ extractErrorFrom200Response: function extractErrorFrom200Response(resp) { - var service = this.service ? this.service : this; - if (!service.is200Error(resp) && !operationsWith200StatusCodeError[resp.request.operation]) { - return; - } + if (!operationsWith200StatusCodeError[resp.request.operation]) return; var httpResponse = resp.httpResponse; - var bodyString = httpResponse.body && httpResponse.body.toString() || ''; - if (bodyString && bodyString.indexOf('') === bodyString.length - 8) { + if (httpResponse.body && httpResponse.body.toString().match('')) { // Response body with '...' indicates an exception. // Get S3 client object. In ManagedUpload, this.service refers to // S3 client object. resp.data = null; + var service = this.service ? this.service : this; service.extractError(resp); - resp.error.is200Error = true; throw resp.error; - } else if (!httpResponse.body || !bodyString.match(/<[\w_]/)) { + } else if (!httpResponse.body || !httpResponse.body.toString().match(/<[\w_]/)) { // When body is empty or incomplete, S3 might stop the request on detecting client // side aborting the request. resp.data = null; @@ -33188,54 +32256,13 @@ AWS.util.update(AWS.S3.prototype, { } }, - /** - * @api private - * @param resp - to evaluate. - * @return true if the response has status code 200 but is an error. - */ - is200Error: function is200Error(resp) { - var code = resp && resp.httpResponse && resp.httpResponse.statusCode; - if (code !== 200) { - return false; - } - try { - var req = resp.request; - var outputMembers = req.service.api.operations[req.operation].output.members; - var keys = Object.keys(outputMembers); - for (var i = 0; i < keys.length; ++i) { - var member = outputMembers[keys[i]]; - if (member.type === 'binary' && member.isStreaming) { - return false; - } - } - - var body = resp.httpResponse.body; - if (body && body.byteLength !== undefined) { - if (body.byteLength < 15 || body.byteLength > 3000) { - // body is too short or long to be an error message. - return false; - } - } - if (!body) { - return false; - } - var bodyString = body.toString(); - if (bodyString.indexOf('') === bodyString.length - 8) { - return true; - } - } catch (e) { - return false; - } - return false; - }, - /** * @return [Boolean] whether the error can be retried * @api private */ retryableError: function retryableError(error, request) { - if (error.is200Error || - (operationsWith200StatusCodeError[request.operation] && error.statusCode === 200)) { + if (operationsWith200StatusCodeError[request.operation] && + error.statusCode === 200) { return true; } else if (request._requestRegionForBucket && request.service.bucketRegionCache[request._requestRegionForBucket]) { @@ -33895,11 +32922,7 @@ AWS.util.update(AWS.S3.prototype, { // mutate params object argument passed in by user var copiedParams = AWS.util.copy(params); - if ( - this.config.region !== 'us-east-1' - && hostname !== this.api.globalEndpoint - && !params.CreateBucketConfiguration - ) { + if (hostname !== this.api.globalEndpoint && !params.CreateBucketConfiguration) { copiedParams.CreateBucketConfiguration = { LocationConstraint: this.config.region }; } return this.makeRequest('createBucket', copiedParams, callback); @@ -33966,30 +32989,6 @@ AWS.util.update(AWS.S3.prototype, { var uploader = new AWS.S3.ManagedUpload(options); if (typeof callback === 'function') uploader.send(callback); return uploader; - }, - - /** - * @api private - */ - setExpiresString: function setExpiresString(response) { - // Check if response contains Expires value, and populate ExpiresString. - if (response && response.httpResponse && response.httpResponse.headers) { - if ('expires' in response.httpResponse.headers) { - response.httpResponse.headers.expiresstring = response.httpResponse.headers.expires; - } - } - - // Check if value in Expires is not a Date using parseTimestamp. - try { - if (response && response.httpResponse && response.httpResponse.headers) { - if ('expires' in response.httpResponse.headers) { - AWS.util.date.parseTimestamp(response.httpResponse.headers.expires); - } - } - } catch (e) { - console.log('AWS SDK', '(warning)', e); - delete response.httpResponse.headers.expires; - } } }); @@ -34889,6 +33888,9 @@ AWS.IniLoader = AWS.util.inherit({ return this.resolvedSsoSessions[filename]; }, + /** + * @api private + */ getDefaultFilePath: function getDefaultFilePath(isConfig) { return path.join( this.getHomeDir(), @@ -34897,6 +33899,9 @@ AWS.IniLoader = AWS.util.inherit({ ); }, + /** + * @api private + */ getHomeDir: function getHomeDir() { var env = process.env; var home = env.HOME || @@ -37458,7 +36463,7 @@ var util = { */ uuid: { v4: function uuidV4() { - return (__nccwpck_require__(75840).v4)(); + return (__nccwpck_require__(57821).v4)(); } }, @@ -38003,3279 +37008,3723 @@ module.exports = { /***/ }), -/***/ 96323: -/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { +/***/ 35827: +/***/ ((__unused_webpack_module, exports) => { "use strict"; -var __webpack_unused_export__; -__webpack_unused_export__ = ({ value: true }); -var LRU_1 = __nccwpck_require__(77710); -var CACHE_SIZE = 1000; + +Object.defineProperty(exports, "__esModule", ({ + value: true +})); +exports["default"] = void 0; + /** - * Inspired node-lru-cache[https://github.com/isaacs/node-lru-cache] + * Convert array of 16 byte values to UUID string format of the form: + * XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX */ -var EndpointCache = /** @class */ (function () { - function EndpointCache(maxSize) { - if (maxSize === void 0) { maxSize = CACHE_SIZE; } - this.maxSize = maxSize; - this.cache = new LRU_1.LRUCache(maxSize); - } - ; - Object.defineProperty(EndpointCache.prototype, "size", { - get: function () { - return this.cache.length; - }, - enumerable: true, - configurable: true - }); - EndpointCache.prototype.put = function (key, value) { - var keyString = typeof key !== 'string' ? EndpointCache.getKeyString(key) : key; - var endpointRecord = this.populateValue(value); - this.cache.put(keyString, endpointRecord); - }; - EndpointCache.prototype.get = function (key) { - var keyString = typeof key !== 'string' ? EndpointCache.getKeyString(key) : key; - var now = Date.now(); - var records = this.cache.get(keyString); - if (records) { - for (var i = records.length-1; i >= 0; i--) { - var record = records[i]; - if (record.Expire < now) { - records.splice(i, 1); - } - } - if (records.length === 0) { - this.cache.remove(keyString); - return undefined; - } - } - return records; - }; - EndpointCache.getKeyString = function (key) { - var identifiers = []; - var identifierNames = Object.keys(key).sort(); - for (var i = 0; i < identifierNames.length; i++) { - var identifierName = identifierNames[i]; - if (key[identifierName] === undefined) - continue; - identifiers.push(key[identifierName]); - } - return identifiers.join(' '); - }; - EndpointCache.prototype.populateValue = function (endpoints) { - var now = Date.now(); - return endpoints.map(function (endpoint) { return ({ - Address: endpoint.Address || '', - Expire: now + (endpoint.CachePeriodInMinutes || 1) * 60 * 1000 - }); }); - }; - EndpointCache.prototype.empty = function () { - this.cache.empty(); - }; - EndpointCache.prototype.remove = function (key) { - var keyString = typeof key !== 'string' ? EndpointCache.getKeyString(key) : key; - this.cache.remove(keyString); - }; - return EndpointCache; -}()); -exports.$ = EndpointCache; +var byteToHex = []; -/***/ }), +for (var i = 0; i < 256; ++i) { + byteToHex[i] = (i + 0x100).toString(16).substr(1); +} -/***/ 77710: -/***/ ((__unused_webpack_module, exports) => { +function bytesToUuid(buf, offset) { + var i = offset || 0; + var bth = byteToHex; // join used to fix memory issue caused by concatenation: https://bugs.chromium.org/p/v8/issues/detail?id=3175#c4 -"use strict"; + return [bth[buf[i++]], bth[buf[i++]], bth[buf[i++]], bth[buf[i++]], '-', bth[buf[i++]], bth[buf[i++]], '-', bth[buf[i++]], bth[buf[i++]], '-', bth[buf[i++]], bth[buf[i++]], '-', bth[buf[i++]], bth[buf[i++]], bth[buf[i++]], bth[buf[i++]], bth[buf[i++]], bth[buf[i++]]].join(''); +} -Object.defineProperty(exports, "__esModule", ({ value: true })); -var LinkedListNode = /** @class */ (function () { - function LinkedListNode(key, value) { - this.key = key; - this.value = value; - } - return LinkedListNode; -}()); -var LRUCache = /** @class */ (function () { - function LRUCache(size) { - this.nodeMap = {}; - this.size = 0; - if (typeof size !== 'number' || size < 1) { - throw new Error('Cache size can only be positive number'); - } - this.sizeLimit = size; - } - Object.defineProperty(LRUCache.prototype, "length", { - get: function () { - return this.size; - }, - enumerable: true, - configurable: true - }); - LRUCache.prototype.prependToList = function (node) { - if (!this.headerNode) { - this.tailNode = node; - } - else { - this.headerNode.prev = node; - node.next = this.headerNode; - } - this.headerNode = node; - this.size++; - }; - LRUCache.prototype.removeFromTail = function () { - if (!this.tailNode) { - return undefined; - } - var node = this.tailNode; - var prevNode = node.prev; - if (prevNode) { - prevNode.next = undefined; - } - node.prev = undefined; - this.tailNode = prevNode; - this.size--; - return node; - }; - LRUCache.prototype.detachFromList = function (node) { - if (this.headerNode === node) { - this.headerNode = node.next; - } - if (this.tailNode === node) { - this.tailNode = node.prev; - } - if (node.prev) { - node.prev.next = node.next; - } - if (node.next) { - node.next.prev = node.prev; - } - node.next = undefined; - node.prev = undefined; - this.size--; - }; - LRUCache.prototype.get = function (key) { - if (this.nodeMap[key]) { - var node = this.nodeMap[key]; - this.detachFromList(node); - this.prependToList(node); - return node.value; - } - }; - LRUCache.prototype.remove = function (key) { - if (this.nodeMap[key]) { - var node = this.nodeMap[key]; - this.detachFromList(node); - delete this.nodeMap[key]; - } - }; - LRUCache.prototype.put = function (key, value) { - if (this.nodeMap[key]) { - this.remove(key); - } - else if (this.size === this.sizeLimit) { - var tailNode = this.removeFromTail(); - var key_1 = tailNode.key; - delete this.nodeMap[key_1]; - } - var newNode = new LinkedListNode(key, value); - this.nodeMap[key] = newNode; - this.prependToList(newNode); - }; - LRUCache.prototype.empty = function () { - var keys = Object.keys(this.nodeMap); - for (var i = 0; i < keys.length; i++) { - var key = keys[i]; - var node = this.nodeMap[key]; - this.detachFromList(node); - delete this.nodeMap[key]; - } - }; - return LRUCache; -}()); -exports.LRUCache = LRUCache; +var _default = bytesToUuid; +exports["default"] = _default; /***/ }), -/***/ 83682: -/***/ ((module, __unused_webpack_exports, __nccwpck_require__) => { +/***/ 57821: +/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { -var register = __nccwpck_require__(44670); -var addHook = __nccwpck_require__(5549); -var removeHook = __nccwpck_require__(6819); +"use strict"; +var __webpack_unused_export__; -// bind with array of arguments: https://stackoverflow.com/a/21792913 -var bind = Function.bind; -var bindable = bind.bind(bind); -function bindApi(hook, state, name) { - var removeHookRef = bindable(removeHook, null).apply( - null, - name ? [state, name] : [state] - ); - hook.api = { remove: removeHookRef }; - hook.remove = removeHookRef; - ["before", "error", "after", "wrap"].forEach(function (kind) { - var args = name ? [state, kind, name] : [state, kind]; - hook[kind] = hook.api[kind] = bindable(addHook, null).apply(null, args); - }); -} +__webpack_unused_export__ = ({ + value: true +}); +__webpack_unused_export__ = ({ + enumerable: true, + get: function () { + return _v.default; + } +}); +__webpack_unused_export__ = ({ + enumerable: true, + get: function () { + return _v2.default; + } +}); +Object.defineProperty(exports, "v4", ({ + enumerable: true, + get: function () { + return _v3.default; + } +})); +__webpack_unused_export__ = ({ + enumerable: true, + get: function () { + return _v4.default; + } +}); -function HookSingular() { - var singularHookName = "h"; - var singularHookState = { - registry: {}, - }; - var singularHook = register.bind(null, singularHookState, singularHookName); - bindApi(singularHook, singularHookState, singularHookName); - return singularHook; -} +var _v = _interopRequireDefault(__nccwpck_require__(67668)); -function HookCollection() { - var state = { - registry: {}, - }; +var _v2 = _interopRequireDefault(__nccwpck_require__(98573)); - var hook = register.bind(null, state); - bindApi(hook, state); +var _v3 = _interopRequireDefault(__nccwpck_require__(7811)); - return hook; -} +var _v4 = _interopRequireDefault(__nccwpck_require__(46508)); -var collectionHookDeprecationMessageDisplayed = false; -function Hook() { - if (!collectionHookDeprecationMessageDisplayed) { - console.warn( - '[before-after-hook]: "Hook()" repurposing warning, use "Hook.Collection()". Read more: https://git.io/upgrade-before-after-hook-to-1.4' - ); - collectionHookDeprecationMessageDisplayed = true; - } - return HookCollection(); -} +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } -Hook.Singular = HookSingular.bind(); -Hook.Collection = HookCollection.bind(); +/***/ }), -module.exports = Hook; -// expose constructors as a named property for TypeScript -module.exports.Hook = Hook; -module.exports.Singular = Hook.Singular; -module.exports.Collection = Hook.Collection; +/***/ 93525: +/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { +"use strict"; -/***/ }), -/***/ 5549: -/***/ ((module) => { - -module.exports = addHook; - -function addHook(state, kind, name, hook) { - var orig = hook; - if (!state.registry[name]) { - state.registry[name] = []; - } +Object.defineProperty(exports, "__esModule", ({ + value: true +})); +exports["default"] = void 0; - if (kind === "before") { - hook = function (method, options) { - return Promise.resolve() - .then(orig.bind(null, options)) - .then(method.bind(null, options)); - }; - } +var _crypto = _interopRequireDefault(__nccwpck_require__(6113)); - if (kind === "after") { - hook = function (method, options) { - var result; - return Promise.resolve() - .then(method.bind(null, options)) - .then(function (result_) { - result = result_; - return orig(result, options); - }) - .then(function () { - return result; - }); - }; - } +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } - if (kind === "error") { - hook = function (method, options) { - return Promise.resolve() - .then(method.bind(null, options)) - .catch(function (error) { - return orig(error, options); - }); - }; +function md5(bytes) { + if (Array.isArray(bytes)) { + bytes = Buffer.from(bytes); + } else if (typeof bytes === 'string') { + bytes = Buffer.from(bytes, 'utf8'); } - state.registry[name].push({ - hook: hook, - orig: orig, - }); + return _crypto.default.createHash('md5').update(bytes).digest(); } +var _default = md5; +exports["default"] = _default; /***/ }), -/***/ 44670: -/***/ ((module) => { +/***/ 49788: +/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { -module.exports = register; +"use strict"; -function register(state, name, method, options) { - if (typeof method !== "function") { - throw new Error("method for before hook must be a function"); - } - if (!options) { - options = {}; - } +Object.defineProperty(exports, "__esModule", ({ + value: true +})); +exports["default"] = rng; - if (Array.isArray(name)) { - return name.reverse().reduce(function (callback, name) { - return register.bind(null, state, name, callback, options); - }, method)(); - } +var _crypto = _interopRequireDefault(__nccwpck_require__(6113)); - return Promise.resolve().then(function () { - if (!state.registry[name]) { - return method(options); - } +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } - return state.registry[name].reduce(function (method, registered) { - return registered.hook.bind(null, method, options); - }, method)(); - }); +function rng() { + return _crypto.default.randomBytes(16); } - /***/ }), -/***/ 6819: -/***/ ((module) => { +/***/ 7387: +/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { -module.exports = removeHook; +"use strict"; -function removeHook(state, name, method) { - if (!state.registry[name]) { - return; - } - var index = state.registry[name] - .map(function (registered) { - return registered.orig; - }) - .indexOf(method); +Object.defineProperty(exports, "__esModule", ({ + value: true +})); +exports["default"] = void 0; - if (index === -1) { - return; +var _crypto = _interopRequireDefault(__nccwpck_require__(6113)); + +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } + +function sha1(bytes) { + if (Array.isArray(bytes)) { + bytes = Buffer.from(bytes); + } else if (typeof bytes === 'string') { + bytes = Buffer.from(bytes, 'utf8'); } - state.registry[name].splice(index, 1); + return _crypto.default.createHash('sha1').update(bytes).digest(); } +var _default = sha1; +exports["default"] = _default; /***/ }), -/***/ 58932: -/***/ ((__unused_webpack_module, exports) => { +/***/ 67668: +/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { "use strict"; -Object.defineProperty(exports, "__esModule", ({ value: true })); - -class Deprecation extends Error { - constructor(message) { - super(message); // Maintains proper stack trace (only available on V8) +Object.defineProperty(exports, "__esModule", ({ + value: true +})); +exports["default"] = void 0; - /* istanbul ignore next */ +var _rng = _interopRequireDefault(__nccwpck_require__(49788)); - if (Error.captureStackTrace) { - Error.captureStackTrace(this, this.constructor); - } +var _bytesToUuid = _interopRequireDefault(__nccwpck_require__(35827)); - this.name = 'Deprecation'; - } +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } -} +// **`v1()` - Generate time-based UUID** +// +// Inspired by https://github.com/LiosK/UUID.js +// and http://docs.python.org/library/uuid.html +var _nodeId; -exports.Deprecation = Deprecation; +var _clockseq; // Previous uuid creation time -/***/ }), +var _lastMSecs = 0; +var _lastNSecs = 0; // See https://github.com/uuidjs/uuid for API details -/***/ 87783: -/***/ ((__unused_webpack_module, exports) => { +function v1(options, buf, offset) { + var i = buf && offset || 0; + var b = buf || []; + options = options || {}; + var node = options.node || _nodeId; + var clockseq = options.clockseq !== undefined ? options.clockseq : _clockseq; // node and clockseq need to be initialized to random values if they're not + // specified. We do this lazily to minimize issues related to insufficient + // system entropy. See #189 -(function(exports) { - "use strict"; + if (node == null || clockseq == null) { + var seedBytes = options.random || (options.rng || _rng.default)(); - function isArray(obj) { - if (obj !== null) { - return Object.prototype.toString.call(obj) === "[object Array]"; - } else { - return false; + if (node == null) { + // Per 4.5, create and 48-bit node id, (47 random bits + multicast bit = 1) + node = _nodeId = [seedBytes[0] | 0x01, seedBytes[1], seedBytes[2], seedBytes[3], seedBytes[4], seedBytes[5]]; } - } - function isObject(obj) { - if (obj !== null) { - return Object.prototype.toString.call(obj) === "[object Object]"; - } else { - return false; + if (clockseq == null) { + // Per 4.2.2, randomize (14 bit) clockseq + clockseq = _clockseq = (seedBytes[6] << 8 | seedBytes[7]) & 0x3fff; } - } + } // UUID timestamps are 100 nano-second units since the Gregorian epoch, + // (1582-10-15 00:00). JSNumbers aren't precise enough for this, so + // time is handled internally as 'msecs' (integer milliseconds) and 'nsecs' + // (100-nanoseconds offset from msecs) since unix epoch, 1970-01-01 00:00. - function strictDeepEqual(first, second) { - // Check the scalar case first. - if (first === second) { - return true; - } - // Check if they are the same type. - var firstType = Object.prototype.toString.call(first); - if (firstType !== Object.prototype.toString.call(second)) { - return false; - } - // We know that first and second have the same type so we can just check the - // first type from now on. - if (isArray(first) === true) { - // Short circuit if they're not the same length; - if (first.length !== second.length) { - return false; - } - for (var i = 0; i < first.length; i++) { - if (strictDeepEqual(first[i], second[i]) === false) { - return false; - } - } - return true; - } - if (isObject(first) === true) { - // An object is equal if it has the same key/value pairs. - var keysSeen = {}; - for (var key in first) { - if (hasOwnProperty.call(first, key)) { - if (strictDeepEqual(first[key], second[key]) === false) { - return false; - } - keysSeen[key] = true; - } - } - // Now check that there aren't any keys in second that weren't - // in first. - for (var key2 in second) { - if (hasOwnProperty.call(second, key2)) { - if (keysSeen[key2] !== true) { - return false; - } - } - } - return true; - } - return false; - } + var msecs = options.msecs !== undefined ? options.msecs : new Date().getTime(); // Per 4.2.1.2, use count of uuid's generated during the current clock + // cycle to simulate higher resolution clock - function isFalse(obj) { - // From the spec: - // A false value corresponds to the following values: - // Empty list - // Empty object - // Empty string - // False boolean - // null value + var nsecs = options.nsecs !== undefined ? options.nsecs : _lastNSecs + 1; // Time since last uuid creation (in msecs) - // First check the scalar values. - if (obj === "" || obj === false || obj === null) { - return true; - } else if (isArray(obj) && obj.length === 0) { - // Check for an empty array. - return true; - } else if (isObject(obj)) { - // Check for an empty object. - for (var key in obj) { - // If there are any keys, then - // the object is not empty so the object - // is not false. - if (obj.hasOwnProperty(key)) { - return false; - } - } - return true; - } else { - return false; - } - } + var dt = msecs - _lastMSecs + (nsecs - _lastNSecs) / 10000; // Per 4.2.1.2, Bump clockseq on clock regression - function objValues(obj) { - var keys = Object.keys(obj); - var values = []; - for (var i = 0; i < keys.length; i++) { - values.push(obj[keys[i]]); - } - return values; - } + if (dt < 0 && options.clockseq === undefined) { + clockseq = clockseq + 1 & 0x3fff; + } // Reset nsecs if clock regresses (new clockseq) or we've moved onto a new + // time interval - function merge(a, b) { - var merged = {}; - for (var key in a) { - merged[key] = a[key]; - } - for (var key2 in b) { - merged[key2] = b[key2]; - } - return merged; - } - var trimLeft; - if (typeof String.prototype.trimLeft === "function") { - trimLeft = function(str) { - return str.trimLeft(); - }; - } else { - trimLeft = function(str) { - return str.match(/^\s*(.*)/)[1]; - }; + if ((dt < 0 || msecs > _lastMSecs) && options.nsecs === undefined) { + nsecs = 0; + } // Per 4.2.1.2 Throw error if too many uuids are requested + + + if (nsecs >= 10000) { + throw new Error("uuid.v1(): Can't create more than 10M uuids/sec"); } - // Type constants used to define functions. - var TYPE_NUMBER = 0; - var TYPE_ANY = 1; - var TYPE_STRING = 2; - var TYPE_ARRAY = 3; - var TYPE_OBJECT = 4; - var TYPE_BOOLEAN = 5; - var TYPE_EXPREF = 6; - var TYPE_NULL = 7; - var TYPE_ARRAY_NUMBER = 8; - var TYPE_ARRAY_STRING = 9; - var TYPE_NAME_TABLE = { - 0: 'number', - 1: 'any', - 2: 'string', - 3: 'array', - 4: 'object', - 5: 'boolean', - 6: 'expression', - 7: 'null', - 8: 'Array', - 9: 'Array' - }; + _lastMSecs = msecs; + _lastNSecs = nsecs; + _clockseq = clockseq; // Per 4.1.4 - Convert from unix epoch to Gregorian epoch - var TOK_EOF = "EOF"; - var TOK_UNQUOTEDIDENTIFIER = "UnquotedIdentifier"; - var TOK_QUOTEDIDENTIFIER = "QuotedIdentifier"; - var TOK_RBRACKET = "Rbracket"; - var TOK_RPAREN = "Rparen"; - var TOK_COMMA = "Comma"; - var TOK_COLON = "Colon"; - var TOK_RBRACE = "Rbrace"; - var TOK_NUMBER = "Number"; - var TOK_CURRENT = "Current"; - var TOK_EXPREF = "Expref"; - var TOK_PIPE = "Pipe"; - var TOK_OR = "Or"; - var TOK_AND = "And"; - var TOK_EQ = "EQ"; - var TOK_GT = "GT"; - var TOK_LT = "LT"; - var TOK_GTE = "GTE"; - var TOK_LTE = "LTE"; - var TOK_NE = "NE"; - var TOK_FLATTEN = "Flatten"; - var TOK_STAR = "Star"; - var TOK_FILTER = "Filter"; - var TOK_DOT = "Dot"; - var TOK_NOT = "Not"; - var TOK_LBRACE = "Lbrace"; - var TOK_LBRACKET = "Lbracket"; - var TOK_LPAREN= "Lparen"; - var TOK_LITERAL= "Literal"; + msecs += 12219292800000; // `time_low` - // The "&", "[", "<", ">" tokens - // are not in basicToken because - // there are two token variants - // ("&&", "[?", "<=", ">="). This is specially handled - // below. + var tl = ((msecs & 0xfffffff) * 10000 + nsecs) % 0x100000000; + b[i++] = tl >>> 24 & 0xff; + b[i++] = tl >>> 16 & 0xff; + b[i++] = tl >>> 8 & 0xff; + b[i++] = tl & 0xff; // `time_mid` - var basicTokens = { - ".": TOK_DOT, - "*": TOK_STAR, - ",": TOK_COMMA, - ":": TOK_COLON, - "{": TOK_LBRACE, - "}": TOK_RBRACE, - "]": TOK_RBRACKET, - "(": TOK_LPAREN, - ")": TOK_RPAREN, - "@": TOK_CURRENT - }; + var tmh = msecs / 0x100000000 * 10000 & 0xfffffff; + b[i++] = tmh >>> 8 & 0xff; + b[i++] = tmh & 0xff; // `time_high_and_version` - var operatorStartToken = { - "<": true, - ">": true, - "=": true, - "!": true - }; + b[i++] = tmh >>> 24 & 0xf | 0x10; // include version - var skipChars = { - " ": true, - "\t": true, - "\n": true - }; + b[i++] = tmh >>> 16 & 0xff; // `clock_seq_hi_and_reserved` (Per 4.2.2 - include variant) + b[i++] = clockseq >>> 8 | 0x80; // `clock_seq_low` - function isAlpha(ch) { - return (ch >= "a" && ch <= "z") || - (ch >= "A" && ch <= "Z") || - ch === "_"; - } + b[i++] = clockseq & 0xff; // `node` - function isNum(ch) { - return (ch >= "0" && ch <= "9") || - ch === "-"; - } - function isAlphaNum(ch) { - return (ch >= "a" && ch <= "z") || - (ch >= "A" && ch <= "Z") || - (ch >= "0" && ch <= "9") || - ch === "_"; + for (var n = 0; n < 6; ++n) { + b[i + n] = node[n]; } - function Lexer() { - } - Lexer.prototype = { - tokenize: function(stream) { - var tokens = []; - this._current = 0; - var start; - var identifier; - var token; - while (this._current < stream.length) { - if (isAlpha(stream[this._current])) { - start = this._current; - identifier = this._consumeUnquotedIdentifier(stream); - tokens.push({type: TOK_UNQUOTEDIDENTIFIER, - value: identifier, - start: start}); - } else if (basicTokens[stream[this._current]] !== undefined) { - tokens.push({type: basicTokens[stream[this._current]], - value: stream[this._current], - start: this._current}); - this._current++; - } else if (isNum(stream[this._current])) { - token = this._consumeNumber(stream); - tokens.push(token); - } else if (stream[this._current] === "[") { - // No need to increment this._current. This happens - // in _consumeLBracket - token = this._consumeLBracket(stream); - tokens.push(token); - } else if (stream[this._current] === "\"") { - start = this._current; - identifier = this._consumeQuotedIdentifier(stream); - tokens.push({type: TOK_QUOTEDIDENTIFIER, - value: identifier, - start: start}); - } else if (stream[this._current] === "'") { - start = this._current; - identifier = this._consumeRawStringLiteral(stream); - tokens.push({type: TOK_LITERAL, - value: identifier, - start: start}); - } else if (stream[this._current] === "`") { - start = this._current; - var literal = this._consumeLiteral(stream); - tokens.push({type: TOK_LITERAL, - value: literal, - start: start}); - } else if (operatorStartToken[stream[this._current]] !== undefined) { - tokens.push(this._consumeOperator(stream)); - } else if (skipChars[stream[this._current]] !== undefined) { - // Ignore whitespace. - this._current++; - } else if (stream[this._current] === "&") { - start = this._current; - this._current++; - if (stream[this._current] === "&") { - this._current++; - tokens.push({type: TOK_AND, value: "&&", start: start}); - } else { - tokens.push({type: TOK_EXPREF, value: "&", start: start}); - } - } else if (stream[this._current] === "|") { - start = this._current; - this._current++; - if (stream[this._current] === "|") { - this._current++; - tokens.push({type: TOK_OR, value: "||", start: start}); - } else { - tokens.push({type: TOK_PIPE, value: "|", start: start}); - } - } else { - var error = new Error("Unknown character:" + stream[this._current]); - error.name = "LexerError"; - throw error; - } - } - return tokens; - }, + return buf ? buf : (0, _bytesToUuid.default)(b); +} - _consumeUnquotedIdentifier: function(stream) { - var start = this._current; - this._current++; - while (this._current < stream.length && isAlphaNum(stream[this._current])) { - this._current++; - } - return stream.slice(start, this._current); - }, +var _default = v1; +exports["default"] = _default; - _consumeQuotedIdentifier: function(stream) { - var start = this._current; - this._current++; - var maxLength = stream.length; - while (stream[this._current] !== "\"" && this._current < maxLength) { - // You can escape a double quote and you can escape an escape. - var current = this._current; - if (stream[current] === "\\" && (stream[current + 1] === "\\" || - stream[current + 1] === "\"")) { - current += 2; - } else { - current++; - } - this._current = current; - } - this._current++; - return JSON.parse(stream.slice(start, this._current)); - }, +/***/ }), - _consumeRawStringLiteral: function(stream) { - var start = this._current; - this._current++; - var maxLength = stream.length; - while (stream[this._current] !== "'" && this._current < maxLength) { - // You can escape a single quote and you can escape an escape. - var current = this._current; - if (stream[current] === "\\" && (stream[current + 1] === "\\" || - stream[current + 1] === "'")) { - current += 2; - } else { - current++; - } - this._current = current; - } - this._current++; - var literal = stream.slice(start + 1, this._current - 1); - return literal.replace("\\'", "'"); - }, +/***/ 98573: +/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { - _consumeNumber: function(stream) { - var start = this._current; - this._current++; - var maxLength = stream.length; - while (isNum(stream[this._current]) && this._current < maxLength) { - this._current++; - } - var value = parseInt(stream.slice(start, this._current)); - return {type: TOK_NUMBER, value: value, start: start}; - }, +"use strict"; - _consumeLBracket: function(stream) { - var start = this._current; - this._current++; - if (stream[this._current] === "?") { - this._current++; - return {type: TOK_FILTER, value: "[?", start: start}; - } else if (stream[this._current] === "]") { - this._current++; - return {type: TOK_FLATTEN, value: "[]", start: start}; - } else { - return {type: TOK_LBRACKET, value: "[", start: start}; - } - }, - _consumeOperator: function(stream) { - var start = this._current; - var startingChar = stream[start]; - this._current++; - if (startingChar === "!") { - if (stream[this._current] === "=") { - this._current++; - return {type: TOK_NE, value: "!=", start: start}; - } else { - return {type: TOK_NOT, value: "!", start: start}; - } - } else if (startingChar === "<") { - if (stream[this._current] === "=") { - this._current++; - return {type: TOK_LTE, value: "<=", start: start}; - } else { - return {type: TOK_LT, value: "<", start: start}; - } - } else if (startingChar === ">") { - if (stream[this._current] === "=") { - this._current++; - return {type: TOK_GTE, value: ">=", start: start}; - } else { - return {type: TOK_GT, value: ">", start: start}; - } - } else if (startingChar === "=") { - if (stream[this._current] === "=") { - this._current++; - return {type: TOK_EQ, value: "==", start: start}; - } - } - }, +Object.defineProperty(exports, "__esModule", ({ + value: true +})); +exports["default"] = void 0; - _consumeLiteral: function(stream) { - this._current++; - var start = this._current; - var maxLength = stream.length; - var literal; - while(stream[this._current] !== "`" && this._current < maxLength) { - // You can escape a literal char or you can escape the escape. - var current = this._current; - if (stream[current] === "\\" && (stream[current + 1] === "\\" || - stream[current + 1] === "`")) { - current += 2; - } else { - current++; - } - this._current = current; - } - var literalString = trimLeft(stream.slice(start, this._current)); - literalString = literalString.replace("\\`", "`"); - if (this._looksLikeJSON(literalString)) { - literal = JSON.parse(literalString); - } else { - // Try to JSON parse it as "" - literal = JSON.parse("\"" + literalString + "\""); - } - // +1 gets us to the ending "`", +1 to move on to the next char. - this._current++; - return literal; - }, +var _v = _interopRequireDefault(__nccwpck_require__(36097)); - _looksLikeJSON: function(literalString) { - var startingChars = "[{\""; - var jsonLiterals = ["true", "false", "null"]; - var numberLooking = "-0123456789"; +var _md = _interopRequireDefault(__nccwpck_require__(93525)); - if (literalString === "") { - return false; - } else if (startingChars.indexOf(literalString[0]) >= 0) { - return true; - } else if (jsonLiterals.indexOf(literalString) >= 0) { - return true; - } else if (numberLooking.indexOf(literalString[0]) >= 0) { - try { - JSON.parse(literalString); - return true; - } catch (ex) { - return false; - } - } else { - return false; - } - } - }; +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } - var bindingPower = {}; - bindingPower[TOK_EOF] = 0; - bindingPower[TOK_UNQUOTEDIDENTIFIER] = 0; - bindingPower[TOK_QUOTEDIDENTIFIER] = 0; - bindingPower[TOK_RBRACKET] = 0; - bindingPower[TOK_RPAREN] = 0; - bindingPower[TOK_COMMA] = 0; - bindingPower[TOK_RBRACE] = 0; - bindingPower[TOK_NUMBER] = 0; - bindingPower[TOK_CURRENT] = 0; - bindingPower[TOK_EXPREF] = 0; - bindingPower[TOK_PIPE] = 1; - bindingPower[TOK_OR] = 2; - bindingPower[TOK_AND] = 3; - bindingPower[TOK_EQ] = 5; - bindingPower[TOK_GT] = 5; - bindingPower[TOK_LT] = 5; - bindingPower[TOK_GTE] = 5; - bindingPower[TOK_LTE] = 5; - bindingPower[TOK_NE] = 5; - bindingPower[TOK_FLATTEN] = 9; - bindingPower[TOK_STAR] = 20; - bindingPower[TOK_FILTER] = 21; - bindingPower[TOK_DOT] = 40; - bindingPower[TOK_NOT] = 45; - bindingPower[TOK_LBRACE] = 50; - bindingPower[TOK_LBRACKET] = 55; - bindingPower[TOK_LPAREN] = 60; +const v3 = (0, _v.default)('v3', 0x30, _md.default); +var _default = v3; +exports["default"] = _default; - function Parser() { - } +/***/ }), - Parser.prototype = { - parse: function(expression) { - this._loadTokens(expression); - this.index = 0; - var ast = this.expression(0); - if (this._lookahead(0) !== TOK_EOF) { - var t = this._lookaheadToken(0); - var error = new Error( - "Unexpected token type: " + t.type + ", value: " + t.value); - error.name = "ParserError"; - throw error; - } - return ast; - }, +/***/ 36097: +/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { - _loadTokens: function(expression) { - var lexer = new Lexer(); - var tokens = lexer.tokenize(expression); - tokens.push({type: TOK_EOF, value: "", start: expression.length}); - this.tokens = tokens; - }, +"use strict"; - expression: function(rbp) { - var leftToken = this._lookaheadToken(0); - this._advance(); - var left = this.nud(leftToken); - var currentToken = this._lookahead(0); - while (rbp < bindingPower[currentToken]) { - this._advance(); - left = this.led(currentToken, left); - currentToken = this._lookahead(0); - } - return left; - }, - _lookahead: function(number) { - return this.tokens[this.index + number].type; - }, +Object.defineProperty(exports, "__esModule", ({ + value: true +})); +exports["default"] = _default; +exports.URL = exports.DNS = void 0; - _lookaheadToken: function(number) { - return this.tokens[this.index + number]; - }, +var _bytesToUuid = _interopRequireDefault(__nccwpck_require__(35827)); - _advance: function() { - this.index++; - }, +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } - nud: function(token) { - var left; - var right; - var expression; - switch (token.type) { - case TOK_LITERAL: - return {type: "Literal", value: token.value}; - case TOK_UNQUOTEDIDENTIFIER: - return {type: "Field", name: token.value}; - case TOK_QUOTEDIDENTIFIER: - var node = {type: "Field", name: token.value}; - if (this._lookahead(0) === TOK_LPAREN) { - throw new Error("Quoted identifier not allowed for function names."); - } - return node; - case TOK_NOT: - right = this.expression(bindingPower.Not); - return {type: "NotExpression", children: [right]}; - case TOK_STAR: - left = {type: "Identity"}; - right = null; - if (this._lookahead(0) === TOK_RBRACKET) { - // This can happen in a multiselect, - // [a, b, *] - right = {type: "Identity"}; - } else { - right = this._parseProjectionRHS(bindingPower.Star); - } - return {type: "ValueProjection", children: [left, right]}; - case TOK_FILTER: - return this.led(token.type, {type: "Identity"}); - case TOK_LBRACE: - return this._parseMultiselectHash(); - case TOK_FLATTEN: - left = {type: TOK_FLATTEN, children: [{type: "Identity"}]}; - right = this._parseProjectionRHS(bindingPower.Flatten); - return {type: "Projection", children: [left, right]}; - case TOK_LBRACKET: - if (this._lookahead(0) === TOK_NUMBER || this._lookahead(0) === TOK_COLON) { - right = this._parseIndexExpression(); - return this._projectIfSlice({type: "Identity"}, right); - } else if (this._lookahead(0) === TOK_STAR && - this._lookahead(1) === TOK_RBRACKET) { - this._advance(); - this._advance(); - right = this._parseProjectionRHS(bindingPower.Star); - return {type: "Projection", - children: [{type: "Identity"}, right]}; - } - return this._parseMultiselectList(); - case TOK_CURRENT: - return {type: TOK_CURRENT}; - case TOK_EXPREF: - expression = this.expression(bindingPower.Expref); - return {type: "ExpressionReference", children: [expression]}; - case TOK_LPAREN: - var args = []; - while (this._lookahead(0) !== TOK_RPAREN) { - if (this._lookahead(0) === TOK_CURRENT) { - expression = {type: TOK_CURRENT}; - this._advance(); - } else { - expression = this.expression(0); - } - args.push(expression); - } - this._match(TOK_RPAREN); - return args[0]; - default: - this._errorToken(token); - } - }, +function uuidToBytes(uuid) { + // Note: We assume we're being passed a valid uuid string + var bytes = []; + uuid.replace(/[a-fA-F0-9]{2}/g, function (hex) { + bytes.push(parseInt(hex, 16)); + }); + return bytes; +} - led: function(tokenName, left) { - var right; - switch(tokenName) { - case TOK_DOT: - var rbp = bindingPower.Dot; - if (this._lookahead(0) !== TOK_STAR) { - right = this._parseDotRHS(rbp); - return {type: "Subexpression", children: [left, right]}; - } - // Creating a projection. - this._advance(); - right = this._parseProjectionRHS(rbp); - return {type: "ValueProjection", children: [left, right]}; - case TOK_PIPE: - right = this.expression(bindingPower.Pipe); - return {type: TOK_PIPE, children: [left, right]}; - case TOK_OR: - right = this.expression(bindingPower.Or); - return {type: "OrExpression", children: [left, right]}; - case TOK_AND: - right = this.expression(bindingPower.And); - return {type: "AndExpression", children: [left, right]}; - case TOK_LPAREN: - var name = left.name; - var args = []; - var expression, node; - while (this._lookahead(0) !== TOK_RPAREN) { - if (this._lookahead(0) === TOK_CURRENT) { - expression = {type: TOK_CURRENT}; - this._advance(); - } else { - expression = this.expression(0); - } - if (this._lookahead(0) === TOK_COMMA) { - this._match(TOK_COMMA); - } - args.push(expression); - } - this._match(TOK_RPAREN); - node = {type: "Function", name: name, children: args}; - return node; - case TOK_FILTER: - var condition = this.expression(0); - this._match(TOK_RBRACKET); - if (this._lookahead(0) === TOK_FLATTEN) { - right = {type: "Identity"}; - } else { - right = this._parseProjectionRHS(bindingPower.Filter); - } - return {type: "FilterProjection", children: [left, right, condition]}; - case TOK_FLATTEN: - var leftNode = {type: TOK_FLATTEN, children: [left]}; - var rightNode = this._parseProjectionRHS(bindingPower.Flatten); - return {type: "Projection", children: [leftNode, rightNode]}; - case TOK_EQ: - case TOK_NE: - case TOK_GT: - case TOK_GTE: - case TOK_LT: - case TOK_LTE: - return this._parseComparator(left, tokenName); - case TOK_LBRACKET: - var token = this._lookaheadToken(0); - if (token.type === TOK_NUMBER || token.type === TOK_COLON) { - right = this._parseIndexExpression(); - return this._projectIfSlice(left, right); - } - this._match(TOK_STAR); - this._match(TOK_RBRACKET); - right = this._parseProjectionRHS(bindingPower.Star); - return {type: "Projection", children: [left, right]}; - default: - this._errorToken(this._lookaheadToken(0)); - } - }, +function stringToBytes(str) { + str = unescape(encodeURIComponent(str)); // UTF8 escape - _match: function(tokenType) { - if (this._lookahead(0) === tokenType) { - this._advance(); - } else { - var t = this._lookaheadToken(0); - var error = new Error("Expected " + tokenType + ", got: " + t.type); - error.name = "ParserError"; - throw error; - } - }, + var bytes = new Array(str.length); - _errorToken: function(token) { - var error = new Error("Invalid token (" + - token.type + "): \"" + - token.value + "\""); - error.name = "ParserError"; - throw error; - }, + for (var i = 0; i < str.length; i++) { + bytes[i] = str.charCodeAt(i); + } + return bytes; +} - _parseIndexExpression: function() { - if (this._lookahead(0) === TOK_COLON || this._lookahead(1) === TOK_COLON) { - return this._parseSliceExpression(); - } else { - var node = { - type: "Index", - value: this._lookaheadToken(0).value}; - this._advance(); - this._match(TOK_RBRACKET); - return node; - } - }, +const DNS = '6ba7b810-9dad-11d1-80b4-00c04fd430c8'; +exports.DNS = DNS; +const URL = '6ba7b811-9dad-11d1-80b4-00c04fd430c8'; +exports.URL = URL; - _projectIfSlice: function(left, right) { - var indexExpr = {type: "IndexExpression", children: [left, right]}; - if (right.type === "Slice") { - return { - type: "Projection", - children: [indexExpr, this._parseProjectionRHS(bindingPower.Star)] - }; - } else { - return indexExpr; - } - }, +function _default(name, version, hashfunc) { + var generateUUID = function (value, namespace, buf, offset) { + var off = buf && offset || 0; + if (typeof value == 'string') value = stringToBytes(value); + if (typeof namespace == 'string') namespace = uuidToBytes(namespace); + if (!Array.isArray(value)) throw TypeError('value must be an array of bytes'); + if (!Array.isArray(namespace) || namespace.length !== 16) throw TypeError('namespace must be uuid string or an Array of 16 byte values'); // Per 4.3 - _parseSliceExpression: function() { - // [start:end:step] where each part is optional, as well as the last - // colon. - var parts = [null, null, null]; - var index = 0; - var currentToken = this._lookahead(0); - while (currentToken !== TOK_RBRACKET && index < 3) { - if (currentToken === TOK_COLON) { - index++; - this._advance(); - } else if (currentToken === TOK_NUMBER) { - parts[index] = this._lookaheadToken(0).value; - this._advance(); - } else { - var t = this._lookahead(0); - var error = new Error("Syntax error, unexpected token: " + - t.value + "(" + t.type + ")"); - error.name = "Parsererror"; - throw error; - } - currentToken = this._lookahead(0); - } - this._match(TOK_RBRACKET); - return { - type: "Slice", - children: parts - }; - }, + var bytes = hashfunc(namespace.concat(value)); + bytes[6] = bytes[6] & 0x0f | version; + bytes[8] = bytes[8] & 0x3f | 0x80; - _parseComparator: function(left, comparator) { - var right = this.expression(bindingPower[comparator]); - return {type: "Comparator", name: comparator, children: [left, right]}; - }, + if (buf) { + for (var idx = 0; idx < 16; ++idx) { + buf[off + idx] = bytes[idx]; + } + } - _parseDotRHS: function(rbp) { - var lookahead = this._lookahead(0); - var exprTokens = [TOK_UNQUOTEDIDENTIFIER, TOK_QUOTEDIDENTIFIER, TOK_STAR]; - if (exprTokens.indexOf(lookahead) >= 0) { - return this.expression(rbp); - } else if (lookahead === TOK_LBRACKET) { - this._match(TOK_LBRACKET); - return this._parseMultiselectList(); - } else if (lookahead === TOK_LBRACE) { - this._match(TOK_LBRACE); - return this._parseMultiselectHash(); - } - }, + return buf || (0, _bytesToUuid.default)(bytes); + }; // Function#name is not settable on some platforms (#270) - _parseProjectionRHS: function(rbp) { - var right; - if (bindingPower[this._lookahead(0)] < 10) { - right = {type: "Identity"}; - } else if (this._lookahead(0) === TOK_LBRACKET) { - right = this.expression(rbp); - } else if (this._lookahead(0) === TOK_FILTER) { - right = this.expression(rbp); - } else if (this._lookahead(0) === TOK_DOT) { - this._match(TOK_DOT); - right = this._parseDotRHS(rbp); - } else { - var t = this._lookaheadToken(0); - var error = new Error("Sytanx error, unexpected token: " + - t.value + "(" + t.type + ")"); - error.name = "ParserError"; - throw error; - } - return right; - }, - _parseMultiselectList: function() { - var expressions = []; - while (this._lookahead(0) !== TOK_RBRACKET) { - var expression = this.expression(0); - expressions.push(expression); - if (this._lookahead(0) === TOK_COMMA) { - this._match(TOK_COMMA); - if (this._lookahead(0) === TOK_RBRACKET) { - throw new Error("Unexpected token Rbracket"); - } - } - } - this._match(TOK_RBRACKET); - return {type: "MultiSelectList", children: expressions}; - }, + try { + generateUUID.name = name; + } catch (err) {} // For CommonJS default export support - _parseMultiselectHash: function() { - var pairs = []; - var identifierTypes = [TOK_UNQUOTEDIDENTIFIER, TOK_QUOTEDIDENTIFIER]; - var keyToken, keyName, value, node; - for (;;) { - keyToken = this._lookaheadToken(0); - if (identifierTypes.indexOf(keyToken.type) < 0) { - throw new Error("Expecting an identifier token, got: " + - keyToken.type); - } - keyName = keyToken.value; - this._advance(); - this._match(TOK_COLON); - value = this.expression(0); - node = {type: "KeyValuePair", name: keyName, value: value}; - pairs.push(node); - if (this._lookahead(0) === TOK_COMMA) { - this._match(TOK_COMMA); - } else if (this._lookahead(0) === TOK_RBRACE) { - this._match(TOK_RBRACE); - break; - } - } - return {type: "MultiSelectHash", children: pairs}; - } - }; + generateUUID.DNS = DNS; + generateUUID.URL = URL; + return generateUUID; +} - function TreeInterpreter(runtime) { - this.runtime = runtime; - } +/***/ }), - TreeInterpreter.prototype = { - search: function(node, value) { - return this.visit(node, value); - }, +/***/ 7811: +/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { - visit: function(node, value) { - var matched, current, result, first, second, field, left, right, collected, i; - switch (node.type) { - case "Field": - if (value !== null && isObject(value)) { - field = value[node.name]; - if (field === undefined) { - return null; - } else { - return field; - } - } - return null; - case "Subexpression": - result = this.visit(node.children[0], value); - for (i = 1; i < node.children.length; i++) { - result = this.visit(node.children[1], result); - if (result === null) { - return null; - } - } - return result; - case "IndexExpression": - left = this.visit(node.children[0], value); - right = this.visit(node.children[1], left); - return right; - case "Index": - if (!isArray(value)) { - return null; - } - var index = node.value; - if (index < 0) { - index = value.length + index; - } - result = value[index]; - if (result === undefined) { - result = null; - } - return result; - case "Slice": - if (!isArray(value)) { - return null; - } - var sliceParams = node.children.slice(0); - var computed = this.computeSliceParams(value.length, sliceParams); - var start = computed[0]; - var stop = computed[1]; - var step = computed[2]; - result = []; - if (step > 0) { - for (i = start; i < stop; i += step) { - result.push(value[i]); - } - } else { - for (i = start; i > stop; i += step) { - result.push(value[i]); - } - } - return result; - case "Projection": - // Evaluate left child. - var base = this.visit(node.children[0], value); - if (!isArray(base)) { - return null; - } - collected = []; - for (i = 0; i < base.length; i++) { - current = this.visit(node.children[1], base[i]); - if (current !== null) { - collected.push(current); - } - } - return collected; - case "ValueProjection": - // Evaluate left child. - base = this.visit(node.children[0], value); - if (!isObject(base)) { - return null; - } - collected = []; - var values = objValues(base); - for (i = 0; i < values.length; i++) { - current = this.visit(node.children[1], values[i]); - if (current !== null) { - collected.push(current); - } - } - return collected; - case "FilterProjection": - base = this.visit(node.children[0], value); - if (!isArray(base)) { - return null; - } - var filtered = []; - var finalResults = []; - for (i = 0; i < base.length; i++) { - matched = this.visit(node.children[2], base[i]); - if (!isFalse(matched)) { - filtered.push(base[i]); - } - } - for (var j = 0; j < filtered.length; j++) { - current = this.visit(node.children[1], filtered[j]); - if (current !== null) { - finalResults.push(current); - } - } - return finalResults; - case "Comparator": - first = this.visit(node.children[0], value); - second = this.visit(node.children[1], value); - switch(node.name) { - case TOK_EQ: - result = strictDeepEqual(first, second); - break; - case TOK_NE: - result = !strictDeepEqual(first, second); - break; - case TOK_GT: - result = first > second; - break; - case TOK_GTE: - result = first >= second; - break; - case TOK_LT: - result = first < second; - break; - case TOK_LTE: - result = first <= second; - break; - default: - throw new Error("Unknown comparator: " + node.name); - } - return result; - case TOK_FLATTEN: - var original = this.visit(node.children[0], value); - if (!isArray(original)) { - return null; - } - var merged = []; - for (i = 0; i < original.length; i++) { - current = original[i]; - if (isArray(current)) { - merged.push.apply(merged, current); - } else { - merged.push(current); - } - } - return merged; - case "Identity": - return value; - case "MultiSelectList": - if (value === null) { - return null; - } - collected = []; - for (i = 0; i < node.children.length; i++) { - collected.push(this.visit(node.children[i], value)); - } - return collected; - case "MultiSelectHash": - if (value === null) { - return null; - } - collected = {}; - var child; - for (i = 0; i < node.children.length; i++) { - child = node.children[i]; - collected[child.name] = this.visit(child.value, value); - } - return collected; - case "OrExpression": - matched = this.visit(node.children[0], value); - if (isFalse(matched)) { - matched = this.visit(node.children[1], value); - } - return matched; - case "AndExpression": - first = this.visit(node.children[0], value); +"use strict"; - if (isFalse(first) === true) { - return first; - } - return this.visit(node.children[1], value); - case "NotExpression": - first = this.visit(node.children[0], value); - return isFalse(first); - case "Literal": - return node.value; - case TOK_PIPE: - left = this.visit(node.children[0], value); - return this.visit(node.children[1], left); - case TOK_CURRENT: - return value; - case "Function": - var resolvedArgs = []; - for (i = 0; i < node.children.length; i++) { - resolvedArgs.push(this.visit(node.children[i], value)); - } - return this.runtime.callFunction(node.name, resolvedArgs); - case "ExpressionReference": - var refNode = node.children[0]; - // Tag the node with a specific attribute so the type - // checker verify the type. - refNode.jmespathType = TOK_EXPREF; - return refNode; - default: - throw new Error("Unknown node type: " + node.type); - } - }, - computeSliceParams: function(arrayLength, sliceParams) { - var start = sliceParams[0]; - var stop = sliceParams[1]; - var step = sliceParams[2]; - var computed = [null, null, null]; - if (step === null) { - step = 1; - } else if (step === 0) { - var error = new Error("Invalid slice, step cannot be 0"); - error.name = "RuntimeError"; - throw error; - } - var stepValueNegative = step < 0 ? true : false; +Object.defineProperty(exports, "__esModule", ({ + value: true +})); +exports["default"] = void 0; - if (start === null) { - start = stepValueNegative ? arrayLength - 1 : 0; - } else { - start = this.capSliceRange(arrayLength, start, step); - } +var _rng = _interopRequireDefault(__nccwpck_require__(49788)); - if (stop === null) { - stop = stepValueNegative ? -1 : arrayLength; - } else { - stop = this.capSliceRange(arrayLength, stop, step); - } - computed[0] = start; - computed[1] = stop; - computed[2] = step; - return computed; - }, +var _bytesToUuid = _interopRequireDefault(__nccwpck_require__(35827)); - capSliceRange: function(arrayLength, actualValue, step) { - if (actualValue < 0) { - actualValue += arrayLength; - if (actualValue < 0) { - actualValue = step < 0 ? -1 : 0; - } - } else if (actualValue >= arrayLength) { - actualValue = step < 0 ? arrayLength - 1 : arrayLength; - } - return actualValue; - } +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } - }; +function v4(options, buf, offset) { + var i = buf && offset || 0; - function Runtime(interpreter) { - this._interpreter = interpreter; - this.functionTable = { - // name: [function, ] - // The can be: - // - // { - // args: [[type1, type2], [type1, type2]], - // variadic: true|false - // } - // - // Each arg in the arg list is a list of valid types - // (if the function is overloaded and supports multiple - // types. If the type is "any" then no type checking - // occurs on the argument. Variadic is optional - // and if not provided is assumed to be false. - abs: {_func: this._functionAbs, _signature: [{types: [TYPE_NUMBER]}]}, - avg: {_func: this._functionAvg, _signature: [{types: [TYPE_ARRAY_NUMBER]}]}, - ceil: {_func: this._functionCeil, _signature: [{types: [TYPE_NUMBER]}]}, - contains: { - _func: this._functionContains, - _signature: [{types: [TYPE_STRING, TYPE_ARRAY]}, - {types: [TYPE_ANY]}]}, - "ends_with": { - _func: this._functionEndsWith, - _signature: [{types: [TYPE_STRING]}, {types: [TYPE_STRING]}]}, - floor: {_func: this._functionFloor, _signature: [{types: [TYPE_NUMBER]}]}, - length: { - _func: this._functionLength, - _signature: [{types: [TYPE_STRING, TYPE_ARRAY, TYPE_OBJECT]}]}, - map: { - _func: this._functionMap, - _signature: [{types: [TYPE_EXPREF]}, {types: [TYPE_ARRAY]}]}, - max: { - _func: this._functionMax, - _signature: [{types: [TYPE_ARRAY_NUMBER, TYPE_ARRAY_STRING]}]}, - "merge": { - _func: this._functionMerge, - _signature: [{types: [TYPE_OBJECT], variadic: true}] - }, - "max_by": { - _func: this._functionMaxBy, - _signature: [{types: [TYPE_ARRAY]}, {types: [TYPE_EXPREF]}] - }, - sum: {_func: this._functionSum, _signature: [{types: [TYPE_ARRAY_NUMBER]}]}, - "starts_with": { - _func: this._functionStartsWith, - _signature: [{types: [TYPE_STRING]}, {types: [TYPE_STRING]}]}, - min: { - _func: this._functionMin, - _signature: [{types: [TYPE_ARRAY_NUMBER, TYPE_ARRAY_STRING]}]}, - "min_by": { - _func: this._functionMinBy, - _signature: [{types: [TYPE_ARRAY]}, {types: [TYPE_EXPREF]}] - }, - type: {_func: this._functionType, _signature: [{types: [TYPE_ANY]}]}, - keys: {_func: this._functionKeys, _signature: [{types: [TYPE_OBJECT]}]}, - values: {_func: this._functionValues, _signature: [{types: [TYPE_OBJECT]}]}, - sort: {_func: this._functionSort, _signature: [{types: [TYPE_ARRAY_STRING, TYPE_ARRAY_NUMBER]}]}, - "sort_by": { - _func: this._functionSortBy, - _signature: [{types: [TYPE_ARRAY]}, {types: [TYPE_EXPREF]}] - }, - join: { - _func: this._functionJoin, - _signature: [ - {types: [TYPE_STRING]}, - {types: [TYPE_ARRAY_STRING]} - ] - }, - reverse: { - _func: this._functionReverse, - _signature: [{types: [TYPE_STRING, TYPE_ARRAY]}]}, - "to_array": {_func: this._functionToArray, _signature: [{types: [TYPE_ANY]}]}, - "to_string": {_func: this._functionToString, _signature: [{types: [TYPE_ANY]}]}, - "to_number": {_func: this._functionToNumber, _signature: [{types: [TYPE_ANY]}]}, - "not_null": { - _func: this._functionNotNull, - _signature: [{types: [TYPE_ANY], variadic: true}] - } - }; + if (typeof options == 'string') { + buf = options === 'binary' ? new Array(16) : null; + options = null; } - Runtime.prototype = { - callFunction: function(name, resolvedArgs) { - var functionEntry = this.functionTable[name]; - if (functionEntry === undefined) { - throw new Error("Unknown function: " + name + "()"); - } - this._validateArgs(name, resolvedArgs, functionEntry._signature); - return functionEntry._func.call(this, resolvedArgs); - }, + options = options || {}; - _validateArgs: function(name, args, signature) { - // Validating the args requires validating - // the correct arity and the correct type of each arg. - // If the last argument is declared as variadic, then we need - // a minimum number of args to be required. Otherwise it has to - // be an exact amount. - var pluralized; - if (signature[signature.length - 1].variadic) { - if (args.length < signature.length) { - pluralized = signature.length === 1 ? " argument" : " arguments"; - throw new Error("ArgumentError: " + name + "() " + - "takes at least" + signature.length + pluralized + - " but received " + args.length); - } - } else if (args.length !== signature.length) { - pluralized = signature.length === 1 ? " argument" : " arguments"; - throw new Error("ArgumentError: " + name + "() " + - "takes " + signature.length + pluralized + - " but received " + args.length); - } - var currentSpec; - var actualType; - var typeMatched; - for (var i = 0; i < signature.length; i++) { - typeMatched = false; - currentSpec = signature[i].types; - actualType = this._getTypeName(args[i]); - for (var j = 0; j < currentSpec.length; j++) { - if (this._typeMatches(actualType, currentSpec[j], args[i])) { - typeMatched = true; - break; - } - } - if (!typeMatched) { - var expected = currentSpec - .map(function(typeIdentifier) { - return TYPE_NAME_TABLE[typeIdentifier]; - }) - .join(','); - throw new Error("TypeError: " + name + "() " + - "expected argument " + (i + 1) + - " to be type " + expected + - " but received type " + - TYPE_NAME_TABLE[actualType] + " instead."); - } - } - }, + var rnds = options.random || (options.rng || _rng.default)(); // Per 4.4, set bits for version and `clock_seq_hi_and_reserved` - _typeMatches: function(actual, expected, argValue) { - if (expected === TYPE_ANY) { - return true; - } - if (expected === TYPE_ARRAY_STRING || - expected === TYPE_ARRAY_NUMBER || - expected === TYPE_ARRAY) { - // The expected type can either just be array, - // or it can require a specific subtype (array of numbers). - // - // The simplest case is if "array" with no subtype is specified. - if (expected === TYPE_ARRAY) { - return actual === TYPE_ARRAY; - } else if (actual === TYPE_ARRAY) { - // Otherwise we need to check subtypes. - // I think this has potential to be improved. - var subtype; - if (expected === TYPE_ARRAY_NUMBER) { - subtype = TYPE_NUMBER; - } else if (expected === TYPE_ARRAY_STRING) { - subtype = TYPE_STRING; - } - for (var i = 0; i < argValue.length; i++) { - if (!this._typeMatches( - this._getTypeName(argValue[i]), subtype, - argValue[i])) { - return false; - } - } - return true; - } - } else { - return actual === expected; - } - }, - _getTypeName: function(obj) { - switch (Object.prototype.toString.call(obj)) { - case "[object String]": - return TYPE_STRING; - case "[object Number]": - return TYPE_NUMBER; - case "[object Array]": - return TYPE_ARRAY; - case "[object Boolean]": - return TYPE_BOOLEAN; - case "[object Null]": - return TYPE_NULL; - case "[object Object]": - // Check if it's an expref. If it has, it's been - // tagged with a jmespathType attr of 'Expref'; - if (obj.jmespathType === TOK_EXPREF) { - return TYPE_EXPREF; - } else { - return TYPE_OBJECT; - } - } - }, - _functionStartsWith: function(resolvedArgs) { - return resolvedArgs[0].lastIndexOf(resolvedArgs[1]) === 0; - }, + rnds[6] = rnds[6] & 0x0f | 0x40; + rnds[8] = rnds[8] & 0x3f | 0x80; // Copy bytes to buffer, if provided - _functionEndsWith: function(resolvedArgs) { - var searchStr = resolvedArgs[0]; - var suffix = resolvedArgs[1]; - return searchStr.indexOf(suffix, searchStr.length - suffix.length) !== -1; - }, + if (buf) { + for (var ii = 0; ii < 16; ++ii) { + buf[i + ii] = rnds[ii]; + } + } - _functionReverse: function(resolvedArgs) { - var typeName = this._getTypeName(resolvedArgs[0]); - if (typeName === TYPE_STRING) { - var originalStr = resolvedArgs[0]; - var reversedStr = ""; - for (var i = originalStr.length - 1; i >= 0; i--) { - reversedStr += originalStr[i]; - } - return reversedStr; - } else { - var reversedArray = resolvedArgs[0].slice(0); - reversedArray.reverse(); - return reversedArray; - } - }, + return buf || (0, _bytesToUuid.default)(rnds); +} - _functionAbs: function(resolvedArgs) { - return Math.abs(resolvedArgs[0]); - }, +var _default = v4; +exports["default"] = _default; - _functionCeil: function(resolvedArgs) { - return Math.ceil(resolvedArgs[0]); - }, +/***/ }), - _functionAvg: function(resolvedArgs) { - var sum = 0; - var inputArray = resolvedArgs[0]; - for (var i = 0; i < inputArray.length; i++) { - sum += inputArray[i]; - } - return sum / inputArray.length; - }, +/***/ 46508: +/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { - _functionContains: function(resolvedArgs) { - return resolvedArgs[0].indexOf(resolvedArgs[1]) >= 0; - }, +"use strict"; - _functionFloor: function(resolvedArgs) { - return Math.floor(resolvedArgs[0]); - }, - _functionLength: function(resolvedArgs) { - if (!isObject(resolvedArgs[0])) { - return resolvedArgs[0].length; - } else { - // As far as I can tell, there's no way to get the length - // of an object without O(n) iteration through the object. - return Object.keys(resolvedArgs[0]).length; - } - }, +Object.defineProperty(exports, "__esModule", ({ + value: true +})); +exports["default"] = void 0; - _functionMap: function(resolvedArgs) { - var mapped = []; - var interpreter = this._interpreter; - var exprefNode = resolvedArgs[0]; - var elements = resolvedArgs[1]; - for (var i = 0; i < elements.length; i++) { - mapped.push(interpreter.visit(exprefNode, elements[i])); - } - return mapped; - }, +var _v = _interopRequireDefault(__nccwpck_require__(36097)); - _functionMerge: function(resolvedArgs) { - var merged = {}; - for (var i = 0; i < resolvedArgs.length; i++) { - var current = resolvedArgs[i]; - for (var key in current) { - merged[key] = current[key]; - } - } - return merged; - }, +var _sha = _interopRequireDefault(__nccwpck_require__(7387)); - _functionMax: function(resolvedArgs) { - if (resolvedArgs[0].length > 0) { - var typeName = this._getTypeName(resolvedArgs[0][0]); - if (typeName === TYPE_NUMBER) { - return Math.max.apply(Math, resolvedArgs[0]); - } else { - var elements = resolvedArgs[0]; - var maxElement = elements[0]; - for (var i = 1; i < elements.length; i++) { - if (maxElement.localeCompare(elements[i]) < 0) { - maxElement = elements[i]; - } - } - return maxElement; - } - } else { - return null; - } - }, +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } - _functionMin: function(resolvedArgs) { - if (resolvedArgs[0].length > 0) { - var typeName = this._getTypeName(resolvedArgs[0][0]); - if (typeName === TYPE_NUMBER) { - return Math.min.apply(Math, resolvedArgs[0]); - } else { - var elements = resolvedArgs[0]; - var minElement = elements[0]; - for (var i = 1; i < elements.length; i++) { - if (elements[i].localeCompare(minElement) < 0) { - minElement = elements[i]; - } - } - return minElement; - } - } else { - return null; - } - }, +const v5 = (0, _v.default)('v5', 0x50, _sha.default); +var _default = v5; +exports["default"] = _default; - _functionSum: function(resolvedArgs) { - var sum = 0; - var listToSum = resolvedArgs[0]; - for (var i = 0; i < listToSum.length; i++) { - sum += listToSum[i]; - } - return sum; - }, +/***/ }), - _functionType: function(resolvedArgs) { - switch (this._getTypeName(resolvedArgs[0])) { - case TYPE_NUMBER: - return "number"; - case TYPE_STRING: - return "string"; - case TYPE_ARRAY: - return "array"; - case TYPE_OBJECT: - return "object"; - case TYPE_BOOLEAN: - return "boolean"; - case TYPE_EXPREF: - return "expref"; - case TYPE_NULL: - return "null"; - } - }, +/***/ 96323: +/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { - _functionKeys: function(resolvedArgs) { - return Object.keys(resolvedArgs[0]); - }, +"use strict"; +var __webpack_unused_export__; - _functionValues: function(resolvedArgs) { - var obj = resolvedArgs[0]; - var keys = Object.keys(obj); - var values = []; - for (var i = 0; i < keys.length; i++) { - values.push(obj[keys[i]]); +__webpack_unused_export__ = ({ value: true }); +var LRU_1 = __nccwpck_require__(77710); +var CACHE_SIZE = 1000; +/** + * Inspired node-lru-cache[https://github.com/isaacs/node-lru-cache] + */ +var EndpointCache = /** @class */ (function () { + function EndpointCache(maxSize) { + if (maxSize === void 0) { maxSize = CACHE_SIZE; } + this.maxSize = maxSize; + this.cache = new LRU_1.LRUCache(maxSize); + } + ; + Object.defineProperty(EndpointCache.prototype, "size", { + get: function () { + return this.cache.length; + }, + enumerable: true, + configurable: true + }); + EndpointCache.prototype.put = function (key, value) { + var keyString = typeof key !== 'string' ? EndpointCache.getKeyString(key) : key; + var endpointRecord = this.populateValue(value); + this.cache.put(keyString, endpointRecord); + }; + EndpointCache.prototype.get = function (key) { + var keyString = typeof key !== 'string' ? EndpointCache.getKeyString(key) : key; + var now = Date.now(); + var records = this.cache.get(keyString); + if (records) { + for (var i = records.length-1; i >= 0; i--) { + var record = records[i]; + if (record.Expire < now) { + records.splice(i, 1); + } + } + if (records.length === 0) { + this.cache.remove(keyString); + return undefined; + } } - return values; - }, + return records; + }; + EndpointCache.getKeyString = function (key) { + var identifiers = []; + var identifierNames = Object.keys(key).sort(); + for (var i = 0; i < identifierNames.length; i++) { + var identifierName = identifierNames[i]; + if (key[identifierName] === undefined) + continue; + identifiers.push(key[identifierName]); + } + return identifiers.join(' '); + }; + EndpointCache.prototype.populateValue = function (endpoints) { + var now = Date.now(); + return endpoints.map(function (endpoint) { return ({ + Address: endpoint.Address || '', + Expire: now + (endpoint.CachePeriodInMinutes || 1) * 60 * 1000 + }); }); + }; + EndpointCache.prototype.empty = function () { + this.cache.empty(); + }; + EndpointCache.prototype.remove = function (key) { + var keyString = typeof key !== 'string' ? EndpointCache.getKeyString(key) : key; + this.cache.remove(keyString); + }; + return EndpointCache; +}()); +exports.$ = EndpointCache; - _functionJoin: function(resolvedArgs) { - var joinChar = resolvedArgs[0]; - var listJoin = resolvedArgs[1]; - return listJoin.join(joinChar); - }, +/***/ }), - _functionToArray: function(resolvedArgs) { - if (this._getTypeName(resolvedArgs[0]) === TYPE_ARRAY) { - return resolvedArgs[0]; - } else { - return [resolvedArgs[0]]; - } - }, +/***/ 77710: +/***/ ((__unused_webpack_module, exports) => { - _functionToString: function(resolvedArgs) { - if (this._getTypeName(resolvedArgs[0]) === TYPE_STRING) { - return resolvedArgs[0]; - } else { - return JSON.stringify(resolvedArgs[0]); - } - }, +"use strict"; - _functionToNumber: function(resolvedArgs) { - var typeName = this._getTypeName(resolvedArgs[0]); - var convertedValue; - if (typeName === TYPE_NUMBER) { - return resolvedArgs[0]; - } else if (typeName === TYPE_STRING) { - convertedValue = +resolvedArgs[0]; - if (!isNaN(convertedValue)) { - return convertedValue; - } +Object.defineProperty(exports, "__esModule", ({ value: true })); +var LinkedListNode = /** @class */ (function () { + function LinkedListNode(key, value) { + this.key = key; + this.value = value; + } + return LinkedListNode; +}()); +var LRUCache = /** @class */ (function () { + function LRUCache(size) { + this.nodeMap = {}; + this.size = 0; + if (typeof size !== 'number' || size < 1) { + throw new Error('Cache size can only be positive number'); } - return null; - }, - - _functionNotNull: function(resolvedArgs) { - for (var i = 0; i < resolvedArgs.length; i++) { - if (this._getTypeName(resolvedArgs[i]) !== TYPE_NULL) { - return resolvedArgs[i]; - } + this.sizeLimit = size; + } + Object.defineProperty(LRUCache.prototype, "length", { + get: function () { + return this.size; + }, + enumerable: true, + configurable: true + }); + LRUCache.prototype.prependToList = function (node) { + if (!this.headerNode) { + this.tailNode = node; } - return null; - }, - - _functionSort: function(resolvedArgs) { - var sortedArray = resolvedArgs[0].slice(0); - sortedArray.sort(); - return sortedArray; - }, - - _functionSortBy: function(resolvedArgs) { - var sortedArray = resolvedArgs[0].slice(0); - if (sortedArray.length === 0) { - return sortedArray; + else { + this.headerNode.prev = node; + node.next = this.headerNode; } - var interpreter = this._interpreter; - var exprefNode = resolvedArgs[1]; - var requiredType = this._getTypeName( - interpreter.visit(exprefNode, sortedArray[0])); - if ([TYPE_NUMBER, TYPE_STRING].indexOf(requiredType) < 0) { - throw new Error("TypeError"); + this.headerNode = node; + this.size++; + }; + LRUCache.prototype.removeFromTail = function () { + if (!this.tailNode) { + return undefined; } - var that = this; - // In order to get a stable sort out of an unstable - // sort algorithm, we decorate/sort/undecorate (DSU) - // by creating a new list of [index, element] pairs. - // In the cmp function, if the evaluated elements are - // equal, then the index will be used as the tiebreaker. - // After the decorated list has been sorted, it will be - // undecorated to extract the original elements. - var decorated = []; - for (var i = 0; i < sortedArray.length; i++) { - decorated.push([i, sortedArray[i]]); + var node = this.tailNode; + var prevNode = node.prev; + if (prevNode) { + prevNode.next = undefined; } - decorated.sort(function(a, b) { - var exprA = interpreter.visit(exprefNode, a[1]); - var exprB = interpreter.visit(exprefNode, b[1]); - if (that._getTypeName(exprA) !== requiredType) { - throw new Error( - "TypeError: expected " + requiredType + ", received " + - that._getTypeName(exprA)); - } else if (that._getTypeName(exprB) !== requiredType) { - throw new Error( - "TypeError: expected " + requiredType + ", received " + - that._getTypeName(exprB)); - } - if (exprA > exprB) { - return 1; - } else if (exprA < exprB) { - return -1; - } else { - // If they're equal compare the items by their - // order to maintain relative order of equal keys - // (i.e. to get a stable sort). - return a[0] - b[0]; - } - }); - // Undecorate: extract out the original list elements. - for (var j = 0; j < decorated.length; j++) { - sortedArray[j] = decorated[j][1]; + node.prev = undefined; + this.tailNode = prevNode; + this.size--; + return node; + }; + LRUCache.prototype.detachFromList = function (node) { + if (this.headerNode === node) { + this.headerNode = node.next; } - return sortedArray; - }, - - _functionMaxBy: function(resolvedArgs) { - var exprefNode = resolvedArgs[1]; - var resolvedArray = resolvedArgs[0]; - var keyFunction = this.createKeyFunction(exprefNode, [TYPE_NUMBER, TYPE_STRING]); - var maxNumber = -Infinity; - var maxRecord; - var current; - for (var i = 0; i < resolvedArray.length; i++) { - current = keyFunction(resolvedArray[i]); - if (current > maxNumber) { - maxNumber = current; - maxRecord = resolvedArray[i]; + if (this.tailNode === node) { + this.tailNode = node.prev; } - } - return maxRecord; - }, - - _functionMinBy: function(resolvedArgs) { - var exprefNode = resolvedArgs[1]; - var resolvedArray = resolvedArgs[0]; - var keyFunction = this.createKeyFunction(exprefNode, [TYPE_NUMBER, TYPE_STRING]); - var minNumber = Infinity; - var minRecord; - var current; - for (var i = 0; i < resolvedArray.length; i++) { - current = keyFunction(resolvedArray[i]); - if (current < minNumber) { - minNumber = current; - minRecord = resolvedArray[i]; + if (node.prev) { + node.prev.next = node.next; } - } - return minRecord; - }, - - createKeyFunction: function(exprefNode, allowedTypes) { - var that = this; - var interpreter = this._interpreter; - var keyFunc = function(x) { - var current = interpreter.visit(exprefNode, x); - if (allowedTypes.indexOf(that._getTypeName(current)) < 0) { - var msg = "TypeError: expected one of " + allowedTypes + - ", received " + that._getTypeName(current); - throw new Error(msg); + if (node.next) { + node.next.prev = node.prev; } - return current; - }; - return keyFunc; - } + node.next = undefined; + node.prev = undefined; + this.size--; + }; + LRUCache.prototype.get = function (key) { + if (this.nodeMap[key]) { + var node = this.nodeMap[key]; + this.detachFromList(node); + this.prependToList(node); + return node.value; + } + }; + LRUCache.prototype.remove = function (key) { + if (this.nodeMap[key]) { + var node = this.nodeMap[key]; + this.detachFromList(node); + delete this.nodeMap[key]; + } + }; + LRUCache.prototype.put = function (key, value) { + if (this.nodeMap[key]) { + this.remove(key); + } + else if (this.size === this.sizeLimit) { + var tailNode = this.removeFromTail(); + var key_1 = tailNode.key; + delete this.nodeMap[key_1]; + } + var newNode = new LinkedListNode(key, value); + this.nodeMap[key] = newNode; + this.prependToList(newNode); + }; + LRUCache.prototype.empty = function () { + var keys = Object.keys(this.nodeMap); + for (var i = 0; i < keys.length; i++) { + var key = keys[i]; + var node = this.nodeMap[key]; + this.detachFromList(node); + delete this.nodeMap[key]; + } + }; + return LRUCache; +}()); +exports.LRUCache = LRUCache; - }; +/***/ }), - function compile(stream) { - var parser = new Parser(); - var ast = parser.parse(stream); - return ast; - } +/***/ 83682: +/***/ ((module, __unused_webpack_exports, __nccwpck_require__) => { - function tokenize(stream) { - var lexer = new Lexer(); - return lexer.tokenize(stream); - } +var register = __nccwpck_require__(44670); +var addHook = __nccwpck_require__(5549); +var removeHook = __nccwpck_require__(6819); - function search(data, expression) { - var parser = new Parser(); - // This needs to be improved. Both the interpreter and runtime depend on - // each other. The runtime needs the interpreter to support exprefs. - // There's likely a clean way to avoid the cyclic dependency. - var runtime = new Runtime(); - var interpreter = new TreeInterpreter(runtime); - runtime._interpreter = interpreter; - var node = parser.parse(expression); - return interpreter.search(node, data); - } +// bind with array of arguments: https://stackoverflow.com/a/21792913 +var bind = Function.bind; +var bindable = bind.bind(bind); - exports.tokenize = tokenize; - exports.compile = compile; - exports.search = search; - exports.strictDeepEqual = strictDeepEqual; -})( false ? 0 : exports); +function bindApi(hook, state, name) { + var removeHookRef = bindable(removeHook, null).apply( + null, + name ? [state, name] : [state] + ); + hook.api = { remove: removeHookRef }; + hook.remove = removeHookRef; + ["before", "error", "after", "wrap"].forEach(function (kind) { + var args = name ? [state, kind, name] : [state, kind]; + hook[kind] = hook.api[kind] = bindable(addHook, null).apply(null, args); + }); +} +function HookSingular() { + var singularHookName = "h"; + var singularHookState = { + registry: {}, + }; + var singularHook = register.bind(null, singularHookState, singularHookName); + bindApi(singularHook, singularHookState, singularHookName); + return singularHook; +} -/***/ }), +function HookCollection() { + var state = { + registry: {}, + }; -/***/ 90250: -/***/ (function(module, exports, __nccwpck_require__) { + var hook = register.bind(null, state); + bindApi(hook, state); -/* module decorator */ module = __nccwpck_require__.nmd(module); -/** - * @license - * Lodash - * Copyright OpenJS Foundation and other contributors - * Released under MIT license - * Based on Underscore.js 1.8.3 - * Copyright Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors - */ -;(function() { + return hook; +} - /** Used as a safe reference for `undefined` in pre-ES5 environments. */ - var undefined; +var collectionHookDeprecationMessageDisplayed = false; +function Hook() { + if (!collectionHookDeprecationMessageDisplayed) { + console.warn( + '[before-after-hook]: "Hook()" repurposing warning, use "Hook.Collection()". Read more: https://git.io/upgrade-before-after-hook-to-1.4' + ); + collectionHookDeprecationMessageDisplayed = true; + } + return HookCollection(); +} - /** Used as the semantic version number. */ - var VERSION = '4.17.21'; +Hook.Singular = HookSingular.bind(); +Hook.Collection = HookCollection.bind(); - /** Used as the size to enable large array optimizations. */ - var LARGE_ARRAY_SIZE = 200; +module.exports = Hook; +// expose constructors as a named property for TypeScript +module.exports.Hook = Hook; +module.exports.Singular = Hook.Singular; +module.exports.Collection = Hook.Collection; - /** Error message constants. */ - var CORE_ERROR_TEXT = 'Unsupported core-js use. Try https://npms.io/search?q=ponyfill.', - FUNC_ERROR_TEXT = 'Expected a function', - INVALID_TEMPL_VAR_ERROR_TEXT = 'Invalid `variable` option passed into `_.template`'; - /** Used to stand-in for `undefined` hash values. */ - var HASH_UNDEFINED = '__lodash_hash_undefined__'; +/***/ }), - /** Used as the maximum memoize cache size. */ - var MAX_MEMOIZE_SIZE = 500; +/***/ 5549: +/***/ ((module) => { - /** Used as the internal argument placeholder. */ - var PLACEHOLDER = '__lodash_placeholder__'; +module.exports = addHook; - /** Used to compose bitmasks for cloning. */ - var CLONE_DEEP_FLAG = 1, - CLONE_FLAT_FLAG = 2, - CLONE_SYMBOLS_FLAG = 4; +function addHook(state, kind, name, hook) { + var orig = hook; + if (!state.registry[name]) { + state.registry[name] = []; + } - /** Used to compose bitmasks for value comparisons. */ - var COMPARE_PARTIAL_FLAG = 1, - COMPARE_UNORDERED_FLAG = 2; + if (kind === "before") { + hook = function (method, options) { + return Promise.resolve() + .then(orig.bind(null, options)) + .then(method.bind(null, options)); + }; + } - /** Used to compose bitmasks for function metadata. */ - var WRAP_BIND_FLAG = 1, - WRAP_BIND_KEY_FLAG = 2, - WRAP_CURRY_BOUND_FLAG = 4, - WRAP_CURRY_FLAG = 8, - WRAP_CURRY_RIGHT_FLAG = 16, - WRAP_PARTIAL_FLAG = 32, - WRAP_PARTIAL_RIGHT_FLAG = 64, - WRAP_ARY_FLAG = 128, - WRAP_REARG_FLAG = 256, - WRAP_FLIP_FLAG = 512; + if (kind === "after") { + hook = function (method, options) { + var result; + return Promise.resolve() + .then(method.bind(null, options)) + .then(function (result_) { + result = result_; + return orig(result, options); + }) + .then(function () { + return result; + }); + }; + } - /** Used as default options for `_.truncate`. */ - var DEFAULT_TRUNC_LENGTH = 30, - DEFAULT_TRUNC_OMISSION = '...'; + if (kind === "error") { + hook = function (method, options) { + return Promise.resolve() + .then(method.bind(null, options)) + .catch(function (error) { + return orig(error, options); + }); + }; + } - /** Used to detect hot functions by number of calls within a span of milliseconds. */ - var HOT_COUNT = 800, - HOT_SPAN = 16; + state.registry[name].push({ + hook: hook, + orig: orig, + }); +} - /** Used to indicate the type of lazy iteratees. */ - var LAZY_FILTER_FLAG = 1, - LAZY_MAP_FLAG = 2, - LAZY_WHILE_FLAG = 3; - /** Used as references for various `Number` constants. */ - var INFINITY = 1 / 0, - MAX_SAFE_INTEGER = 9007199254740991, - MAX_INTEGER = 1.7976931348623157e+308, - NAN = 0 / 0; +/***/ }), - /** Used as references for the maximum length and index of an array. */ - var MAX_ARRAY_LENGTH = 4294967295, - MAX_ARRAY_INDEX = MAX_ARRAY_LENGTH - 1, - HALF_MAX_ARRAY_LENGTH = MAX_ARRAY_LENGTH >>> 1; +/***/ 44670: +/***/ ((module) => { - /** Used to associate wrap methods with their bit flags. */ - var wrapFlags = [ - ['ary', WRAP_ARY_FLAG], - ['bind', WRAP_BIND_FLAG], - ['bindKey', WRAP_BIND_KEY_FLAG], - ['curry', WRAP_CURRY_FLAG], - ['curryRight', WRAP_CURRY_RIGHT_FLAG], - ['flip', WRAP_FLIP_FLAG], - ['partial', WRAP_PARTIAL_FLAG], - ['partialRight', WRAP_PARTIAL_RIGHT_FLAG], - ['rearg', WRAP_REARG_FLAG] - ]; +module.exports = register; - /** `Object#toString` result references. */ - var argsTag = '[object Arguments]', - arrayTag = '[object Array]', - asyncTag = '[object AsyncFunction]', - boolTag = '[object Boolean]', - dateTag = '[object Date]', - domExcTag = '[object DOMException]', - errorTag = '[object Error]', - funcTag = '[object Function]', - genTag = '[object GeneratorFunction]', - mapTag = '[object Map]', - numberTag = '[object Number]', - nullTag = '[object Null]', - objectTag = '[object Object]', - promiseTag = '[object Promise]', - proxyTag = '[object Proxy]', - regexpTag = '[object RegExp]', - setTag = '[object Set]', - stringTag = '[object String]', - symbolTag = '[object Symbol]', - undefinedTag = '[object Undefined]', - weakMapTag = '[object WeakMap]', - weakSetTag = '[object WeakSet]'; +function register(state, name, method, options) { + if (typeof method !== "function") { + throw new Error("method for before hook must be a function"); + } - var arrayBufferTag = '[object ArrayBuffer]', - dataViewTag = '[object DataView]', - float32Tag = '[object Float32Array]', - float64Tag = '[object Float64Array]', - int8Tag = '[object Int8Array]', - int16Tag = '[object Int16Array]', - int32Tag = '[object Int32Array]', - uint8Tag = '[object Uint8Array]', - uint8ClampedTag = '[object Uint8ClampedArray]', - uint16Tag = '[object Uint16Array]', - uint32Tag = '[object Uint32Array]'; + if (!options) { + options = {}; + } - /** Used to match empty string literals in compiled template source. */ - var reEmptyStringLeading = /\b__p \+= '';/g, - reEmptyStringMiddle = /\b(__p \+=) '' \+/g, - reEmptyStringTrailing = /(__e\(.*?\)|\b__t\)) \+\n'';/g; + if (Array.isArray(name)) { + return name.reverse().reduce(function (callback, name) { + return register.bind(null, state, name, callback, options); + }, method)(); + } - /** Used to match HTML entities and HTML characters. */ - var reEscapedHtml = /&(?:amp|lt|gt|quot|#39);/g, - reUnescapedHtml = /[&<>"']/g, - reHasEscapedHtml = RegExp(reEscapedHtml.source), - reHasUnescapedHtml = RegExp(reUnescapedHtml.source); + return Promise.resolve().then(function () { + if (!state.registry[name]) { + return method(options); + } - /** Used to match template delimiters. */ - var reEscape = /<%-([\s\S]+?)%>/g, - reEvaluate = /<%([\s\S]+?)%>/g, - reInterpolate = /<%=([\s\S]+?)%>/g; + return state.registry[name].reduce(function (method, registered) { + return registered.hook.bind(null, method, options); + }, method)(); + }); +} - /** Used to match property names within property paths. */ - var reIsDeepProp = /\.|\[(?:[^[\]]*|(["'])(?:(?!\1)[^\\]|\\.)*?\1)\]/, - reIsPlainProp = /^\w*$/, - rePropName = /[^.[\]]+|\[(?:(-?\d+(?:\.\d+)?)|(["'])((?:(?!\2)[^\\]|\\.)*?)\2)\]|(?=(?:\.|\[\])(?:\.|\[\]|$))/g; - /** - * Used to match `RegExp` - * [syntax characters](http://ecma-international.org/ecma-262/7.0/#sec-patterns). - */ - var reRegExpChar = /[\\^$.*+?()[\]{}|]/g, - reHasRegExpChar = RegExp(reRegExpChar.source); +/***/ }), - /** Used to match leading whitespace. */ - var reTrimStart = /^\s+/; +/***/ 6819: +/***/ ((module) => { - /** Used to match a single whitespace character. */ - var reWhitespace = /\s/; +module.exports = removeHook; - /** Used to match wrap detail comments. */ - var reWrapComment = /\{(?:\n\/\* \[wrapped with .+\] \*\/)?\n?/, - reWrapDetails = /\{\n\/\* \[wrapped with (.+)\] \*/, - reSplitDetails = /,? & /; +function removeHook(state, name, method) { + if (!state.registry[name]) { + return; + } - /** Used to match words composed of alphanumeric characters. */ - var reAsciiWord = /[^\x00-\x2f\x3a-\x40\x5b-\x60\x7b-\x7f]+/g; + var index = state.registry[name] + .map(function (registered) { + return registered.orig; + }) + .indexOf(method); - /** - * Used to validate the `validate` option in `_.template` variable. - * - * Forbids characters which could potentially change the meaning of the function argument definition: - * - "()," (modification of function parameters) - * - "=" (default value) - * - "[]{}" (destructuring of function parameters) - * - "/" (beginning of a comment) - * - whitespace - */ - var reForbiddenIdentifierChars = /[()=,{}\[\]\/\s]/; + if (index === -1) { + return; + } - /** Used to match backslashes in property paths. */ - var reEscapeChar = /\\(\\)?/g; + state.registry[name].splice(index, 1); +} - /** - * Used to match - * [ES template delimiters](http://ecma-international.org/ecma-262/7.0/#sec-template-literal-lexical-components). - */ - var reEsTemplate = /\$\{([^\\}]*(?:\\.[^\\}]*)*)\}/g; - /** Used to match `RegExp` flags from their coerced string values. */ - var reFlags = /\w*$/; +/***/ }), - /** Used to detect bad signed hexadecimal string values. */ - var reIsBadHex = /^[-+]0x[0-9a-f]+$/i; +/***/ 58932: +/***/ ((__unused_webpack_module, exports) => { - /** Used to detect binary string values. */ - var reIsBinary = /^0b[01]+$/i; +"use strict"; - /** Used to detect host constructors (Safari). */ - var reIsHostCtor = /^\[object .+?Constructor\]$/; - /** Used to detect octal string values. */ - var reIsOctal = /^0o[0-7]+$/i; +Object.defineProperty(exports, "__esModule", ({ value: true })); - /** Used to detect unsigned integer values. */ - var reIsUint = /^(?:0|[1-9]\d*)$/; +class Deprecation extends Error { + constructor(message) { + super(message); // Maintains proper stack trace (only available on V8) - /** Used to match Latin Unicode letters (excluding mathematical operators). */ - var reLatin = /[\xc0-\xd6\xd8-\xf6\xf8-\xff\u0100-\u017f]/g; + /* istanbul ignore next */ - /** Used to ensure capturing order of template delimiters. */ - var reNoMatch = /($^)/; + if (Error.captureStackTrace) { + Error.captureStackTrace(this, this.constructor); + } - /** Used to match unescaped characters in compiled string literals. */ - var reUnescapedString = /['\n\r\u2028\u2029\\]/g; + this.name = 'Deprecation'; + } - /** Used to compose unicode character classes. */ - var rsAstralRange = '\\ud800-\\udfff', - rsComboMarksRange = '\\u0300-\\u036f', - reComboHalfMarksRange = '\\ufe20-\\ufe2f', - rsComboSymbolsRange = '\\u20d0-\\u20ff', - rsComboRange = rsComboMarksRange + reComboHalfMarksRange + rsComboSymbolsRange, - rsDingbatRange = '\\u2700-\\u27bf', - rsLowerRange = 'a-z\\xdf-\\xf6\\xf8-\\xff', - rsMathOpRange = '\\xac\\xb1\\xd7\\xf7', - rsNonCharRange = '\\x00-\\x2f\\x3a-\\x40\\x5b-\\x60\\x7b-\\xbf', - rsPunctuationRange = '\\u2000-\\u206f', - rsSpaceRange = ' \\t\\x0b\\f\\xa0\\ufeff\\n\\r\\u2028\\u2029\\u1680\\u180e\\u2000\\u2001\\u2002\\u2003\\u2004\\u2005\\u2006\\u2007\\u2008\\u2009\\u200a\\u202f\\u205f\\u3000', - rsUpperRange = 'A-Z\\xc0-\\xd6\\xd8-\\xde', - rsVarRange = '\\ufe0e\\ufe0f', - rsBreakRange = rsMathOpRange + rsNonCharRange + rsPunctuationRange + rsSpaceRange; +} - /** Used to compose unicode capture groups. */ - var rsApos = "['\u2019]", - rsAstral = '[' + rsAstralRange + ']', - rsBreak = '[' + rsBreakRange + ']', - rsCombo = '[' + rsComboRange + ']', - rsDigits = '\\d+', - rsDingbat = '[' + rsDingbatRange + ']', - rsLower = '[' + rsLowerRange + ']', - rsMisc = '[^' + rsAstralRange + rsBreakRange + rsDigits + rsDingbatRange + rsLowerRange + rsUpperRange + ']', - rsFitz = '\\ud83c[\\udffb-\\udfff]', - rsModifier = '(?:' + rsCombo + '|' + rsFitz + ')', - rsNonAstral = '[^' + rsAstralRange + ']', - rsRegional = '(?:\\ud83c[\\udde6-\\uddff]){2}', - rsSurrPair = '[\\ud800-\\udbff][\\udc00-\\udfff]', - rsUpper = '[' + rsUpperRange + ']', - rsZWJ = '\\u200d'; +exports.Deprecation = Deprecation; - /** Used to compose unicode regexes. */ - var rsMiscLower = '(?:' + rsLower + '|' + rsMisc + ')', - rsMiscUpper = '(?:' + rsUpper + '|' + rsMisc + ')', - rsOptContrLower = '(?:' + rsApos + '(?:d|ll|m|re|s|t|ve))?', - rsOptContrUpper = '(?:' + rsApos + '(?:D|LL|M|RE|S|T|VE))?', - reOptMod = rsModifier + '?', - rsOptVar = '[' + rsVarRange + ']?', - rsOptJoin = '(?:' + rsZWJ + '(?:' + [rsNonAstral, rsRegional, rsSurrPair].join('|') + ')' + rsOptVar + reOptMod + ')*', - rsOrdLower = '\\d*(?:1st|2nd|3rd|(?![123])\\dth)(?=\\b|[A-Z_])', - rsOrdUpper = '\\d*(?:1ST|2ND|3RD|(?![123])\\dTH)(?=\\b|[a-z_])', - rsSeq = rsOptVar + reOptMod + rsOptJoin, - rsEmoji = '(?:' + [rsDingbat, rsRegional, rsSurrPair].join('|') + ')' + rsSeq, - rsSymbol = '(?:' + [rsNonAstral + rsCombo + '?', rsCombo, rsRegional, rsSurrPair, rsAstral].join('|') + ')'; - /** Used to match apostrophes. */ - var reApos = RegExp(rsApos, 'g'); +/***/ }), - /** - * Used to match [combining diacritical marks](https://en.wikipedia.org/wiki/Combining_Diacritical_Marks) and - * [combining diacritical marks for symbols](https://en.wikipedia.org/wiki/Combining_Diacritical_Marks_for_Symbols). - */ - var reComboMark = RegExp(rsCombo, 'g'); +/***/ 87783: +/***/ ((__unused_webpack_module, exports) => { - /** Used to match [string symbols](https://mathiasbynens.be/notes/javascript-unicode). */ - var reUnicode = RegExp(rsFitz + '(?=' + rsFitz + ')|' + rsSymbol + rsSeq, 'g'); +(function(exports) { + "use strict"; - /** Used to match complex or compound words. */ - var reUnicodeWord = RegExp([ - rsUpper + '?' + rsLower + '+' + rsOptContrLower + '(?=' + [rsBreak, rsUpper, '$'].join('|') + ')', - rsMiscUpper + '+' + rsOptContrUpper + '(?=' + [rsBreak, rsUpper + rsMiscLower, '$'].join('|') + ')', - rsUpper + '?' + rsMiscLower + '+' + rsOptContrLower, - rsUpper + '+' + rsOptContrUpper, - rsOrdUpper, - rsOrdLower, - rsDigits, - rsEmoji - ].join('|'), 'g'); + function isArray(obj) { + if (obj !== null) { + return Object.prototype.toString.call(obj) === "[object Array]"; + } else { + return false; + } + } - /** Used to detect strings with [zero-width joiners or code points from the astral planes](http://eev.ee/blog/2015/09/12/dark-corners-of-unicode/). */ - var reHasUnicode = RegExp('[' + rsZWJ + rsAstralRange + rsComboRange + rsVarRange + ']'); + function isObject(obj) { + if (obj !== null) { + return Object.prototype.toString.call(obj) === "[object Object]"; + } else { + return false; + } + } - /** Used to detect strings that need a more robust regexp to match words. */ - var reHasUnicodeWord = /[a-z][A-Z]|[A-Z]{2}[a-z]|[0-9][a-zA-Z]|[a-zA-Z][0-9]|[^a-zA-Z0-9 ]/; + function strictDeepEqual(first, second) { + // Check the scalar case first. + if (first === second) { + return true; + } - /** Used to assign default `context` object properties. */ - var contextProps = [ - 'Array', 'Buffer', 'DataView', 'Date', 'Error', 'Float32Array', 'Float64Array', - 'Function', 'Int8Array', 'Int16Array', 'Int32Array', 'Map', 'Math', 'Object', - 'Promise', 'RegExp', 'Set', 'String', 'Symbol', 'TypeError', 'Uint8Array', - 'Uint8ClampedArray', 'Uint16Array', 'Uint32Array', 'WeakMap', - '_', 'clearTimeout', 'isFinite', 'parseInt', 'setTimeout' - ]; + // Check if they are the same type. + var firstType = Object.prototype.toString.call(first); + if (firstType !== Object.prototype.toString.call(second)) { + return false; + } + // We know that first and second have the same type so we can just check the + // first type from now on. + if (isArray(first) === true) { + // Short circuit if they're not the same length; + if (first.length !== second.length) { + return false; + } + for (var i = 0; i < first.length; i++) { + if (strictDeepEqual(first[i], second[i]) === false) { + return false; + } + } + return true; + } + if (isObject(first) === true) { + // An object is equal if it has the same key/value pairs. + var keysSeen = {}; + for (var key in first) { + if (hasOwnProperty.call(first, key)) { + if (strictDeepEqual(first[key], second[key]) === false) { + return false; + } + keysSeen[key] = true; + } + } + // Now check that there aren't any keys in second that weren't + // in first. + for (var key2 in second) { + if (hasOwnProperty.call(second, key2)) { + if (keysSeen[key2] !== true) { + return false; + } + } + } + return true; + } + return false; + } - /** Used to make template sourceURLs easier to identify. */ - var templateCounter = -1; + function isFalse(obj) { + // From the spec: + // A false value corresponds to the following values: + // Empty list + // Empty object + // Empty string + // False boolean + // null value - /** Used to identify `toStringTag` values of typed arrays. */ - var typedArrayTags = {}; - typedArrayTags[float32Tag] = typedArrayTags[float64Tag] = - typedArrayTags[int8Tag] = typedArrayTags[int16Tag] = - typedArrayTags[int32Tag] = typedArrayTags[uint8Tag] = - typedArrayTags[uint8ClampedTag] = typedArrayTags[uint16Tag] = - typedArrayTags[uint32Tag] = true; - typedArrayTags[argsTag] = typedArrayTags[arrayTag] = - typedArrayTags[arrayBufferTag] = typedArrayTags[boolTag] = - typedArrayTags[dataViewTag] = typedArrayTags[dateTag] = - typedArrayTags[errorTag] = typedArrayTags[funcTag] = - typedArrayTags[mapTag] = typedArrayTags[numberTag] = - typedArrayTags[objectTag] = typedArrayTags[regexpTag] = - typedArrayTags[setTag] = typedArrayTags[stringTag] = - typedArrayTags[weakMapTag] = false; - - /** Used to identify `toStringTag` values supported by `_.clone`. */ - var cloneableTags = {}; - cloneableTags[argsTag] = cloneableTags[arrayTag] = - cloneableTags[arrayBufferTag] = cloneableTags[dataViewTag] = - cloneableTags[boolTag] = cloneableTags[dateTag] = - cloneableTags[float32Tag] = cloneableTags[float64Tag] = - cloneableTags[int8Tag] = cloneableTags[int16Tag] = - cloneableTags[int32Tag] = cloneableTags[mapTag] = - cloneableTags[numberTag] = cloneableTags[objectTag] = - cloneableTags[regexpTag] = cloneableTags[setTag] = - cloneableTags[stringTag] = cloneableTags[symbolTag] = - cloneableTags[uint8Tag] = cloneableTags[uint8ClampedTag] = - cloneableTags[uint16Tag] = cloneableTags[uint32Tag] = true; - cloneableTags[errorTag] = cloneableTags[funcTag] = - cloneableTags[weakMapTag] = false; - - /** Used to map Latin Unicode letters to basic Latin letters. */ - var deburredLetters = { - // Latin-1 Supplement block. - '\xc0': 'A', '\xc1': 'A', '\xc2': 'A', '\xc3': 'A', '\xc4': 'A', '\xc5': 'A', - '\xe0': 'a', '\xe1': 'a', '\xe2': 'a', '\xe3': 'a', '\xe4': 'a', '\xe5': 'a', - '\xc7': 'C', '\xe7': 'c', - '\xd0': 'D', '\xf0': 'd', - '\xc8': 'E', '\xc9': 'E', '\xca': 'E', '\xcb': 'E', - '\xe8': 'e', '\xe9': 'e', '\xea': 'e', '\xeb': 'e', - '\xcc': 'I', '\xcd': 'I', '\xce': 'I', '\xcf': 'I', - '\xec': 'i', '\xed': 'i', '\xee': 'i', '\xef': 'i', - '\xd1': 'N', '\xf1': 'n', - '\xd2': 'O', '\xd3': 'O', '\xd4': 'O', '\xd5': 'O', '\xd6': 'O', '\xd8': 'O', - '\xf2': 'o', '\xf3': 'o', '\xf4': 'o', '\xf5': 'o', '\xf6': 'o', '\xf8': 'o', - '\xd9': 'U', '\xda': 'U', '\xdb': 'U', '\xdc': 'U', - '\xf9': 'u', '\xfa': 'u', '\xfb': 'u', '\xfc': 'u', - '\xdd': 'Y', '\xfd': 'y', '\xff': 'y', - '\xc6': 'Ae', '\xe6': 'ae', - '\xde': 'Th', '\xfe': 'th', - '\xdf': 'ss', - // Latin Extended-A block. - '\u0100': 'A', '\u0102': 'A', '\u0104': 'A', - '\u0101': 'a', '\u0103': 'a', '\u0105': 'a', - '\u0106': 'C', '\u0108': 'C', '\u010a': 'C', '\u010c': 'C', - '\u0107': 'c', '\u0109': 'c', '\u010b': 'c', '\u010d': 'c', - '\u010e': 'D', '\u0110': 'D', '\u010f': 'd', '\u0111': 'd', - '\u0112': 'E', '\u0114': 'E', '\u0116': 'E', '\u0118': 'E', '\u011a': 'E', - '\u0113': 'e', '\u0115': 'e', '\u0117': 'e', '\u0119': 'e', '\u011b': 'e', - '\u011c': 'G', '\u011e': 'G', '\u0120': 'G', '\u0122': 'G', - '\u011d': 'g', '\u011f': 'g', '\u0121': 'g', '\u0123': 'g', - '\u0124': 'H', '\u0126': 'H', '\u0125': 'h', '\u0127': 'h', - '\u0128': 'I', '\u012a': 'I', '\u012c': 'I', '\u012e': 'I', '\u0130': 'I', - '\u0129': 'i', '\u012b': 'i', '\u012d': 'i', '\u012f': 'i', '\u0131': 'i', - '\u0134': 'J', '\u0135': 'j', - '\u0136': 'K', '\u0137': 'k', '\u0138': 'k', - '\u0139': 'L', '\u013b': 'L', '\u013d': 'L', '\u013f': 'L', '\u0141': 'L', - '\u013a': 'l', '\u013c': 'l', '\u013e': 'l', '\u0140': 'l', '\u0142': 'l', - '\u0143': 'N', '\u0145': 'N', '\u0147': 'N', '\u014a': 'N', - '\u0144': 'n', '\u0146': 'n', '\u0148': 'n', '\u014b': 'n', - '\u014c': 'O', '\u014e': 'O', '\u0150': 'O', - '\u014d': 'o', '\u014f': 'o', '\u0151': 'o', - '\u0154': 'R', '\u0156': 'R', '\u0158': 'R', - '\u0155': 'r', '\u0157': 'r', '\u0159': 'r', - '\u015a': 'S', '\u015c': 'S', '\u015e': 'S', '\u0160': 'S', - '\u015b': 's', '\u015d': 's', '\u015f': 's', '\u0161': 's', - '\u0162': 'T', '\u0164': 'T', '\u0166': 'T', - '\u0163': 't', '\u0165': 't', '\u0167': 't', - '\u0168': 'U', '\u016a': 'U', '\u016c': 'U', '\u016e': 'U', '\u0170': 'U', '\u0172': 'U', - '\u0169': 'u', '\u016b': 'u', '\u016d': 'u', '\u016f': 'u', '\u0171': 'u', '\u0173': 'u', - '\u0174': 'W', '\u0175': 'w', - '\u0176': 'Y', '\u0177': 'y', '\u0178': 'Y', - '\u0179': 'Z', '\u017b': 'Z', '\u017d': 'Z', - '\u017a': 'z', '\u017c': 'z', '\u017e': 'z', - '\u0132': 'IJ', '\u0133': 'ij', - '\u0152': 'Oe', '\u0153': 'oe', - '\u0149': "'n", '\u017f': 's' - }; - - /** Used to map characters to HTML entities. */ - var htmlEscapes = { - '&': '&', - '<': '<', - '>': '>', - '"': '"', - "'": ''' - }; - - /** Used to map HTML entities to characters. */ - var htmlUnescapes = { - '&': '&', - '<': '<', - '>': '>', - '"': '"', - ''': "'" - }; - - /** Used to escape characters for inclusion in compiled string literals. */ - var stringEscapes = { - '\\': '\\', - "'": "'", - '\n': 'n', - '\r': 'r', - '\u2028': 'u2028', - '\u2029': 'u2029' - }; - - /** Built-in method references without a dependency on `root`. */ - var freeParseFloat = parseFloat, - freeParseInt = parseInt; - - /** Detect free variable `global` from Node.js. */ - var freeGlobal = typeof global == 'object' && global && global.Object === Object && global; - - /** Detect free variable `self`. */ - var freeSelf = typeof self == 'object' && self && self.Object === Object && self; - - /** Used as a reference to the global object. */ - var root = freeGlobal || freeSelf || Function('return this')(); - - /** Detect free variable `exports`. */ - var freeExports = true && exports && !exports.nodeType && exports; - - /** Detect free variable `module`. */ - var freeModule = freeExports && "object" == 'object' && module && !module.nodeType && module; - - /** Detect the popular CommonJS extension `module.exports`. */ - var moduleExports = freeModule && freeModule.exports === freeExports; - - /** Detect free variable `process` from Node.js. */ - var freeProcess = moduleExports && freeGlobal.process; - - /** Used to access faster Node.js helpers. */ - var nodeUtil = (function() { - try { - // Use `util.types` for Node.js 10+. - var types = freeModule && freeModule.require && freeModule.require('util').types; - - if (types) { - return types; - } - - // Legacy `process.binding('util')` for Node.js < 10. - return freeProcess && freeProcess.binding && freeProcess.binding('util'); - } catch (e) {} - }()); - - /* Node.js helper references. */ - var nodeIsArrayBuffer = nodeUtil && nodeUtil.isArrayBuffer, - nodeIsDate = nodeUtil && nodeUtil.isDate, - nodeIsMap = nodeUtil && nodeUtil.isMap, - nodeIsRegExp = nodeUtil && nodeUtil.isRegExp, - nodeIsSet = nodeUtil && nodeUtil.isSet, - nodeIsTypedArray = nodeUtil && nodeUtil.isTypedArray; - - /*--------------------------------------------------------------------------*/ - - /** - * A faster alternative to `Function#apply`, this function invokes `func` - * with the `this` binding of `thisArg` and the arguments of `args`. - * - * @private - * @param {Function} func The function to invoke. - * @param {*} thisArg The `this` binding of `func`. - * @param {Array} args The arguments to invoke `func` with. - * @returns {*} Returns the result of `func`. - */ - function apply(func, thisArg, args) { - switch (args.length) { - case 0: return func.call(thisArg); - case 1: return func.call(thisArg, args[0]); - case 2: return func.call(thisArg, args[0], args[1]); - case 3: return func.call(thisArg, args[0], args[1], args[2]); + // First check the scalar values. + if (obj === "" || obj === false || obj === null) { + return true; + } else if (isArray(obj) && obj.length === 0) { + // Check for an empty array. + return true; + } else if (isObject(obj)) { + // Check for an empty object. + for (var key in obj) { + // If there are any keys, then + // the object is not empty so the object + // is not false. + if (obj.hasOwnProperty(key)) { + return false; + } + } + return true; + } else { + return false; } - return func.apply(thisArg, args); } - /** - * A specialized version of `baseAggregator` for arrays. - * - * @private - * @param {Array} [array] The array to iterate over. - * @param {Function} setter The function to set `accumulator` values. - * @param {Function} iteratee The iteratee to transform keys. - * @param {Object} accumulator The initial aggregated object. - * @returns {Function} Returns `accumulator`. - */ - function arrayAggregator(array, setter, iteratee, accumulator) { - var index = -1, - length = array == null ? 0 : array.length; - - while (++index < length) { - var value = array[index]; - setter(accumulator, value, iteratee(value), array); + function objValues(obj) { + var keys = Object.keys(obj); + var values = []; + for (var i = 0; i < keys.length; i++) { + values.push(obj[keys[i]]); } - return accumulator; + return values; } - /** - * A specialized version of `_.forEach` for arrays without support for - * iteratee shorthands. - * - * @private - * @param {Array} [array] The array to iterate over. - * @param {Function} iteratee The function invoked per iteration. - * @returns {Array} Returns `array`. - */ - function arrayEach(array, iteratee) { - var index = -1, - length = array == null ? 0 : array.length; - - while (++index < length) { - if (iteratee(array[index], index, array) === false) { - break; + function merge(a, b) { + var merged = {}; + for (var key in a) { + merged[key] = a[key]; } - } - return array; - } - - /** - * A specialized version of `_.forEachRight` for arrays without support for - * iteratee shorthands. - * - * @private - * @param {Array} [array] The array to iterate over. - * @param {Function} iteratee The function invoked per iteration. - * @returns {Array} Returns `array`. - */ - function arrayEachRight(array, iteratee) { - var length = array == null ? 0 : array.length; - - while (length--) { - if (iteratee(array[length], length, array) === false) { - break; + for (var key2 in b) { + merged[key2] = b[key2]; } - } - return array; + return merged; } - /** - * A specialized version of `_.every` for arrays without support for - * iteratee shorthands. - * - * @private - * @param {Array} [array] The array to iterate over. - * @param {Function} predicate The function invoked per iteration. - * @returns {boolean} Returns `true` if all elements pass the predicate check, - * else `false`. - */ - function arrayEvery(array, predicate) { - var index = -1, - length = array == null ? 0 : array.length; - - while (++index < length) { - if (!predicate(array[index], index, array)) { - return false; - } - } - return true; + var trimLeft; + if (typeof String.prototype.trimLeft === "function") { + trimLeft = function(str) { + return str.trimLeft(); + }; + } else { + trimLeft = function(str) { + return str.match(/^\s*(.*)/)[1]; + }; } - /** - * A specialized version of `_.filter` for arrays without support for - * iteratee shorthands. - * - * @private - * @param {Array} [array] The array to iterate over. - * @param {Function} predicate The function invoked per iteration. - * @returns {Array} Returns the new filtered array. - */ - function arrayFilter(array, predicate) { - var index = -1, - length = array == null ? 0 : array.length, - resIndex = 0, - result = []; - - while (++index < length) { - var value = array[index]; - if (predicate(value, index, array)) { - result[resIndex++] = value; - } - } - return result; - } + // Type constants used to define functions. + var TYPE_NUMBER = 0; + var TYPE_ANY = 1; + var TYPE_STRING = 2; + var TYPE_ARRAY = 3; + var TYPE_OBJECT = 4; + var TYPE_BOOLEAN = 5; + var TYPE_EXPREF = 6; + var TYPE_NULL = 7; + var TYPE_ARRAY_NUMBER = 8; + var TYPE_ARRAY_STRING = 9; + var TYPE_NAME_TABLE = { + 0: 'number', + 1: 'any', + 2: 'string', + 3: 'array', + 4: 'object', + 5: 'boolean', + 6: 'expression', + 7: 'null', + 8: 'Array', + 9: 'Array' + }; - /** - * A specialized version of `_.includes` for arrays without support for - * specifying an index to search from. - * - * @private - * @param {Array} [array] The array to inspect. - * @param {*} target The value to search for. - * @returns {boolean} Returns `true` if `target` is found, else `false`. - */ - function arrayIncludes(array, value) { - var length = array == null ? 0 : array.length; - return !!length && baseIndexOf(array, value, 0) > -1; - } + var TOK_EOF = "EOF"; + var TOK_UNQUOTEDIDENTIFIER = "UnquotedIdentifier"; + var TOK_QUOTEDIDENTIFIER = "QuotedIdentifier"; + var TOK_RBRACKET = "Rbracket"; + var TOK_RPAREN = "Rparen"; + var TOK_COMMA = "Comma"; + var TOK_COLON = "Colon"; + var TOK_RBRACE = "Rbrace"; + var TOK_NUMBER = "Number"; + var TOK_CURRENT = "Current"; + var TOK_EXPREF = "Expref"; + var TOK_PIPE = "Pipe"; + var TOK_OR = "Or"; + var TOK_AND = "And"; + var TOK_EQ = "EQ"; + var TOK_GT = "GT"; + var TOK_LT = "LT"; + var TOK_GTE = "GTE"; + var TOK_LTE = "LTE"; + var TOK_NE = "NE"; + var TOK_FLATTEN = "Flatten"; + var TOK_STAR = "Star"; + var TOK_FILTER = "Filter"; + var TOK_DOT = "Dot"; + var TOK_NOT = "Not"; + var TOK_LBRACE = "Lbrace"; + var TOK_LBRACKET = "Lbracket"; + var TOK_LPAREN= "Lparen"; + var TOK_LITERAL= "Literal"; - /** - * This function is like `arrayIncludes` except that it accepts a comparator. - * - * @private - * @param {Array} [array] The array to inspect. - * @param {*} target The value to search for. - * @param {Function} comparator The comparator invoked per element. - * @returns {boolean} Returns `true` if `target` is found, else `false`. - */ - function arrayIncludesWith(array, value, comparator) { - var index = -1, - length = array == null ? 0 : array.length; + // The "&", "[", "<", ">" tokens + // are not in basicToken because + // there are two token variants + // ("&&", "[?", "<=", ">="). This is specially handled + // below. - while (++index < length) { - if (comparator(value, array[index])) { - return true; - } - } - return false; - } + var basicTokens = { + ".": TOK_DOT, + "*": TOK_STAR, + ",": TOK_COMMA, + ":": TOK_COLON, + "{": TOK_LBRACE, + "}": TOK_RBRACE, + "]": TOK_RBRACKET, + "(": TOK_LPAREN, + ")": TOK_RPAREN, + "@": TOK_CURRENT + }; - /** - * A specialized version of `_.map` for arrays without support for iteratee - * shorthands. - * - * @private - * @param {Array} [array] The array to iterate over. - * @param {Function} iteratee The function invoked per iteration. - * @returns {Array} Returns the new mapped array. - */ - function arrayMap(array, iteratee) { - var index = -1, - length = array == null ? 0 : array.length, - result = Array(length); + var operatorStartToken = { + "<": true, + ">": true, + "=": true, + "!": true + }; - while (++index < length) { - result[index] = iteratee(array[index], index, array); - } - return result; - } + var skipChars = { + " ": true, + "\t": true, + "\n": true + }; - /** - * Appends the elements of `values` to `array`. - * - * @private - * @param {Array} array The array to modify. - * @param {Array} values The values to append. - * @returns {Array} Returns `array`. - */ - function arrayPush(array, values) { - var index = -1, - length = values.length, - offset = array.length; - while (++index < length) { - array[offset + index] = values[index]; - } - return array; + function isAlpha(ch) { + return (ch >= "a" && ch <= "z") || + (ch >= "A" && ch <= "Z") || + ch === "_"; } - /** - * A specialized version of `_.reduce` for arrays without support for - * iteratee shorthands. - * - * @private - * @param {Array} [array] The array to iterate over. - * @param {Function} iteratee The function invoked per iteration. - * @param {*} [accumulator] The initial value. - * @param {boolean} [initAccum] Specify using the first element of `array` as - * the initial value. - * @returns {*} Returns the accumulated value. - */ - function arrayReduce(array, iteratee, accumulator, initAccum) { - var index = -1, - length = array == null ? 0 : array.length; - - if (initAccum && length) { - accumulator = array[++index]; - } - while (++index < length) { - accumulator = iteratee(accumulator, array[index], index, array); - } - return accumulator; + function isNum(ch) { + return (ch >= "0" && ch <= "9") || + ch === "-"; + } + function isAlphaNum(ch) { + return (ch >= "a" && ch <= "z") || + (ch >= "A" && ch <= "Z") || + (ch >= "0" && ch <= "9") || + ch === "_"; } - /** - * A specialized version of `_.reduceRight` for arrays without support for - * iteratee shorthands. - * - * @private - * @param {Array} [array] The array to iterate over. - * @param {Function} iteratee The function invoked per iteration. - * @param {*} [accumulator] The initial value. - * @param {boolean} [initAccum] Specify using the last element of `array` as - * the initial value. - * @returns {*} Returns the accumulated value. - */ - function arrayReduceRight(array, iteratee, accumulator, initAccum) { - var length = array == null ? 0 : array.length; - if (initAccum && length) { - accumulator = array[--length]; - } - while (length--) { - accumulator = iteratee(accumulator, array[length], length, array); - } - return accumulator; - } - - /** - * A specialized version of `_.some` for arrays without support for iteratee - * shorthands. - * - * @private - * @param {Array} [array] The array to iterate over. - * @param {Function} predicate The function invoked per iteration. - * @returns {boolean} Returns `true` if any element passes the predicate check, - * else `false`. - */ - function arraySome(array, predicate) { - var index = -1, - length = array == null ? 0 : array.length; - - while (++index < length) { - if (predicate(array[index], index, array)) { - return true; - } - } - return false; + function Lexer() { } + Lexer.prototype = { + tokenize: function(stream) { + var tokens = []; + this._current = 0; + var start; + var identifier; + var token; + while (this._current < stream.length) { + if (isAlpha(stream[this._current])) { + start = this._current; + identifier = this._consumeUnquotedIdentifier(stream); + tokens.push({type: TOK_UNQUOTEDIDENTIFIER, + value: identifier, + start: start}); + } else if (basicTokens[stream[this._current]] !== undefined) { + tokens.push({type: basicTokens[stream[this._current]], + value: stream[this._current], + start: this._current}); + this._current++; + } else if (isNum(stream[this._current])) { + token = this._consumeNumber(stream); + tokens.push(token); + } else if (stream[this._current] === "[") { + // No need to increment this._current. This happens + // in _consumeLBracket + token = this._consumeLBracket(stream); + tokens.push(token); + } else if (stream[this._current] === "\"") { + start = this._current; + identifier = this._consumeQuotedIdentifier(stream); + tokens.push({type: TOK_QUOTEDIDENTIFIER, + value: identifier, + start: start}); + } else if (stream[this._current] === "'") { + start = this._current; + identifier = this._consumeRawStringLiteral(stream); + tokens.push({type: TOK_LITERAL, + value: identifier, + start: start}); + } else if (stream[this._current] === "`") { + start = this._current; + var literal = this._consumeLiteral(stream); + tokens.push({type: TOK_LITERAL, + value: literal, + start: start}); + } else if (operatorStartToken[stream[this._current]] !== undefined) { + tokens.push(this._consumeOperator(stream)); + } else if (skipChars[stream[this._current]] !== undefined) { + // Ignore whitespace. + this._current++; + } else if (stream[this._current] === "&") { + start = this._current; + this._current++; + if (stream[this._current] === "&") { + this._current++; + tokens.push({type: TOK_AND, value: "&&", start: start}); + } else { + tokens.push({type: TOK_EXPREF, value: "&", start: start}); + } + } else if (stream[this._current] === "|") { + start = this._current; + this._current++; + if (stream[this._current] === "|") { + this._current++; + tokens.push({type: TOK_OR, value: "||", start: start}); + } else { + tokens.push({type: TOK_PIPE, value: "|", start: start}); + } + } else { + var error = new Error("Unknown character:" + stream[this._current]); + error.name = "LexerError"; + throw error; + } + } + return tokens; + }, - /** - * Gets the size of an ASCII `string`. - * - * @private - * @param {string} string The string inspect. - * @returns {number} Returns the string size. - */ - var asciiSize = baseProperty('length'); + _consumeUnquotedIdentifier: function(stream) { + var start = this._current; + this._current++; + while (this._current < stream.length && isAlphaNum(stream[this._current])) { + this._current++; + } + return stream.slice(start, this._current); + }, - /** - * Converts an ASCII `string` to an array. - * - * @private - * @param {string} string The string to convert. - * @returns {Array} Returns the converted array. - */ - function asciiToArray(string) { - return string.split(''); - } + _consumeQuotedIdentifier: function(stream) { + var start = this._current; + this._current++; + var maxLength = stream.length; + while (stream[this._current] !== "\"" && this._current < maxLength) { + // You can escape a double quote and you can escape an escape. + var current = this._current; + if (stream[current] === "\\" && (stream[current + 1] === "\\" || + stream[current + 1] === "\"")) { + current += 2; + } else { + current++; + } + this._current = current; + } + this._current++; + return JSON.parse(stream.slice(start, this._current)); + }, - /** - * Splits an ASCII `string` into an array of its words. - * - * @private - * @param {string} The string to inspect. - * @returns {Array} Returns the words of `string`. - */ - function asciiWords(string) { - return string.match(reAsciiWord) || []; - } + _consumeRawStringLiteral: function(stream) { + var start = this._current; + this._current++; + var maxLength = stream.length; + while (stream[this._current] !== "'" && this._current < maxLength) { + // You can escape a single quote and you can escape an escape. + var current = this._current; + if (stream[current] === "\\" && (stream[current + 1] === "\\" || + stream[current + 1] === "'")) { + current += 2; + } else { + current++; + } + this._current = current; + } + this._current++; + var literal = stream.slice(start + 1, this._current - 1); + return literal.replace("\\'", "'"); + }, - /** - * The base implementation of methods like `_.findKey` and `_.findLastKey`, - * without support for iteratee shorthands, which iterates over `collection` - * using `eachFunc`. - * - * @private - * @param {Array|Object} collection The collection to inspect. - * @param {Function} predicate The function invoked per iteration. - * @param {Function} eachFunc The function to iterate over `collection`. - * @returns {*} Returns the found element or its key, else `undefined`. - */ - function baseFindKey(collection, predicate, eachFunc) { - var result; - eachFunc(collection, function(value, key, collection) { - if (predicate(value, key, collection)) { - result = key; - return false; - } - }); - return result; - } + _consumeNumber: function(stream) { + var start = this._current; + this._current++; + var maxLength = stream.length; + while (isNum(stream[this._current]) && this._current < maxLength) { + this._current++; + } + var value = parseInt(stream.slice(start, this._current)); + return {type: TOK_NUMBER, value: value, start: start}; + }, - /** - * The base implementation of `_.findIndex` and `_.findLastIndex` without - * support for iteratee shorthands. - * - * @private - * @param {Array} array The array to inspect. - * @param {Function} predicate The function invoked per iteration. - * @param {number} fromIndex The index to search from. - * @param {boolean} [fromRight] Specify iterating from right to left. - * @returns {number} Returns the index of the matched value, else `-1`. - */ - function baseFindIndex(array, predicate, fromIndex, fromRight) { - var length = array.length, - index = fromIndex + (fromRight ? 1 : -1); + _consumeLBracket: function(stream) { + var start = this._current; + this._current++; + if (stream[this._current] === "?") { + this._current++; + return {type: TOK_FILTER, value: "[?", start: start}; + } else if (stream[this._current] === "]") { + this._current++; + return {type: TOK_FLATTEN, value: "[]", start: start}; + } else { + return {type: TOK_LBRACKET, value: "[", start: start}; + } + }, - while ((fromRight ? index-- : ++index < length)) { - if (predicate(array[index], index, array)) { - return index; - } - } - return -1; - } + _consumeOperator: function(stream) { + var start = this._current; + var startingChar = stream[start]; + this._current++; + if (startingChar === "!") { + if (stream[this._current] === "=") { + this._current++; + return {type: TOK_NE, value: "!=", start: start}; + } else { + return {type: TOK_NOT, value: "!", start: start}; + } + } else if (startingChar === "<") { + if (stream[this._current] === "=") { + this._current++; + return {type: TOK_LTE, value: "<=", start: start}; + } else { + return {type: TOK_LT, value: "<", start: start}; + } + } else if (startingChar === ">") { + if (stream[this._current] === "=") { + this._current++; + return {type: TOK_GTE, value: ">=", start: start}; + } else { + return {type: TOK_GT, value: ">", start: start}; + } + } else if (startingChar === "=") { + if (stream[this._current] === "=") { + this._current++; + return {type: TOK_EQ, value: "==", start: start}; + } + } + }, - /** - * The base implementation of `_.indexOf` without `fromIndex` bounds checks. - * - * @private - * @param {Array} array The array to inspect. - * @param {*} value The value to search for. - * @param {number} fromIndex The index to search from. - * @returns {number} Returns the index of the matched value, else `-1`. - */ - function baseIndexOf(array, value, fromIndex) { - return value === value - ? strictIndexOf(array, value, fromIndex) - : baseFindIndex(array, baseIsNaN, fromIndex); - } + _consumeLiteral: function(stream) { + this._current++; + var start = this._current; + var maxLength = stream.length; + var literal; + while(stream[this._current] !== "`" && this._current < maxLength) { + // You can escape a literal char or you can escape the escape. + var current = this._current; + if (stream[current] === "\\" && (stream[current + 1] === "\\" || + stream[current + 1] === "`")) { + current += 2; + } else { + current++; + } + this._current = current; + } + var literalString = trimLeft(stream.slice(start, this._current)); + literalString = literalString.replace("\\`", "`"); + if (this._looksLikeJSON(literalString)) { + literal = JSON.parse(literalString); + } else { + // Try to JSON parse it as "" + literal = JSON.parse("\"" + literalString + "\""); + } + // +1 gets us to the ending "`", +1 to move on to the next char. + this._current++; + return literal; + }, - /** - * This function is like `baseIndexOf` except that it accepts a comparator. - * - * @private - * @param {Array} array The array to inspect. - * @param {*} value The value to search for. - * @param {number} fromIndex The index to search from. - * @param {Function} comparator The comparator invoked per element. - * @returns {number} Returns the index of the matched value, else `-1`. - */ - function baseIndexOfWith(array, value, fromIndex, comparator) { - var index = fromIndex - 1, - length = array.length; + _looksLikeJSON: function(literalString) { + var startingChars = "[{\""; + var jsonLiterals = ["true", "false", "null"]; + var numberLooking = "-0123456789"; - while (++index < length) { - if (comparator(array[index], value)) { - return index; + if (literalString === "") { + return false; + } else if (startingChars.indexOf(literalString[0]) >= 0) { + return true; + } else if (jsonLiterals.indexOf(literalString) >= 0) { + return true; + } else if (numberLooking.indexOf(literalString[0]) >= 0) { + try { + JSON.parse(literalString); + return true; + } catch (ex) { + return false; + } + } else { + return false; + } } - } - return -1; - } + }; - /** - * The base implementation of `_.isNaN` without support for number objects. - * - * @private - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is `NaN`, else `false`. - */ - function baseIsNaN(value) { - return value !== value; - } + var bindingPower = {}; + bindingPower[TOK_EOF] = 0; + bindingPower[TOK_UNQUOTEDIDENTIFIER] = 0; + bindingPower[TOK_QUOTEDIDENTIFIER] = 0; + bindingPower[TOK_RBRACKET] = 0; + bindingPower[TOK_RPAREN] = 0; + bindingPower[TOK_COMMA] = 0; + bindingPower[TOK_RBRACE] = 0; + bindingPower[TOK_NUMBER] = 0; + bindingPower[TOK_CURRENT] = 0; + bindingPower[TOK_EXPREF] = 0; + bindingPower[TOK_PIPE] = 1; + bindingPower[TOK_OR] = 2; + bindingPower[TOK_AND] = 3; + bindingPower[TOK_EQ] = 5; + bindingPower[TOK_GT] = 5; + bindingPower[TOK_LT] = 5; + bindingPower[TOK_GTE] = 5; + bindingPower[TOK_LTE] = 5; + bindingPower[TOK_NE] = 5; + bindingPower[TOK_FLATTEN] = 9; + bindingPower[TOK_STAR] = 20; + bindingPower[TOK_FILTER] = 21; + bindingPower[TOK_DOT] = 40; + bindingPower[TOK_NOT] = 45; + bindingPower[TOK_LBRACE] = 50; + bindingPower[TOK_LBRACKET] = 55; + bindingPower[TOK_LPAREN] = 60; - /** - * The base implementation of `_.mean` and `_.meanBy` without support for - * iteratee shorthands. - * - * @private - * @param {Array} array The array to iterate over. - * @param {Function} iteratee The function invoked per iteration. - * @returns {number} Returns the mean. - */ - function baseMean(array, iteratee) { - var length = array == null ? 0 : array.length; - return length ? (baseSum(array, iteratee) / length) : NAN; + function Parser() { } - /** - * The base implementation of `_.property` without support for deep paths. - * - * @private - * @param {string} key The key of the property to get. - * @returns {Function} Returns the new accessor function. - */ - function baseProperty(key) { - return function(object) { - return object == null ? undefined : object[key]; - }; - } + Parser.prototype = { + parse: function(expression) { + this._loadTokens(expression); + this.index = 0; + var ast = this.expression(0); + if (this._lookahead(0) !== TOK_EOF) { + var t = this._lookaheadToken(0); + var error = new Error( + "Unexpected token type: " + t.type + ", value: " + t.value); + error.name = "ParserError"; + throw error; + } + return ast; + }, - /** - * The base implementation of `_.propertyOf` without support for deep paths. - * - * @private - * @param {Object} object The object to query. - * @returns {Function} Returns the new accessor function. - */ - function basePropertyOf(object) { - return function(key) { - return object == null ? undefined : object[key]; - }; - } + _loadTokens: function(expression) { + var lexer = new Lexer(); + var tokens = lexer.tokenize(expression); + tokens.push({type: TOK_EOF, value: "", start: expression.length}); + this.tokens = tokens; + }, - /** - * The base implementation of `_.reduce` and `_.reduceRight`, without support - * for iteratee shorthands, which iterates over `collection` using `eachFunc`. - * - * @private - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} iteratee The function invoked per iteration. - * @param {*} accumulator The initial value. - * @param {boolean} initAccum Specify using the first or last element of - * `collection` as the initial value. - * @param {Function} eachFunc The function to iterate over `collection`. - * @returns {*} Returns the accumulated value. - */ - function baseReduce(collection, iteratee, accumulator, initAccum, eachFunc) { - eachFunc(collection, function(value, index, collection) { - accumulator = initAccum - ? (initAccum = false, value) - : iteratee(accumulator, value, index, collection); - }); - return accumulator; - } + expression: function(rbp) { + var leftToken = this._lookaheadToken(0); + this._advance(); + var left = this.nud(leftToken); + var currentToken = this._lookahead(0); + while (rbp < bindingPower[currentToken]) { + this._advance(); + left = this.led(currentToken, left); + currentToken = this._lookahead(0); + } + return left; + }, - /** - * The base implementation of `_.sortBy` which uses `comparer` to define the - * sort order of `array` and replaces criteria objects with their corresponding - * values. - * - * @private - * @param {Array} array The array to sort. - * @param {Function} comparer The function to define sort order. - * @returns {Array} Returns `array`. - */ - function baseSortBy(array, comparer) { - var length = array.length; + _lookahead: function(number) { + return this.tokens[this.index + number].type; + }, - array.sort(comparer); - while (length--) { - array[length] = array[length].value; - } - return array; - } + _lookaheadToken: function(number) { + return this.tokens[this.index + number]; + }, - /** - * The base implementation of `_.sum` and `_.sumBy` without support for - * iteratee shorthands. - * - * @private - * @param {Array} array The array to iterate over. - * @param {Function} iteratee The function invoked per iteration. - * @returns {number} Returns the sum. - */ - function baseSum(array, iteratee) { - var result, - index = -1, - length = array.length; - - while (++index < length) { - var current = iteratee(array[index]); - if (current !== undefined) { - result = result === undefined ? current : (result + current); - } - } - return result; - } + _advance: function() { + this.index++; + }, - /** - * The base implementation of `_.times` without support for iteratee shorthands - * or max array length checks. - * - * @private - * @param {number} n The number of times to invoke `iteratee`. - * @param {Function} iteratee The function invoked per iteration. - * @returns {Array} Returns the array of results. - */ - function baseTimes(n, iteratee) { - var index = -1, - result = Array(n); + nud: function(token) { + var left; + var right; + var expression; + switch (token.type) { + case TOK_LITERAL: + return {type: "Literal", value: token.value}; + case TOK_UNQUOTEDIDENTIFIER: + return {type: "Field", name: token.value}; + case TOK_QUOTEDIDENTIFIER: + var node = {type: "Field", name: token.value}; + if (this._lookahead(0) === TOK_LPAREN) { + throw new Error("Quoted identifier not allowed for function names."); + } + return node; + case TOK_NOT: + right = this.expression(bindingPower.Not); + return {type: "NotExpression", children: [right]}; + case TOK_STAR: + left = {type: "Identity"}; + right = null; + if (this._lookahead(0) === TOK_RBRACKET) { + // This can happen in a multiselect, + // [a, b, *] + right = {type: "Identity"}; + } else { + right = this._parseProjectionRHS(bindingPower.Star); + } + return {type: "ValueProjection", children: [left, right]}; + case TOK_FILTER: + return this.led(token.type, {type: "Identity"}); + case TOK_LBRACE: + return this._parseMultiselectHash(); + case TOK_FLATTEN: + left = {type: TOK_FLATTEN, children: [{type: "Identity"}]}; + right = this._parseProjectionRHS(bindingPower.Flatten); + return {type: "Projection", children: [left, right]}; + case TOK_LBRACKET: + if (this._lookahead(0) === TOK_NUMBER || this._lookahead(0) === TOK_COLON) { + right = this._parseIndexExpression(); + return this._projectIfSlice({type: "Identity"}, right); + } else if (this._lookahead(0) === TOK_STAR && + this._lookahead(1) === TOK_RBRACKET) { + this._advance(); + this._advance(); + right = this._parseProjectionRHS(bindingPower.Star); + return {type: "Projection", + children: [{type: "Identity"}, right]}; + } + return this._parseMultiselectList(); + case TOK_CURRENT: + return {type: TOK_CURRENT}; + case TOK_EXPREF: + expression = this.expression(bindingPower.Expref); + return {type: "ExpressionReference", children: [expression]}; + case TOK_LPAREN: + var args = []; + while (this._lookahead(0) !== TOK_RPAREN) { + if (this._lookahead(0) === TOK_CURRENT) { + expression = {type: TOK_CURRENT}; + this._advance(); + } else { + expression = this.expression(0); + } + args.push(expression); + } + this._match(TOK_RPAREN); + return args[0]; + default: + this._errorToken(token); + } + }, - while (++index < n) { - result[index] = iteratee(index); - } - return result; - } + led: function(tokenName, left) { + var right; + switch(tokenName) { + case TOK_DOT: + var rbp = bindingPower.Dot; + if (this._lookahead(0) !== TOK_STAR) { + right = this._parseDotRHS(rbp); + return {type: "Subexpression", children: [left, right]}; + } + // Creating a projection. + this._advance(); + right = this._parseProjectionRHS(rbp); + return {type: "ValueProjection", children: [left, right]}; + case TOK_PIPE: + right = this.expression(bindingPower.Pipe); + return {type: TOK_PIPE, children: [left, right]}; + case TOK_OR: + right = this.expression(bindingPower.Or); + return {type: "OrExpression", children: [left, right]}; + case TOK_AND: + right = this.expression(bindingPower.And); + return {type: "AndExpression", children: [left, right]}; + case TOK_LPAREN: + var name = left.name; + var args = []; + var expression, node; + while (this._lookahead(0) !== TOK_RPAREN) { + if (this._lookahead(0) === TOK_CURRENT) { + expression = {type: TOK_CURRENT}; + this._advance(); + } else { + expression = this.expression(0); + } + if (this._lookahead(0) === TOK_COMMA) { + this._match(TOK_COMMA); + } + args.push(expression); + } + this._match(TOK_RPAREN); + node = {type: "Function", name: name, children: args}; + return node; + case TOK_FILTER: + var condition = this.expression(0); + this._match(TOK_RBRACKET); + if (this._lookahead(0) === TOK_FLATTEN) { + right = {type: "Identity"}; + } else { + right = this._parseProjectionRHS(bindingPower.Filter); + } + return {type: "FilterProjection", children: [left, right, condition]}; + case TOK_FLATTEN: + var leftNode = {type: TOK_FLATTEN, children: [left]}; + var rightNode = this._parseProjectionRHS(bindingPower.Flatten); + return {type: "Projection", children: [leftNode, rightNode]}; + case TOK_EQ: + case TOK_NE: + case TOK_GT: + case TOK_GTE: + case TOK_LT: + case TOK_LTE: + return this._parseComparator(left, tokenName); + case TOK_LBRACKET: + var token = this._lookaheadToken(0); + if (token.type === TOK_NUMBER || token.type === TOK_COLON) { + right = this._parseIndexExpression(); + return this._projectIfSlice(left, right); + } + this._match(TOK_STAR); + this._match(TOK_RBRACKET); + right = this._parseProjectionRHS(bindingPower.Star); + return {type: "Projection", children: [left, right]}; + default: + this._errorToken(this._lookaheadToken(0)); + } + }, - /** - * The base implementation of `_.toPairs` and `_.toPairsIn` which creates an array - * of key-value pairs for `object` corresponding to the property names of `props`. - * - * @private - * @param {Object} object The object to query. - * @param {Array} props The property names to get values for. - * @returns {Object} Returns the key-value pairs. - */ - function baseToPairs(object, props) { - return arrayMap(props, function(key) { - return [key, object[key]]; - }); - } + _match: function(tokenType) { + if (this._lookahead(0) === tokenType) { + this._advance(); + } else { + var t = this._lookaheadToken(0); + var error = new Error("Expected " + tokenType + ", got: " + t.type); + error.name = "ParserError"; + throw error; + } + }, - /** - * The base implementation of `_.trim`. - * - * @private - * @param {string} string The string to trim. - * @returns {string} Returns the trimmed string. - */ - function baseTrim(string) { - return string - ? string.slice(0, trimmedEndIndex(string) + 1).replace(reTrimStart, '') - : string; - } + _errorToken: function(token) { + var error = new Error("Invalid token (" + + token.type + "): \"" + + token.value + "\""); + error.name = "ParserError"; + throw error; + }, - /** - * The base implementation of `_.unary` without support for storing metadata. - * - * @private - * @param {Function} func The function to cap arguments for. - * @returns {Function} Returns the new capped function. - */ - function baseUnary(func) { - return function(value) { - return func(value); - }; - } - /** - * The base implementation of `_.values` and `_.valuesIn` which creates an - * array of `object` property values corresponding to the property names - * of `props`. - * - * @private - * @param {Object} object The object to query. - * @param {Array} props The property names to get values for. - * @returns {Object} Returns the array of property values. - */ - function baseValues(object, props) { - return arrayMap(props, function(key) { - return object[key]; - }); - } + _parseIndexExpression: function() { + if (this._lookahead(0) === TOK_COLON || this._lookahead(1) === TOK_COLON) { + return this._parseSliceExpression(); + } else { + var node = { + type: "Index", + value: this._lookaheadToken(0).value}; + this._advance(); + this._match(TOK_RBRACKET); + return node; + } + }, - /** - * Checks if a `cache` value for `key` exists. - * - * @private - * @param {Object} cache The cache to query. - * @param {string} key The key of the entry to check. - * @returns {boolean} Returns `true` if an entry for `key` exists, else `false`. - */ - function cacheHas(cache, key) { - return cache.has(key); - } + _projectIfSlice: function(left, right) { + var indexExpr = {type: "IndexExpression", children: [left, right]}; + if (right.type === "Slice") { + return { + type: "Projection", + children: [indexExpr, this._parseProjectionRHS(bindingPower.Star)] + }; + } else { + return indexExpr; + } + }, - /** - * Used by `_.trim` and `_.trimStart` to get the index of the first string symbol - * that is not found in the character symbols. - * - * @private - * @param {Array} strSymbols The string symbols to inspect. - * @param {Array} chrSymbols The character symbols to find. - * @returns {number} Returns the index of the first unmatched string symbol. - */ - function charsStartIndex(strSymbols, chrSymbols) { - var index = -1, - length = strSymbols.length; + _parseSliceExpression: function() { + // [start:end:step] where each part is optional, as well as the last + // colon. + var parts = [null, null, null]; + var index = 0; + var currentToken = this._lookahead(0); + while (currentToken !== TOK_RBRACKET && index < 3) { + if (currentToken === TOK_COLON) { + index++; + this._advance(); + } else if (currentToken === TOK_NUMBER) { + parts[index] = this._lookaheadToken(0).value; + this._advance(); + } else { + var t = this._lookahead(0); + var error = new Error("Syntax error, unexpected token: " + + t.value + "(" + t.type + ")"); + error.name = "Parsererror"; + throw error; + } + currentToken = this._lookahead(0); + } + this._match(TOK_RBRACKET); + return { + type: "Slice", + children: parts + }; + }, - while (++index < length && baseIndexOf(chrSymbols, strSymbols[index], 0) > -1) {} - return index; - } + _parseComparator: function(left, comparator) { + var right = this.expression(bindingPower[comparator]); + return {type: "Comparator", name: comparator, children: [left, right]}; + }, - /** - * Used by `_.trim` and `_.trimEnd` to get the index of the last string symbol - * that is not found in the character symbols. - * - * @private - * @param {Array} strSymbols The string symbols to inspect. - * @param {Array} chrSymbols The character symbols to find. - * @returns {number} Returns the index of the last unmatched string symbol. - */ - function charsEndIndex(strSymbols, chrSymbols) { - var index = strSymbols.length; + _parseDotRHS: function(rbp) { + var lookahead = this._lookahead(0); + var exprTokens = [TOK_UNQUOTEDIDENTIFIER, TOK_QUOTEDIDENTIFIER, TOK_STAR]; + if (exprTokens.indexOf(lookahead) >= 0) { + return this.expression(rbp); + } else if (lookahead === TOK_LBRACKET) { + this._match(TOK_LBRACKET); + return this._parseMultiselectList(); + } else if (lookahead === TOK_LBRACE) { + this._match(TOK_LBRACE); + return this._parseMultiselectHash(); + } + }, - while (index-- && baseIndexOf(chrSymbols, strSymbols[index], 0) > -1) {} - return index; - } + _parseProjectionRHS: function(rbp) { + var right; + if (bindingPower[this._lookahead(0)] < 10) { + right = {type: "Identity"}; + } else if (this._lookahead(0) === TOK_LBRACKET) { + right = this.expression(rbp); + } else if (this._lookahead(0) === TOK_FILTER) { + right = this.expression(rbp); + } else if (this._lookahead(0) === TOK_DOT) { + this._match(TOK_DOT); + right = this._parseDotRHS(rbp); + } else { + var t = this._lookaheadToken(0); + var error = new Error("Sytanx error, unexpected token: " + + t.value + "(" + t.type + ")"); + error.name = "ParserError"; + throw error; + } + return right; + }, - /** - * Gets the number of `placeholder` occurrences in `array`. - * - * @private - * @param {Array} array The array to inspect. - * @param {*} placeholder The placeholder to search for. - * @returns {number} Returns the placeholder count. - */ - function countHolders(array, placeholder) { - var length = array.length, - result = 0; + _parseMultiselectList: function() { + var expressions = []; + while (this._lookahead(0) !== TOK_RBRACKET) { + var expression = this.expression(0); + expressions.push(expression); + if (this._lookahead(0) === TOK_COMMA) { + this._match(TOK_COMMA); + if (this._lookahead(0) === TOK_RBRACKET) { + throw new Error("Unexpected token Rbracket"); + } + } + } + this._match(TOK_RBRACKET); + return {type: "MultiSelectList", children: expressions}; + }, - while (length--) { - if (array[length] === placeholder) { - ++result; + _parseMultiselectHash: function() { + var pairs = []; + var identifierTypes = [TOK_UNQUOTEDIDENTIFIER, TOK_QUOTEDIDENTIFIER]; + var keyToken, keyName, value, node; + for (;;) { + keyToken = this._lookaheadToken(0); + if (identifierTypes.indexOf(keyToken.type) < 0) { + throw new Error("Expecting an identifier token, got: " + + keyToken.type); + } + keyName = keyToken.value; + this._advance(); + this._match(TOK_COLON); + value = this.expression(0); + node = {type: "KeyValuePair", name: keyName, value: value}; + pairs.push(node); + if (this._lookahead(0) === TOK_COMMA) { + this._match(TOK_COMMA); + } else if (this._lookahead(0) === TOK_RBRACE) { + this._match(TOK_RBRACE); + break; + } + } + return {type: "MultiSelectHash", children: pairs}; } - } - return result; - } - - /** - * Used by `_.deburr` to convert Latin-1 Supplement and Latin Extended-A - * letters to basic Latin letters. - * - * @private - * @param {string} letter The matched letter to deburr. - * @returns {string} Returns the deburred letter. - */ - var deburrLetter = basePropertyOf(deburredLetters); - - /** - * Used by `_.escape` to convert characters to HTML entities. - * - * @private - * @param {string} chr The matched character to escape. - * @returns {string} Returns the escaped character. - */ - var escapeHtmlChar = basePropertyOf(htmlEscapes); - - /** - * Used by `_.template` to escape characters for inclusion in compiled string literals. - * - * @private - * @param {string} chr The matched character to escape. - * @returns {string} Returns the escaped character. - */ - function escapeStringChar(chr) { - return '\\' + stringEscapes[chr]; - } + }; - /** - * Gets the value at `key` of `object`. - * - * @private - * @param {Object} [object] The object to query. - * @param {string} key The key of the property to get. - * @returns {*} Returns the property value. - */ - function getValue(object, key) { - return object == null ? undefined : object[key]; - } - /** - * Checks if `string` contains Unicode symbols. - * - * @private - * @param {string} string The string to inspect. - * @returns {boolean} Returns `true` if a symbol is found, else `false`. - */ - function hasUnicode(string) { - return reHasUnicode.test(string); + function TreeInterpreter(runtime) { + this.runtime = runtime; } - /** - * Checks if `string` contains a word composed of Unicode symbols. - * - * @private - * @param {string} string The string to inspect. - * @returns {boolean} Returns `true` if a word is found, else `false`. - */ - function hasUnicodeWord(string) { - return reHasUnicodeWord.test(string); - } + TreeInterpreter.prototype = { + search: function(node, value) { + return this.visit(node, value); + }, - /** - * Converts `iterator` to an array. - * - * @private - * @param {Object} iterator The iterator to convert. - * @returns {Array} Returns the converted array. - */ + visit: function(node, value) { + var matched, current, result, first, second, field, left, right, collected, i; + switch (node.type) { + case "Field": + if (value !== null && isObject(value)) { + field = value[node.name]; + if (field === undefined) { + return null; + } else { + return field; + } + } + return null; + case "Subexpression": + result = this.visit(node.children[0], value); + for (i = 1; i < node.children.length; i++) { + result = this.visit(node.children[1], result); + if (result === null) { + return null; + } + } + return result; + case "IndexExpression": + left = this.visit(node.children[0], value); + right = this.visit(node.children[1], left); + return right; + case "Index": + if (!isArray(value)) { + return null; + } + var index = node.value; + if (index < 0) { + index = value.length + index; + } + result = value[index]; + if (result === undefined) { + result = null; + } + return result; + case "Slice": + if (!isArray(value)) { + return null; + } + var sliceParams = node.children.slice(0); + var computed = this.computeSliceParams(value.length, sliceParams); + var start = computed[0]; + var stop = computed[1]; + var step = computed[2]; + result = []; + if (step > 0) { + for (i = start; i < stop; i += step) { + result.push(value[i]); + } + } else { + for (i = start; i > stop; i += step) { + result.push(value[i]); + } + } + return result; + case "Projection": + // Evaluate left child. + var base = this.visit(node.children[0], value); + if (!isArray(base)) { + return null; + } + collected = []; + for (i = 0; i < base.length; i++) { + current = this.visit(node.children[1], base[i]); + if (current !== null) { + collected.push(current); + } + } + return collected; + case "ValueProjection": + // Evaluate left child. + base = this.visit(node.children[0], value); + if (!isObject(base)) { + return null; + } + collected = []; + var values = objValues(base); + for (i = 0; i < values.length; i++) { + current = this.visit(node.children[1], values[i]); + if (current !== null) { + collected.push(current); + } + } + return collected; + case "FilterProjection": + base = this.visit(node.children[0], value); + if (!isArray(base)) { + return null; + } + var filtered = []; + var finalResults = []; + for (i = 0; i < base.length; i++) { + matched = this.visit(node.children[2], base[i]); + if (!isFalse(matched)) { + filtered.push(base[i]); + } + } + for (var j = 0; j < filtered.length; j++) { + current = this.visit(node.children[1], filtered[j]); + if (current !== null) { + finalResults.push(current); + } + } + return finalResults; + case "Comparator": + first = this.visit(node.children[0], value); + second = this.visit(node.children[1], value); + switch(node.name) { + case TOK_EQ: + result = strictDeepEqual(first, second); + break; + case TOK_NE: + result = !strictDeepEqual(first, second); + break; + case TOK_GT: + result = first > second; + break; + case TOK_GTE: + result = first >= second; + break; + case TOK_LT: + result = first < second; + break; + case TOK_LTE: + result = first <= second; + break; + default: + throw new Error("Unknown comparator: " + node.name); + } + return result; + case TOK_FLATTEN: + var original = this.visit(node.children[0], value); + if (!isArray(original)) { + return null; + } + var merged = []; + for (i = 0; i < original.length; i++) { + current = original[i]; + if (isArray(current)) { + merged.push.apply(merged, current); + } else { + merged.push(current); + } + } + return merged; + case "Identity": + return value; + case "MultiSelectList": + if (value === null) { + return null; + } + collected = []; + for (i = 0; i < node.children.length; i++) { + collected.push(this.visit(node.children[i], value)); + } + return collected; + case "MultiSelectHash": + if (value === null) { + return null; + } + collected = {}; + var child; + for (i = 0; i < node.children.length; i++) { + child = node.children[i]; + collected[child.name] = this.visit(child.value, value); + } + return collected; + case "OrExpression": + matched = this.visit(node.children[0], value); + if (isFalse(matched)) { + matched = this.visit(node.children[1], value); + } + return matched; + case "AndExpression": + first = this.visit(node.children[0], value); + + if (isFalse(first) === true) { + return first; + } + return this.visit(node.children[1], value); + case "NotExpression": + first = this.visit(node.children[0], value); + return isFalse(first); + case "Literal": + return node.value; + case TOK_PIPE: + left = this.visit(node.children[0], value); + return this.visit(node.children[1], left); + case TOK_CURRENT: + return value; + case "Function": + var resolvedArgs = []; + for (i = 0; i < node.children.length; i++) { + resolvedArgs.push(this.visit(node.children[i], value)); + } + return this.runtime.callFunction(node.name, resolvedArgs); + case "ExpressionReference": + var refNode = node.children[0]; + // Tag the node with a specific attribute so the type + // checker verify the type. + refNode.jmespathType = TOK_EXPREF; + return refNode; + default: + throw new Error("Unknown node type: " + node.type); + } + }, + + computeSliceParams: function(arrayLength, sliceParams) { + var start = sliceParams[0]; + var stop = sliceParams[1]; + var step = sliceParams[2]; + var computed = [null, null, null]; + if (step === null) { + step = 1; + } else if (step === 0) { + var error = new Error("Invalid slice, step cannot be 0"); + error.name = "RuntimeError"; + throw error; + } + var stepValueNegative = step < 0 ? true : false; + + if (start === null) { + start = stepValueNegative ? arrayLength - 1 : 0; + } else { + start = this.capSliceRange(arrayLength, start, step); + } + + if (stop === null) { + stop = stepValueNegative ? -1 : arrayLength; + } else { + stop = this.capSliceRange(arrayLength, stop, step); + } + computed[0] = start; + computed[1] = stop; + computed[2] = step; + return computed; + }, + + capSliceRange: function(arrayLength, actualValue, step) { + if (actualValue < 0) { + actualValue += arrayLength; + if (actualValue < 0) { + actualValue = step < 0 ? -1 : 0; + } + } else if (actualValue >= arrayLength) { + actualValue = step < 0 ? arrayLength - 1 : arrayLength; + } + return actualValue; + } + + }; + + function Runtime(interpreter) { + this._interpreter = interpreter; + this.functionTable = { + // name: [function, ] + // The can be: + // + // { + // args: [[type1, type2], [type1, type2]], + // variadic: true|false + // } + // + // Each arg in the arg list is a list of valid types + // (if the function is overloaded and supports multiple + // types. If the type is "any" then no type checking + // occurs on the argument. Variadic is optional + // and if not provided is assumed to be false. + abs: {_func: this._functionAbs, _signature: [{types: [TYPE_NUMBER]}]}, + avg: {_func: this._functionAvg, _signature: [{types: [TYPE_ARRAY_NUMBER]}]}, + ceil: {_func: this._functionCeil, _signature: [{types: [TYPE_NUMBER]}]}, + contains: { + _func: this._functionContains, + _signature: [{types: [TYPE_STRING, TYPE_ARRAY]}, + {types: [TYPE_ANY]}]}, + "ends_with": { + _func: this._functionEndsWith, + _signature: [{types: [TYPE_STRING]}, {types: [TYPE_STRING]}]}, + floor: {_func: this._functionFloor, _signature: [{types: [TYPE_NUMBER]}]}, + length: { + _func: this._functionLength, + _signature: [{types: [TYPE_STRING, TYPE_ARRAY, TYPE_OBJECT]}]}, + map: { + _func: this._functionMap, + _signature: [{types: [TYPE_EXPREF]}, {types: [TYPE_ARRAY]}]}, + max: { + _func: this._functionMax, + _signature: [{types: [TYPE_ARRAY_NUMBER, TYPE_ARRAY_STRING]}]}, + "merge": { + _func: this._functionMerge, + _signature: [{types: [TYPE_OBJECT], variadic: true}] + }, + "max_by": { + _func: this._functionMaxBy, + _signature: [{types: [TYPE_ARRAY]}, {types: [TYPE_EXPREF]}] + }, + sum: {_func: this._functionSum, _signature: [{types: [TYPE_ARRAY_NUMBER]}]}, + "starts_with": { + _func: this._functionStartsWith, + _signature: [{types: [TYPE_STRING]}, {types: [TYPE_STRING]}]}, + min: { + _func: this._functionMin, + _signature: [{types: [TYPE_ARRAY_NUMBER, TYPE_ARRAY_STRING]}]}, + "min_by": { + _func: this._functionMinBy, + _signature: [{types: [TYPE_ARRAY]}, {types: [TYPE_EXPREF]}] + }, + type: {_func: this._functionType, _signature: [{types: [TYPE_ANY]}]}, + keys: {_func: this._functionKeys, _signature: [{types: [TYPE_OBJECT]}]}, + values: {_func: this._functionValues, _signature: [{types: [TYPE_OBJECT]}]}, + sort: {_func: this._functionSort, _signature: [{types: [TYPE_ARRAY_STRING, TYPE_ARRAY_NUMBER]}]}, + "sort_by": { + _func: this._functionSortBy, + _signature: [{types: [TYPE_ARRAY]}, {types: [TYPE_EXPREF]}] + }, + join: { + _func: this._functionJoin, + _signature: [ + {types: [TYPE_STRING]}, + {types: [TYPE_ARRAY_STRING]} + ] + }, + reverse: { + _func: this._functionReverse, + _signature: [{types: [TYPE_STRING, TYPE_ARRAY]}]}, + "to_array": {_func: this._functionToArray, _signature: [{types: [TYPE_ANY]}]}, + "to_string": {_func: this._functionToString, _signature: [{types: [TYPE_ANY]}]}, + "to_number": {_func: this._functionToNumber, _signature: [{types: [TYPE_ANY]}]}, + "not_null": { + _func: this._functionNotNull, + _signature: [{types: [TYPE_ANY], variadic: true}] + } + }; + } + + Runtime.prototype = { + callFunction: function(name, resolvedArgs) { + var functionEntry = this.functionTable[name]; + if (functionEntry === undefined) { + throw new Error("Unknown function: " + name + "()"); + } + this._validateArgs(name, resolvedArgs, functionEntry._signature); + return functionEntry._func.call(this, resolvedArgs); + }, + + _validateArgs: function(name, args, signature) { + // Validating the args requires validating + // the correct arity and the correct type of each arg. + // If the last argument is declared as variadic, then we need + // a minimum number of args to be required. Otherwise it has to + // be an exact amount. + var pluralized; + if (signature[signature.length - 1].variadic) { + if (args.length < signature.length) { + pluralized = signature.length === 1 ? " argument" : " arguments"; + throw new Error("ArgumentError: " + name + "() " + + "takes at least" + signature.length + pluralized + + " but received " + args.length); + } + } else if (args.length !== signature.length) { + pluralized = signature.length === 1 ? " argument" : " arguments"; + throw new Error("ArgumentError: " + name + "() " + + "takes " + signature.length + pluralized + + " but received " + args.length); + } + var currentSpec; + var actualType; + var typeMatched; + for (var i = 0; i < signature.length; i++) { + typeMatched = false; + currentSpec = signature[i].types; + actualType = this._getTypeName(args[i]); + for (var j = 0; j < currentSpec.length; j++) { + if (this._typeMatches(actualType, currentSpec[j], args[i])) { + typeMatched = true; + break; + } + } + if (!typeMatched) { + var expected = currentSpec + .map(function(typeIdentifier) { + return TYPE_NAME_TABLE[typeIdentifier]; + }) + .join(','); + throw new Error("TypeError: " + name + "() " + + "expected argument " + (i + 1) + + " to be type " + expected + + " but received type " + + TYPE_NAME_TABLE[actualType] + " instead."); + } + } + }, + + _typeMatches: function(actual, expected, argValue) { + if (expected === TYPE_ANY) { + return true; + } + if (expected === TYPE_ARRAY_STRING || + expected === TYPE_ARRAY_NUMBER || + expected === TYPE_ARRAY) { + // The expected type can either just be array, + // or it can require a specific subtype (array of numbers). + // + // The simplest case is if "array" with no subtype is specified. + if (expected === TYPE_ARRAY) { + return actual === TYPE_ARRAY; + } else if (actual === TYPE_ARRAY) { + // Otherwise we need to check subtypes. + // I think this has potential to be improved. + var subtype; + if (expected === TYPE_ARRAY_NUMBER) { + subtype = TYPE_NUMBER; + } else if (expected === TYPE_ARRAY_STRING) { + subtype = TYPE_STRING; + } + for (var i = 0; i < argValue.length; i++) { + if (!this._typeMatches( + this._getTypeName(argValue[i]), subtype, + argValue[i])) { + return false; + } + } + return true; + } + } else { + return actual === expected; + } + }, + _getTypeName: function(obj) { + switch (Object.prototype.toString.call(obj)) { + case "[object String]": + return TYPE_STRING; + case "[object Number]": + return TYPE_NUMBER; + case "[object Array]": + return TYPE_ARRAY; + case "[object Boolean]": + return TYPE_BOOLEAN; + case "[object Null]": + return TYPE_NULL; + case "[object Object]": + // Check if it's an expref. If it has, it's been + // tagged with a jmespathType attr of 'Expref'; + if (obj.jmespathType === TOK_EXPREF) { + return TYPE_EXPREF; + } else { + return TYPE_OBJECT; + } + } + }, + + _functionStartsWith: function(resolvedArgs) { + return resolvedArgs[0].lastIndexOf(resolvedArgs[1]) === 0; + }, + + _functionEndsWith: function(resolvedArgs) { + var searchStr = resolvedArgs[0]; + var suffix = resolvedArgs[1]; + return searchStr.indexOf(suffix, searchStr.length - suffix.length) !== -1; + }, + + _functionReverse: function(resolvedArgs) { + var typeName = this._getTypeName(resolvedArgs[0]); + if (typeName === TYPE_STRING) { + var originalStr = resolvedArgs[0]; + var reversedStr = ""; + for (var i = originalStr.length - 1; i >= 0; i--) { + reversedStr += originalStr[i]; + } + return reversedStr; + } else { + var reversedArray = resolvedArgs[0].slice(0); + reversedArray.reverse(); + return reversedArray; + } + }, + + _functionAbs: function(resolvedArgs) { + return Math.abs(resolvedArgs[0]); + }, + + _functionCeil: function(resolvedArgs) { + return Math.ceil(resolvedArgs[0]); + }, + + _functionAvg: function(resolvedArgs) { + var sum = 0; + var inputArray = resolvedArgs[0]; + for (var i = 0; i < inputArray.length; i++) { + sum += inputArray[i]; + } + return sum / inputArray.length; + }, + + _functionContains: function(resolvedArgs) { + return resolvedArgs[0].indexOf(resolvedArgs[1]) >= 0; + }, + + _functionFloor: function(resolvedArgs) { + return Math.floor(resolvedArgs[0]); + }, + + _functionLength: function(resolvedArgs) { + if (!isObject(resolvedArgs[0])) { + return resolvedArgs[0].length; + } else { + // As far as I can tell, there's no way to get the length + // of an object without O(n) iteration through the object. + return Object.keys(resolvedArgs[0]).length; + } + }, + + _functionMap: function(resolvedArgs) { + var mapped = []; + var interpreter = this._interpreter; + var exprefNode = resolvedArgs[0]; + var elements = resolvedArgs[1]; + for (var i = 0; i < elements.length; i++) { + mapped.push(interpreter.visit(exprefNode, elements[i])); + } + return mapped; + }, + + _functionMerge: function(resolvedArgs) { + var merged = {}; + for (var i = 0; i < resolvedArgs.length; i++) { + var current = resolvedArgs[i]; + for (var key in current) { + merged[key] = current[key]; + } + } + return merged; + }, + + _functionMax: function(resolvedArgs) { + if (resolvedArgs[0].length > 0) { + var typeName = this._getTypeName(resolvedArgs[0][0]); + if (typeName === TYPE_NUMBER) { + return Math.max.apply(Math, resolvedArgs[0]); + } else { + var elements = resolvedArgs[0]; + var maxElement = elements[0]; + for (var i = 1; i < elements.length; i++) { + if (maxElement.localeCompare(elements[i]) < 0) { + maxElement = elements[i]; + } + } + return maxElement; + } + } else { + return null; + } + }, + + _functionMin: function(resolvedArgs) { + if (resolvedArgs[0].length > 0) { + var typeName = this._getTypeName(resolvedArgs[0][0]); + if (typeName === TYPE_NUMBER) { + return Math.min.apply(Math, resolvedArgs[0]); + } else { + var elements = resolvedArgs[0]; + var minElement = elements[0]; + for (var i = 1; i < elements.length; i++) { + if (elements[i].localeCompare(minElement) < 0) { + minElement = elements[i]; + } + } + return minElement; + } + } else { + return null; + } + }, + + _functionSum: function(resolvedArgs) { + var sum = 0; + var listToSum = resolvedArgs[0]; + for (var i = 0; i < listToSum.length; i++) { + sum += listToSum[i]; + } + return sum; + }, + + _functionType: function(resolvedArgs) { + switch (this._getTypeName(resolvedArgs[0])) { + case TYPE_NUMBER: + return "number"; + case TYPE_STRING: + return "string"; + case TYPE_ARRAY: + return "array"; + case TYPE_OBJECT: + return "object"; + case TYPE_BOOLEAN: + return "boolean"; + case TYPE_EXPREF: + return "expref"; + case TYPE_NULL: + return "null"; + } + }, + + _functionKeys: function(resolvedArgs) { + return Object.keys(resolvedArgs[0]); + }, + + _functionValues: function(resolvedArgs) { + var obj = resolvedArgs[0]; + var keys = Object.keys(obj); + var values = []; + for (var i = 0; i < keys.length; i++) { + values.push(obj[keys[i]]); + } + return values; + }, + + _functionJoin: function(resolvedArgs) { + var joinChar = resolvedArgs[0]; + var listJoin = resolvedArgs[1]; + return listJoin.join(joinChar); + }, + + _functionToArray: function(resolvedArgs) { + if (this._getTypeName(resolvedArgs[0]) === TYPE_ARRAY) { + return resolvedArgs[0]; + } else { + return [resolvedArgs[0]]; + } + }, + + _functionToString: function(resolvedArgs) { + if (this._getTypeName(resolvedArgs[0]) === TYPE_STRING) { + return resolvedArgs[0]; + } else { + return JSON.stringify(resolvedArgs[0]); + } + }, + + _functionToNumber: function(resolvedArgs) { + var typeName = this._getTypeName(resolvedArgs[0]); + var convertedValue; + if (typeName === TYPE_NUMBER) { + return resolvedArgs[0]; + } else if (typeName === TYPE_STRING) { + convertedValue = +resolvedArgs[0]; + if (!isNaN(convertedValue)) { + return convertedValue; + } + } + return null; + }, + + _functionNotNull: function(resolvedArgs) { + for (var i = 0; i < resolvedArgs.length; i++) { + if (this._getTypeName(resolvedArgs[i]) !== TYPE_NULL) { + return resolvedArgs[i]; + } + } + return null; + }, + + _functionSort: function(resolvedArgs) { + var sortedArray = resolvedArgs[0].slice(0); + sortedArray.sort(); + return sortedArray; + }, + + _functionSortBy: function(resolvedArgs) { + var sortedArray = resolvedArgs[0].slice(0); + if (sortedArray.length === 0) { + return sortedArray; + } + var interpreter = this._interpreter; + var exprefNode = resolvedArgs[1]; + var requiredType = this._getTypeName( + interpreter.visit(exprefNode, sortedArray[0])); + if ([TYPE_NUMBER, TYPE_STRING].indexOf(requiredType) < 0) { + throw new Error("TypeError"); + } + var that = this; + // In order to get a stable sort out of an unstable + // sort algorithm, we decorate/sort/undecorate (DSU) + // by creating a new list of [index, element] pairs. + // In the cmp function, if the evaluated elements are + // equal, then the index will be used as the tiebreaker. + // After the decorated list has been sorted, it will be + // undecorated to extract the original elements. + var decorated = []; + for (var i = 0; i < sortedArray.length; i++) { + decorated.push([i, sortedArray[i]]); + } + decorated.sort(function(a, b) { + var exprA = interpreter.visit(exprefNode, a[1]); + var exprB = interpreter.visit(exprefNode, b[1]); + if (that._getTypeName(exprA) !== requiredType) { + throw new Error( + "TypeError: expected " + requiredType + ", received " + + that._getTypeName(exprA)); + } else if (that._getTypeName(exprB) !== requiredType) { + throw new Error( + "TypeError: expected " + requiredType + ", received " + + that._getTypeName(exprB)); + } + if (exprA > exprB) { + return 1; + } else if (exprA < exprB) { + return -1; + } else { + // If they're equal compare the items by their + // order to maintain relative order of equal keys + // (i.e. to get a stable sort). + return a[0] - b[0]; + } + }); + // Undecorate: extract out the original list elements. + for (var j = 0; j < decorated.length; j++) { + sortedArray[j] = decorated[j][1]; + } + return sortedArray; + }, + + _functionMaxBy: function(resolvedArgs) { + var exprefNode = resolvedArgs[1]; + var resolvedArray = resolvedArgs[0]; + var keyFunction = this.createKeyFunction(exprefNode, [TYPE_NUMBER, TYPE_STRING]); + var maxNumber = -Infinity; + var maxRecord; + var current; + for (var i = 0; i < resolvedArray.length; i++) { + current = keyFunction(resolvedArray[i]); + if (current > maxNumber) { + maxNumber = current; + maxRecord = resolvedArray[i]; + } + } + return maxRecord; + }, + + _functionMinBy: function(resolvedArgs) { + var exprefNode = resolvedArgs[1]; + var resolvedArray = resolvedArgs[0]; + var keyFunction = this.createKeyFunction(exprefNode, [TYPE_NUMBER, TYPE_STRING]); + var minNumber = Infinity; + var minRecord; + var current; + for (var i = 0; i < resolvedArray.length; i++) { + current = keyFunction(resolvedArray[i]); + if (current < minNumber) { + minNumber = current; + minRecord = resolvedArray[i]; + } + } + return minRecord; + }, + + createKeyFunction: function(exprefNode, allowedTypes) { + var that = this; + var interpreter = this._interpreter; + var keyFunc = function(x) { + var current = interpreter.visit(exprefNode, x); + if (allowedTypes.indexOf(that._getTypeName(current)) < 0) { + var msg = "TypeError: expected one of " + allowedTypes + + ", received " + that._getTypeName(current); + throw new Error(msg); + } + return current; + }; + return keyFunc; + } + + }; + + function compile(stream) { + var parser = new Parser(); + var ast = parser.parse(stream); + return ast; + } + + function tokenize(stream) { + var lexer = new Lexer(); + return lexer.tokenize(stream); + } + + function search(data, expression) { + var parser = new Parser(); + // This needs to be improved. Both the interpreter and runtime depend on + // each other. The runtime needs the interpreter to support exprefs. + // There's likely a clean way to avoid the cyclic dependency. + var runtime = new Runtime(); + var interpreter = new TreeInterpreter(runtime); + runtime._interpreter = interpreter; + var node = parser.parse(expression); + return interpreter.search(node, data); + } + + exports.tokenize = tokenize; + exports.compile = compile; + exports.search = search; + exports.strictDeepEqual = strictDeepEqual; +})( false ? 0 : exports); + + +/***/ }), + +/***/ 90250: +/***/ (function(module, exports, __nccwpck_require__) { + +/* module decorator */ module = __nccwpck_require__.nmd(module); +/** + * @license + * Lodash + * Copyright OpenJS Foundation and other contributors + * Released under MIT license + * Based on Underscore.js 1.8.3 + * Copyright Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors + */ +;(function() { + + /** Used as a safe reference for `undefined` in pre-ES5 environments. */ + var undefined; + + /** Used as the semantic version number. */ + var VERSION = '4.17.21'; + + /** Used as the size to enable large array optimizations. */ + var LARGE_ARRAY_SIZE = 200; + + /** Error message constants. */ + var CORE_ERROR_TEXT = 'Unsupported core-js use. Try https://npms.io/search?q=ponyfill.', + FUNC_ERROR_TEXT = 'Expected a function', + INVALID_TEMPL_VAR_ERROR_TEXT = 'Invalid `variable` option passed into `_.template`'; + + /** Used to stand-in for `undefined` hash values. */ + var HASH_UNDEFINED = '__lodash_hash_undefined__'; + + /** Used as the maximum memoize cache size. */ + var MAX_MEMOIZE_SIZE = 500; + + /** Used as the internal argument placeholder. */ + var PLACEHOLDER = '__lodash_placeholder__'; + + /** Used to compose bitmasks for cloning. */ + var CLONE_DEEP_FLAG = 1, + CLONE_FLAT_FLAG = 2, + CLONE_SYMBOLS_FLAG = 4; + + /** Used to compose bitmasks for value comparisons. */ + var COMPARE_PARTIAL_FLAG = 1, + COMPARE_UNORDERED_FLAG = 2; + + /** Used to compose bitmasks for function metadata. */ + var WRAP_BIND_FLAG = 1, + WRAP_BIND_KEY_FLAG = 2, + WRAP_CURRY_BOUND_FLAG = 4, + WRAP_CURRY_FLAG = 8, + WRAP_CURRY_RIGHT_FLAG = 16, + WRAP_PARTIAL_FLAG = 32, + WRAP_PARTIAL_RIGHT_FLAG = 64, + WRAP_ARY_FLAG = 128, + WRAP_REARG_FLAG = 256, + WRAP_FLIP_FLAG = 512; + + /** Used as default options for `_.truncate`. */ + var DEFAULT_TRUNC_LENGTH = 30, + DEFAULT_TRUNC_OMISSION = '...'; + + /** Used to detect hot functions by number of calls within a span of milliseconds. */ + var HOT_COUNT = 800, + HOT_SPAN = 16; + + /** Used to indicate the type of lazy iteratees. */ + var LAZY_FILTER_FLAG = 1, + LAZY_MAP_FLAG = 2, + LAZY_WHILE_FLAG = 3; + + /** Used as references for various `Number` constants. */ + var INFINITY = 1 / 0, + MAX_SAFE_INTEGER = 9007199254740991, + MAX_INTEGER = 1.7976931348623157e+308, + NAN = 0 / 0; + + /** Used as references for the maximum length and index of an array. */ + var MAX_ARRAY_LENGTH = 4294967295, + MAX_ARRAY_INDEX = MAX_ARRAY_LENGTH - 1, + HALF_MAX_ARRAY_LENGTH = MAX_ARRAY_LENGTH >>> 1; + + /** Used to associate wrap methods with their bit flags. */ + var wrapFlags = [ + ['ary', WRAP_ARY_FLAG], + ['bind', WRAP_BIND_FLAG], + ['bindKey', WRAP_BIND_KEY_FLAG], + ['curry', WRAP_CURRY_FLAG], + ['curryRight', WRAP_CURRY_RIGHT_FLAG], + ['flip', WRAP_FLIP_FLAG], + ['partial', WRAP_PARTIAL_FLAG], + ['partialRight', WRAP_PARTIAL_RIGHT_FLAG], + ['rearg', WRAP_REARG_FLAG] + ]; + + /** `Object#toString` result references. */ + var argsTag = '[object Arguments]', + arrayTag = '[object Array]', + asyncTag = '[object AsyncFunction]', + boolTag = '[object Boolean]', + dateTag = '[object Date]', + domExcTag = '[object DOMException]', + errorTag = '[object Error]', + funcTag = '[object Function]', + genTag = '[object GeneratorFunction]', + mapTag = '[object Map]', + numberTag = '[object Number]', + nullTag = '[object Null]', + objectTag = '[object Object]', + promiseTag = '[object Promise]', + proxyTag = '[object Proxy]', + regexpTag = '[object RegExp]', + setTag = '[object Set]', + stringTag = '[object String]', + symbolTag = '[object Symbol]', + undefinedTag = '[object Undefined]', + weakMapTag = '[object WeakMap]', + weakSetTag = '[object WeakSet]'; + + var arrayBufferTag = '[object ArrayBuffer]', + dataViewTag = '[object DataView]', + float32Tag = '[object Float32Array]', + float64Tag = '[object Float64Array]', + int8Tag = '[object Int8Array]', + int16Tag = '[object Int16Array]', + int32Tag = '[object Int32Array]', + uint8Tag = '[object Uint8Array]', + uint8ClampedTag = '[object Uint8ClampedArray]', + uint16Tag = '[object Uint16Array]', + uint32Tag = '[object Uint32Array]'; + + /** Used to match empty string literals in compiled template source. */ + var reEmptyStringLeading = /\b__p \+= '';/g, + reEmptyStringMiddle = /\b(__p \+=) '' \+/g, + reEmptyStringTrailing = /(__e\(.*?\)|\b__t\)) \+\n'';/g; + + /** Used to match HTML entities and HTML characters. */ + var reEscapedHtml = /&(?:amp|lt|gt|quot|#39);/g, + reUnescapedHtml = /[&<>"']/g, + reHasEscapedHtml = RegExp(reEscapedHtml.source), + reHasUnescapedHtml = RegExp(reUnescapedHtml.source); + + /** Used to match template delimiters. */ + var reEscape = /<%-([\s\S]+?)%>/g, + reEvaluate = /<%([\s\S]+?)%>/g, + reInterpolate = /<%=([\s\S]+?)%>/g; + + /** Used to match property names within property paths. */ + var reIsDeepProp = /\.|\[(?:[^[\]]*|(["'])(?:(?!\1)[^\\]|\\.)*?\1)\]/, + reIsPlainProp = /^\w*$/, + rePropName = /[^.[\]]+|\[(?:(-?\d+(?:\.\d+)?)|(["'])((?:(?!\2)[^\\]|\\.)*?)\2)\]|(?=(?:\.|\[\])(?:\.|\[\]|$))/g; + + /** + * Used to match `RegExp` + * [syntax characters](http://ecma-international.org/ecma-262/7.0/#sec-patterns). + */ + var reRegExpChar = /[\\^$.*+?()[\]{}|]/g, + reHasRegExpChar = RegExp(reRegExpChar.source); + + /** Used to match leading whitespace. */ + var reTrimStart = /^\s+/; + + /** Used to match a single whitespace character. */ + var reWhitespace = /\s/; + + /** Used to match wrap detail comments. */ + var reWrapComment = /\{(?:\n\/\* \[wrapped with .+\] \*\/)?\n?/, + reWrapDetails = /\{\n\/\* \[wrapped with (.+)\] \*/, + reSplitDetails = /,? & /; + + /** Used to match words composed of alphanumeric characters. */ + var reAsciiWord = /[^\x00-\x2f\x3a-\x40\x5b-\x60\x7b-\x7f]+/g; + + /** + * Used to validate the `validate` option in `_.template` variable. + * + * Forbids characters which could potentially change the meaning of the function argument definition: + * - "()," (modification of function parameters) + * - "=" (default value) + * - "[]{}" (destructuring of function parameters) + * - "/" (beginning of a comment) + * - whitespace + */ + var reForbiddenIdentifierChars = /[()=,{}\[\]\/\s]/; + + /** Used to match backslashes in property paths. */ + var reEscapeChar = /\\(\\)?/g; + + /** + * Used to match + * [ES template delimiters](http://ecma-international.org/ecma-262/7.0/#sec-template-literal-lexical-components). + */ + var reEsTemplate = /\$\{([^\\}]*(?:\\.[^\\}]*)*)\}/g; + + /** Used to match `RegExp` flags from their coerced string values. */ + var reFlags = /\w*$/; + + /** Used to detect bad signed hexadecimal string values. */ + var reIsBadHex = /^[-+]0x[0-9a-f]+$/i; + + /** Used to detect binary string values. */ + var reIsBinary = /^0b[01]+$/i; + + /** Used to detect host constructors (Safari). */ + var reIsHostCtor = /^\[object .+?Constructor\]$/; + + /** Used to detect octal string values. */ + var reIsOctal = /^0o[0-7]+$/i; + + /** Used to detect unsigned integer values. */ + var reIsUint = /^(?:0|[1-9]\d*)$/; + + /** Used to match Latin Unicode letters (excluding mathematical operators). */ + var reLatin = /[\xc0-\xd6\xd8-\xf6\xf8-\xff\u0100-\u017f]/g; + + /** Used to ensure capturing order of template delimiters. */ + var reNoMatch = /($^)/; + + /** Used to match unescaped characters in compiled string literals. */ + var reUnescapedString = /['\n\r\u2028\u2029\\]/g; + + /** Used to compose unicode character classes. */ + var rsAstralRange = '\\ud800-\\udfff', + rsComboMarksRange = '\\u0300-\\u036f', + reComboHalfMarksRange = '\\ufe20-\\ufe2f', + rsComboSymbolsRange = '\\u20d0-\\u20ff', + rsComboRange = rsComboMarksRange + reComboHalfMarksRange + rsComboSymbolsRange, + rsDingbatRange = '\\u2700-\\u27bf', + rsLowerRange = 'a-z\\xdf-\\xf6\\xf8-\\xff', + rsMathOpRange = '\\xac\\xb1\\xd7\\xf7', + rsNonCharRange = '\\x00-\\x2f\\x3a-\\x40\\x5b-\\x60\\x7b-\\xbf', + rsPunctuationRange = '\\u2000-\\u206f', + rsSpaceRange = ' \\t\\x0b\\f\\xa0\\ufeff\\n\\r\\u2028\\u2029\\u1680\\u180e\\u2000\\u2001\\u2002\\u2003\\u2004\\u2005\\u2006\\u2007\\u2008\\u2009\\u200a\\u202f\\u205f\\u3000', + rsUpperRange = 'A-Z\\xc0-\\xd6\\xd8-\\xde', + rsVarRange = '\\ufe0e\\ufe0f', + rsBreakRange = rsMathOpRange + rsNonCharRange + rsPunctuationRange + rsSpaceRange; + + /** Used to compose unicode capture groups. */ + var rsApos = "['\u2019]", + rsAstral = '[' + rsAstralRange + ']', + rsBreak = '[' + rsBreakRange + ']', + rsCombo = '[' + rsComboRange + ']', + rsDigits = '\\d+', + rsDingbat = '[' + rsDingbatRange + ']', + rsLower = '[' + rsLowerRange + ']', + rsMisc = '[^' + rsAstralRange + rsBreakRange + rsDigits + rsDingbatRange + rsLowerRange + rsUpperRange + ']', + rsFitz = '\\ud83c[\\udffb-\\udfff]', + rsModifier = '(?:' + rsCombo + '|' + rsFitz + ')', + rsNonAstral = '[^' + rsAstralRange + ']', + rsRegional = '(?:\\ud83c[\\udde6-\\uddff]){2}', + rsSurrPair = '[\\ud800-\\udbff][\\udc00-\\udfff]', + rsUpper = '[' + rsUpperRange + ']', + rsZWJ = '\\u200d'; + + /** Used to compose unicode regexes. */ + var rsMiscLower = '(?:' + rsLower + '|' + rsMisc + ')', + rsMiscUpper = '(?:' + rsUpper + '|' + rsMisc + ')', + rsOptContrLower = '(?:' + rsApos + '(?:d|ll|m|re|s|t|ve))?', + rsOptContrUpper = '(?:' + rsApos + '(?:D|LL|M|RE|S|T|VE))?', + reOptMod = rsModifier + '?', + rsOptVar = '[' + rsVarRange + ']?', + rsOptJoin = '(?:' + rsZWJ + '(?:' + [rsNonAstral, rsRegional, rsSurrPair].join('|') + ')' + rsOptVar + reOptMod + ')*', + rsOrdLower = '\\d*(?:1st|2nd|3rd|(?![123])\\dth)(?=\\b|[A-Z_])', + rsOrdUpper = '\\d*(?:1ST|2ND|3RD|(?![123])\\dTH)(?=\\b|[a-z_])', + rsSeq = rsOptVar + reOptMod + rsOptJoin, + rsEmoji = '(?:' + [rsDingbat, rsRegional, rsSurrPair].join('|') + ')' + rsSeq, + rsSymbol = '(?:' + [rsNonAstral + rsCombo + '?', rsCombo, rsRegional, rsSurrPair, rsAstral].join('|') + ')'; + + /** Used to match apostrophes. */ + var reApos = RegExp(rsApos, 'g'); + + /** + * Used to match [combining diacritical marks](https://en.wikipedia.org/wiki/Combining_Diacritical_Marks) and + * [combining diacritical marks for symbols](https://en.wikipedia.org/wiki/Combining_Diacritical_Marks_for_Symbols). + */ + var reComboMark = RegExp(rsCombo, 'g'); + + /** Used to match [string symbols](https://mathiasbynens.be/notes/javascript-unicode). */ + var reUnicode = RegExp(rsFitz + '(?=' + rsFitz + ')|' + rsSymbol + rsSeq, 'g'); + + /** Used to match complex or compound words. */ + var reUnicodeWord = RegExp([ + rsUpper + '?' + rsLower + '+' + rsOptContrLower + '(?=' + [rsBreak, rsUpper, '$'].join('|') + ')', + rsMiscUpper + '+' + rsOptContrUpper + '(?=' + [rsBreak, rsUpper + rsMiscLower, '$'].join('|') + ')', + rsUpper + '?' + rsMiscLower + '+' + rsOptContrLower, + rsUpper + '+' + rsOptContrUpper, + rsOrdUpper, + rsOrdLower, + rsDigits, + rsEmoji + ].join('|'), 'g'); + + /** Used to detect strings with [zero-width joiners or code points from the astral planes](http://eev.ee/blog/2015/09/12/dark-corners-of-unicode/). */ + var reHasUnicode = RegExp('[' + rsZWJ + rsAstralRange + rsComboRange + rsVarRange + ']'); + + /** Used to detect strings that need a more robust regexp to match words. */ + var reHasUnicodeWord = /[a-z][A-Z]|[A-Z]{2}[a-z]|[0-9][a-zA-Z]|[a-zA-Z][0-9]|[^a-zA-Z0-9 ]/; + + /** Used to assign default `context` object properties. */ + var contextProps = [ + 'Array', 'Buffer', 'DataView', 'Date', 'Error', 'Float32Array', 'Float64Array', + 'Function', 'Int8Array', 'Int16Array', 'Int32Array', 'Map', 'Math', 'Object', + 'Promise', 'RegExp', 'Set', 'String', 'Symbol', 'TypeError', 'Uint8Array', + 'Uint8ClampedArray', 'Uint16Array', 'Uint32Array', 'WeakMap', + '_', 'clearTimeout', 'isFinite', 'parseInt', 'setTimeout' + ]; + + /** Used to make template sourceURLs easier to identify. */ + var templateCounter = -1; + + /** Used to identify `toStringTag` values of typed arrays. */ + var typedArrayTags = {}; + typedArrayTags[float32Tag] = typedArrayTags[float64Tag] = + typedArrayTags[int8Tag] = typedArrayTags[int16Tag] = + typedArrayTags[int32Tag] = typedArrayTags[uint8Tag] = + typedArrayTags[uint8ClampedTag] = typedArrayTags[uint16Tag] = + typedArrayTags[uint32Tag] = true; + typedArrayTags[argsTag] = typedArrayTags[arrayTag] = + typedArrayTags[arrayBufferTag] = typedArrayTags[boolTag] = + typedArrayTags[dataViewTag] = typedArrayTags[dateTag] = + typedArrayTags[errorTag] = typedArrayTags[funcTag] = + typedArrayTags[mapTag] = typedArrayTags[numberTag] = + typedArrayTags[objectTag] = typedArrayTags[regexpTag] = + typedArrayTags[setTag] = typedArrayTags[stringTag] = + typedArrayTags[weakMapTag] = false; + + /** Used to identify `toStringTag` values supported by `_.clone`. */ + var cloneableTags = {}; + cloneableTags[argsTag] = cloneableTags[arrayTag] = + cloneableTags[arrayBufferTag] = cloneableTags[dataViewTag] = + cloneableTags[boolTag] = cloneableTags[dateTag] = + cloneableTags[float32Tag] = cloneableTags[float64Tag] = + cloneableTags[int8Tag] = cloneableTags[int16Tag] = + cloneableTags[int32Tag] = cloneableTags[mapTag] = + cloneableTags[numberTag] = cloneableTags[objectTag] = + cloneableTags[regexpTag] = cloneableTags[setTag] = + cloneableTags[stringTag] = cloneableTags[symbolTag] = + cloneableTags[uint8Tag] = cloneableTags[uint8ClampedTag] = + cloneableTags[uint16Tag] = cloneableTags[uint32Tag] = true; + cloneableTags[errorTag] = cloneableTags[funcTag] = + cloneableTags[weakMapTag] = false; + + /** Used to map Latin Unicode letters to basic Latin letters. */ + var deburredLetters = { + // Latin-1 Supplement block. + '\xc0': 'A', '\xc1': 'A', '\xc2': 'A', '\xc3': 'A', '\xc4': 'A', '\xc5': 'A', + '\xe0': 'a', '\xe1': 'a', '\xe2': 'a', '\xe3': 'a', '\xe4': 'a', '\xe5': 'a', + '\xc7': 'C', '\xe7': 'c', + '\xd0': 'D', '\xf0': 'd', + '\xc8': 'E', '\xc9': 'E', '\xca': 'E', '\xcb': 'E', + '\xe8': 'e', '\xe9': 'e', '\xea': 'e', '\xeb': 'e', + '\xcc': 'I', '\xcd': 'I', '\xce': 'I', '\xcf': 'I', + '\xec': 'i', '\xed': 'i', '\xee': 'i', '\xef': 'i', + '\xd1': 'N', '\xf1': 'n', + '\xd2': 'O', '\xd3': 'O', '\xd4': 'O', '\xd5': 'O', '\xd6': 'O', '\xd8': 'O', + '\xf2': 'o', '\xf3': 'o', '\xf4': 'o', '\xf5': 'o', '\xf6': 'o', '\xf8': 'o', + '\xd9': 'U', '\xda': 'U', '\xdb': 'U', '\xdc': 'U', + '\xf9': 'u', '\xfa': 'u', '\xfb': 'u', '\xfc': 'u', + '\xdd': 'Y', '\xfd': 'y', '\xff': 'y', + '\xc6': 'Ae', '\xe6': 'ae', + '\xde': 'Th', '\xfe': 'th', + '\xdf': 'ss', + // Latin Extended-A block. + '\u0100': 'A', '\u0102': 'A', '\u0104': 'A', + '\u0101': 'a', '\u0103': 'a', '\u0105': 'a', + '\u0106': 'C', '\u0108': 'C', '\u010a': 'C', '\u010c': 'C', + '\u0107': 'c', '\u0109': 'c', '\u010b': 'c', '\u010d': 'c', + '\u010e': 'D', '\u0110': 'D', '\u010f': 'd', '\u0111': 'd', + '\u0112': 'E', '\u0114': 'E', '\u0116': 'E', '\u0118': 'E', '\u011a': 'E', + '\u0113': 'e', '\u0115': 'e', '\u0117': 'e', '\u0119': 'e', '\u011b': 'e', + '\u011c': 'G', '\u011e': 'G', '\u0120': 'G', '\u0122': 'G', + '\u011d': 'g', '\u011f': 'g', '\u0121': 'g', '\u0123': 'g', + '\u0124': 'H', '\u0126': 'H', '\u0125': 'h', '\u0127': 'h', + '\u0128': 'I', '\u012a': 'I', '\u012c': 'I', '\u012e': 'I', '\u0130': 'I', + '\u0129': 'i', '\u012b': 'i', '\u012d': 'i', '\u012f': 'i', '\u0131': 'i', + '\u0134': 'J', '\u0135': 'j', + '\u0136': 'K', '\u0137': 'k', '\u0138': 'k', + '\u0139': 'L', '\u013b': 'L', '\u013d': 'L', '\u013f': 'L', '\u0141': 'L', + '\u013a': 'l', '\u013c': 'l', '\u013e': 'l', '\u0140': 'l', '\u0142': 'l', + '\u0143': 'N', '\u0145': 'N', '\u0147': 'N', '\u014a': 'N', + '\u0144': 'n', '\u0146': 'n', '\u0148': 'n', '\u014b': 'n', + '\u014c': 'O', '\u014e': 'O', '\u0150': 'O', + '\u014d': 'o', '\u014f': 'o', '\u0151': 'o', + '\u0154': 'R', '\u0156': 'R', '\u0158': 'R', + '\u0155': 'r', '\u0157': 'r', '\u0159': 'r', + '\u015a': 'S', '\u015c': 'S', '\u015e': 'S', '\u0160': 'S', + '\u015b': 's', '\u015d': 's', '\u015f': 's', '\u0161': 's', + '\u0162': 'T', '\u0164': 'T', '\u0166': 'T', + '\u0163': 't', '\u0165': 't', '\u0167': 't', + '\u0168': 'U', '\u016a': 'U', '\u016c': 'U', '\u016e': 'U', '\u0170': 'U', '\u0172': 'U', + '\u0169': 'u', '\u016b': 'u', '\u016d': 'u', '\u016f': 'u', '\u0171': 'u', '\u0173': 'u', + '\u0174': 'W', '\u0175': 'w', + '\u0176': 'Y', '\u0177': 'y', '\u0178': 'Y', + '\u0179': 'Z', '\u017b': 'Z', '\u017d': 'Z', + '\u017a': 'z', '\u017c': 'z', '\u017e': 'z', + '\u0132': 'IJ', '\u0133': 'ij', + '\u0152': 'Oe', '\u0153': 'oe', + '\u0149': "'n", '\u017f': 's' + }; + + /** Used to map characters to HTML entities. */ + var htmlEscapes = { + '&': '&', + '<': '<', + '>': '>', + '"': '"', + "'": ''' + }; + + /** Used to map HTML entities to characters. */ + var htmlUnescapes = { + '&': '&', + '<': '<', + '>': '>', + '"': '"', + ''': "'" + }; + + /** Used to escape characters for inclusion in compiled string literals. */ + var stringEscapes = { + '\\': '\\', + "'": "'", + '\n': 'n', + '\r': 'r', + '\u2028': 'u2028', + '\u2029': 'u2029' + }; + + /** Built-in method references without a dependency on `root`. */ + var freeParseFloat = parseFloat, + freeParseInt = parseInt; + + /** Detect free variable `global` from Node.js. */ + var freeGlobal = typeof global == 'object' && global && global.Object === Object && global; + + /** Detect free variable `self`. */ + var freeSelf = typeof self == 'object' && self && self.Object === Object && self; + + /** Used as a reference to the global object. */ + var root = freeGlobal || freeSelf || Function('return this')(); + + /** Detect free variable `exports`. */ + var freeExports = true && exports && !exports.nodeType && exports; + + /** Detect free variable `module`. */ + var freeModule = freeExports && "object" == 'object' && module && !module.nodeType && module; + + /** Detect the popular CommonJS extension `module.exports`. */ + var moduleExports = freeModule && freeModule.exports === freeExports; + + /** Detect free variable `process` from Node.js. */ + var freeProcess = moduleExports && freeGlobal.process; + + /** Used to access faster Node.js helpers. */ + var nodeUtil = (function() { + try { + // Use `util.types` for Node.js 10+. + var types = freeModule && freeModule.require && freeModule.require('util').types; + + if (types) { + return types; + } + + // Legacy `process.binding('util')` for Node.js < 10. + return freeProcess && freeProcess.binding && freeProcess.binding('util'); + } catch (e) {} + }()); + + /* Node.js helper references. */ + var nodeIsArrayBuffer = nodeUtil && nodeUtil.isArrayBuffer, + nodeIsDate = nodeUtil && nodeUtil.isDate, + nodeIsMap = nodeUtil && nodeUtil.isMap, + nodeIsRegExp = nodeUtil && nodeUtil.isRegExp, + nodeIsSet = nodeUtil && nodeUtil.isSet, + nodeIsTypedArray = nodeUtil && nodeUtil.isTypedArray; + + /*--------------------------------------------------------------------------*/ + + /** + * A faster alternative to `Function#apply`, this function invokes `func` + * with the `this` binding of `thisArg` and the arguments of `args`. + * + * @private + * @param {Function} func The function to invoke. + * @param {*} thisArg The `this` binding of `func`. + * @param {Array} args The arguments to invoke `func` with. + * @returns {*} Returns the result of `func`. + */ + function apply(func, thisArg, args) { + switch (args.length) { + case 0: return func.call(thisArg); + case 1: return func.call(thisArg, args[0]); + case 2: return func.call(thisArg, args[0], args[1]); + case 3: return func.call(thisArg, args[0], args[1], args[2]); + } + return func.apply(thisArg, args); + } + + /** + * A specialized version of `baseAggregator` for arrays. + * + * @private + * @param {Array} [array] The array to iterate over. + * @param {Function} setter The function to set `accumulator` values. + * @param {Function} iteratee The iteratee to transform keys. + * @param {Object} accumulator The initial aggregated object. + * @returns {Function} Returns `accumulator`. + */ + function arrayAggregator(array, setter, iteratee, accumulator) { + var index = -1, + length = array == null ? 0 : array.length; + + while (++index < length) { + var value = array[index]; + setter(accumulator, value, iteratee(value), array); + } + return accumulator; + } + + /** + * A specialized version of `_.forEach` for arrays without support for + * iteratee shorthands. + * + * @private + * @param {Array} [array] The array to iterate over. + * @param {Function} iteratee The function invoked per iteration. + * @returns {Array} Returns `array`. + */ + function arrayEach(array, iteratee) { + var index = -1, + length = array == null ? 0 : array.length; + + while (++index < length) { + if (iteratee(array[index], index, array) === false) { + break; + } + } + return array; + } + + /** + * A specialized version of `_.forEachRight` for arrays without support for + * iteratee shorthands. + * + * @private + * @param {Array} [array] The array to iterate over. + * @param {Function} iteratee The function invoked per iteration. + * @returns {Array} Returns `array`. + */ + function arrayEachRight(array, iteratee) { + var length = array == null ? 0 : array.length; + + while (length--) { + if (iteratee(array[length], length, array) === false) { + break; + } + } + return array; + } + + /** + * A specialized version of `_.every` for arrays without support for + * iteratee shorthands. + * + * @private + * @param {Array} [array] The array to iterate over. + * @param {Function} predicate The function invoked per iteration. + * @returns {boolean} Returns `true` if all elements pass the predicate check, + * else `false`. + */ + function arrayEvery(array, predicate) { + var index = -1, + length = array == null ? 0 : array.length; + + while (++index < length) { + if (!predicate(array[index], index, array)) { + return false; + } + } + return true; + } + + /** + * A specialized version of `_.filter` for arrays without support for + * iteratee shorthands. + * + * @private + * @param {Array} [array] The array to iterate over. + * @param {Function} predicate The function invoked per iteration. + * @returns {Array} Returns the new filtered array. + */ + function arrayFilter(array, predicate) { + var index = -1, + length = array == null ? 0 : array.length, + resIndex = 0, + result = []; + + while (++index < length) { + var value = array[index]; + if (predicate(value, index, array)) { + result[resIndex++] = value; + } + } + return result; + } + + /** + * A specialized version of `_.includes` for arrays without support for + * specifying an index to search from. + * + * @private + * @param {Array} [array] The array to inspect. + * @param {*} target The value to search for. + * @returns {boolean} Returns `true` if `target` is found, else `false`. + */ + function arrayIncludes(array, value) { + var length = array == null ? 0 : array.length; + return !!length && baseIndexOf(array, value, 0) > -1; + } + + /** + * This function is like `arrayIncludes` except that it accepts a comparator. + * + * @private + * @param {Array} [array] The array to inspect. + * @param {*} target The value to search for. + * @param {Function} comparator The comparator invoked per element. + * @returns {boolean} Returns `true` if `target` is found, else `false`. + */ + function arrayIncludesWith(array, value, comparator) { + var index = -1, + length = array == null ? 0 : array.length; + + while (++index < length) { + if (comparator(value, array[index])) { + return true; + } + } + return false; + } + + /** + * A specialized version of `_.map` for arrays without support for iteratee + * shorthands. + * + * @private + * @param {Array} [array] The array to iterate over. + * @param {Function} iteratee The function invoked per iteration. + * @returns {Array} Returns the new mapped array. + */ + function arrayMap(array, iteratee) { + var index = -1, + length = array == null ? 0 : array.length, + result = Array(length); + + while (++index < length) { + result[index] = iteratee(array[index], index, array); + } + return result; + } + + /** + * Appends the elements of `values` to `array`. + * + * @private + * @param {Array} array The array to modify. + * @param {Array} values The values to append. + * @returns {Array} Returns `array`. + */ + function arrayPush(array, values) { + var index = -1, + length = values.length, + offset = array.length; + + while (++index < length) { + array[offset + index] = values[index]; + } + return array; + } + + /** + * A specialized version of `_.reduce` for arrays without support for + * iteratee shorthands. + * + * @private + * @param {Array} [array] The array to iterate over. + * @param {Function} iteratee The function invoked per iteration. + * @param {*} [accumulator] The initial value. + * @param {boolean} [initAccum] Specify using the first element of `array` as + * the initial value. + * @returns {*} Returns the accumulated value. + */ + function arrayReduce(array, iteratee, accumulator, initAccum) { + var index = -1, + length = array == null ? 0 : array.length; + + if (initAccum && length) { + accumulator = array[++index]; + } + while (++index < length) { + accumulator = iteratee(accumulator, array[index], index, array); + } + return accumulator; + } + + /** + * A specialized version of `_.reduceRight` for arrays without support for + * iteratee shorthands. + * + * @private + * @param {Array} [array] The array to iterate over. + * @param {Function} iteratee The function invoked per iteration. + * @param {*} [accumulator] The initial value. + * @param {boolean} [initAccum] Specify using the last element of `array` as + * the initial value. + * @returns {*} Returns the accumulated value. + */ + function arrayReduceRight(array, iteratee, accumulator, initAccum) { + var length = array == null ? 0 : array.length; + if (initAccum && length) { + accumulator = array[--length]; + } + while (length--) { + accumulator = iteratee(accumulator, array[length], length, array); + } + return accumulator; + } + + /** + * A specialized version of `_.some` for arrays without support for iteratee + * shorthands. + * + * @private + * @param {Array} [array] The array to iterate over. + * @param {Function} predicate The function invoked per iteration. + * @returns {boolean} Returns `true` if any element passes the predicate check, + * else `false`. + */ + function arraySome(array, predicate) { + var index = -1, + length = array == null ? 0 : array.length; + + while (++index < length) { + if (predicate(array[index], index, array)) { + return true; + } + } + return false; + } + + /** + * Gets the size of an ASCII `string`. + * + * @private + * @param {string} string The string inspect. + * @returns {number} Returns the string size. + */ + var asciiSize = baseProperty('length'); + + /** + * Converts an ASCII `string` to an array. + * + * @private + * @param {string} string The string to convert. + * @returns {Array} Returns the converted array. + */ + function asciiToArray(string) { + return string.split(''); + } + + /** + * Splits an ASCII `string` into an array of its words. + * + * @private + * @param {string} The string to inspect. + * @returns {Array} Returns the words of `string`. + */ + function asciiWords(string) { + return string.match(reAsciiWord) || []; + } + + /** + * The base implementation of methods like `_.findKey` and `_.findLastKey`, + * without support for iteratee shorthands, which iterates over `collection` + * using `eachFunc`. + * + * @private + * @param {Array|Object} collection The collection to inspect. + * @param {Function} predicate The function invoked per iteration. + * @param {Function} eachFunc The function to iterate over `collection`. + * @returns {*} Returns the found element or its key, else `undefined`. + */ + function baseFindKey(collection, predicate, eachFunc) { + var result; + eachFunc(collection, function(value, key, collection) { + if (predicate(value, key, collection)) { + result = key; + return false; + } + }); + return result; + } + + /** + * The base implementation of `_.findIndex` and `_.findLastIndex` without + * support for iteratee shorthands. + * + * @private + * @param {Array} array The array to inspect. + * @param {Function} predicate The function invoked per iteration. + * @param {number} fromIndex The index to search from. + * @param {boolean} [fromRight] Specify iterating from right to left. + * @returns {number} Returns the index of the matched value, else `-1`. + */ + function baseFindIndex(array, predicate, fromIndex, fromRight) { + var length = array.length, + index = fromIndex + (fromRight ? 1 : -1); + + while ((fromRight ? index-- : ++index < length)) { + if (predicate(array[index], index, array)) { + return index; + } + } + return -1; + } + + /** + * The base implementation of `_.indexOf` without `fromIndex` bounds checks. + * + * @private + * @param {Array} array The array to inspect. + * @param {*} value The value to search for. + * @param {number} fromIndex The index to search from. + * @returns {number} Returns the index of the matched value, else `-1`. + */ + function baseIndexOf(array, value, fromIndex) { + return value === value + ? strictIndexOf(array, value, fromIndex) + : baseFindIndex(array, baseIsNaN, fromIndex); + } + + /** + * This function is like `baseIndexOf` except that it accepts a comparator. + * + * @private + * @param {Array} array The array to inspect. + * @param {*} value The value to search for. + * @param {number} fromIndex The index to search from. + * @param {Function} comparator The comparator invoked per element. + * @returns {number} Returns the index of the matched value, else `-1`. + */ + function baseIndexOfWith(array, value, fromIndex, comparator) { + var index = fromIndex - 1, + length = array.length; + + while (++index < length) { + if (comparator(array[index], value)) { + return index; + } + } + return -1; + } + + /** + * The base implementation of `_.isNaN` without support for number objects. + * + * @private + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is `NaN`, else `false`. + */ + function baseIsNaN(value) { + return value !== value; + } + + /** + * The base implementation of `_.mean` and `_.meanBy` without support for + * iteratee shorthands. + * + * @private + * @param {Array} array The array to iterate over. + * @param {Function} iteratee The function invoked per iteration. + * @returns {number} Returns the mean. + */ + function baseMean(array, iteratee) { + var length = array == null ? 0 : array.length; + return length ? (baseSum(array, iteratee) / length) : NAN; + } + + /** + * The base implementation of `_.property` without support for deep paths. + * + * @private + * @param {string} key The key of the property to get. + * @returns {Function} Returns the new accessor function. + */ + function baseProperty(key) { + return function(object) { + return object == null ? undefined : object[key]; + }; + } + + /** + * The base implementation of `_.propertyOf` without support for deep paths. + * + * @private + * @param {Object} object The object to query. + * @returns {Function} Returns the new accessor function. + */ + function basePropertyOf(object) { + return function(key) { + return object == null ? undefined : object[key]; + }; + } + + /** + * The base implementation of `_.reduce` and `_.reduceRight`, without support + * for iteratee shorthands, which iterates over `collection` using `eachFunc`. + * + * @private + * @param {Array|Object} collection The collection to iterate over. + * @param {Function} iteratee The function invoked per iteration. + * @param {*} accumulator The initial value. + * @param {boolean} initAccum Specify using the first or last element of + * `collection` as the initial value. + * @param {Function} eachFunc The function to iterate over `collection`. + * @returns {*} Returns the accumulated value. + */ + function baseReduce(collection, iteratee, accumulator, initAccum, eachFunc) { + eachFunc(collection, function(value, index, collection) { + accumulator = initAccum + ? (initAccum = false, value) + : iteratee(accumulator, value, index, collection); + }); + return accumulator; + } + + /** + * The base implementation of `_.sortBy` which uses `comparer` to define the + * sort order of `array` and replaces criteria objects with their corresponding + * values. + * + * @private + * @param {Array} array The array to sort. + * @param {Function} comparer The function to define sort order. + * @returns {Array} Returns `array`. + */ + function baseSortBy(array, comparer) { + var length = array.length; + + array.sort(comparer); + while (length--) { + array[length] = array[length].value; + } + return array; + } + + /** + * The base implementation of `_.sum` and `_.sumBy` without support for + * iteratee shorthands. + * + * @private + * @param {Array} array The array to iterate over. + * @param {Function} iteratee The function invoked per iteration. + * @returns {number} Returns the sum. + */ + function baseSum(array, iteratee) { + var result, + index = -1, + length = array.length; + + while (++index < length) { + var current = iteratee(array[index]); + if (current !== undefined) { + result = result === undefined ? current : (result + current); + } + } + return result; + } + + /** + * The base implementation of `_.times` without support for iteratee shorthands + * or max array length checks. + * + * @private + * @param {number} n The number of times to invoke `iteratee`. + * @param {Function} iteratee The function invoked per iteration. + * @returns {Array} Returns the array of results. + */ + function baseTimes(n, iteratee) { + var index = -1, + result = Array(n); + + while (++index < n) { + result[index] = iteratee(index); + } + return result; + } + + /** + * The base implementation of `_.toPairs` and `_.toPairsIn` which creates an array + * of key-value pairs for `object` corresponding to the property names of `props`. + * + * @private + * @param {Object} object The object to query. + * @param {Array} props The property names to get values for. + * @returns {Object} Returns the key-value pairs. + */ + function baseToPairs(object, props) { + return arrayMap(props, function(key) { + return [key, object[key]]; + }); + } + + /** + * The base implementation of `_.trim`. + * + * @private + * @param {string} string The string to trim. + * @returns {string} Returns the trimmed string. + */ + function baseTrim(string) { + return string + ? string.slice(0, trimmedEndIndex(string) + 1).replace(reTrimStart, '') + : string; + } + + /** + * The base implementation of `_.unary` without support for storing metadata. + * + * @private + * @param {Function} func The function to cap arguments for. + * @returns {Function} Returns the new capped function. + */ + function baseUnary(func) { + return function(value) { + return func(value); + }; + } + + /** + * The base implementation of `_.values` and `_.valuesIn` which creates an + * array of `object` property values corresponding to the property names + * of `props`. + * + * @private + * @param {Object} object The object to query. + * @param {Array} props The property names to get values for. + * @returns {Object} Returns the array of property values. + */ + function baseValues(object, props) { + return arrayMap(props, function(key) { + return object[key]; + }); + } + + /** + * Checks if a `cache` value for `key` exists. + * + * @private + * @param {Object} cache The cache to query. + * @param {string} key The key of the entry to check. + * @returns {boolean} Returns `true` if an entry for `key` exists, else `false`. + */ + function cacheHas(cache, key) { + return cache.has(key); + } + + /** + * Used by `_.trim` and `_.trimStart` to get the index of the first string symbol + * that is not found in the character symbols. + * + * @private + * @param {Array} strSymbols The string symbols to inspect. + * @param {Array} chrSymbols The character symbols to find. + * @returns {number} Returns the index of the first unmatched string symbol. + */ + function charsStartIndex(strSymbols, chrSymbols) { + var index = -1, + length = strSymbols.length; + + while (++index < length && baseIndexOf(chrSymbols, strSymbols[index], 0) > -1) {} + return index; + } + + /** + * Used by `_.trim` and `_.trimEnd` to get the index of the last string symbol + * that is not found in the character symbols. + * + * @private + * @param {Array} strSymbols The string symbols to inspect. + * @param {Array} chrSymbols The character symbols to find. + * @returns {number} Returns the index of the last unmatched string symbol. + */ + function charsEndIndex(strSymbols, chrSymbols) { + var index = strSymbols.length; + + while (index-- && baseIndexOf(chrSymbols, strSymbols[index], 0) > -1) {} + return index; + } + + /** + * Gets the number of `placeholder` occurrences in `array`. + * + * @private + * @param {Array} array The array to inspect. + * @param {*} placeholder The placeholder to search for. + * @returns {number} Returns the placeholder count. + */ + function countHolders(array, placeholder) { + var length = array.length, + result = 0; + + while (length--) { + if (array[length] === placeholder) { + ++result; + } + } + return result; + } + + /** + * Used by `_.deburr` to convert Latin-1 Supplement and Latin Extended-A + * letters to basic Latin letters. + * + * @private + * @param {string} letter The matched letter to deburr. + * @returns {string} Returns the deburred letter. + */ + var deburrLetter = basePropertyOf(deburredLetters); + + /** + * Used by `_.escape` to convert characters to HTML entities. + * + * @private + * @param {string} chr The matched character to escape. + * @returns {string} Returns the escaped character. + */ + var escapeHtmlChar = basePropertyOf(htmlEscapes); + + /** + * Used by `_.template` to escape characters for inclusion in compiled string literals. + * + * @private + * @param {string} chr The matched character to escape. + * @returns {string} Returns the escaped character. + */ + function escapeStringChar(chr) { + return '\\' + stringEscapes[chr]; + } + + /** + * Gets the value at `key` of `object`. + * + * @private + * @param {Object} [object] The object to query. + * @param {string} key The key of the property to get. + * @returns {*} Returns the property value. + */ + function getValue(object, key) { + return object == null ? undefined : object[key]; + } + + /** + * Checks if `string` contains Unicode symbols. + * + * @private + * @param {string} string The string to inspect. + * @returns {boolean} Returns `true` if a symbol is found, else `false`. + */ + function hasUnicode(string) { + return reHasUnicode.test(string); + } + + /** + * Checks if `string` contains a word composed of Unicode symbols. + * + * @private + * @param {string} string The string to inspect. + * @returns {boolean} Returns `true` if a word is found, else `false`. + */ + function hasUnicodeWord(string) { + return reHasUnicodeWord.test(string); + } + + /** + * Converts `iterator` to an array. + * + * @private + * @param {Object} iterator The iterator to convert. + * @returns {Array} Returns the converted array. + */ function iteratorToArray(iterator) { var data, result = []; @@ -41815,5853 +41264,6662 @@ exports.Deprecation = Deprecation; }()); /** - * The function whose prototype chain sequence wrappers inherit from. + * The function whose prototype chain sequence wrappers inherit from. + * + * @private + */ + function baseLodash() { + // No operation performed. + } + + /** + * The base constructor for creating `lodash` wrapper objects. + * + * @private + * @param {*} value The value to wrap. + * @param {boolean} [chainAll] Enable explicit method chain sequences. + */ + function LodashWrapper(value, chainAll) { + this.__wrapped__ = value; + this.__actions__ = []; + this.__chain__ = !!chainAll; + this.__index__ = 0; + this.__values__ = undefined; + } + + /** + * By default, the template delimiters used by lodash are like those in + * embedded Ruby (ERB) as well as ES2015 template strings. Change the + * following template settings to use alternative delimiters. + * + * @static + * @memberOf _ + * @type {Object} + */ + lodash.templateSettings = { + + /** + * Used to detect `data` property values to be HTML-escaped. + * + * @memberOf _.templateSettings + * @type {RegExp} + */ + 'escape': reEscape, + + /** + * Used to detect code to be evaluated. + * + * @memberOf _.templateSettings + * @type {RegExp} + */ + 'evaluate': reEvaluate, + + /** + * Used to detect `data` property values to inject. + * + * @memberOf _.templateSettings + * @type {RegExp} + */ + 'interpolate': reInterpolate, + + /** + * Used to reference the data object in the template text. + * + * @memberOf _.templateSettings + * @type {string} + */ + 'variable': '', + + /** + * Used to import variables into the compiled template. + * + * @memberOf _.templateSettings + * @type {Object} + */ + 'imports': { + + /** + * A reference to the `lodash` function. + * + * @memberOf _.templateSettings.imports + * @type {Function} + */ + '_': lodash + } + }; + + // Ensure wrappers are instances of `baseLodash`. + lodash.prototype = baseLodash.prototype; + lodash.prototype.constructor = lodash; + + LodashWrapper.prototype = baseCreate(baseLodash.prototype); + LodashWrapper.prototype.constructor = LodashWrapper; + + /*------------------------------------------------------------------------*/ + + /** + * Creates a lazy wrapper object which wraps `value` to enable lazy evaluation. + * + * @private + * @constructor + * @param {*} value The value to wrap. + */ + function LazyWrapper(value) { + this.__wrapped__ = value; + this.__actions__ = []; + this.__dir__ = 1; + this.__filtered__ = false; + this.__iteratees__ = []; + this.__takeCount__ = MAX_ARRAY_LENGTH; + this.__views__ = []; + } + + /** + * Creates a clone of the lazy wrapper object. + * + * @private + * @name clone + * @memberOf LazyWrapper + * @returns {Object} Returns the cloned `LazyWrapper` object. + */ + function lazyClone() { + var result = new LazyWrapper(this.__wrapped__); + result.__actions__ = copyArray(this.__actions__); + result.__dir__ = this.__dir__; + result.__filtered__ = this.__filtered__; + result.__iteratees__ = copyArray(this.__iteratees__); + result.__takeCount__ = this.__takeCount__; + result.__views__ = copyArray(this.__views__); + return result; + } + + /** + * Reverses the direction of lazy iteration. + * + * @private + * @name reverse + * @memberOf LazyWrapper + * @returns {Object} Returns the new reversed `LazyWrapper` object. + */ + function lazyReverse() { + if (this.__filtered__) { + var result = new LazyWrapper(this); + result.__dir__ = -1; + result.__filtered__ = true; + } else { + result = this.clone(); + result.__dir__ *= -1; + } + return result; + } + + /** + * Extracts the unwrapped value from its lazy wrapper. + * + * @private + * @name value + * @memberOf LazyWrapper + * @returns {*} Returns the unwrapped value. + */ + function lazyValue() { + var array = this.__wrapped__.value(), + dir = this.__dir__, + isArr = isArray(array), + isRight = dir < 0, + arrLength = isArr ? array.length : 0, + view = getView(0, arrLength, this.__views__), + start = view.start, + end = view.end, + length = end - start, + index = isRight ? end : (start - 1), + iteratees = this.__iteratees__, + iterLength = iteratees.length, + resIndex = 0, + takeCount = nativeMin(length, this.__takeCount__); + + if (!isArr || (!isRight && arrLength == length && takeCount == length)) { + return baseWrapperValue(array, this.__actions__); + } + var result = []; + + outer: + while (length-- && resIndex < takeCount) { + index += dir; + + var iterIndex = -1, + value = array[index]; + + while (++iterIndex < iterLength) { + var data = iteratees[iterIndex], + iteratee = data.iteratee, + type = data.type, + computed = iteratee(value); + + if (type == LAZY_MAP_FLAG) { + value = computed; + } else if (!computed) { + if (type == LAZY_FILTER_FLAG) { + continue outer; + } else { + break outer; + } + } + } + result[resIndex++] = value; + } + return result; + } + + // Ensure `LazyWrapper` is an instance of `baseLodash`. + LazyWrapper.prototype = baseCreate(baseLodash.prototype); + LazyWrapper.prototype.constructor = LazyWrapper; + + /*------------------------------------------------------------------------*/ + + /** + * Creates a hash object. + * + * @private + * @constructor + * @param {Array} [entries] The key-value pairs to cache. + */ + function Hash(entries) { + var index = -1, + length = entries == null ? 0 : entries.length; + + this.clear(); + while (++index < length) { + var entry = entries[index]; + this.set(entry[0], entry[1]); + } + } + + /** + * Removes all key-value entries from the hash. + * + * @private + * @name clear + * @memberOf Hash + */ + function hashClear() { + this.__data__ = nativeCreate ? nativeCreate(null) : {}; + this.size = 0; + } + + /** + * Removes `key` and its value from the hash. + * + * @private + * @name delete + * @memberOf Hash + * @param {Object} hash The hash to modify. + * @param {string} key The key of the value to remove. + * @returns {boolean} Returns `true` if the entry was removed, else `false`. + */ + function hashDelete(key) { + var result = this.has(key) && delete this.__data__[key]; + this.size -= result ? 1 : 0; + return result; + } + + /** + * Gets the hash value for `key`. + * + * @private + * @name get + * @memberOf Hash + * @param {string} key The key of the value to get. + * @returns {*} Returns the entry value. + */ + function hashGet(key) { + var data = this.__data__; + if (nativeCreate) { + var result = data[key]; + return result === HASH_UNDEFINED ? undefined : result; + } + return hasOwnProperty.call(data, key) ? data[key] : undefined; + } + + /** + * Checks if a hash value for `key` exists. + * + * @private + * @name has + * @memberOf Hash + * @param {string} key The key of the entry to check. + * @returns {boolean} Returns `true` if an entry for `key` exists, else `false`. + */ + function hashHas(key) { + var data = this.__data__; + return nativeCreate ? (data[key] !== undefined) : hasOwnProperty.call(data, key); + } + + /** + * Sets the hash `key` to `value`. + * + * @private + * @name set + * @memberOf Hash + * @param {string} key The key of the value to set. + * @param {*} value The value to set. + * @returns {Object} Returns the hash instance. + */ + function hashSet(key, value) { + var data = this.__data__; + this.size += this.has(key) ? 0 : 1; + data[key] = (nativeCreate && value === undefined) ? HASH_UNDEFINED : value; + return this; + } + + // Add methods to `Hash`. + Hash.prototype.clear = hashClear; + Hash.prototype['delete'] = hashDelete; + Hash.prototype.get = hashGet; + Hash.prototype.has = hashHas; + Hash.prototype.set = hashSet; + + /*------------------------------------------------------------------------*/ + + /** + * Creates an list cache object. + * + * @private + * @constructor + * @param {Array} [entries] The key-value pairs to cache. + */ + function ListCache(entries) { + var index = -1, + length = entries == null ? 0 : entries.length; + + this.clear(); + while (++index < length) { + var entry = entries[index]; + this.set(entry[0], entry[1]); + } + } + + /** + * Removes all key-value entries from the list cache. + * + * @private + * @name clear + * @memberOf ListCache + */ + function listCacheClear() { + this.__data__ = []; + this.size = 0; + } + + /** + * Removes `key` and its value from the list cache. + * + * @private + * @name delete + * @memberOf ListCache + * @param {string} key The key of the value to remove. + * @returns {boolean} Returns `true` if the entry was removed, else `false`. + */ + function listCacheDelete(key) { + var data = this.__data__, + index = assocIndexOf(data, key); + + if (index < 0) { + return false; + } + var lastIndex = data.length - 1; + if (index == lastIndex) { + data.pop(); + } else { + splice.call(data, index, 1); + } + --this.size; + return true; + } + + /** + * Gets the list cache value for `key`. + * + * @private + * @name get + * @memberOf ListCache + * @param {string} key The key of the value to get. + * @returns {*} Returns the entry value. + */ + function listCacheGet(key) { + var data = this.__data__, + index = assocIndexOf(data, key); + + return index < 0 ? undefined : data[index][1]; + } + + /** + * Checks if a list cache value for `key` exists. + * + * @private + * @name has + * @memberOf ListCache + * @param {string} key The key of the entry to check. + * @returns {boolean} Returns `true` if an entry for `key` exists, else `false`. + */ + function listCacheHas(key) { + return assocIndexOf(this.__data__, key) > -1; + } + + /** + * Sets the list cache `key` to `value`. + * + * @private + * @name set + * @memberOf ListCache + * @param {string} key The key of the value to set. + * @param {*} value The value to set. + * @returns {Object} Returns the list cache instance. + */ + function listCacheSet(key, value) { + var data = this.__data__, + index = assocIndexOf(data, key); + + if (index < 0) { + ++this.size; + data.push([key, value]); + } else { + data[index][1] = value; + } + return this; + } + + // Add methods to `ListCache`. + ListCache.prototype.clear = listCacheClear; + ListCache.prototype['delete'] = listCacheDelete; + ListCache.prototype.get = listCacheGet; + ListCache.prototype.has = listCacheHas; + ListCache.prototype.set = listCacheSet; + + /*------------------------------------------------------------------------*/ + + /** + * Creates a map cache object to store key-value pairs. + * + * @private + * @constructor + * @param {Array} [entries] The key-value pairs to cache. + */ + function MapCache(entries) { + var index = -1, + length = entries == null ? 0 : entries.length; + + this.clear(); + while (++index < length) { + var entry = entries[index]; + this.set(entry[0], entry[1]); + } + } + + /** + * Removes all key-value entries from the map. + * + * @private + * @name clear + * @memberOf MapCache + */ + function mapCacheClear() { + this.size = 0; + this.__data__ = { + 'hash': new Hash, + 'map': new (Map || ListCache), + 'string': new Hash + }; + } + + /** + * Removes `key` and its value from the map. + * + * @private + * @name delete + * @memberOf MapCache + * @param {string} key The key of the value to remove. + * @returns {boolean} Returns `true` if the entry was removed, else `false`. + */ + function mapCacheDelete(key) { + var result = getMapData(this, key)['delete'](key); + this.size -= result ? 1 : 0; + return result; + } + + /** + * Gets the map value for `key`. * * @private + * @name get + * @memberOf MapCache + * @param {string} key The key of the value to get. + * @returns {*} Returns the entry value. */ - function baseLodash() { - // No operation performed. + function mapCacheGet(key) { + return getMapData(this, key).get(key); } /** - * The base constructor for creating `lodash` wrapper objects. + * Checks if a map value for `key` exists. * * @private - * @param {*} value The value to wrap. - * @param {boolean} [chainAll] Enable explicit method chain sequences. + * @name has + * @memberOf MapCache + * @param {string} key The key of the entry to check. + * @returns {boolean} Returns `true` if an entry for `key` exists, else `false`. */ - function LodashWrapper(value, chainAll) { - this.__wrapped__ = value; - this.__actions__ = []; - this.__chain__ = !!chainAll; - this.__index__ = 0; - this.__values__ = undefined; + function mapCacheHas(key) { + return getMapData(this, key).has(key); } /** - * By default, the template delimiters used by lodash are like those in - * embedded Ruby (ERB) as well as ES2015 template strings. Change the - * following template settings to use alternative delimiters. + * Sets the map `key` to `value`. * - * @static - * @memberOf _ - * @type {Object} + * @private + * @name set + * @memberOf MapCache + * @param {string} key The key of the value to set. + * @param {*} value The value to set. + * @returns {Object} Returns the map cache instance. */ - lodash.templateSettings = { - - /** - * Used to detect `data` property values to be HTML-escaped. - * - * @memberOf _.templateSettings - * @type {RegExp} - */ - 'escape': reEscape, + function mapCacheSet(key, value) { + var data = getMapData(this, key), + size = data.size; - /** - * Used to detect code to be evaluated. - * - * @memberOf _.templateSettings - * @type {RegExp} - */ - 'evaluate': reEvaluate, + data.set(key, value); + this.size += data.size == size ? 0 : 1; + return this; + } - /** - * Used to detect `data` property values to inject. - * - * @memberOf _.templateSettings - * @type {RegExp} - */ - 'interpolate': reInterpolate, + // Add methods to `MapCache`. + MapCache.prototype.clear = mapCacheClear; + MapCache.prototype['delete'] = mapCacheDelete; + MapCache.prototype.get = mapCacheGet; + MapCache.prototype.has = mapCacheHas; + MapCache.prototype.set = mapCacheSet; - /** - * Used to reference the data object in the template text. - * - * @memberOf _.templateSettings - * @type {string} - */ - 'variable': '', + /*------------------------------------------------------------------------*/ - /** - * Used to import variables into the compiled template. - * - * @memberOf _.templateSettings - * @type {Object} - */ - 'imports': { + /** + * + * Creates an array cache object to store unique values. + * + * @private + * @constructor + * @param {Array} [values] The values to cache. + */ + function SetCache(values) { + var index = -1, + length = values == null ? 0 : values.length; - /** - * A reference to the `lodash` function. - * - * @memberOf _.templateSettings.imports - * @type {Function} - */ - '_': lodash + this.__data__ = new MapCache; + while (++index < length) { + this.add(values[index]); } - }; + } - // Ensure wrappers are instances of `baseLodash`. - lodash.prototype = baseLodash.prototype; - lodash.prototype.constructor = lodash; + /** + * Adds `value` to the array cache. + * + * @private + * @name add + * @memberOf SetCache + * @alias push + * @param {*} value The value to cache. + * @returns {Object} Returns the cache instance. + */ + function setCacheAdd(value) { + this.__data__.set(value, HASH_UNDEFINED); + return this; + } - LodashWrapper.prototype = baseCreate(baseLodash.prototype); - LodashWrapper.prototype.constructor = LodashWrapper; + /** + * Checks if `value` is in the array cache. + * + * @private + * @name has + * @memberOf SetCache + * @param {*} value The value to search for. + * @returns {number} Returns `true` if `value` is found, else `false`. + */ + function setCacheHas(value) { + return this.__data__.has(value); + } + + // Add methods to `SetCache`. + SetCache.prototype.add = SetCache.prototype.push = setCacheAdd; + SetCache.prototype.has = setCacheHas; /*------------------------------------------------------------------------*/ /** - * Creates a lazy wrapper object which wraps `value` to enable lazy evaluation. + * Creates a stack cache object to store key-value pairs. * * @private * @constructor - * @param {*} value The value to wrap. + * @param {Array} [entries] The key-value pairs to cache. */ - function LazyWrapper(value) { - this.__wrapped__ = value; - this.__actions__ = []; - this.__dir__ = 1; - this.__filtered__ = false; - this.__iteratees__ = []; - this.__takeCount__ = MAX_ARRAY_LENGTH; - this.__views__ = []; + function Stack(entries) { + var data = this.__data__ = new ListCache(entries); + this.size = data.size; } /** - * Creates a clone of the lazy wrapper object. + * Removes all key-value entries from the stack. * * @private - * @name clone - * @memberOf LazyWrapper - * @returns {Object} Returns the cloned `LazyWrapper` object. + * @name clear + * @memberOf Stack */ - function lazyClone() { - var result = new LazyWrapper(this.__wrapped__); - result.__actions__ = copyArray(this.__actions__); - result.__dir__ = this.__dir__; - result.__filtered__ = this.__filtered__; - result.__iteratees__ = copyArray(this.__iteratees__); - result.__takeCount__ = this.__takeCount__; - result.__views__ = copyArray(this.__views__); - return result; + function stackClear() { + this.__data__ = new ListCache; + this.size = 0; } /** - * Reverses the direction of lazy iteration. + * Removes `key` and its value from the stack. * * @private - * @name reverse - * @memberOf LazyWrapper - * @returns {Object} Returns the new reversed `LazyWrapper` object. + * @name delete + * @memberOf Stack + * @param {string} key The key of the value to remove. + * @returns {boolean} Returns `true` if the entry was removed, else `false`. */ - function lazyReverse() { - if (this.__filtered__) { - var result = new LazyWrapper(this); - result.__dir__ = -1; - result.__filtered__ = true; - } else { - result = this.clone(); - result.__dir__ *= -1; - } + function stackDelete(key) { + var data = this.__data__, + result = data['delete'](key); + + this.size = data.size; return result; } /** - * Extracts the unwrapped value from its lazy wrapper. + * Gets the stack value for `key`. * * @private - * @name value - * @memberOf LazyWrapper - * @returns {*} Returns the unwrapped value. + * @name get + * @memberOf Stack + * @param {string} key The key of the value to get. + * @returns {*} Returns the entry value. */ - function lazyValue() { - var array = this.__wrapped__.value(), - dir = this.__dir__, - isArr = isArray(array), - isRight = dir < 0, - arrLength = isArr ? array.length : 0, - view = getView(0, arrLength, this.__views__), - start = view.start, - end = view.end, - length = end - start, - index = isRight ? end : (start - 1), - iteratees = this.__iteratees__, - iterLength = iteratees.length, - resIndex = 0, - takeCount = nativeMin(length, this.__takeCount__); + function stackGet(key) { + return this.__data__.get(key); + } - if (!isArr || (!isRight && arrLength == length && takeCount == length)) { - return baseWrapperValue(array, this.__actions__); + /** + * Checks if a stack value for `key` exists. + * + * @private + * @name has + * @memberOf Stack + * @param {string} key The key of the entry to check. + * @returns {boolean} Returns `true` if an entry for `key` exists, else `false`. + */ + function stackHas(key) { + return this.__data__.has(key); + } + + /** + * Sets the stack `key` to `value`. + * + * @private + * @name set + * @memberOf Stack + * @param {string} key The key of the value to set. + * @param {*} value The value to set. + * @returns {Object} Returns the stack cache instance. + */ + function stackSet(key, value) { + var data = this.__data__; + if (data instanceof ListCache) { + var pairs = data.__data__; + if (!Map || (pairs.length < LARGE_ARRAY_SIZE - 1)) { + pairs.push([key, value]); + this.size = ++data.size; + return this; + } + data = this.__data__ = new MapCache(pairs); } - var result = []; + data.set(key, value); + this.size = data.size; + return this; + } - outer: - while (length-- && resIndex < takeCount) { - index += dir; + // Add methods to `Stack`. + Stack.prototype.clear = stackClear; + Stack.prototype['delete'] = stackDelete; + Stack.prototype.get = stackGet; + Stack.prototype.has = stackHas; + Stack.prototype.set = stackSet; - var iterIndex = -1, - value = array[index]; + /*------------------------------------------------------------------------*/ - while (++iterIndex < iterLength) { - var data = iteratees[iterIndex], - iteratee = data.iteratee, - type = data.type, - computed = iteratee(value); + /** + * Creates an array of the enumerable property names of the array-like `value`. + * + * @private + * @param {*} value The value to query. + * @param {boolean} inherited Specify returning inherited property names. + * @returns {Array} Returns the array of property names. + */ + function arrayLikeKeys(value, inherited) { + var isArr = isArray(value), + isArg = !isArr && isArguments(value), + isBuff = !isArr && !isArg && isBuffer(value), + isType = !isArr && !isArg && !isBuff && isTypedArray(value), + skipIndexes = isArr || isArg || isBuff || isType, + result = skipIndexes ? baseTimes(value.length, String) : [], + length = result.length; - if (type == LAZY_MAP_FLAG) { - value = computed; - } else if (!computed) { - if (type == LAZY_FILTER_FLAG) { - continue outer; - } else { - break outer; - } - } + for (var key in value) { + if ((inherited || hasOwnProperty.call(value, key)) && + !(skipIndexes && ( + // Safari 9 has enumerable `arguments.length` in strict mode. + key == 'length' || + // Node.js 0.10 has enumerable non-index properties on buffers. + (isBuff && (key == 'offset' || key == 'parent')) || + // PhantomJS 2 has enumerable non-index properties on typed arrays. + (isType && (key == 'buffer' || key == 'byteLength' || key == 'byteOffset')) || + // Skip index properties. + isIndex(key, length) + ))) { + result.push(key); } - result[resIndex++] = value; } return result; } - // Ensure `LazyWrapper` is an instance of `baseLodash`. - LazyWrapper.prototype = baseCreate(baseLodash.prototype); - LazyWrapper.prototype.constructor = LazyWrapper; + /** + * A specialized version of `_.sample` for arrays. + * + * @private + * @param {Array} array The array to sample. + * @returns {*} Returns the random element. + */ + function arraySample(array) { + var length = array.length; + return length ? array[baseRandom(0, length - 1)] : undefined; + } - /*------------------------------------------------------------------------*/ + /** + * A specialized version of `_.sampleSize` for arrays. + * + * @private + * @param {Array} array The array to sample. + * @param {number} n The number of elements to sample. + * @returns {Array} Returns the random elements. + */ + function arraySampleSize(array, n) { + return shuffleSelf(copyArray(array), baseClamp(n, 0, array.length)); + } /** - * Creates a hash object. + * A specialized version of `_.shuffle` for arrays. * * @private - * @constructor - * @param {Array} [entries] The key-value pairs to cache. + * @param {Array} array The array to shuffle. + * @returns {Array} Returns the new shuffled array. */ - function Hash(entries) { - var index = -1, - length = entries == null ? 0 : entries.length; + function arrayShuffle(array) { + return shuffleSelf(copyArray(array)); + } - this.clear(); - while (++index < length) { - var entry = entries[index]; - this.set(entry[0], entry[1]); + /** + * This function is like `assignValue` except that it doesn't assign + * `undefined` values. + * + * @private + * @param {Object} object The object to modify. + * @param {string} key The key of the property to assign. + * @param {*} value The value to assign. + */ + function assignMergeValue(object, key, value) { + if ((value !== undefined && !eq(object[key], value)) || + (value === undefined && !(key in object))) { + baseAssignValue(object, key, value); } } /** - * Removes all key-value entries from the hash. + * Assigns `value` to `key` of `object` if the existing value is not equivalent + * using [`SameValueZero`](http://ecma-international.org/ecma-262/7.0/#sec-samevaluezero) + * for equality comparisons. * * @private - * @name clear - * @memberOf Hash + * @param {Object} object The object to modify. + * @param {string} key The key of the property to assign. + * @param {*} value The value to assign. */ - function hashClear() { - this.__data__ = nativeCreate ? nativeCreate(null) : {}; - this.size = 0; + function assignValue(object, key, value) { + var objValue = object[key]; + if (!(hasOwnProperty.call(object, key) && eq(objValue, value)) || + (value === undefined && !(key in object))) { + baseAssignValue(object, key, value); + } } /** - * Removes `key` and its value from the hash. + * Gets the index at which the `key` is found in `array` of key-value pairs. * * @private - * @name delete - * @memberOf Hash - * @param {Object} hash The hash to modify. - * @param {string} key The key of the value to remove. - * @returns {boolean} Returns `true` if the entry was removed, else `false`. + * @param {Array} array The array to inspect. + * @param {*} key The key to search for. + * @returns {number} Returns the index of the matched value, else `-1`. */ - function hashDelete(key) { - var result = this.has(key) && delete this.__data__[key]; - this.size -= result ? 1 : 0; - return result; + function assocIndexOf(array, key) { + var length = array.length; + while (length--) { + if (eq(array[length][0], key)) { + return length; + } + } + return -1; } /** - * Gets the hash value for `key`. + * Aggregates elements of `collection` on `accumulator` with keys transformed + * by `iteratee` and values set by `setter`. * * @private - * @name get - * @memberOf Hash - * @param {string} key The key of the value to get. - * @returns {*} Returns the entry value. + * @param {Array|Object} collection The collection to iterate over. + * @param {Function} setter The function to set `accumulator` values. + * @param {Function} iteratee The iteratee to transform keys. + * @param {Object} accumulator The initial aggregated object. + * @returns {Function} Returns `accumulator`. */ - function hashGet(key) { - var data = this.__data__; - if (nativeCreate) { - var result = data[key]; - return result === HASH_UNDEFINED ? undefined : result; - } - return hasOwnProperty.call(data, key) ? data[key] : undefined; + function baseAggregator(collection, setter, iteratee, accumulator) { + baseEach(collection, function(value, key, collection) { + setter(accumulator, value, iteratee(value), collection); + }); + return accumulator; } /** - * Checks if a hash value for `key` exists. + * The base implementation of `_.assign` without support for multiple sources + * or `customizer` functions. * * @private - * @name has - * @memberOf Hash - * @param {string} key The key of the entry to check. - * @returns {boolean} Returns `true` if an entry for `key` exists, else `false`. + * @param {Object} object The destination object. + * @param {Object} source The source object. + * @returns {Object} Returns `object`. */ - function hashHas(key) { - var data = this.__data__; - return nativeCreate ? (data[key] !== undefined) : hasOwnProperty.call(data, key); + function baseAssign(object, source) { + return object && copyObject(source, keys(source), object); } /** - * Sets the hash `key` to `value`. + * The base implementation of `_.assignIn` without support for multiple sources + * or `customizer` functions. * * @private - * @name set - * @memberOf Hash - * @param {string} key The key of the value to set. - * @param {*} value The value to set. - * @returns {Object} Returns the hash instance. + * @param {Object} object The destination object. + * @param {Object} source The source object. + * @returns {Object} Returns `object`. */ - function hashSet(key, value) { - var data = this.__data__; - this.size += this.has(key) ? 0 : 1; - data[key] = (nativeCreate && value === undefined) ? HASH_UNDEFINED : value; - return this; + function baseAssignIn(object, source) { + return object && copyObject(source, keysIn(source), object); + } + + /** + * The base implementation of `assignValue` and `assignMergeValue` without + * value checks. + * + * @private + * @param {Object} object The object to modify. + * @param {string} key The key of the property to assign. + * @param {*} value The value to assign. + */ + function baseAssignValue(object, key, value) { + if (key == '__proto__' && defineProperty) { + defineProperty(object, key, { + 'configurable': true, + 'enumerable': true, + 'value': value, + 'writable': true + }); + } else { + object[key] = value; + } } - // Add methods to `Hash`. - Hash.prototype.clear = hashClear; - Hash.prototype['delete'] = hashDelete; - Hash.prototype.get = hashGet; - Hash.prototype.has = hashHas; - Hash.prototype.set = hashSet; - - /*------------------------------------------------------------------------*/ - /** - * Creates an list cache object. + * The base implementation of `_.at` without support for individual paths. * * @private - * @constructor - * @param {Array} [entries] The key-value pairs to cache. + * @param {Object} object The object to iterate over. + * @param {string[]} paths The property paths to pick. + * @returns {Array} Returns the picked elements. */ - function ListCache(entries) { + function baseAt(object, paths) { var index = -1, - length = entries == null ? 0 : entries.length; + length = paths.length, + result = Array(length), + skip = object == null; - this.clear(); while (++index < length) { - var entry = entries[index]; - this.set(entry[0], entry[1]); + result[index] = skip ? undefined : get(object, paths[index]); } + return result; } /** - * Removes all key-value entries from the list cache. + * The base implementation of `_.clamp` which doesn't coerce arguments. * * @private - * @name clear - * @memberOf ListCache + * @param {number} number The number to clamp. + * @param {number} [lower] The lower bound. + * @param {number} upper The upper bound. + * @returns {number} Returns the clamped number. */ - function listCacheClear() { - this.__data__ = []; - this.size = 0; + function baseClamp(number, lower, upper) { + if (number === number) { + if (upper !== undefined) { + number = number <= upper ? number : upper; + } + if (lower !== undefined) { + number = number >= lower ? number : lower; + } + } + return number; } /** - * Removes `key` and its value from the list cache. + * The base implementation of `_.clone` and `_.cloneDeep` which tracks + * traversed objects. * * @private - * @name delete - * @memberOf ListCache - * @param {string} key The key of the value to remove. - * @returns {boolean} Returns `true` if the entry was removed, else `false`. + * @param {*} value The value to clone. + * @param {boolean} bitmask The bitmask flags. + * 1 - Deep clone + * 2 - Flatten inherited properties + * 4 - Clone symbols + * @param {Function} [customizer] The function to customize cloning. + * @param {string} [key] The key of `value`. + * @param {Object} [object] The parent object of `value`. + * @param {Object} [stack] Tracks traversed objects and their clone counterparts. + * @returns {*} Returns the cloned value. */ - function listCacheDelete(key) { - var data = this.__data__, - index = assocIndexOf(data, key); + function baseClone(value, bitmask, customizer, key, object, stack) { + var result, + isDeep = bitmask & CLONE_DEEP_FLAG, + isFlat = bitmask & CLONE_FLAT_FLAG, + isFull = bitmask & CLONE_SYMBOLS_FLAG; - if (index < 0) { - return false; + if (customizer) { + result = object ? customizer(value, key, object, stack) : customizer(value); } - var lastIndex = data.length - 1; - if (index == lastIndex) { - data.pop(); + if (result !== undefined) { + return result; + } + if (!isObject(value)) { + return value; + } + var isArr = isArray(value); + if (isArr) { + result = initCloneArray(value); + if (!isDeep) { + return copyArray(value, result); + } } else { - splice.call(data, index, 1); + var tag = getTag(value), + isFunc = tag == funcTag || tag == genTag; + + if (isBuffer(value)) { + return cloneBuffer(value, isDeep); + } + if (tag == objectTag || tag == argsTag || (isFunc && !object)) { + result = (isFlat || isFunc) ? {} : initCloneObject(value); + if (!isDeep) { + return isFlat + ? copySymbolsIn(value, baseAssignIn(result, value)) + : copySymbols(value, baseAssign(result, value)); + } + } else { + if (!cloneableTags[tag]) { + return object ? value : {}; + } + result = initCloneByTag(value, tag, isDeep); + } } - --this.size; - return true; + // Check for circular references and return its corresponding clone. + stack || (stack = new Stack); + var stacked = stack.get(value); + if (stacked) { + return stacked; + } + stack.set(value, result); + + if (isSet(value)) { + value.forEach(function(subValue) { + result.add(baseClone(subValue, bitmask, customizer, subValue, value, stack)); + }); + } else if (isMap(value)) { + value.forEach(function(subValue, key) { + result.set(key, baseClone(subValue, bitmask, customizer, key, value, stack)); + }); + } + + var keysFunc = isFull + ? (isFlat ? getAllKeysIn : getAllKeys) + : (isFlat ? keysIn : keys); + + var props = isArr ? undefined : keysFunc(value); + arrayEach(props || value, function(subValue, key) { + if (props) { + key = subValue; + subValue = value[key]; + } + // Recursively populate clone (susceptible to call stack limits). + assignValue(result, key, baseClone(subValue, bitmask, customizer, key, value, stack)); + }); + return result; } /** - * Gets the list cache value for `key`. + * The base implementation of `_.conforms` which doesn't clone `source`. * * @private - * @name get - * @memberOf ListCache - * @param {string} key The key of the value to get. - * @returns {*} Returns the entry value. + * @param {Object} source The object of property predicates to conform to. + * @returns {Function} Returns the new spec function. */ - function listCacheGet(key) { - var data = this.__data__, - index = assocIndexOf(data, key); - - return index < 0 ? undefined : data[index][1]; + function baseConforms(source) { + var props = keys(source); + return function(object) { + return baseConformsTo(object, source, props); + }; } /** - * Checks if a list cache value for `key` exists. + * The base implementation of `_.conformsTo` which accepts `props` to check. * * @private - * @name has - * @memberOf ListCache - * @param {string} key The key of the entry to check. - * @returns {boolean} Returns `true` if an entry for `key` exists, else `false`. + * @param {Object} object The object to inspect. + * @param {Object} source The object of property predicates to conform to. + * @returns {boolean} Returns `true` if `object` conforms, else `false`. */ - function listCacheHas(key) { - return assocIndexOf(this.__data__, key) > -1; + function baseConformsTo(object, source, props) { + var length = props.length; + if (object == null) { + return !length; + } + object = Object(object); + while (length--) { + var key = props[length], + predicate = source[key], + value = object[key]; + + if ((value === undefined && !(key in object)) || !predicate(value)) { + return false; + } + } + return true; } /** - * Sets the list cache `key` to `value`. + * The base implementation of `_.delay` and `_.defer` which accepts `args` + * to provide to `func`. * * @private - * @name set - * @memberOf ListCache - * @param {string} key The key of the value to set. - * @param {*} value The value to set. - * @returns {Object} Returns the list cache instance. + * @param {Function} func The function to delay. + * @param {number} wait The number of milliseconds to delay invocation. + * @param {Array} args The arguments to provide to `func`. + * @returns {number|Object} Returns the timer id or timeout object. */ - function listCacheSet(key, value) { - var data = this.__data__, - index = assocIndexOf(data, key); - - if (index < 0) { - ++this.size; - data.push([key, value]); - } else { - data[index][1] = value; + function baseDelay(func, wait, args) { + if (typeof func != 'function') { + throw new TypeError(FUNC_ERROR_TEXT); } - return this; + return setTimeout(function() { func.apply(undefined, args); }, wait); } - // Add methods to `ListCache`. - ListCache.prototype.clear = listCacheClear; - ListCache.prototype['delete'] = listCacheDelete; - ListCache.prototype.get = listCacheGet; - ListCache.prototype.has = listCacheHas; - ListCache.prototype.set = listCacheSet; - - /*------------------------------------------------------------------------*/ - /** - * Creates a map cache object to store key-value pairs. + * The base implementation of methods like `_.difference` without support + * for excluding multiple arrays or iteratee shorthands. * * @private - * @constructor - * @param {Array} [entries] The key-value pairs to cache. + * @param {Array} array The array to inspect. + * @param {Array} values The values to exclude. + * @param {Function} [iteratee] The iteratee invoked per element. + * @param {Function} [comparator] The comparator invoked per element. + * @returns {Array} Returns the new array of filtered values. */ - function MapCache(entries) { + function baseDifference(array, values, iteratee, comparator) { var index = -1, - length = entries == null ? 0 : entries.length; + includes = arrayIncludes, + isCommon = true, + length = array.length, + result = [], + valuesLength = values.length; - this.clear(); + if (!length) { + return result; + } + if (iteratee) { + values = arrayMap(values, baseUnary(iteratee)); + } + if (comparator) { + includes = arrayIncludesWith; + isCommon = false; + } + else if (values.length >= LARGE_ARRAY_SIZE) { + includes = cacheHas; + isCommon = false; + values = new SetCache(values); + } + outer: while (++index < length) { - var entry = entries[index]; - this.set(entry[0], entry[1]); + var value = array[index], + computed = iteratee == null ? value : iteratee(value); + + value = (comparator || value !== 0) ? value : 0; + if (isCommon && computed === computed) { + var valuesIndex = valuesLength; + while (valuesIndex--) { + if (values[valuesIndex] === computed) { + continue outer; + } + } + result.push(value); + } + else if (!includes(values, computed, comparator)) { + result.push(value); + } } + return result; } /** - * Removes all key-value entries from the map. + * The base implementation of `_.forEach` without support for iteratee shorthands. * * @private - * @name clear - * @memberOf MapCache + * @param {Array|Object} collection The collection to iterate over. + * @param {Function} iteratee The function invoked per iteration. + * @returns {Array|Object} Returns `collection`. */ - function mapCacheClear() { - this.size = 0; - this.__data__ = { - 'hash': new Hash, - 'map': new (Map || ListCache), - 'string': new Hash - }; - } + var baseEach = createBaseEach(baseForOwn); /** - * Removes `key` and its value from the map. + * The base implementation of `_.forEachRight` without support for iteratee shorthands. * * @private - * @name delete - * @memberOf MapCache - * @param {string} key The key of the value to remove. - * @returns {boolean} Returns `true` if the entry was removed, else `false`. + * @param {Array|Object} collection The collection to iterate over. + * @param {Function} iteratee The function invoked per iteration. + * @returns {Array|Object} Returns `collection`. */ - function mapCacheDelete(key) { - var result = getMapData(this, key)['delete'](key); - this.size -= result ? 1 : 0; - return result; - } + var baseEachRight = createBaseEach(baseForOwnRight, true); /** - * Gets the map value for `key`. + * The base implementation of `_.every` without support for iteratee shorthands. * * @private - * @name get - * @memberOf MapCache - * @param {string} key The key of the value to get. - * @returns {*} Returns the entry value. + * @param {Array|Object} collection The collection to iterate over. + * @param {Function} predicate The function invoked per iteration. + * @returns {boolean} Returns `true` if all elements pass the predicate check, + * else `false` */ - function mapCacheGet(key) { - return getMapData(this, key).get(key); + function baseEvery(collection, predicate) { + var result = true; + baseEach(collection, function(value, index, collection) { + result = !!predicate(value, index, collection); + return result; + }); + return result; } /** - * Checks if a map value for `key` exists. + * The base implementation of methods like `_.max` and `_.min` which accepts a + * `comparator` to determine the extremum value. * * @private - * @name has - * @memberOf MapCache - * @param {string} key The key of the entry to check. - * @returns {boolean} Returns `true` if an entry for `key` exists, else `false`. + * @param {Array} array The array to iterate over. + * @param {Function} iteratee The iteratee invoked per iteration. + * @param {Function} comparator The comparator used to compare values. + * @returns {*} Returns the extremum value. */ - function mapCacheHas(key) { - return getMapData(this, key).has(key); - } + function baseExtremum(array, iteratee, comparator) { + var index = -1, + length = array.length; - /** - * Sets the map `key` to `value`. - * - * @private - * @name set - * @memberOf MapCache - * @param {string} key The key of the value to set. - * @param {*} value The value to set. - * @returns {Object} Returns the map cache instance. - */ - function mapCacheSet(key, value) { - var data = getMapData(this, key), - size = data.size; + while (++index < length) { + var value = array[index], + current = iteratee(value); - data.set(key, value); - this.size += data.size == size ? 0 : 1; - return this; + if (current != null && (computed === undefined + ? (current === current && !isSymbol(current)) + : comparator(current, computed) + )) { + var computed = current, + result = value; + } + } + return result; } - // Add methods to `MapCache`. - MapCache.prototype.clear = mapCacheClear; - MapCache.prototype['delete'] = mapCacheDelete; - MapCache.prototype.get = mapCacheGet; - MapCache.prototype.has = mapCacheHas; - MapCache.prototype.set = mapCacheSet; - - /*------------------------------------------------------------------------*/ - /** - * - * Creates an array cache object to store unique values. + * The base implementation of `_.fill` without an iteratee call guard. * * @private - * @constructor - * @param {Array} [values] The values to cache. + * @param {Array} array The array to fill. + * @param {*} value The value to fill `array` with. + * @param {number} [start=0] The start position. + * @param {number} [end=array.length] The end position. + * @returns {Array} Returns `array`. */ - function SetCache(values) { - var index = -1, - length = values == null ? 0 : values.length; + function baseFill(array, value, start, end) { + var length = array.length; - this.__data__ = new MapCache; - while (++index < length) { - this.add(values[index]); + start = toInteger(start); + if (start < 0) { + start = -start > length ? 0 : (length + start); + } + end = (end === undefined || end > length) ? length : toInteger(end); + if (end < 0) { + end += length; + } + end = start > end ? 0 : toLength(end); + while (start < end) { + array[start++] = value; } + return array; } /** - * Adds `value` to the array cache. + * The base implementation of `_.filter` without support for iteratee shorthands. * * @private - * @name add - * @memberOf SetCache - * @alias push - * @param {*} value The value to cache. - * @returns {Object} Returns the cache instance. + * @param {Array|Object} collection The collection to iterate over. + * @param {Function} predicate The function invoked per iteration. + * @returns {Array} Returns the new filtered array. */ - function setCacheAdd(value) { - this.__data__.set(value, HASH_UNDEFINED); - return this; + function baseFilter(collection, predicate) { + var result = []; + baseEach(collection, function(value, index, collection) { + if (predicate(value, index, collection)) { + result.push(value); + } + }); + return result; } /** - * Checks if `value` is in the array cache. + * The base implementation of `_.flatten` with support for restricting flattening. * * @private - * @name has - * @memberOf SetCache - * @param {*} value The value to search for. - * @returns {number} Returns `true` if `value` is found, else `false`. + * @param {Array} array The array to flatten. + * @param {number} depth The maximum recursion depth. + * @param {boolean} [predicate=isFlattenable] The function invoked per iteration. + * @param {boolean} [isStrict] Restrict to values that pass `predicate` checks. + * @param {Array} [result=[]] The initial result value. + * @returns {Array} Returns the new flattened array. */ - function setCacheHas(value) { - return this.__data__.has(value); - } + function baseFlatten(array, depth, predicate, isStrict, result) { + var index = -1, + length = array.length; - // Add methods to `SetCache`. - SetCache.prototype.add = SetCache.prototype.push = setCacheAdd; - SetCache.prototype.has = setCacheHas; + predicate || (predicate = isFlattenable); + result || (result = []); - /*------------------------------------------------------------------------*/ + while (++index < length) { + var value = array[index]; + if (depth > 0 && predicate(value)) { + if (depth > 1) { + // Recursively flatten arrays (susceptible to call stack limits). + baseFlatten(value, depth - 1, predicate, isStrict, result); + } else { + arrayPush(result, value); + } + } else if (!isStrict) { + result[result.length] = value; + } + } + return result; + } /** - * Creates a stack cache object to store key-value pairs. + * The base implementation of `baseForOwn` which iterates over `object` + * properties returned by `keysFunc` and invokes `iteratee` for each property. + * Iteratee functions may exit iteration early by explicitly returning `false`. * * @private - * @constructor - * @param {Array} [entries] The key-value pairs to cache. + * @param {Object} object The object to iterate over. + * @param {Function} iteratee The function invoked per iteration. + * @param {Function} keysFunc The function to get the keys of `object`. + * @returns {Object} Returns `object`. */ - function Stack(entries) { - var data = this.__data__ = new ListCache(entries); - this.size = data.size; - } + var baseFor = createBaseFor(); /** - * Removes all key-value entries from the stack. + * This function is like `baseFor` except that it iterates over properties + * in the opposite order. * * @private - * @name clear - * @memberOf Stack + * @param {Object} object The object to iterate over. + * @param {Function} iteratee The function invoked per iteration. + * @param {Function} keysFunc The function to get the keys of `object`. + * @returns {Object} Returns `object`. */ - function stackClear() { - this.__data__ = new ListCache; - this.size = 0; - } + var baseForRight = createBaseFor(true); /** - * Removes `key` and its value from the stack. + * The base implementation of `_.forOwn` without support for iteratee shorthands. * * @private - * @name delete - * @memberOf Stack - * @param {string} key The key of the value to remove. - * @returns {boolean} Returns `true` if the entry was removed, else `false`. + * @param {Object} object The object to iterate over. + * @param {Function} iteratee The function invoked per iteration. + * @returns {Object} Returns `object`. */ - function stackDelete(key) { - var data = this.__data__, - result = data['delete'](key); - - this.size = data.size; - return result; + function baseForOwn(object, iteratee) { + return object && baseFor(object, iteratee, keys); } /** - * Gets the stack value for `key`. + * The base implementation of `_.forOwnRight` without support for iteratee shorthands. * * @private - * @name get - * @memberOf Stack - * @param {string} key The key of the value to get. - * @returns {*} Returns the entry value. + * @param {Object} object The object to iterate over. + * @param {Function} iteratee The function invoked per iteration. + * @returns {Object} Returns `object`. */ - function stackGet(key) { - return this.__data__.get(key); + function baseForOwnRight(object, iteratee) { + return object && baseForRight(object, iteratee, keys); } /** - * Checks if a stack value for `key` exists. + * The base implementation of `_.functions` which creates an array of + * `object` function property names filtered from `props`. * * @private - * @name has - * @memberOf Stack - * @param {string} key The key of the entry to check. - * @returns {boolean} Returns `true` if an entry for `key` exists, else `false`. + * @param {Object} object The object to inspect. + * @param {Array} props The property names to filter. + * @returns {Array} Returns the function names. */ - function stackHas(key) { - return this.__data__.has(key); + function baseFunctions(object, props) { + return arrayFilter(props, function(key) { + return isFunction(object[key]); + }); } /** - * Sets the stack `key` to `value`. + * The base implementation of `_.get` without support for default values. * * @private - * @name set - * @memberOf Stack - * @param {string} key The key of the value to set. - * @param {*} value The value to set. - * @returns {Object} Returns the stack cache instance. + * @param {Object} object The object to query. + * @param {Array|string} path The path of the property to get. + * @returns {*} Returns the resolved value. */ - function stackSet(key, value) { - var data = this.__data__; - if (data instanceof ListCache) { - var pairs = data.__data__; - if (!Map || (pairs.length < LARGE_ARRAY_SIZE - 1)) { - pairs.push([key, value]); - this.size = ++data.size; - return this; - } - data = this.__data__ = new MapCache(pairs); - } - data.set(key, value); - this.size = data.size; - return this; - } + function baseGet(object, path) { + path = castPath(path, object); - // Add methods to `Stack`. - Stack.prototype.clear = stackClear; - Stack.prototype['delete'] = stackDelete; - Stack.prototype.get = stackGet; - Stack.prototype.has = stackHas; - Stack.prototype.set = stackSet; + var index = 0, + length = path.length; - /*------------------------------------------------------------------------*/ + while (object != null && index < length) { + object = object[toKey(path[index++])]; + } + return (index && index == length) ? object : undefined; + } /** - * Creates an array of the enumerable property names of the array-like `value`. + * The base implementation of `getAllKeys` and `getAllKeysIn` which uses + * `keysFunc` and `symbolsFunc` to get the enumerable property names and + * symbols of `object`. * * @private - * @param {*} value The value to query. - * @param {boolean} inherited Specify returning inherited property names. - * @returns {Array} Returns the array of property names. + * @param {Object} object The object to query. + * @param {Function} keysFunc The function to get the keys of `object`. + * @param {Function} symbolsFunc The function to get the symbols of `object`. + * @returns {Array} Returns the array of property names and symbols. */ - function arrayLikeKeys(value, inherited) { - var isArr = isArray(value), - isArg = !isArr && isArguments(value), - isBuff = !isArr && !isArg && isBuffer(value), - isType = !isArr && !isArg && !isBuff && isTypedArray(value), - skipIndexes = isArr || isArg || isBuff || isType, - result = skipIndexes ? baseTimes(value.length, String) : [], - length = result.length; - - for (var key in value) { - if ((inherited || hasOwnProperty.call(value, key)) && - !(skipIndexes && ( - // Safari 9 has enumerable `arguments.length` in strict mode. - key == 'length' || - // Node.js 0.10 has enumerable non-index properties on buffers. - (isBuff && (key == 'offset' || key == 'parent')) || - // PhantomJS 2 has enumerable non-index properties on typed arrays. - (isType && (key == 'buffer' || key == 'byteLength' || key == 'byteOffset')) || - // Skip index properties. - isIndex(key, length) - ))) { - result.push(key); - } - } - return result; + function baseGetAllKeys(object, keysFunc, symbolsFunc) { + var result = keysFunc(object); + return isArray(object) ? result : arrayPush(result, symbolsFunc(object)); } /** - * A specialized version of `_.sample` for arrays. + * The base implementation of `getTag` without fallbacks for buggy environments. * * @private - * @param {Array} array The array to sample. - * @returns {*} Returns the random element. + * @param {*} value The value to query. + * @returns {string} Returns the `toStringTag`. */ - function arraySample(array) { - var length = array.length; - return length ? array[baseRandom(0, length - 1)] : undefined; + function baseGetTag(value) { + if (value == null) { + return value === undefined ? undefinedTag : nullTag; + } + return (symToStringTag && symToStringTag in Object(value)) + ? getRawTag(value) + : objectToString(value); } /** - * A specialized version of `_.sampleSize` for arrays. + * The base implementation of `_.gt` which doesn't coerce arguments. * * @private - * @param {Array} array The array to sample. - * @param {number} n The number of elements to sample. - * @returns {Array} Returns the random elements. + * @param {*} value The value to compare. + * @param {*} other The other value to compare. + * @returns {boolean} Returns `true` if `value` is greater than `other`, + * else `false`. */ - function arraySampleSize(array, n) { - return shuffleSelf(copyArray(array), baseClamp(n, 0, array.length)); + function baseGt(value, other) { + return value > other; } /** - * A specialized version of `_.shuffle` for arrays. + * The base implementation of `_.has` without support for deep paths. * * @private - * @param {Array} array The array to shuffle. - * @returns {Array} Returns the new shuffled array. + * @param {Object} [object] The object to query. + * @param {Array|string} key The key to check. + * @returns {boolean} Returns `true` if `key` exists, else `false`. */ - function arrayShuffle(array) { - return shuffleSelf(copyArray(array)); + function baseHas(object, key) { + return object != null && hasOwnProperty.call(object, key); } /** - * This function is like `assignValue` except that it doesn't assign - * `undefined` values. + * The base implementation of `_.hasIn` without support for deep paths. * * @private - * @param {Object} object The object to modify. - * @param {string} key The key of the property to assign. - * @param {*} value The value to assign. + * @param {Object} [object] The object to query. + * @param {Array|string} key The key to check. + * @returns {boolean} Returns `true` if `key` exists, else `false`. */ - function assignMergeValue(object, key, value) { - if ((value !== undefined && !eq(object[key], value)) || - (value === undefined && !(key in object))) { - baseAssignValue(object, key, value); - } + function baseHasIn(object, key) { + return object != null && key in Object(object); } /** - * Assigns `value` to `key` of `object` if the existing value is not equivalent - * using [`SameValueZero`](http://ecma-international.org/ecma-262/7.0/#sec-samevaluezero) - * for equality comparisons. + * The base implementation of `_.inRange` which doesn't coerce arguments. * * @private - * @param {Object} object The object to modify. - * @param {string} key The key of the property to assign. - * @param {*} value The value to assign. + * @param {number} number The number to check. + * @param {number} start The start of the range. + * @param {number} end The end of the range. + * @returns {boolean} Returns `true` if `number` is in the range, else `false`. */ - function assignValue(object, key, value) { - var objValue = object[key]; - if (!(hasOwnProperty.call(object, key) && eq(objValue, value)) || - (value === undefined && !(key in object))) { - baseAssignValue(object, key, value); - } + function baseInRange(number, start, end) { + return number >= nativeMin(start, end) && number < nativeMax(start, end); } /** - * Gets the index at which the `key` is found in `array` of key-value pairs. + * The base implementation of methods like `_.intersection`, without support + * for iteratee shorthands, that accepts an array of arrays to inspect. * * @private - * @param {Array} array The array to inspect. - * @param {*} key The key to search for. - * @returns {number} Returns the index of the matched value, else `-1`. + * @param {Array} arrays The arrays to inspect. + * @param {Function} [iteratee] The iteratee invoked per element. + * @param {Function} [comparator] The comparator invoked per element. + * @returns {Array} Returns the new array of shared values. */ - function assocIndexOf(array, key) { - var length = array.length; - while (length--) { - if (eq(array[length][0], key)) { - return length; + function baseIntersection(arrays, iteratee, comparator) { + var includes = comparator ? arrayIncludesWith : arrayIncludes, + length = arrays[0].length, + othLength = arrays.length, + othIndex = othLength, + caches = Array(othLength), + maxLength = Infinity, + result = []; + + while (othIndex--) { + var array = arrays[othIndex]; + if (othIndex && iteratee) { + array = arrayMap(array, baseUnary(iteratee)); } + maxLength = nativeMin(array.length, maxLength); + caches[othIndex] = !comparator && (iteratee || (length >= 120 && array.length >= 120)) + ? new SetCache(othIndex && array) + : undefined; } - return -1; + array = arrays[0]; + + var index = -1, + seen = caches[0]; + + outer: + while (++index < length && result.length < maxLength) { + var value = array[index], + computed = iteratee ? iteratee(value) : value; + + value = (comparator || value !== 0) ? value : 0; + if (!(seen + ? cacheHas(seen, computed) + : includes(result, computed, comparator) + )) { + othIndex = othLength; + while (--othIndex) { + var cache = caches[othIndex]; + if (!(cache + ? cacheHas(cache, computed) + : includes(arrays[othIndex], computed, comparator)) + ) { + continue outer; + } + } + if (seen) { + seen.push(computed); + } + result.push(value); + } + } + return result; } /** - * Aggregates elements of `collection` on `accumulator` with keys transformed - * by `iteratee` and values set by `setter`. + * The base implementation of `_.invert` and `_.invertBy` which inverts + * `object` with values transformed by `iteratee` and set by `setter`. * * @private - * @param {Array|Object} collection The collection to iterate over. + * @param {Object} object The object to iterate over. * @param {Function} setter The function to set `accumulator` values. - * @param {Function} iteratee The iteratee to transform keys. - * @param {Object} accumulator The initial aggregated object. + * @param {Function} iteratee The iteratee to transform values. + * @param {Object} accumulator The initial inverted object. * @returns {Function} Returns `accumulator`. */ - function baseAggregator(collection, setter, iteratee, accumulator) { - baseEach(collection, function(value, key, collection) { - setter(accumulator, value, iteratee(value), collection); + function baseInverter(object, setter, iteratee, accumulator) { + baseForOwn(object, function(value, key, object) { + setter(accumulator, iteratee(value), key, object); }); return accumulator; } /** - * The base implementation of `_.assign` without support for multiple sources - * or `customizer` functions. - * - * @private - * @param {Object} object The destination object. - * @param {Object} source The source object. - * @returns {Object} Returns `object`. - */ - function baseAssign(object, source) { - return object && copyObject(source, keys(source), object); - } - - /** - * The base implementation of `_.assignIn` without support for multiple sources - * or `customizer` functions. + * The base implementation of `_.invoke` without support for individual + * method arguments. * * @private - * @param {Object} object The destination object. - * @param {Object} source The source object. - * @returns {Object} Returns `object`. + * @param {Object} object The object to query. + * @param {Array|string} path The path of the method to invoke. + * @param {Array} args The arguments to invoke the method with. + * @returns {*} Returns the result of the invoked method. */ - function baseAssignIn(object, source) { - return object && copyObject(source, keysIn(source), object); + function baseInvoke(object, path, args) { + path = castPath(path, object); + object = parent(object, path); + var func = object == null ? object : object[toKey(last(path))]; + return func == null ? undefined : apply(func, object, args); } /** - * The base implementation of `assignValue` and `assignMergeValue` without - * value checks. + * The base implementation of `_.isArguments`. * * @private - * @param {Object} object The object to modify. - * @param {string} key The key of the property to assign. - * @param {*} value The value to assign. + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is an `arguments` object, */ - function baseAssignValue(object, key, value) { - if (key == '__proto__' && defineProperty) { - defineProperty(object, key, { - 'configurable': true, - 'enumerable': true, - 'value': value, - 'writable': true - }); - } else { - object[key] = value; - } + function baseIsArguments(value) { + return isObjectLike(value) && baseGetTag(value) == argsTag; } /** - * The base implementation of `_.at` without support for individual paths. + * The base implementation of `_.isArrayBuffer` without Node.js optimizations. * * @private - * @param {Object} object The object to iterate over. - * @param {string[]} paths The property paths to pick. - * @returns {Array} Returns the picked elements. + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is an array buffer, else `false`. */ - function baseAt(object, paths) { - var index = -1, - length = paths.length, - result = Array(length), - skip = object == null; - - while (++index < length) { - result[index] = skip ? undefined : get(object, paths[index]); - } - return result; + function baseIsArrayBuffer(value) { + return isObjectLike(value) && baseGetTag(value) == arrayBufferTag; } /** - * The base implementation of `_.clamp` which doesn't coerce arguments. + * The base implementation of `_.isDate` without Node.js optimizations. * * @private - * @param {number} number The number to clamp. - * @param {number} [lower] The lower bound. - * @param {number} upper The upper bound. - * @returns {number} Returns the clamped number. + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is a date object, else `false`. */ - function baseClamp(number, lower, upper) { - if (number === number) { - if (upper !== undefined) { - number = number <= upper ? number : upper; - } - if (lower !== undefined) { - number = number >= lower ? number : lower; - } - } - return number; + function baseIsDate(value) { + return isObjectLike(value) && baseGetTag(value) == dateTag; } /** - * The base implementation of `_.clone` and `_.cloneDeep` which tracks - * traversed objects. + * The base implementation of `_.isEqual` which supports partial comparisons + * and tracks traversed objects. * * @private - * @param {*} value The value to clone. + * @param {*} value The value to compare. + * @param {*} other The other value to compare. * @param {boolean} bitmask The bitmask flags. - * 1 - Deep clone - * 2 - Flatten inherited properties - * 4 - Clone symbols - * @param {Function} [customizer] The function to customize cloning. - * @param {string} [key] The key of `value`. - * @param {Object} [object] The parent object of `value`. - * @param {Object} [stack] Tracks traversed objects and their clone counterparts. - * @returns {*} Returns the cloned value. + * 1 - Unordered comparison + * 2 - Partial comparison + * @param {Function} [customizer] The function to customize comparisons. + * @param {Object} [stack] Tracks traversed `value` and `other` objects. + * @returns {boolean} Returns `true` if the values are equivalent, else `false`. */ - function baseClone(value, bitmask, customizer, key, object, stack) { - var result, - isDeep = bitmask & CLONE_DEEP_FLAG, - isFlat = bitmask & CLONE_FLAT_FLAG, - isFull = bitmask & CLONE_SYMBOLS_FLAG; - - if (customizer) { - result = object ? customizer(value, key, object, stack) : customizer(value); - } - if (result !== undefined) { - return result; - } - if (!isObject(value)) { - return value; - } - var isArr = isArray(value); - if (isArr) { - result = initCloneArray(value); - if (!isDeep) { - return copyArray(value, result); - } - } else { - var tag = getTag(value), - isFunc = tag == funcTag || tag == genTag; - - if (isBuffer(value)) { - return cloneBuffer(value, isDeep); - } - if (tag == objectTag || tag == argsTag || (isFunc && !object)) { - result = (isFlat || isFunc) ? {} : initCloneObject(value); - if (!isDeep) { - return isFlat - ? copySymbolsIn(value, baseAssignIn(result, value)) - : copySymbols(value, baseAssign(result, value)); - } - } else { - if (!cloneableTags[tag]) { - return object ? value : {}; - } - result = initCloneByTag(value, tag, isDeep); - } - } - // Check for circular references and return its corresponding clone. - stack || (stack = new Stack); - var stacked = stack.get(value); - if (stacked) { - return stacked; + function baseIsEqual(value, other, bitmask, customizer, stack) { + if (value === other) { + return true; } - stack.set(value, result); - - if (isSet(value)) { - value.forEach(function(subValue) { - result.add(baseClone(subValue, bitmask, customizer, subValue, value, stack)); - }); - } else if (isMap(value)) { - value.forEach(function(subValue, key) { - result.set(key, baseClone(subValue, bitmask, customizer, key, value, stack)); - }); + if (value == null || other == null || (!isObjectLike(value) && !isObjectLike(other))) { + return value !== value && other !== other; } - - var keysFunc = isFull - ? (isFlat ? getAllKeysIn : getAllKeys) - : (isFlat ? keysIn : keys); - - var props = isArr ? undefined : keysFunc(value); - arrayEach(props || value, function(subValue, key) { - if (props) { - key = subValue; - subValue = value[key]; - } - // Recursively populate clone (susceptible to call stack limits). - assignValue(result, key, baseClone(subValue, bitmask, customizer, key, value, stack)); - }); - return result; + return baseIsEqualDeep(value, other, bitmask, customizer, baseIsEqual, stack); } /** - * The base implementation of `_.conforms` which doesn't clone `source`. + * A specialized version of `baseIsEqual` for arrays and objects which performs + * deep comparisons and tracks traversed objects enabling objects with circular + * references to be compared. * * @private - * @param {Object} source The object of property predicates to conform to. - * @returns {Function} Returns the new spec function. + * @param {Object} object The object to compare. + * @param {Object} other The other object to compare. + * @param {number} bitmask The bitmask flags. See `baseIsEqual` for more details. + * @param {Function} customizer The function to customize comparisons. + * @param {Function} equalFunc The function to determine equivalents of values. + * @param {Object} [stack] Tracks traversed `object` and `other` objects. + * @returns {boolean} Returns `true` if the objects are equivalent, else `false`. */ - function baseConforms(source) { - var props = keys(source); - return function(object) { - return baseConformsTo(object, source, props); - }; - } + function baseIsEqualDeep(object, other, bitmask, customizer, equalFunc, stack) { + var objIsArr = isArray(object), + othIsArr = isArray(other), + objTag = objIsArr ? arrayTag : getTag(object), + othTag = othIsArr ? arrayTag : getTag(other); - /** - * The base implementation of `_.conformsTo` which accepts `props` to check. - * - * @private - * @param {Object} object The object to inspect. - * @param {Object} source The object of property predicates to conform to. - * @returns {boolean} Returns `true` if `object` conforms, else `false`. - */ - function baseConformsTo(object, source, props) { - var length = props.length; - if (object == null) { - return !length; - } - object = Object(object); - while (length--) { - var key = props[length], - predicate = source[key], - value = object[key]; + objTag = objTag == argsTag ? objectTag : objTag; + othTag = othTag == argsTag ? objectTag : othTag; + + var objIsObj = objTag == objectTag, + othIsObj = othTag == objectTag, + isSameTag = objTag == othTag; - if ((value === undefined && !(key in object)) || !predicate(value)) { + if (isSameTag && isBuffer(object)) { + if (!isBuffer(other)) { return false; } + objIsArr = true; + objIsObj = false; } - return true; + if (isSameTag && !objIsObj) { + stack || (stack = new Stack); + return (objIsArr || isTypedArray(object)) + ? equalArrays(object, other, bitmask, customizer, equalFunc, stack) + : equalByTag(object, other, objTag, bitmask, customizer, equalFunc, stack); + } + if (!(bitmask & COMPARE_PARTIAL_FLAG)) { + var objIsWrapped = objIsObj && hasOwnProperty.call(object, '__wrapped__'), + othIsWrapped = othIsObj && hasOwnProperty.call(other, '__wrapped__'); + + if (objIsWrapped || othIsWrapped) { + var objUnwrapped = objIsWrapped ? object.value() : object, + othUnwrapped = othIsWrapped ? other.value() : other; + + stack || (stack = new Stack); + return equalFunc(objUnwrapped, othUnwrapped, bitmask, customizer, stack); + } + } + if (!isSameTag) { + return false; + } + stack || (stack = new Stack); + return equalObjects(object, other, bitmask, customizer, equalFunc, stack); } /** - * The base implementation of `_.delay` and `_.defer` which accepts `args` - * to provide to `func`. + * The base implementation of `_.isMap` without Node.js optimizations. * * @private - * @param {Function} func The function to delay. - * @param {number} wait The number of milliseconds to delay invocation. - * @param {Array} args The arguments to provide to `func`. - * @returns {number|Object} Returns the timer id or timeout object. + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is a map, else `false`. */ - function baseDelay(func, wait, args) { - if (typeof func != 'function') { - throw new TypeError(FUNC_ERROR_TEXT); - } - return setTimeout(function() { func.apply(undefined, args); }, wait); + function baseIsMap(value) { + return isObjectLike(value) && getTag(value) == mapTag; } /** - * The base implementation of methods like `_.difference` without support - * for excluding multiple arrays or iteratee shorthands. + * The base implementation of `_.isMatch` without support for iteratee shorthands. * * @private - * @param {Array} array The array to inspect. - * @param {Array} values The values to exclude. - * @param {Function} [iteratee] The iteratee invoked per element. - * @param {Function} [comparator] The comparator invoked per element. - * @returns {Array} Returns the new array of filtered values. + * @param {Object} object The object to inspect. + * @param {Object} source The object of property values to match. + * @param {Array} matchData The property names, values, and compare flags to match. + * @param {Function} [customizer] The function to customize comparisons. + * @returns {boolean} Returns `true` if `object` is a match, else `false`. */ - function baseDifference(array, values, iteratee, comparator) { - var index = -1, - includes = arrayIncludes, - isCommon = true, - length = array.length, - result = [], - valuesLength = values.length; + function baseIsMatch(object, source, matchData, customizer) { + var index = matchData.length, + length = index, + noCustomizer = !customizer; - if (!length) { - return result; - } - if (iteratee) { - values = arrayMap(values, baseUnary(iteratee)); - } - if (comparator) { - includes = arrayIncludesWith; - isCommon = false; + if (object == null) { + return !length; } - else if (values.length >= LARGE_ARRAY_SIZE) { - includes = cacheHas; - isCommon = false; - values = new SetCache(values); + object = Object(object); + while (index--) { + var data = matchData[index]; + if ((noCustomizer && data[2]) + ? data[1] !== object[data[0]] + : !(data[0] in object) + ) { + return false; + } } - outer: while (++index < length) { - var value = array[index], - computed = iteratee == null ? value : iteratee(value); + data = matchData[index]; + var key = data[0], + objValue = object[key], + srcValue = data[1]; - value = (comparator || value !== 0) ? value : 0; - if (isCommon && computed === computed) { - var valuesIndex = valuesLength; - while (valuesIndex--) { - if (values[valuesIndex] === computed) { - continue outer; - } + if (noCustomizer && data[2]) { + if (objValue === undefined && !(key in object)) { + return false; + } + } else { + var stack = new Stack; + if (customizer) { + var result = customizer(objValue, srcValue, key, object, source, stack); + } + if (!(result === undefined + ? baseIsEqual(srcValue, objValue, COMPARE_PARTIAL_FLAG | COMPARE_UNORDERED_FLAG, customizer, stack) + : result + )) { + return false; } - result.push(value); - } - else if (!includes(values, computed, comparator)) { - result.push(value); } } - return result; + return true; } /** - * The base implementation of `_.forEach` without support for iteratee shorthands. + * The base implementation of `_.isNative` without bad shim checks. * * @private - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} iteratee The function invoked per iteration. - * @returns {Array|Object} Returns `collection`. + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is a native function, + * else `false`. */ - var baseEach = createBaseEach(baseForOwn); + function baseIsNative(value) { + if (!isObject(value) || isMasked(value)) { + return false; + } + var pattern = isFunction(value) ? reIsNative : reIsHostCtor; + return pattern.test(toSource(value)); + } /** - * The base implementation of `_.forEachRight` without support for iteratee shorthands. + * The base implementation of `_.isRegExp` without Node.js optimizations. * * @private - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} iteratee The function invoked per iteration. - * @returns {Array|Object} Returns `collection`. + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is a regexp, else `false`. */ - var baseEachRight = createBaseEach(baseForOwnRight, true); + function baseIsRegExp(value) { + return isObjectLike(value) && baseGetTag(value) == regexpTag; + } /** - * The base implementation of `_.every` without support for iteratee shorthands. + * The base implementation of `_.isSet` without Node.js optimizations. * * @private - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} predicate The function invoked per iteration. - * @returns {boolean} Returns `true` if all elements pass the predicate check, - * else `false` + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is a set, else `false`. */ - function baseEvery(collection, predicate) { - var result = true; - baseEach(collection, function(value, index, collection) { - result = !!predicate(value, index, collection); - return result; - }); - return result; + function baseIsSet(value) { + return isObjectLike(value) && getTag(value) == setTag; } /** - * The base implementation of methods like `_.max` and `_.min` which accepts a - * `comparator` to determine the extremum value. + * The base implementation of `_.isTypedArray` without Node.js optimizations. * * @private - * @param {Array} array The array to iterate over. - * @param {Function} iteratee The iteratee invoked per iteration. - * @param {Function} comparator The comparator used to compare values. - * @returns {*} Returns the extremum value. + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is a typed array, else `false`. */ - function baseExtremum(array, iteratee, comparator) { - var index = -1, - length = array.length; - - while (++index < length) { - var value = array[index], - current = iteratee(value); - - if (current != null && (computed === undefined - ? (current === current && !isSymbol(current)) - : comparator(current, computed) - )) { - var computed = current, - result = value; - } - } - return result; + function baseIsTypedArray(value) { + return isObjectLike(value) && + isLength(value.length) && !!typedArrayTags[baseGetTag(value)]; } /** - * The base implementation of `_.fill` without an iteratee call guard. + * The base implementation of `_.iteratee`. * * @private - * @param {Array} array The array to fill. - * @param {*} value The value to fill `array` with. - * @param {number} [start=0] The start position. - * @param {number} [end=array.length] The end position. - * @returns {Array} Returns `array`. + * @param {*} [value=_.identity] The value to convert to an iteratee. + * @returns {Function} Returns the iteratee. */ - function baseFill(array, value, start, end) { - var length = array.length; - - start = toInteger(start); - if (start < 0) { - start = -start > length ? 0 : (length + start); + function baseIteratee(value) { + // Don't store the `typeof` result in a variable to avoid a JIT bug in Safari 9. + // See https://bugs.webkit.org/show_bug.cgi?id=156034 for more details. + if (typeof value == 'function') { + return value; } - end = (end === undefined || end > length) ? length : toInteger(end); - if (end < 0) { - end += length; + if (value == null) { + return identity; } - end = start > end ? 0 : toLength(end); - while (start < end) { - array[start++] = value; + if (typeof value == 'object') { + return isArray(value) + ? baseMatchesProperty(value[0], value[1]) + : baseMatches(value); } - return array; + return property(value); } /** - * The base implementation of `_.filter` without support for iteratee shorthands. + * The base implementation of `_.keys` which doesn't treat sparse arrays as dense. * * @private - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} predicate The function invoked per iteration. - * @returns {Array} Returns the new filtered array. + * @param {Object} object The object to query. + * @returns {Array} Returns the array of property names. */ - function baseFilter(collection, predicate) { + function baseKeys(object) { + if (!isPrototype(object)) { + return nativeKeys(object); + } var result = []; - baseEach(collection, function(value, index, collection) { - if (predicate(value, index, collection)) { - result.push(value); + for (var key in Object(object)) { + if (hasOwnProperty.call(object, key) && key != 'constructor') { + result.push(key); } - }); + } return result; } /** - * The base implementation of `_.flatten` with support for restricting flattening. + * The base implementation of `_.keysIn` which doesn't treat sparse arrays as dense. * * @private - * @param {Array} array The array to flatten. - * @param {number} depth The maximum recursion depth. - * @param {boolean} [predicate=isFlattenable] The function invoked per iteration. - * @param {boolean} [isStrict] Restrict to values that pass `predicate` checks. - * @param {Array} [result=[]] The initial result value. - * @returns {Array} Returns the new flattened array. + * @param {Object} object The object to query. + * @returns {Array} Returns the array of property names. */ - function baseFlatten(array, depth, predicate, isStrict, result) { - var index = -1, - length = array.length; - - predicate || (predicate = isFlattenable); - result || (result = []); + function baseKeysIn(object) { + if (!isObject(object)) { + return nativeKeysIn(object); + } + var isProto = isPrototype(object), + result = []; - while (++index < length) { - var value = array[index]; - if (depth > 0 && predicate(value)) { - if (depth > 1) { - // Recursively flatten arrays (susceptible to call stack limits). - baseFlatten(value, depth - 1, predicate, isStrict, result); - } else { - arrayPush(result, value); - } - } else if (!isStrict) { - result[result.length] = value; + for (var key in object) { + if (!(key == 'constructor' && (isProto || !hasOwnProperty.call(object, key)))) { + result.push(key); } } return result; } /** - * The base implementation of `baseForOwn` which iterates over `object` - * properties returned by `keysFunc` and invokes `iteratee` for each property. - * Iteratee functions may exit iteration early by explicitly returning `false`. + * The base implementation of `_.lt` which doesn't coerce arguments. * * @private - * @param {Object} object The object to iterate over. - * @param {Function} iteratee The function invoked per iteration. - * @param {Function} keysFunc The function to get the keys of `object`. - * @returns {Object} Returns `object`. + * @param {*} value The value to compare. + * @param {*} other The other value to compare. + * @returns {boolean} Returns `true` if `value` is less than `other`, + * else `false`. */ - var baseFor = createBaseFor(); + function baseLt(value, other) { + return value < other; + } /** - * This function is like `baseFor` except that it iterates over properties - * in the opposite order. + * The base implementation of `_.map` without support for iteratee shorthands. * * @private - * @param {Object} object The object to iterate over. + * @param {Array|Object} collection The collection to iterate over. * @param {Function} iteratee The function invoked per iteration. - * @param {Function} keysFunc The function to get the keys of `object`. - * @returns {Object} Returns `object`. + * @returns {Array} Returns the new mapped array. */ - var baseForRight = createBaseFor(true); + function baseMap(collection, iteratee) { + var index = -1, + result = isArrayLike(collection) ? Array(collection.length) : []; + + baseEach(collection, function(value, key, collection) { + result[++index] = iteratee(value, key, collection); + }); + return result; + } /** - * The base implementation of `_.forOwn` without support for iteratee shorthands. + * The base implementation of `_.matches` which doesn't clone `source`. * * @private - * @param {Object} object The object to iterate over. - * @param {Function} iteratee The function invoked per iteration. - * @returns {Object} Returns `object`. + * @param {Object} source The object of property values to match. + * @returns {Function} Returns the new spec function. */ - function baseForOwn(object, iteratee) { - return object && baseFor(object, iteratee, keys); + function baseMatches(source) { + var matchData = getMatchData(source); + if (matchData.length == 1 && matchData[0][2]) { + return matchesStrictComparable(matchData[0][0], matchData[0][1]); + } + return function(object) { + return object === source || baseIsMatch(object, source, matchData); + }; } /** - * The base implementation of `_.forOwnRight` without support for iteratee shorthands. + * The base implementation of `_.matchesProperty` which doesn't clone `srcValue`. * * @private - * @param {Object} object The object to iterate over. - * @param {Function} iteratee The function invoked per iteration. - * @returns {Object} Returns `object`. + * @param {string} path The path of the property to get. + * @param {*} srcValue The value to match. + * @returns {Function} Returns the new spec function. */ - function baseForOwnRight(object, iteratee) { - return object && baseForRight(object, iteratee, keys); + function baseMatchesProperty(path, srcValue) { + if (isKey(path) && isStrictComparable(srcValue)) { + return matchesStrictComparable(toKey(path), srcValue); + } + return function(object) { + var objValue = get(object, path); + return (objValue === undefined && objValue === srcValue) + ? hasIn(object, path) + : baseIsEqual(srcValue, objValue, COMPARE_PARTIAL_FLAG | COMPARE_UNORDERED_FLAG); + }; } /** - * The base implementation of `_.functions` which creates an array of - * `object` function property names filtered from `props`. + * The base implementation of `_.merge` without support for multiple sources. * * @private - * @param {Object} object The object to inspect. - * @param {Array} props The property names to filter. - * @returns {Array} Returns the function names. + * @param {Object} object The destination object. + * @param {Object} source The source object. + * @param {number} srcIndex The index of `source`. + * @param {Function} [customizer] The function to customize merged values. + * @param {Object} [stack] Tracks traversed source values and their merged + * counterparts. */ - function baseFunctions(object, props) { - return arrayFilter(props, function(key) { - return isFunction(object[key]); - }); + function baseMerge(object, source, srcIndex, customizer, stack) { + if (object === source) { + return; + } + baseFor(source, function(srcValue, key) { + stack || (stack = new Stack); + if (isObject(srcValue)) { + baseMergeDeep(object, source, key, srcIndex, baseMerge, customizer, stack); + } + else { + var newValue = customizer + ? customizer(safeGet(object, key), srcValue, (key + ''), object, source, stack) + : undefined; + + if (newValue === undefined) { + newValue = srcValue; + } + assignMergeValue(object, key, newValue); + } + }, keysIn); } /** - * The base implementation of `_.get` without support for default values. + * A specialized version of `baseMerge` for arrays and objects which performs + * deep merges and tracks traversed objects enabling objects with circular + * references to be merged. * * @private - * @param {Object} object The object to query. - * @param {Array|string} path The path of the property to get. - * @returns {*} Returns the resolved value. + * @param {Object} object The destination object. + * @param {Object} source The source object. + * @param {string} key The key of the value to merge. + * @param {number} srcIndex The index of `source`. + * @param {Function} mergeFunc The function to merge values. + * @param {Function} [customizer] The function to customize assigned values. + * @param {Object} [stack] Tracks traversed source values and their merged + * counterparts. */ - function baseGet(object, path) { - path = castPath(path, object); + function baseMergeDeep(object, source, key, srcIndex, mergeFunc, customizer, stack) { + var objValue = safeGet(object, key), + srcValue = safeGet(source, key), + stacked = stack.get(srcValue); - var index = 0, - length = path.length; + if (stacked) { + assignMergeValue(object, key, stacked); + return; + } + var newValue = customizer + ? customizer(objValue, srcValue, (key + ''), object, source, stack) + : undefined; - while (object != null && index < length) { - object = object[toKey(path[index++])]; + var isCommon = newValue === undefined; + + if (isCommon) { + var isArr = isArray(srcValue), + isBuff = !isArr && isBuffer(srcValue), + isTyped = !isArr && !isBuff && isTypedArray(srcValue); + + newValue = srcValue; + if (isArr || isBuff || isTyped) { + if (isArray(objValue)) { + newValue = objValue; + } + else if (isArrayLikeObject(objValue)) { + newValue = copyArray(objValue); + } + else if (isBuff) { + isCommon = false; + newValue = cloneBuffer(srcValue, true); + } + else if (isTyped) { + isCommon = false; + newValue = cloneTypedArray(srcValue, true); + } + else { + newValue = []; + } + } + else if (isPlainObject(srcValue) || isArguments(srcValue)) { + newValue = objValue; + if (isArguments(objValue)) { + newValue = toPlainObject(objValue); + } + else if (!isObject(objValue) || isFunction(objValue)) { + newValue = initCloneObject(srcValue); + } + } + else { + isCommon = false; + } } - return (index && index == length) ? object : undefined; + if (isCommon) { + // Recursively merge objects and arrays (susceptible to call stack limits). + stack.set(srcValue, newValue); + mergeFunc(newValue, srcValue, srcIndex, customizer, stack); + stack['delete'](srcValue); + } + assignMergeValue(object, key, newValue); } /** - * The base implementation of `getAllKeys` and `getAllKeysIn` which uses - * `keysFunc` and `symbolsFunc` to get the enumerable property names and - * symbols of `object`. + * The base implementation of `_.nth` which doesn't coerce arguments. * * @private - * @param {Object} object The object to query. - * @param {Function} keysFunc The function to get the keys of `object`. - * @param {Function} symbolsFunc The function to get the symbols of `object`. - * @returns {Array} Returns the array of property names and symbols. + * @param {Array} array The array to query. + * @param {number} n The index of the element to return. + * @returns {*} Returns the nth element of `array`. */ - function baseGetAllKeys(object, keysFunc, symbolsFunc) { - var result = keysFunc(object); - return isArray(object) ? result : arrayPush(result, symbolsFunc(object)); + function baseNth(array, n) { + var length = array.length; + if (!length) { + return; + } + n += n < 0 ? length : 0; + return isIndex(n, length) ? array[n] : undefined; } /** - * The base implementation of `getTag` without fallbacks for buggy environments. + * The base implementation of `_.orderBy` without param guards. * * @private - * @param {*} value The value to query. - * @returns {string} Returns the `toStringTag`. + * @param {Array|Object} collection The collection to iterate over. + * @param {Function[]|Object[]|string[]} iteratees The iteratees to sort by. + * @param {string[]} orders The sort orders of `iteratees`. + * @returns {Array} Returns the new sorted array. */ - function baseGetTag(value) { - if (value == null) { - return value === undefined ? undefinedTag : nullTag; + function baseOrderBy(collection, iteratees, orders) { + if (iteratees.length) { + iteratees = arrayMap(iteratees, function(iteratee) { + if (isArray(iteratee)) { + return function(value) { + return baseGet(value, iteratee.length === 1 ? iteratee[0] : iteratee); + } + } + return iteratee; + }); + } else { + iteratees = [identity]; } - return (symToStringTag && symToStringTag in Object(value)) - ? getRawTag(value) - : objectToString(value); - } - /** - * The base implementation of `_.gt` which doesn't coerce arguments. - * - * @private - * @param {*} value The value to compare. - * @param {*} other The other value to compare. - * @returns {boolean} Returns `true` if `value` is greater than `other`, - * else `false`. - */ - function baseGt(value, other) { - return value > other; + var index = -1; + iteratees = arrayMap(iteratees, baseUnary(getIteratee())); + + var result = baseMap(collection, function(value, key, collection) { + var criteria = arrayMap(iteratees, function(iteratee) { + return iteratee(value); + }); + return { 'criteria': criteria, 'index': ++index, 'value': value }; + }); + + return baseSortBy(result, function(object, other) { + return compareMultiple(object, other, orders); + }); } /** - * The base implementation of `_.has` without support for deep paths. + * The base implementation of `_.pick` without support for individual + * property identifiers. * * @private - * @param {Object} [object] The object to query. - * @param {Array|string} key The key to check. - * @returns {boolean} Returns `true` if `key` exists, else `false`. + * @param {Object} object The source object. + * @param {string[]} paths The property paths to pick. + * @returns {Object} Returns the new object. */ - function baseHas(object, key) { - return object != null && hasOwnProperty.call(object, key); + function basePick(object, paths) { + return basePickBy(object, paths, function(value, path) { + return hasIn(object, path); + }); } /** - * The base implementation of `_.hasIn` without support for deep paths. + * The base implementation of `_.pickBy` without support for iteratee shorthands. * * @private - * @param {Object} [object] The object to query. - * @param {Array|string} key The key to check. - * @returns {boolean} Returns `true` if `key` exists, else `false`. + * @param {Object} object The source object. + * @param {string[]} paths The property paths to pick. + * @param {Function} predicate The function invoked per property. + * @returns {Object} Returns the new object. */ - function baseHasIn(object, key) { - return object != null && key in Object(object); + function basePickBy(object, paths, predicate) { + var index = -1, + length = paths.length, + result = {}; + + while (++index < length) { + var path = paths[index], + value = baseGet(object, path); + + if (predicate(value, path)) { + baseSet(result, castPath(path, object), value); + } + } + return result; } /** - * The base implementation of `_.inRange` which doesn't coerce arguments. + * A specialized version of `baseProperty` which supports deep paths. * * @private - * @param {number} number The number to check. - * @param {number} start The start of the range. - * @param {number} end The end of the range. - * @returns {boolean} Returns `true` if `number` is in the range, else `false`. + * @param {Array|string} path The path of the property to get. + * @returns {Function} Returns the new accessor function. */ - function baseInRange(number, start, end) { - return number >= nativeMin(start, end) && number < nativeMax(start, end); + function basePropertyDeep(path) { + return function(object) { + return baseGet(object, path); + }; } /** - * The base implementation of methods like `_.intersection`, without support - * for iteratee shorthands, that accepts an array of arrays to inspect. + * The base implementation of `_.pullAllBy` without support for iteratee + * shorthands. * * @private - * @param {Array} arrays The arrays to inspect. + * @param {Array} array The array to modify. + * @param {Array} values The values to remove. * @param {Function} [iteratee] The iteratee invoked per element. * @param {Function} [comparator] The comparator invoked per element. - * @returns {Array} Returns the new array of shared values. + * @returns {Array} Returns `array`. */ - function baseIntersection(arrays, iteratee, comparator) { - var includes = comparator ? arrayIncludesWith : arrayIncludes, - length = arrays[0].length, - othLength = arrays.length, - othIndex = othLength, - caches = Array(othLength), - maxLength = Infinity, - result = []; + function basePullAll(array, values, iteratee, comparator) { + var indexOf = comparator ? baseIndexOfWith : baseIndexOf, + index = -1, + length = values.length, + seen = array; - while (othIndex--) { - var array = arrays[othIndex]; - if (othIndex && iteratee) { - array = arrayMap(array, baseUnary(iteratee)); - } - maxLength = nativeMin(array.length, maxLength); - caches[othIndex] = !comparator && (iteratee || (length >= 120 && array.length >= 120)) - ? new SetCache(othIndex && array) - : undefined; + if (array === values) { + values = copyArray(values); } - array = arrays[0]; - - var index = -1, - seen = caches[0]; - - outer: - while (++index < length && result.length < maxLength) { - var value = array[index], + if (iteratee) { + seen = arrayMap(array, baseUnary(iteratee)); + } + while (++index < length) { + var fromIndex = 0, + value = values[index], computed = iteratee ? iteratee(value) : value; - value = (comparator || value !== 0) ? value : 0; - if (!(seen - ? cacheHas(seen, computed) - : includes(result, computed, comparator) - )) { - othIndex = othLength; - while (--othIndex) { - var cache = caches[othIndex]; - if (!(cache - ? cacheHas(cache, computed) - : includes(arrays[othIndex], computed, comparator)) - ) { - continue outer; - } - } - if (seen) { - seen.push(computed); + while ((fromIndex = indexOf(seen, computed, fromIndex, comparator)) > -1) { + if (seen !== array) { + splice.call(seen, fromIndex, 1); } - result.push(value); + splice.call(array, fromIndex, 1); } } - return result; + return array; } /** - * The base implementation of `_.invert` and `_.invertBy` which inverts - * `object` with values transformed by `iteratee` and set by `setter`. + * The base implementation of `_.pullAt` without support for individual + * indexes or capturing the removed elements. * * @private - * @param {Object} object The object to iterate over. - * @param {Function} setter The function to set `accumulator` values. - * @param {Function} iteratee The iteratee to transform values. - * @param {Object} accumulator The initial inverted object. - * @returns {Function} Returns `accumulator`. + * @param {Array} array The array to modify. + * @param {number[]} indexes The indexes of elements to remove. + * @returns {Array} Returns `array`. */ - function baseInverter(object, setter, iteratee, accumulator) { - baseForOwn(object, function(value, key, object) { - setter(accumulator, iteratee(value), key, object); - }); - return accumulator; - } + function basePullAt(array, indexes) { + var length = array ? indexes.length : 0, + lastIndex = length - 1; - /** - * The base implementation of `_.invoke` without support for individual - * method arguments. - * - * @private - * @param {Object} object The object to query. - * @param {Array|string} path The path of the method to invoke. - * @param {Array} args The arguments to invoke the method with. - * @returns {*} Returns the result of the invoked method. - */ - function baseInvoke(object, path, args) { - path = castPath(path, object); - object = parent(object, path); - var func = object == null ? object : object[toKey(last(path))]; - return func == null ? undefined : apply(func, object, args); + while (length--) { + var index = indexes[length]; + if (length == lastIndex || index !== previous) { + var previous = index; + if (isIndex(index)) { + splice.call(array, index, 1); + } else { + baseUnset(array, index); + } + } + } + return array; } /** - * The base implementation of `_.isArguments`. + * The base implementation of `_.random` without support for returning + * floating-point numbers. * * @private - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is an `arguments` object, + * @param {number} lower The lower bound. + * @param {number} upper The upper bound. + * @returns {number} Returns the random number. */ - function baseIsArguments(value) { - return isObjectLike(value) && baseGetTag(value) == argsTag; + function baseRandom(lower, upper) { + return lower + nativeFloor(nativeRandom() * (upper - lower + 1)); } /** - * The base implementation of `_.isArrayBuffer` without Node.js optimizations. + * The base implementation of `_.range` and `_.rangeRight` which doesn't + * coerce arguments. * * @private - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is an array buffer, else `false`. + * @param {number} start The start of the range. + * @param {number} end The end of the range. + * @param {number} step The value to increment or decrement by. + * @param {boolean} [fromRight] Specify iterating from right to left. + * @returns {Array} Returns the range of numbers. */ - function baseIsArrayBuffer(value) { - return isObjectLike(value) && baseGetTag(value) == arrayBufferTag; + function baseRange(start, end, step, fromRight) { + var index = -1, + length = nativeMax(nativeCeil((end - start) / (step || 1)), 0), + result = Array(length); + + while (length--) { + result[fromRight ? length : ++index] = start; + start += step; + } + return result; } /** - * The base implementation of `_.isDate` without Node.js optimizations. + * The base implementation of `_.repeat` which doesn't coerce arguments. * * @private - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a date object, else `false`. + * @param {string} string The string to repeat. + * @param {number} n The number of times to repeat the string. + * @returns {string} Returns the repeated string. */ - function baseIsDate(value) { - return isObjectLike(value) && baseGetTag(value) == dateTag; + function baseRepeat(string, n) { + var result = ''; + if (!string || n < 1 || n > MAX_SAFE_INTEGER) { + return result; + } + // Leverage the exponentiation by squaring algorithm for a faster repeat. + // See https://en.wikipedia.org/wiki/Exponentiation_by_squaring for more details. + do { + if (n % 2) { + result += string; + } + n = nativeFloor(n / 2); + if (n) { + string += string; + } + } while (n); + + return result; } /** - * The base implementation of `_.isEqual` which supports partial comparisons - * and tracks traversed objects. + * The base implementation of `_.rest` which doesn't validate or coerce arguments. * * @private - * @param {*} value The value to compare. - * @param {*} other The other value to compare. - * @param {boolean} bitmask The bitmask flags. - * 1 - Unordered comparison - * 2 - Partial comparison - * @param {Function} [customizer] The function to customize comparisons. - * @param {Object} [stack] Tracks traversed `value` and `other` objects. - * @returns {boolean} Returns `true` if the values are equivalent, else `false`. + * @param {Function} func The function to apply a rest parameter to. + * @param {number} [start=func.length-1] The start position of the rest parameter. + * @returns {Function} Returns the new function. */ - function baseIsEqual(value, other, bitmask, customizer, stack) { - if (value === other) { - return true; - } - if (value == null || other == null || (!isObjectLike(value) && !isObjectLike(other))) { - return value !== value && other !== other; - } - return baseIsEqualDeep(value, other, bitmask, customizer, baseIsEqual, stack); + function baseRest(func, start) { + return setToString(overRest(func, start, identity), func + ''); } /** - * A specialized version of `baseIsEqual` for arrays and objects which performs - * deep comparisons and tracks traversed objects enabling objects with circular - * references to be compared. + * The base implementation of `_.sample`. * * @private - * @param {Object} object The object to compare. - * @param {Object} other The other object to compare. - * @param {number} bitmask The bitmask flags. See `baseIsEqual` for more details. - * @param {Function} customizer The function to customize comparisons. - * @param {Function} equalFunc The function to determine equivalents of values. - * @param {Object} [stack] Tracks traversed `object` and `other` objects. - * @returns {boolean} Returns `true` if the objects are equivalent, else `false`. + * @param {Array|Object} collection The collection to sample. + * @returns {*} Returns the random element. */ - function baseIsEqualDeep(object, other, bitmask, customizer, equalFunc, stack) { - var objIsArr = isArray(object), - othIsArr = isArray(other), - objTag = objIsArr ? arrayTag : getTag(object), - othTag = othIsArr ? arrayTag : getTag(other); - - objTag = objTag == argsTag ? objectTag : objTag; - othTag = othTag == argsTag ? objectTag : othTag; - - var objIsObj = objTag == objectTag, - othIsObj = othTag == objectTag, - isSameTag = objTag == othTag; - - if (isSameTag && isBuffer(object)) { - if (!isBuffer(other)) { - return false; - } - objIsArr = true; - objIsObj = false; - } - if (isSameTag && !objIsObj) { - stack || (stack = new Stack); - return (objIsArr || isTypedArray(object)) - ? equalArrays(object, other, bitmask, customizer, equalFunc, stack) - : equalByTag(object, other, objTag, bitmask, customizer, equalFunc, stack); - } - if (!(bitmask & COMPARE_PARTIAL_FLAG)) { - var objIsWrapped = objIsObj && hasOwnProperty.call(object, '__wrapped__'), - othIsWrapped = othIsObj && hasOwnProperty.call(other, '__wrapped__'); - - if (objIsWrapped || othIsWrapped) { - var objUnwrapped = objIsWrapped ? object.value() : object, - othUnwrapped = othIsWrapped ? other.value() : other; - - stack || (stack = new Stack); - return equalFunc(objUnwrapped, othUnwrapped, bitmask, customizer, stack); - } - } - if (!isSameTag) { - return false; - } - stack || (stack = new Stack); - return equalObjects(object, other, bitmask, customizer, equalFunc, stack); + function baseSample(collection) { + return arraySample(values(collection)); } /** - * The base implementation of `_.isMap` without Node.js optimizations. + * The base implementation of `_.sampleSize` without param guards. * * @private - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a map, else `false`. + * @param {Array|Object} collection The collection to sample. + * @param {number} n The number of elements to sample. + * @returns {Array} Returns the random elements. */ - function baseIsMap(value) { - return isObjectLike(value) && getTag(value) == mapTag; + function baseSampleSize(collection, n) { + var array = values(collection); + return shuffleSelf(array, baseClamp(n, 0, array.length)); } /** - * The base implementation of `_.isMatch` without support for iteratee shorthands. + * The base implementation of `_.set`. * * @private - * @param {Object} object The object to inspect. - * @param {Object} source The object of property values to match. - * @param {Array} matchData The property names, values, and compare flags to match. - * @param {Function} [customizer] The function to customize comparisons. - * @returns {boolean} Returns `true` if `object` is a match, else `false`. + * @param {Object} object The object to modify. + * @param {Array|string} path The path of the property to set. + * @param {*} value The value to set. + * @param {Function} [customizer] The function to customize path creation. + * @returns {Object} Returns `object`. */ - function baseIsMatch(object, source, matchData, customizer) { - var index = matchData.length, - length = index, - noCustomizer = !customizer; - - if (object == null) { - return !length; + function baseSet(object, path, value, customizer) { + if (!isObject(object)) { + return object; } - object = Object(object); - while (index--) { - var data = matchData[index]; - if ((noCustomizer && data[2]) - ? data[1] !== object[data[0]] - : !(data[0] in object) - ) { - return false; + path = castPath(path, object); + + var index = -1, + length = path.length, + lastIndex = length - 1, + nested = object; + + while (nested != null && ++index < length) { + var key = toKey(path[index]), + newValue = value; + + if (key === '__proto__' || key === 'constructor' || key === 'prototype') { + return object; } - } - while (++index < length) { - data = matchData[index]; - var key = data[0], - objValue = object[key], - srcValue = data[1]; - if (noCustomizer && data[2]) { - if (objValue === undefined && !(key in object)) { - return false; - } - } else { - var stack = new Stack; - if (customizer) { - var result = customizer(objValue, srcValue, key, object, source, stack); - } - if (!(result === undefined - ? baseIsEqual(srcValue, objValue, COMPARE_PARTIAL_FLAG | COMPARE_UNORDERED_FLAG, customizer, stack) - : result - )) { - return false; + if (index != lastIndex) { + var objValue = nested[key]; + newValue = customizer ? customizer(objValue, key, nested) : undefined; + if (newValue === undefined) { + newValue = isObject(objValue) + ? objValue + : (isIndex(path[index + 1]) ? [] : {}); } } + assignValue(nested, key, newValue); + nested = nested[key]; } - return true; + return object; } /** - * The base implementation of `_.isNative` without bad shim checks. + * The base implementation of `setData` without support for hot loop shorting. * * @private - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a native function, - * else `false`. + * @param {Function} func The function to associate metadata with. + * @param {*} data The metadata. + * @returns {Function} Returns `func`. */ - function baseIsNative(value) { - if (!isObject(value) || isMasked(value)) { - return false; - } - var pattern = isFunction(value) ? reIsNative : reIsHostCtor; - return pattern.test(toSource(value)); - } + var baseSetData = !metaMap ? identity : function(func, data) { + metaMap.set(func, data); + return func; + }; /** - * The base implementation of `_.isRegExp` without Node.js optimizations. + * The base implementation of `setToString` without support for hot loop shorting. * * @private - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a regexp, else `false`. + * @param {Function} func The function to modify. + * @param {Function} string The `toString` result. + * @returns {Function} Returns `func`. */ - function baseIsRegExp(value) { - return isObjectLike(value) && baseGetTag(value) == regexpTag; - } + var baseSetToString = !defineProperty ? identity : function(func, string) { + return defineProperty(func, 'toString', { + 'configurable': true, + 'enumerable': false, + 'value': constant(string), + 'writable': true + }); + }; /** - * The base implementation of `_.isSet` without Node.js optimizations. + * The base implementation of `_.shuffle`. * * @private - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a set, else `false`. + * @param {Array|Object} collection The collection to shuffle. + * @returns {Array} Returns the new shuffled array. */ - function baseIsSet(value) { - return isObjectLike(value) && getTag(value) == setTag; + function baseShuffle(collection) { + return shuffleSelf(values(collection)); } /** - * The base implementation of `_.isTypedArray` without Node.js optimizations. + * The base implementation of `_.slice` without an iteratee call guard. * * @private - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a typed array, else `false`. + * @param {Array} array The array to slice. + * @param {number} [start=0] The start position. + * @param {number} [end=array.length] The end position. + * @returns {Array} Returns the slice of `array`. */ - function baseIsTypedArray(value) { - return isObjectLike(value) && - isLength(value.length) && !!typedArrayTags[baseGetTag(value)]; - } + function baseSlice(array, start, end) { + var index = -1, + length = array.length; - /** - * The base implementation of `_.iteratee`. - * - * @private - * @param {*} [value=_.identity] The value to convert to an iteratee. - * @returns {Function} Returns the iteratee. - */ - function baseIteratee(value) { - // Don't store the `typeof` result in a variable to avoid a JIT bug in Safari 9. - // See https://bugs.webkit.org/show_bug.cgi?id=156034 for more details. - if (typeof value == 'function') { - return value; + if (start < 0) { + start = -start > length ? 0 : (length + start); } - if (value == null) { - return identity; + end = end > length ? length : end; + if (end < 0) { + end += length; } - if (typeof value == 'object') { - return isArray(value) - ? baseMatchesProperty(value[0], value[1]) - : baseMatches(value); + length = start > end ? 0 : ((end - start) >>> 0); + start >>>= 0; + + var result = Array(length); + while (++index < length) { + result[index] = array[index + start]; } - return property(value); + return result; } /** - * The base implementation of `_.keys` which doesn't treat sparse arrays as dense. + * The base implementation of `_.some` without support for iteratee shorthands. * * @private - * @param {Object} object The object to query. - * @returns {Array} Returns the array of property names. + * @param {Array|Object} collection The collection to iterate over. + * @param {Function} predicate The function invoked per iteration. + * @returns {boolean} Returns `true` if any element passes the predicate check, + * else `false`. */ - function baseKeys(object) { - if (!isPrototype(object)) { - return nativeKeys(object); - } - var result = []; - for (var key in Object(object)) { - if (hasOwnProperty.call(object, key) && key != 'constructor') { - result.push(key); - } - } - return result; + function baseSome(collection, predicate) { + var result; + + baseEach(collection, function(value, index, collection) { + result = predicate(value, index, collection); + return !result; + }); + return !!result; } /** - * The base implementation of `_.keysIn` which doesn't treat sparse arrays as dense. + * The base implementation of `_.sortedIndex` and `_.sortedLastIndex` which + * performs a binary search of `array` to determine the index at which `value` + * should be inserted into `array` in order to maintain its sort order. * * @private - * @param {Object} object The object to query. - * @returns {Array} Returns the array of property names. + * @param {Array} array The sorted array to inspect. + * @param {*} value The value to evaluate. + * @param {boolean} [retHighest] Specify returning the highest qualified index. + * @returns {number} Returns the index at which `value` should be inserted + * into `array`. */ - function baseKeysIn(object) { - if (!isObject(object)) { - return nativeKeysIn(object); - } - var isProto = isPrototype(object), - result = []; + function baseSortedIndex(array, value, retHighest) { + var low = 0, + high = array == null ? low : array.length; - for (var key in object) { - if (!(key == 'constructor' && (isProto || !hasOwnProperty.call(object, key)))) { - result.push(key); + if (typeof value == 'number' && value === value && high <= HALF_MAX_ARRAY_LENGTH) { + while (low < high) { + var mid = (low + high) >>> 1, + computed = array[mid]; + + if (computed !== null && !isSymbol(computed) && + (retHighest ? (computed <= value) : (computed < value))) { + low = mid + 1; + } else { + high = mid; + } } + return high; } - return result; + return baseSortedIndexBy(array, value, identity, retHighest); } /** - * The base implementation of `_.lt` which doesn't coerce arguments. + * The base implementation of `_.sortedIndexBy` and `_.sortedLastIndexBy` + * which invokes `iteratee` for `value` and each element of `array` to compute + * their sort ranking. The iteratee is invoked with one argument; (value). * * @private - * @param {*} value The value to compare. - * @param {*} other The other value to compare. - * @returns {boolean} Returns `true` if `value` is less than `other`, - * else `false`. + * @param {Array} array The sorted array to inspect. + * @param {*} value The value to evaluate. + * @param {Function} iteratee The iteratee invoked per element. + * @param {boolean} [retHighest] Specify returning the highest qualified index. + * @returns {number} Returns the index at which `value` should be inserted + * into `array`. */ - function baseLt(value, other) { - return value < other; + function baseSortedIndexBy(array, value, iteratee, retHighest) { + var low = 0, + high = array == null ? 0 : array.length; + if (high === 0) { + return 0; + } + + value = iteratee(value); + var valIsNaN = value !== value, + valIsNull = value === null, + valIsSymbol = isSymbol(value), + valIsUndefined = value === undefined; + + while (low < high) { + var mid = nativeFloor((low + high) / 2), + computed = iteratee(array[mid]), + othIsDefined = computed !== undefined, + othIsNull = computed === null, + othIsReflexive = computed === computed, + othIsSymbol = isSymbol(computed); + + if (valIsNaN) { + var setLow = retHighest || othIsReflexive; + } else if (valIsUndefined) { + setLow = othIsReflexive && (retHighest || othIsDefined); + } else if (valIsNull) { + setLow = othIsReflexive && othIsDefined && (retHighest || !othIsNull); + } else if (valIsSymbol) { + setLow = othIsReflexive && othIsDefined && !othIsNull && (retHighest || !othIsSymbol); + } else if (othIsNull || othIsSymbol) { + setLow = false; + } else { + setLow = retHighest ? (computed <= value) : (computed < value); + } + if (setLow) { + low = mid + 1; + } else { + high = mid; + } + } + return nativeMin(high, MAX_ARRAY_INDEX); } /** - * The base implementation of `_.map` without support for iteratee shorthands. + * The base implementation of `_.sortedUniq` and `_.sortedUniqBy` without + * support for iteratee shorthands. * * @private - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} iteratee The function invoked per iteration. - * @returns {Array} Returns the new mapped array. + * @param {Array} array The array to inspect. + * @param {Function} [iteratee] The iteratee invoked per element. + * @returns {Array} Returns the new duplicate free array. */ - function baseMap(collection, iteratee) { + function baseSortedUniq(array, iteratee) { var index = -1, - result = isArrayLike(collection) ? Array(collection.length) : []; + length = array.length, + resIndex = 0, + result = []; - baseEach(collection, function(value, key, collection) { - result[++index] = iteratee(value, key, collection); - }); + while (++index < length) { + var value = array[index], + computed = iteratee ? iteratee(value) : value; + + if (!index || !eq(computed, seen)) { + var seen = computed; + result[resIndex++] = value === 0 ? 0 : value; + } + } return result; } /** - * The base implementation of `_.matches` which doesn't clone `source`. + * The base implementation of `_.toNumber` which doesn't ensure correct + * conversions of binary, hexadecimal, or octal string values. * * @private - * @param {Object} source The object of property values to match. - * @returns {Function} Returns the new spec function. + * @param {*} value The value to process. + * @returns {number} Returns the number. */ - function baseMatches(source) { - var matchData = getMatchData(source); - if (matchData.length == 1 && matchData[0][2]) { - return matchesStrictComparable(matchData[0][0], matchData[0][1]); + function baseToNumber(value) { + if (typeof value == 'number') { + return value; } - return function(object) { - return object === source || baseIsMatch(object, source, matchData); - }; + if (isSymbol(value)) { + return NAN; + } + return +value; } /** - * The base implementation of `_.matchesProperty` which doesn't clone `srcValue`. + * The base implementation of `_.toString` which doesn't convert nullish + * values to empty strings. * * @private - * @param {string} path The path of the property to get. - * @param {*} srcValue The value to match. - * @returns {Function} Returns the new spec function. + * @param {*} value The value to process. + * @returns {string} Returns the string. */ - function baseMatchesProperty(path, srcValue) { - if (isKey(path) && isStrictComparable(srcValue)) { - return matchesStrictComparable(toKey(path), srcValue); + function baseToString(value) { + // Exit early for strings to avoid a performance hit in some environments. + if (typeof value == 'string') { + return value; } - return function(object) { - var objValue = get(object, path); - return (objValue === undefined && objValue === srcValue) - ? hasIn(object, path) - : baseIsEqual(srcValue, objValue, COMPARE_PARTIAL_FLAG | COMPARE_UNORDERED_FLAG); - }; + if (isArray(value)) { + // Recursively convert values (susceptible to call stack limits). + return arrayMap(value, baseToString) + ''; + } + if (isSymbol(value)) { + return symbolToString ? symbolToString.call(value) : ''; + } + var result = (value + ''); + return (result == '0' && (1 / value) == -INFINITY) ? '-0' : result; } /** - * The base implementation of `_.merge` without support for multiple sources. + * The base implementation of `_.uniqBy` without support for iteratee shorthands. * * @private - * @param {Object} object The destination object. - * @param {Object} source The source object. - * @param {number} srcIndex The index of `source`. - * @param {Function} [customizer] The function to customize merged values. - * @param {Object} [stack] Tracks traversed source values and their merged - * counterparts. + * @param {Array} array The array to inspect. + * @param {Function} [iteratee] The iteratee invoked per element. + * @param {Function} [comparator] The comparator invoked per element. + * @returns {Array} Returns the new duplicate free array. */ - function baseMerge(object, source, srcIndex, customizer, stack) { - if (object === source) { - return; + function baseUniq(array, iteratee, comparator) { + var index = -1, + includes = arrayIncludes, + length = array.length, + isCommon = true, + result = [], + seen = result; + + if (comparator) { + isCommon = false; + includes = arrayIncludesWith; } - baseFor(source, function(srcValue, key) { - stack || (stack = new Stack); - if (isObject(srcValue)) { - baseMergeDeep(object, source, key, srcIndex, baseMerge, customizer, stack); + else if (length >= LARGE_ARRAY_SIZE) { + var set = iteratee ? null : createSet(array); + if (set) { + return setToArray(set); } - else { - var newValue = customizer - ? customizer(safeGet(object, key), srcValue, (key + ''), object, source, stack) - : undefined; + isCommon = false; + includes = cacheHas; + seen = new SetCache; + } + else { + seen = iteratee ? [] : result; + } + outer: + while (++index < length) { + var value = array[index], + computed = iteratee ? iteratee(value) : value; - if (newValue === undefined) { - newValue = srcValue; + value = (comparator || value !== 0) ? value : 0; + if (isCommon && computed === computed) { + var seenIndex = seen.length; + while (seenIndex--) { + if (seen[seenIndex] === computed) { + continue outer; + } } - assignMergeValue(object, key, newValue); + if (iteratee) { + seen.push(computed); + } + result.push(value); } - }, keysIn); + else if (!includes(seen, computed, comparator)) { + if (seen !== result) { + seen.push(computed); + } + result.push(value); + } + } + return result; } /** - * A specialized version of `baseMerge` for arrays and objects which performs - * deep merges and tracks traversed objects enabling objects with circular - * references to be merged. + * The base implementation of `_.unset`. * * @private - * @param {Object} object The destination object. - * @param {Object} source The source object. - * @param {string} key The key of the value to merge. - * @param {number} srcIndex The index of `source`. - * @param {Function} mergeFunc The function to merge values. - * @param {Function} [customizer] The function to customize assigned values. - * @param {Object} [stack] Tracks traversed source values and their merged - * counterparts. + * @param {Object} object The object to modify. + * @param {Array|string} path The property path to unset. + * @returns {boolean} Returns `true` if the property is deleted, else `false`. */ - function baseMergeDeep(object, source, key, srcIndex, mergeFunc, customizer, stack) { - var objValue = safeGet(object, key), - srcValue = safeGet(source, key), - stacked = stack.get(srcValue); - - if (stacked) { - assignMergeValue(object, key, stacked); - return; - } - var newValue = customizer - ? customizer(objValue, srcValue, (key + ''), object, source, stack) - : undefined; - - var isCommon = newValue === undefined; - - if (isCommon) { - var isArr = isArray(srcValue), - isBuff = !isArr && isBuffer(srcValue), - isTyped = !isArr && !isBuff && isTypedArray(srcValue); - - newValue = srcValue; - if (isArr || isBuff || isTyped) { - if (isArray(objValue)) { - newValue = objValue; - } - else if (isArrayLikeObject(objValue)) { - newValue = copyArray(objValue); - } - else if (isBuff) { - isCommon = false; - newValue = cloneBuffer(srcValue, true); - } - else if (isTyped) { - isCommon = false; - newValue = cloneTypedArray(srcValue, true); - } - else { - newValue = []; - } - } - else if (isPlainObject(srcValue) || isArguments(srcValue)) { - newValue = objValue; - if (isArguments(objValue)) { - newValue = toPlainObject(objValue); - } - else if (!isObject(objValue) || isFunction(objValue)) { - newValue = initCloneObject(srcValue); - } - } - else { - isCommon = false; - } - } - if (isCommon) { - // Recursively merge objects and arrays (susceptible to call stack limits). - stack.set(srcValue, newValue); - mergeFunc(newValue, srcValue, srcIndex, customizer, stack); - stack['delete'](srcValue); - } - assignMergeValue(object, key, newValue); + function baseUnset(object, path) { + path = castPath(path, object); + object = parent(object, path); + return object == null || delete object[toKey(last(path))]; } /** - * The base implementation of `_.nth` which doesn't coerce arguments. + * The base implementation of `_.update`. * * @private - * @param {Array} array The array to query. - * @param {number} n The index of the element to return. - * @returns {*} Returns the nth element of `array`. + * @param {Object} object The object to modify. + * @param {Array|string} path The path of the property to update. + * @param {Function} updater The function to produce the updated value. + * @param {Function} [customizer] The function to customize path creation. + * @returns {Object} Returns `object`. */ - function baseNth(array, n) { - var length = array.length; - if (!length) { - return; - } - n += n < 0 ? length : 0; - return isIndex(n, length) ? array[n] : undefined; + function baseUpdate(object, path, updater, customizer) { + return baseSet(object, path, updater(baseGet(object, path)), customizer); } /** - * The base implementation of `_.orderBy` without param guards. + * The base implementation of methods like `_.dropWhile` and `_.takeWhile` + * without support for iteratee shorthands. * * @private - * @param {Array|Object} collection The collection to iterate over. - * @param {Function[]|Object[]|string[]} iteratees The iteratees to sort by. - * @param {string[]} orders The sort orders of `iteratees`. - * @returns {Array} Returns the new sorted array. + * @param {Array} array The array to query. + * @param {Function} predicate The function invoked per iteration. + * @param {boolean} [isDrop] Specify dropping elements instead of taking them. + * @param {boolean} [fromRight] Specify iterating from right to left. + * @returns {Array} Returns the slice of `array`. */ - function baseOrderBy(collection, iteratees, orders) { - if (iteratees.length) { - iteratees = arrayMap(iteratees, function(iteratee) { - if (isArray(iteratee)) { - return function(value) { - return baseGet(value, iteratee.length === 1 ? iteratee[0] : iteratee); - } - } - return iteratee; - }); - } else { - iteratees = [identity]; - } + function baseWhile(array, predicate, isDrop, fromRight) { + var length = array.length, + index = fromRight ? length : -1; - var index = -1; - iteratees = arrayMap(iteratees, baseUnary(getIteratee())); + while ((fromRight ? index-- : ++index < length) && + predicate(array[index], index, array)) {} - var result = baseMap(collection, function(value, key, collection) { - var criteria = arrayMap(iteratees, function(iteratee) { - return iteratee(value); - }); - return { 'criteria': criteria, 'index': ++index, 'value': value }; - }); + return isDrop + ? baseSlice(array, (fromRight ? 0 : index), (fromRight ? index + 1 : length)) + : baseSlice(array, (fromRight ? index + 1 : 0), (fromRight ? length : index)); + } - return baseSortBy(result, function(object, other) { - return compareMultiple(object, other, orders); - }); + /** + * The base implementation of `wrapperValue` which returns the result of + * performing a sequence of actions on the unwrapped `value`, where each + * successive action is supplied the return value of the previous. + * + * @private + * @param {*} value The unwrapped value. + * @param {Array} actions Actions to perform to resolve the unwrapped value. + * @returns {*} Returns the resolved value. + */ + function baseWrapperValue(value, actions) { + var result = value; + if (result instanceof LazyWrapper) { + result = result.value(); + } + return arrayReduce(actions, function(result, action) { + return action.func.apply(action.thisArg, arrayPush([result], action.args)); + }, result); } /** - * The base implementation of `_.pick` without support for individual - * property identifiers. + * The base implementation of methods like `_.xor`, without support for + * iteratee shorthands, that accepts an array of arrays to inspect. * * @private - * @param {Object} object The source object. - * @param {string[]} paths The property paths to pick. - * @returns {Object} Returns the new object. + * @param {Array} arrays The arrays to inspect. + * @param {Function} [iteratee] The iteratee invoked per element. + * @param {Function} [comparator] The comparator invoked per element. + * @returns {Array} Returns the new array of values. */ - function basePick(object, paths) { - return basePickBy(object, paths, function(value, path) { - return hasIn(object, path); - }); + function baseXor(arrays, iteratee, comparator) { + var length = arrays.length; + if (length < 2) { + return length ? baseUniq(arrays[0]) : []; + } + var index = -1, + result = Array(length); + + while (++index < length) { + var array = arrays[index], + othIndex = -1; + + while (++othIndex < length) { + if (othIndex != index) { + result[index] = baseDifference(result[index] || array, arrays[othIndex], iteratee, comparator); + } + } + } + return baseUniq(baseFlatten(result, 1), iteratee, comparator); } /** - * The base implementation of `_.pickBy` without support for iteratee shorthands. + * This base implementation of `_.zipObject` which assigns values using `assignFunc`. * * @private - * @param {Object} object The source object. - * @param {string[]} paths The property paths to pick. - * @param {Function} predicate The function invoked per property. + * @param {Array} props The property identifiers. + * @param {Array} values The property values. + * @param {Function} assignFunc The function to assign values. * @returns {Object} Returns the new object. */ - function basePickBy(object, paths, predicate) { + function baseZipObject(props, values, assignFunc) { var index = -1, - length = paths.length, + length = props.length, + valsLength = values.length, result = {}; while (++index < length) { - var path = paths[index], - value = baseGet(object, path); - - if (predicate(value, path)) { - baseSet(result, castPath(path, object), value); - } + var value = index < valsLength ? values[index] : undefined; + assignFunc(result, props[index], value); } return result; } /** - * A specialized version of `baseProperty` which supports deep paths. + * Casts `value` to an empty array if it's not an array like object. + * + * @private + * @param {*} value The value to inspect. + * @returns {Array|Object} Returns the cast array-like object. + */ + function castArrayLikeObject(value) { + return isArrayLikeObject(value) ? value : []; + } + + /** + * Casts `value` to `identity` if it's not a function. * * @private - * @param {Array|string} path The path of the property to get. - * @returns {Function} Returns the new accessor function. + * @param {*} value The value to inspect. + * @returns {Function} Returns cast function. */ - function basePropertyDeep(path) { - return function(object) { - return baseGet(object, path); - }; + function castFunction(value) { + return typeof value == 'function' ? value : identity; } /** - * The base implementation of `_.pullAllBy` without support for iteratee - * shorthands. + * Casts `value` to a path array if it's not one. * * @private - * @param {Array} array The array to modify. - * @param {Array} values The values to remove. - * @param {Function} [iteratee] The iteratee invoked per element. - * @param {Function} [comparator] The comparator invoked per element. - * @returns {Array} Returns `array`. + * @param {*} value The value to inspect. + * @param {Object} [object] The object to query keys on. + * @returns {Array} Returns the cast property path array. */ - function basePullAll(array, values, iteratee, comparator) { - var indexOf = comparator ? baseIndexOfWith : baseIndexOf, - index = -1, - length = values.length, - seen = array; - - if (array === values) { - values = copyArray(values); - } - if (iteratee) { - seen = arrayMap(array, baseUnary(iteratee)); - } - while (++index < length) { - var fromIndex = 0, - value = values[index], - computed = iteratee ? iteratee(value) : value; - - while ((fromIndex = indexOf(seen, computed, fromIndex, comparator)) > -1) { - if (seen !== array) { - splice.call(seen, fromIndex, 1); - } - splice.call(array, fromIndex, 1); - } + function castPath(value, object) { + if (isArray(value)) { + return value; } - return array; + return isKey(value, object) ? [value] : stringToPath(toString(value)); } /** - * The base implementation of `_.pullAt` without support for individual - * indexes or capturing the removed elements. + * A `baseRest` alias which can be replaced with `identity` by module + * replacement plugins. * * @private - * @param {Array} array The array to modify. - * @param {number[]} indexes The indexes of elements to remove. - * @returns {Array} Returns `array`. + * @type {Function} + * @param {Function} func The function to apply a rest parameter to. + * @returns {Function} Returns the new function. */ - function basePullAt(array, indexes) { - var length = array ? indexes.length : 0, - lastIndex = length - 1; - - while (length--) { - var index = indexes[length]; - if (length == lastIndex || index !== previous) { - var previous = index; - if (isIndex(index)) { - splice.call(array, index, 1); - } else { - baseUnset(array, index); - } - } - } - return array; - } + var castRest = baseRest; /** - * The base implementation of `_.random` without support for returning - * floating-point numbers. + * Casts `array` to a slice if it's needed. * * @private - * @param {number} lower The lower bound. - * @param {number} upper The upper bound. - * @returns {number} Returns the random number. + * @param {Array} array The array to inspect. + * @param {number} start The start position. + * @param {number} [end=array.length] The end position. + * @returns {Array} Returns the cast slice. */ - function baseRandom(lower, upper) { - return lower + nativeFloor(nativeRandom() * (upper - lower + 1)); + function castSlice(array, start, end) { + var length = array.length; + end = end === undefined ? length : end; + return (!start && end >= length) ? array : baseSlice(array, start, end); } /** - * The base implementation of `_.range` and `_.rangeRight` which doesn't - * coerce arguments. + * A simple wrapper around the global [`clearTimeout`](https://mdn.io/clearTimeout). * * @private - * @param {number} start The start of the range. - * @param {number} end The end of the range. - * @param {number} step The value to increment or decrement by. - * @param {boolean} [fromRight] Specify iterating from right to left. - * @returns {Array} Returns the range of numbers. + * @param {number|Object} id The timer id or timeout object of the timer to clear. */ - function baseRange(start, end, step, fromRight) { - var index = -1, - length = nativeMax(nativeCeil((end - start) / (step || 1)), 0), - result = Array(length); + var clearTimeout = ctxClearTimeout || function(id) { + return root.clearTimeout(id); + }; - while (length--) { - result[fromRight ? length : ++index] = start; - start += step; + /** + * Creates a clone of `buffer`. + * + * @private + * @param {Buffer} buffer The buffer to clone. + * @param {boolean} [isDeep] Specify a deep clone. + * @returns {Buffer} Returns the cloned buffer. + */ + function cloneBuffer(buffer, isDeep) { + if (isDeep) { + return buffer.slice(); } + var length = buffer.length, + result = allocUnsafe ? allocUnsafe(length) : new buffer.constructor(length); + + buffer.copy(result); return result; } /** - * The base implementation of `_.repeat` which doesn't coerce arguments. + * Creates a clone of `arrayBuffer`. * * @private - * @param {string} string The string to repeat. - * @param {number} n The number of times to repeat the string. - * @returns {string} Returns the repeated string. + * @param {ArrayBuffer} arrayBuffer The array buffer to clone. + * @returns {ArrayBuffer} Returns the cloned array buffer. */ - function baseRepeat(string, n) { - var result = ''; - if (!string || n < 1 || n > MAX_SAFE_INTEGER) { - return result; - } - // Leverage the exponentiation by squaring algorithm for a faster repeat. - // See https://en.wikipedia.org/wiki/Exponentiation_by_squaring for more details. - do { - if (n % 2) { - result += string; - } - n = nativeFloor(n / 2); - if (n) { - string += string; - } - } while (n); - + function cloneArrayBuffer(arrayBuffer) { + var result = new arrayBuffer.constructor(arrayBuffer.byteLength); + new Uint8Array(result).set(new Uint8Array(arrayBuffer)); return result; } /** - * The base implementation of `_.rest` which doesn't validate or coerce arguments. + * Creates a clone of `dataView`. * * @private - * @param {Function} func The function to apply a rest parameter to. - * @param {number} [start=func.length-1] The start position of the rest parameter. - * @returns {Function} Returns the new function. + * @param {Object} dataView The data view to clone. + * @param {boolean} [isDeep] Specify a deep clone. + * @returns {Object} Returns the cloned data view. */ - function baseRest(func, start) { - return setToString(overRest(func, start, identity), func + ''); + function cloneDataView(dataView, isDeep) { + var buffer = isDeep ? cloneArrayBuffer(dataView.buffer) : dataView.buffer; + return new dataView.constructor(buffer, dataView.byteOffset, dataView.byteLength); } /** - * The base implementation of `_.sample`. + * Creates a clone of `regexp`. * * @private - * @param {Array|Object} collection The collection to sample. - * @returns {*} Returns the random element. + * @param {Object} regexp The regexp to clone. + * @returns {Object} Returns the cloned regexp. */ - function baseSample(collection) { - return arraySample(values(collection)); + function cloneRegExp(regexp) { + var result = new regexp.constructor(regexp.source, reFlags.exec(regexp)); + result.lastIndex = regexp.lastIndex; + return result; } /** - * The base implementation of `_.sampleSize` without param guards. + * Creates a clone of the `symbol` object. * * @private - * @param {Array|Object} collection The collection to sample. - * @param {number} n The number of elements to sample. - * @returns {Array} Returns the random elements. + * @param {Object} symbol The symbol object to clone. + * @returns {Object} Returns the cloned symbol object. */ - function baseSampleSize(collection, n) { - var array = values(collection); - return shuffleSelf(array, baseClamp(n, 0, array.length)); + function cloneSymbol(symbol) { + return symbolValueOf ? Object(symbolValueOf.call(symbol)) : {}; } /** - * The base implementation of `_.set`. + * Creates a clone of `typedArray`. * * @private - * @param {Object} object The object to modify. - * @param {Array|string} path The path of the property to set. - * @param {*} value The value to set. - * @param {Function} [customizer] The function to customize path creation. - * @returns {Object} Returns `object`. + * @param {Object} typedArray The typed array to clone. + * @param {boolean} [isDeep] Specify a deep clone. + * @returns {Object} Returns the cloned typed array. */ - function baseSet(object, path, value, customizer) { - if (!isObject(object)) { - return object; - } - path = castPath(path, object); + function cloneTypedArray(typedArray, isDeep) { + var buffer = isDeep ? cloneArrayBuffer(typedArray.buffer) : typedArray.buffer; + return new typedArray.constructor(buffer, typedArray.byteOffset, typedArray.length); + } - var index = -1, - length = path.length, - lastIndex = length - 1, - nested = object; + /** + * Compares values to sort them in ascending order. + * + * @private + * @param {*} value The value to compare. + * @param {*} other The other value to compare. + * @returns {number} Returns the sort order indicator for `value`. + */ + function compareAscending(value, other) { + if (value !== other) { + var valIsDefined = value !== undefined, + valIsNull = value === null, + valIsReflexive = value === value, + valIsSymbol = isSymbol(value); - while (nested != null && ++index < length) { - var key = toKey(path[index]), - newValue = value; + var othIsDefined = other !== undefined, + othIsNull = other === null, + othIsReflexive = other === other, + othIsSymbol = isSymbol(other); - if (key === '__proto__' || key === 'constructor' || key === 'prototype') { - return object; + if ((!othIsNull && !othIsSymbol && !valIsSymbol && value > other) || + (valIsSymbol && othIsDefined && othIsReflexive && !othIsNull && !othIsSymbol) || + (valIsNull && othIsDefined && othIsReflexive) || + (!valIsDefined && othIsReflexive) || + !valIsReflexive) { + return 1; } - - if (index != lastIndex) { - var objValue = nested[key]; - newValue = customizer ? customizer(objValue, key, nested) : undefined; - if (newValue === undefined) { - newValue = isObject(objValue) - ? objValue - : (isIndex(path[index + 1]) ? [] : {}); - } + if ((!valIsNull && !valIsSymbol && !othIsSymbol && value < other) || + (othIsSymbol && valIsDefined && valIsReflexive && !valIsNull && !valIsSymbol) || + (othIsNull && valIsDefined && valIsReflexive) || + (!othIsDefined && valIsReflexive) || + !othIsReflexive) { + return -1; } - assignValue(nested, key, newValue); - nested = nested[key]; } - return object; + return 0; } /** - * The base implementation of `setData` without support for hot loop shorting. + * Used by `_.orderBy` to compare multiple properties of a value to another + * and stable sort them. + * + * If `orders` is unspecified, all values are sorted in ascending order. Otherwise, + * specify an order of "desc" for descending or "asc" for ascending sort order + * of corresponding values. * * @private - * @param {Function} func The function to associate metadata with. - * @param {*} data The metadata. - * @returns {Function} Returns `func`. + * @param {Object} object The object to compare. + * @param {Object} other The other object to compare. + * @param {boolean[]|string[]} orders The order to sort by for each property. + * @returns {number} Returns the sort order indicator for `object`. */ - var baseSetData = !metaMap ? identity : function(func, data) { - metaMap.set(func, data); - return func; - }; + function compareMultiple(object, other, orders) { + var index = -1, + objCriteria = object.criteria, + othCriteria = other.criteria, + length = objCriteria.length, + ordersLength = orders.length; + + while (++index < length) { + var result = compareAscending(objCriteria[index], othCriteria[index]); + if (result) { + if (index >= ordersLength) { + return result; + } + var order = orders[index]; + return result * (order == 'desc' ? -1 : 1); + } + } + // Fixes an `Array#sort` bug in the JS engine embedded in Adobe applications + // that causes it, under certain circumstances, to provide the same value for + // `object` and `other`. See https://github.com/jashkenas/underscore/pull/1247 + // for more details. + // + // This also ensures a stable sort in V8 and other engines. + // See https://bugs.chromium.org/p/v8/issues/detail?id=90 for more details. + return object.index - other.index; + } /** - * The base implementation of `setToString` without support for hot loop shorting. + * Creates an array that is the composition of partially applied arguments, + * placeholders, and provided arguments into a single array of arguments. * * @private - * @param {Function} func The function to modify. - * @param {Function} string The `toString` result. - * @returns {Function} Returns `func`. + * @param {Array} args The provided arguments. + * @param {Array} partials The arguments to prepend to those provided. + * @param {Array} holders The `partials` placeholder indexes. + * @params {boolean} [isCurried] Specify composing for a curried function. + * @returns {Array} Returns the new array of composed arguments. */ - var baseSetToString = !defineProperty ? identity : function(func, string) { - return defineProperty(func, 'toString', { - 'configurable': true, - 'enumerable': false, - 'value': constant(string), - 'writable': true - }); - }; + function composeArgs(args, partials, holders, isCurried) { + var argsIndex = -1, + argsLength = args.length, + holdersLength = holders.length, + leftIndex = -1, + leftLength = partials.length, + rangeLength = nativeMax(argsLength - holdersLength, 0), + result = Array(leftLength + rangeLength), + isUncurried = !isCurried; + + while (++leftIndex < leftLength) { + result[leftIndex] = partials[leftIndex]; + } + while (++argsIndex < holdersLength) { + if (isUncurried || argsIndex < argsLength) { + result[holders[argsIndex]] = args[argsIndex]; + } + } + while (rangeLength--) { + result[leftIndex++] = args[argsIndex++]; + } + return result; + } /** - * The base implementation of `_.shuffle`. + * This function is like `composeArgs` except that the arguments composition + * is tailored for `_.partialRight`. * * @private - * @param {Array|Object} collection The collection to shuffle. - * @returns {Array} Returns the new shuffled array. + * @param {Array} args The provided arguments. + * @param {Array} partials The arguments to append to those provided. + * @param {Array} holders The `partials` placeholder indexes. + * @params {boolean} [isCurried] Specify composing for a curried function. + * @returns {Array} Returns the new array of composed arguments. */ - function baseShuffle(collection) { - return shuffleSelf(values(collection)); + function composeArgsRight(args, partials, holders, isCurried) { + var argsIndex = -1, + argsLength = args.length, + holdersIndex = -1, + holdersLength = holders.length, + rightIndex = -1, + rightLength = partials.length, + rangeLength = nativeMax(argsLength - holdersLength, 0), + result = Array(rangeLength + rightLength), + isUncurried = !isCurried; + + while (++argsIndex < rangeLength) { + result[argsIndex] = args[argsIndex]; + } + var offset = argsIndex; + while (++rightIndex < rightLength) { + result[offset + rightIndex] = partials[rightIndex]; + } + while (++holdersIndex < holdersLength) { + if (isUncurried || argsIndex < argsLength) { + result[offset + holders[holdersIndex]] = args[argsIndex++]; + } + } + return result; } /** - * The base implementation of `_.slice` without an iteratee call guard. + * Copies the values of `source` to `array`. * * @private - * @param {Array} array The array to slice. - * @param {number} [start=0] The start position. - * @param {number} [end=array.length] The end position. - * @returns {Array} Returns the slice of `array`. + * @param {Array} source The array to copy values from. + * @param {Array} [array=[]] The array to copy values to. + * @returns {Array} Returns `array`. */ - function baseSlice(array, start, end) { + function copyArray(source, array) { var index = -1, - length = array.length; - - if (start < 0) { - start = -start > length ? 0 : (length + start); - } - end = end > length ? length : end; - if (end < 0) { - end += length; - } - length = start > end ? 0 : ((end - start) >>> 0); - start >>>= 0; + length = source.length; - var result = Array(length); + array || (array = Array(length)); while (++index < length) { - result[index] = array[index + start]; + array[index] = source[index]; } - return result; + return array; } /** - * The base implementation of `_.some` without support for iteratee shorthands. + * Copies properties of `source` to `object`. * * @private - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} predicate The function invoked per iteration. - * @returns {boolean} Returns `true` if any element passes the predicate check, - * else `false`. + * @param {Object} source The object to copy properties from. + * @param {Array} props The property identifiers to copy. + * @param {Object} [object={}] The object to copy properties to. + * @param {Function} [customizer] The function to customize copied values. + * @returns {Object} Returns `object`. */ - function baseSome(collection, predicate) { - var result; + function copyObject(source, props, object, customizer) { + var isNew = !object; + object || (object = {}); - baseEach(collection, function(value, index, collection) { - result = predicate(value, index, collection); - return !result; - }); - return !!result; + var index = -1, + length = props.length; + + while (++index < length) { + var key = props[index]; + + var newValue = customizer + ? customizer(object[key], source[key], key, object, source) + : undefined; + + if (newValue === undefined) { + newValue = source[key]; + } + if (isNew) { + baseAssignValue(object, key, newValue); + } else { + assignValue(object, key, newValue); + } + } + return object; } /** - * The base implementation of `_.sortedIndex` and `_.sortedLastIndex` which - * performs a binary search of `array` to determine the index at which `value` - * should be inserted into `array` in order to maintain its sort order. + * Copies own symbols of `source` to `object`. * * @private - * @param {Array} array The sorted array to inspect. - * @param {*} value The value to evaluate. - * @param {boolean} [retHighest] Specify returning the highest qualified index. - * @returns {number} Returns the index at which `value` should be inserted - * into `array`. + * @param {Object} source The object to copy symbols from. + * @param {Object} [object={}] The object to copy symbols to. + * @returns {Object} Returns `object`. */ - function baseSortedIndex(array, value, retHighest) { - var low = 0, - high = array == null ? low : array.length; + function copySymbols(source, object) { + return copyObject(source, getSymbols(source), object); + } - if (typeof value == 'number' && value === value && high <= HALF_MAX_ARRAY_LENGTH) { - while (low < high) { - var mid = (low + high) >>> 1, - computed = array[mid]; + /** + * Copies own and inherited symbols of `source` to `object`. + * + * @private + * @param {Object} source The object to copy symbols from. + * @param {Object} [object={}] The object to copy symbols to. + * @returns {Object} Returns `object`. + */ + function copySymbolsIn(source, object) { + return copyObject(source, getSymbolsIn(source), object); + } - if (computed !== null && !isSymbol(computed) && - (retHighest ? (computed <= value) : (computed < value))) { - low = mid + 1; - } else { - high = mid; - } - } - return high; - } - return baseSortedIndexBy(array, value, identity, retHighest); + /** + * Creates a function like `_.groupBy`. + * + * @private + * @param {Function} setter The function to set accumulator values. + * @param {Function} [initializer] The accumulator object initializer. + * @returns {Function} Returns the new aggregator function. + */ + function createAggregator(setter, initializer) { + return function(collection, iteratee) { + var func = isArray(collection) ? arrayAggregator : baseAggregator, + accumulator = initializer ? initializer() : {}; + + return func(collection, setter, getIteratee(iteratee, 2), accumulator); + }; } /** - * The base implementation of `_.sortedIndexBy` and `_.sortedLastIndexBy` - * which invokes `iteratee` for `value` and each element of `array` to compute - * their sort ranking. The iteratee is invoked with one argument; (value). + * Creates a function like `_.assign`. * * @private - * @param {Array} array The sorted array to inspect. - * @param {*} value The value to evaluate. - * @param {Function} iteratee The iteratee invoked per element. - * @param {boolean} [retHighest] Specify returning the highest qualified index. - * @returns {number} Returns the index at which `value` should be inserted - * into `array`. + * @param {Function} assigner The function to assign values. + * @returns {Function} Returns the new assigner function. */ - function baseSortedIndexBy(array, value, iteratee, retHighest) { - var low = 0, - high = array == null ? 0 : array.length; - if (high === 0) { - return 0; - } - - value = iteratee(value); - var valIsNaN = value !== value, - valIsNull = value === null, - valIsSymbol = isSymbol(value), - valIsUndefined = value === undefined; + function createAssigner(assigner) { + return baseRest(function(object, sources) { + var index = -1, + length = sources.length, + customizer = length > 1 ? sources[length - 1] : undefined, + guard = length > 2 ? sources[2] : undefined; - while (low < high) { - var mid = nativeFloor((low + high) / 2), - computed = iteratee(array[mid]), - othIsDefined = computed !== undefined, - othIsNull = computed === null, - othIsReflexive = computed === computed, - othIsSymbol = isSymbol(computed); + customizer = (assigner.length > 3 && typeof customizer == 'function') + ? (length--, customizer) + : undefined; - if (valIsNaN) { - var setLow = retHighest || othIsReflexive; - } else if (valIsUndefined) { - setLow = othIsReflexive && (retHighest || othIsDefined); - } else if (valIsNull) { - setLow = othIsReflexive && othIsDefined && (retHighest || !othIsNull); - } else if (valIsSymbol) { - setLow = othIsReflexive && othIsDefined && !othIsNull && (retHighest || !othIsSymbol); - } else if (othIsNull || othIsSymbol) { - setLow = false; - } else { - setLow = retHighest ? (computed <= value) : (computed < value); + if (guard && isIterateeCall(sources[0], sources[1], guard)) { + customizer = length < 3 ? undefined : customizer; + length = 1; } - if (setLow) { - low = mid + 1; - } else { - high = mid; + object = Object(object); + while (++index < length) { + var source = sources[index]; + if (source) { + assigner(object, source, index, customizer); + } } - } - return nativeMin(high, MAX_ARRAY_INDEX); + return object; + }); } /** - * The base implementation of `_.sortedUniq` and `_.sortedUniqBy` without - * support for iteratee shorthands. + * Creates a `baseEach` or `baseEachRight` function. * * @private - * @param {Array} array The array to inspect. - * @param {Function} [iteratee] The iteratee invoked per element. - * @returns {Array} Returns the new duplicate free array. + * @param {Function} eachFunc The function to iterate over a collection. + * @param {boolean} [fromRight] Specify iterating from right to left. + * @returns {Function} Returns the new base function. */ - function baseSortedUniq(array, iteratee) { - var index = -1, - length = array.length, - resIndex = 0, - result = []; - - while (++index < length) { - var value = array[index], - computed = iteratee ? iteratee(value) : value; + function createBaseEach(eachFunc, fromRight) { + return function(collection, iteratee) { + if (collection == null) { + return collection; + } + if (!isArrayLike(collection)) { + return eachFunc(collection, iteratee); + } + var length = collection.length, + index = fromRight ? length : -1, + iterable = Object(collection); - if (!index || !eq(computed, seen)) { - var seen = computed; - result[resIndex++] = value === 0 ? 0 : value; + while ((fromRight ? index-- : ++index < length)) { + if (iteratee(iterable[index], index, iterable) === false) { + break; + } } - } - return result; + return collection; + }; } /** - * The base implementation of `_.toNumber` which doesn't ensure correct - * conversions of binary, hexadecimal, or octal string values. + * Creates a base function for methods like `_.forIn` and `_.forOwn`. * * @private - * @param {*} value The value to process. - * @returns {number} Returns the number. + * @param {boolean} [fromRight] Specify iterating from right to left. + * @returns {Function} Returns the new base function. */ - function baseToNumber(value) { - if (typeof value == 'number') { - return value; - } - if (isSymbol(value)) { - return NAN; - } - return +value; + function createBaseFor(fromRight) { + return function(object, iteratee, keysFunc) { + var index = -1, + iterable = Object(object), + props = keysFunc(object), + length = props.length; + + while (length--) { + var key = props[fromRight ? length : ++index]; + if (iteratee(iterable[key], key, iterable) === false) { + break; + } + } + return object; + }; } /** - * The base implementation of `_.toString` which doesn't convert nullish - * values to empty strings. + * Creates a function that wraps `func` to invoke it with the optional `this` + * binding of `thisArg`. * * @private - * @param {*} value The value to process. - * @returns {string} Returns the string. + * @param {Function} func The function to wrap. + * @param {number} bitmask The bitmask flags. See `createWrap` for more details. + * @param {*} [thisArg] The `this` binding of `func`. + * @returns {Function} Returns the new wrapped function. */ - function baseToString(value) { - // Exit early for strings to avoid a performance hit in some environments. - if (typeof value == 'string') { - return value; - } - if (isArray(value)) { - // Recursively convert values (susceptible to call stack limits). - return arrayMap(value, baseToString) + ''; - } - if (isSymbol(value)) { - return symbolToString ? symbolToString.call(value) : ''; + function createBind(func, bitmask, thisArg) { + var isBind = bitmask & WRAP_BIND_FLAG, + Ctor = createCtor(func); + + function wrapper() { + var fn = (this && this !== root && this instanceof wrapper) ? Ctor : func; + return fn.apply(isBind ? thisArg : this, arguments); } - var result = (value + ''); - return (result == '0' && (1 / value) == -INFINITY) ? '-0' : result; + return wrapper; } /** - * The base implementation of `_.uniqBy` without support for iteratee shorthands. + * Creates a function like `_.lowerFirst`. * * @private - * @param {Array} array The array to inspect. - * @param {Function} [iteratee] The iteratee invoked per element. - * @param {Function} [comparator] The comparator invoked per element. - * @returns {Array} Returns the new duplicate free array. + * @param {string} methodName The name of the `String` case method to use. + * @returns {Function} Returns the new case function. */ - function baseUniq(array, iteratee, comparator) { - var index = -1, - includes = arrayIncludes, - length = array.length, - isCommon = true, - result = [], - seen = result; + function createCaseFirst(methodName) { + return function(string) { + string = toString(string); - if (comparator) { - isCommon = false; - includes = arrayIncludesWith; - } - else if (length >= LARGE_ARRAY_SIZE) { - var set = iteratee ? null : createSet(array); - if (set) { - return setToArray(set); - } - isCommon = false; - includes = cacheHas; - seen = new SetCache; - } - else { - seen = iteratee ? [] : result; - } - outer: - while (++index < length) { - var value = array[index], - computed = iteratee ? iteratee(value) : value; + var strSymbols = hasUnicode(string) + ? stringToArray(string) + : undefined; - value = (comparator || value !== 0) ? value : 0; - if (isCommon && computed === computed) { - var seenIndex = seen.length; - while (seenIndex--) { - if (seen[seenIndex] === computed) { - continue outer; - } - } - if (iteratee) { - seen.push(computed); - } - result.push(value); - } - else if (!includes(seen, computed, comparator)) { - if (seen !== result) { - seen.push(computed); - } - result.push(value); - } - } - return result; + var chr = strSymbols + ? strSymbols[0] + : string.charAt(0); + + var trailing = strSymbols + ? castSlice(strSymbols, 1).join('') + : string.slice(1); + + return chr[methodName]() + trailing; + }; } /** - * The base implementation of `_.unset`. + * Creates a function like `_.camelCase`. * * @private - * @param {Object} object The object to modify. - * @param {Array|string} path The property path to unset. - * @returns {boolean} Returns `true` if the property is deleted, else `false`. + * @param {Function} callback The function to combine each word. + * @returns {Function} Returns the new compounder function. */ - function baseUnset(object, path) { - path = castPath(path, object); - object = parent(object, path); - return object == null || delete object[toKey(last(path))]; + function createCompounder(callback) { + return function(string) { + return arrayReduce(words(deburr(string).replace(reApos, '')), callback, ''); + }; } /** - * The base implementation of `_.update`. + * Creates a function that produces an instance of `Ctor` regardless of + * whether it was invoked as part of a `new` expression or by `call` or `apply`. * * @private - * @param {Object} object The object to modify. - * @param {Array|string} path The path of the property to update. - * @param {Function} updater The function to produce the updated value. - * @param {Function} [customizer] The function to customize path creation. - * @returns {Object} Returns `object`. + * @param {Function} Ctor The constructor to wrap. + * @returns {Function} Returns the new wrapped function. */ - function baseUpdate(object, path, updater, customizer) { - return baseSet(object, path, updater(baseGet(object, path)), customizer); + function createCtor(Ctor) { + return function() { + // Use a `switch` statement to work with class constructors. See + // http://ecma-international.org/ecma-262/7.0/#sec-ecmascript-function-objects-call-thisargument-argumentslist + // for more details. + var args = arguments; + switch (args.length) { + case 0: return new Ctor; + case 1: return new Ctor(args[0]); + case 2: return new Ctor(args[0], args[1]); + case 3: return new Ctor(args[0], args[1], args[2]); + case 4: return new Ctor(args[0], args[1], args[2], args[3]); + case 5: return new Ctor(args[0], args[1], args[2], args[3], args[4]); + case 6: return new Ctor(args[0], args[1], args[2], args[3], args[4], args[5]); + case 7: return new Ctor(args[0], args[1], args[2], args[3], args[4], args[5], args[6]); + } + var thisBinding = baseCreate(Ctor.prototype), + result = Ctor.apply(thisBinding, args); + + // Mimic the constructor's `return` behavior. + // See https://es5.github.io/#x13.2.2 for more details. + return isObject(result) ? result : thisBinding; + }; } /** - * The base implementation of methods like `_.dropWhile` and `_.takeWhile` - * without support for iteratee shorthands. + * Creates a function that wraps `func` to enable currying. * * @private - * @param {Array} array The array to query. - * @param {Function} predicate The function invoked per iteration. - * @param {boolean} [isDrop] Specify dropping elements instead of taking them. - * @param {boolean} [fromRight] Specify iterating from right to left. - * @returns {Array} Returns the slice of `array`. + * @param {Function} func The function to wrap. + * @param {number} bitmask The bitmask flags. See `createWrap` for more details. + * @param {number} arity The arity of `func`. + * @returns {Function} Returns the new wrapped function. */ - function baseWhile(array, predicate, isDrop, fromRight) { - var length = array.length, - index = fromRight ? length : -1; + function createCurry(func, bitmask, arity) { + var Ctor = createCtor(func); - while ((fromRight ? index-- : ++index < length) && - predicate(array[index], index, array)) {} + function wrapper() { + var length = arguments.length, + args = Array(length), + index = length, + placeholder = getHolder(wrapper); - return isDrop - ? baseSlice(array, (fromRight ? 0 : index), (fromRight ? index + 1 : length)) - : baseSlice(array, (fromRight ? index + 1 : 0), (fromRight ? length : index)); + while (index--) { + args[index] = arguments[index]; + } + var holders = (length < 3 && args[0] !== placeholder && args[length - 1] !== placeholder) + ? [] + : replaceHolders(args, placeholder); + + length -= holders.length; + if (length < arity) { + return createRecurry( + func, bitmask, createHybrid, wrapper.placeholder, undefined, + args, holders, undefined, undefined, arity - length); + } + var fn = (this && this !== root && this instanceof wrapper) ? Ctor : func; + return apply(fn, this, args); + } + return wrapper; } /** - * The base implementation of `wrapperValue` which returns the result of - * performing a sequence of actions on the unwrapped `value`, where each - * successive action is supplied the return value of the previous. + * Creates a `_.find` or `_.findLast` function. * * @private - * @param {*} value The unwrapped value. - * @param {Array} actions Actions to perform to resolve the unwrapped value. - * @returns {*} Returns the resolved value. + * @param {Function} findIndexFunc The function to find the collection index. + * @returns {Function} Returns the new find function. */ - function baseWrapperValue(value, actions) { - var result = value; - if (result instanceof LazyWrapper) { - result = result.value(); - } - return arrayReduce(actions, function(result, action) { - return action.func.apply(action.thisArg, arrayPush([result], action.args)); - }, result); + function createFind(findIndexFunc) { + return function(collection, predicate, fromIndex) { + var iterable = Object(collection); + if (!isArrayLike(collection)) { + var iteratee = getIteratee(predicate, 3); + collection = keys(collection); + predicate = function(key) { return iteratee(iterable[key], key, iterable); }; + } + var index = findIndexFunc(collection, predicate, fromIndex); + return index > -1 ? iterable[iteratee ? collection[index] : index] : undefined; + }; } /** - * The base implementation of methods like `_.xor`, without support for - * iteratee shorthands, that accepts an array of arrays to inspect. + * Creates a `_.flow` or `_.flowRight` function. * * @private - * @param {Array} arrays The arrays to inspect. - * @param {Function} [iteratee] The iteratee invoked per element. - * @param {Function} [comparator] The comparator invoked per element. - * @returns {Array} Returns the new array of values. + * @param {boolean} [fromRight] Specify iterating from right to left. + * @returns {Function} Returns the new flow function. */ - function baseXor(arrays, iteratee, comparator) { - var length = arrays.length; - if (length < 2) { - return length ? baseUniq(arrays[0]) : []; - } - var index = -1, - result = Array(length); + function createFlow(fromRight) { + return flatRest(function(funcs) { + var length = funcs.length, + index = length, + prereq = LodashWrapper.prototype.thru; - while (++index < length) { - var array = arrays[index], - othIndex = -1; + if (fromRight) { + funcs.reverse(); + } + while (index--) { + var func = funcs[index]; + if (typeof func != 'function') { + throw new TypeError(FUNC_ERROR_TEXT); + } + if (prereq && !wrapper && getFuncName(func) == 'wrapper') { + var wrapper = new LodashWrapper([], true); + } + } + index = wrapper ? index : length; + while (++index < length) { + func = funcs[index]; - while (++othIndex < length) { - if (othIndex != index) { - result[index] = baseDifference(result[index] || array, arrays[othIndex], iteratee, comparator); + var funcName = getFuncName(func), + data = funcName == 'wrapper' ? getData(func) : undefined; + + if (data && isLaziable(data[0]) && + data[1] == (WRAP_ARY_FLAG | WRAP_CURRY_FLAG | WRAP_PARTIAL_FLAG | WRAP_REARG_FLAG) && + !data[4].length && data[9] == 1 + ) { + wrapper = wrapper[getFuncName(data[0])].apply(wrapper, data[3]); + } else { + wrapper = (func.length == 1 && isLaziable(func)) + ? wrapper[funcName]() + : wrapper.thru(func); } } - } - return baseUniq(baseFlatten(result, 1), iteratee, comparator); + return function() { + var args = arguments, + value = args[0]; + + if (wrapper && args.length == 1 && isArray(value)) { + return wrapper.plant(value).value(); + } + var index = 0, + result = length ? funcs[index].apply(this, args) : value; + + while (++index < length) { + result = funcs[index].call(this, result); + } + return result; + }; + }); } /** - * This base implementation of `_.zipObject` which assigns values using `assignFunc`. + * Creates a function that wraps `func` to invoke it with optional `this` + * binding of `thisArg`, partial application, and currying. * * @private - * @param {Array} props The property identifiers. - * @param {Array} values The property values. - * @param {Function} assignFunc The function to assign values. - * @returns {Object} Returns the new object. + * @param {Function|string} func The function or method name to wrap. + * @param {number} bitmask The bitmask flags. See `createWrap` for more details. + * @param {*} [thisArg] The `this` binding of `func`. + * @param {Array} [partials] The arguments to prepend to those provided to + * the new function. + * @param {Array} [holders] The `partials` placeholder indexes. + * @param {Array} [partialsRight] The arguments to append to those provided + * to the new function. + * @param {Array} [holdersRight] The `partialsRight` placeholder indexes. + * @param {Array} [argPos] The argument positions of the new function. + * @param {number} [ary] The arity cap of `func`. + * @param {number} [arity] The arity of `func`. + * @returns {Function} Returns the new wrapped function. */ - function baseZipObject(props, values, assignFunc) { - var index = -1, - length = props.length, - valsLength = values.length, - result = {}; + function createHybrid(func, bitmask, thisArg, partials, holders, partialsRight, holdersRight, argPos, ary, arity) { + var isAry = bitmask & WRAP_ARY_FLAG, + isBind = bitmask & WRAP_BIND_FLAG, + isBindKey = bitmask & WRAP_BIND_KEY_FLAG, + isCurried = bitmask & (WRAP_CURRY_FLAG | WRAP_CURRY_RIGHT_FLAG), + isFlip = bitmask & WRAP_FLIP_FLAG, + Ctor = isBindKey ? undefined : createCtor(func); - while (++index < length) { - var value = index < valsLength ? values[index] : undefined; - assignFunc(result, props[index], value); + function wrapper() { + var length = arguments.length, + args = Array(length), + index = length; + + while (index--) { + args[index] = arguments[index]; + } + if (isCurried) { + var placeholder = getHolder(wrapper), + holdersCount = countHolders(args, placeholder); + } + if (partials) { + args = composeArgs(args, partials, holders, isCurried); + } + if (partialsRight) { + args = composeArgsRight(args, partialsRight, holdersRight, isCurried); + } + length -= holdersCount; + if (isCurried && length < arity) { + var newHolders = replaceHolders(args, placeholder); + return createRecurry( + func, bitmask, createHybrid, wrapper.placeholder, thisArg, + args, newHolders, argPos, ary, arity - length + ); + } + var thisBinding = isBind ? thisArg : this, + fn = isBindKey ? thisBinding[func] : func; + + length = args.length; + if (argPos) { + args = reorder(args, argPos); + } else if (isFlip && length > 1) { + args.reverse(); + } + if (isAry && ary < length) { + args.length = ary; + } + if (this && this !== root && this instanceof wrapper) { + fn = Ctor || createCtor(fn); + } + return fn.apply(thisBinding, args); } - return result; + return wrapper; } /** - * Casts `value` to an empty array if it's not an array like object. + * Creates a function like `_.invertBy`. * * @private - * @param {*} value The value to inspect. - * @returns {Array|Object} Returns the cast array-like object. + * @param {Function} setter The function to set accumulator values. + * @param {Function} toIteratee The function to resolve iteratees. + * @returns {Function} Returns the new inverter function. */ - function castArrayLikeObject(value) { - return isArrayLikeObject(value) ? value : []; + function createInverter(setter, toIteratee) { + return function(object, iteratee) { + return baseInverter(object, setter, toIteratee(iteratee), {}); + }; } /** - * Casts `value` to `identity` if it's not a function. + * Creates a function that performs a mathematical operation on two values. * * @private - * @param {*} value The value to inspect. - * @returns {Function} Returns cast function. + * @param {Function} operator The function to perform the operation. + * @param {number} [defaultValue] The value used for `undefined` arguments. + * @returns {Function} Returns the new mathematical operation function. */ - function castFunction(value) { - return typeof value == 'function' ? value : identity; + function createMathOperation(operator, defaultValue) { + return function(value, other) { + var result; + if (value === undefined && other === undefined) { + return defaultValue; + } + if (value !== undefined) { + result = value; + } + if (other !== undefined) { + if (result === undefined) { + return other; + } + if (typeof value == 'string' || typeof other == 'string') { + value = baseToString(value); + other = baseToString(other); + } else { + value = baseToNumber(value); + other = baseToNumber(other); + } + result = operator(value, other); + } + return result; + }; } /** - * Casts `value` to a path array if it's not one. + * Creates a function like `_.over`. * * @private - * @param {*} value The value to inspect. - * @param {Object} [object] The object to query keys on. - * @returns {Array} Returns the cast property path array. + * @param {Function} arrayFunc The function to iterate over iteratees. + * @returns {Function} Returns the new over function. */ - function castPath(value, object) { - if (isArray(value)) { - return value; - } - return isKey(value, object) ? [value] : stringToPath(toString(value)); + function createOver(arrayFunc) { + return flatRest(function(iteratees) { + iteratees = arrayMap(iteratees, baseUnary(getIteratee())); + return baseRest(function(args) { + var thisArg = this; + return arrayFunc(iteratees, function(iteratee) { + return apply(iteratee, thisArg, args); + }); + }); + }); } /** - * A `baseRest` alias which can be replaced with `identity` by module - * replacement plugins. + * Creates the padding for `string` based on `length`. The `chars` string + * is truncated if the number of characters exceeds `length`. * * @private - * @type {Function} - * @param {Function} func The function to apply a rest parameter to. - * @returns {Function} Returns the new function. + * @param {number} length The padding length. + * @param {string} [chars=' '] The string used as padding. + * @returns {string} Returns the padding for `string`. */ - var castRest = baseRest; + function createPadding(length, chars) { + chars = chars === undefined ? ' ' : baseToString(chars); - /** - * Casts `array` to a slice if it's needed. - * - * @private - * @param {Array} array The array to inspect. - * @param {number} start The start position. - * @param {number} [end=array.length] The end position. - * @returns {Array} Returns the cast slice. - */ - function castSlice(array, start, end) { - var length = array.length; - end = end === undefined ? length : end; - return (!start && end >= length) ? array : baseSlice(array, start, end); + var charsLength = chars.length; + if (charsLength < 2) { + return charsLength ? baseRepeat(chars, length) : chars; + } + var result = baseRepeat(chars, nativeCeil(length / stringSize(chars))); + return hasUnicode(chars) + ? castSlice(stringToArray(result), 0, length).join('') + : result.slice(0, length); } /** - * A simple wrapper around the global [`clearTimeout`](https://mdn.io/clearTimeout). + * Creates a function that wraps `func` to invoke it with the `this` binding + * of `thisArg` and `partials` prepended to the arguments it receives. * * @private - * @param {number|Object} id The timer id or timeout object of the timer to clear. + * @param {Function} func The function to wrap. + * @param {number} bitmask The bitmask flags. See `createWrap` for more details. + * @param {*} thisArg The `this` binding of `func`. + * @param {Array} partials The arguments to prepend to those provided to + * the new function. + * @returns {Function} Returns the new wrapped function. */ - var clearTimeout = ctxClearTimeout || function(id) { - return root.clearTimeout(id); - }; + function createPartial(func, bitmask, thisArg, partials) { + var isBind = bitmask & WRAP_BIND_FLAG, + Ctor = createCtor(func); - /** - * Creates a clone of `buffer`. - * - * @private - * @param {Buffer} buffer The buffer to clone. - * @param {boolean} [isDeep] Specify a deep clone. - * @returns {Buffer} Returns the cloned buffer. - */ - function cloneBuffer(buffer, isDeep) { - if (isDeep) { - return buffer.slice(); - } - var length = buffer.length, - result = allocUnsafe ? allocUnsafe(length) : new buffer.constructor(length); + function wrapper() { + var argsIndex = -1, + argsLength = arguments.length, + leftIndex = -1, + leftLength = partials.length, + args = Array(leftLength + argsLength), + fn = (this && this !== root && this instanceof wrapper) ? Ctor : func; - buffer.copy(result); - return result; + while (++leftIndex < leftLength) { + args[leftIndex] = partials[leftIndex]; + } + while (argsLength--) { + args[leftIndex++] = arguments[++argsIndex]; + } + return apply(fn, isBind ? thisArg : this, args); + } + return wrapper; } /** - * Creates a clone of `arrayBuffer`. + * Creates a `_.range` or `_.rangeRight` function. * * @private - * @param {ArrayBuffer} arrayBuffer The array buffer to clone. - * @returns {ArrayBuffer} Returns the cloned array buffer. + * @param {boolean} [fromRight] Specify iterating from right to left. + * @returns {Function} Returns the new range function. */ - function cloneArrayBuffer(arrayBuffer) { - var result = new arrayBuffer.constructor(arrayBuffer.byteLength); - new Uint8Array(result).set(new Uint8Array(arrayBuffer)); - return result; + function createRange(fromRight) { + return function(start, end, step) { + if (step && typeof step != 'number' && isIterateeCall(start, end, step)) { + end = step = undefined; + } + // Ensure the sign of `-0` is preserved. + start = toFinite(start); + if (end === undefined) { + end = start; + start = 0; + } else { + end = toFinite(end); + } + step = step === undefined ? (start < end ? 1 : -1) : toFinite(step); + return baseRange(start, end, step, fromRight); + }; } /** - * Creates a clone of `dataView`. + * Creates a function that performs a relational operation on two values. * * @private - * @param {Object} dataView The data view to clone. - * @param {boolean} [isDeep] Specify a deep clone. - * @returns {Object} Returns the cloned data view. + * @param {Function} operator The function to perform the operation. + * @returns {Function} Returns the new relational operation function. */ - function cloneDataView(dataView, isDeep) { - var buffer = isDeep ? cloneArrayBuffer(dataView.buffer) : dataView.buffer; - return new dataView.constructor(buffer, dataView.byteOffset, dataView.byteLength); + function createRelationalOperation(operator) { + return function(value, other) { + if (!(typeof value == 'string' && typeof other == 'string')) { + value = toNumber(value); + other = toNumber(other); + } + return operator(value, other); + }; } /** - * Creates a clone of `regexp`. + * Creates a function that wraps `func` to continue currying. * * @private - * @param {Object} regexp The regexp to clone. - * @returns {Object} Returns the cloned regexp. + * @param {Function} func The function to wrap. + * @param {number} bitmask The bitmask flags. See `createWrap` for more details. + * @param {Function} wrapFunc The function to create the `func` wrapper. + * @param {*} placeholder The placeholder value. + * @param {*} [thisArg] The `this` binding of `func`. + * @param {Array} [partials] The arguments to prepend to those provided to + * the new function. + * @param {Array} [holders] The `partials` placeholder indexes. + * @param {Array} [argPos] The argument positions of the new function. + * @param {number} [ary] The arity cap of `func`. + * @param {number} [arity] The arity of `func`. + * @returns {Function} Returns the new wrapped function. */ - function cloneRegExp(regexp) { - var result = new regexp.constructor(regexp.source, reFlags.exec(regexp)); - result.lastIndex = regexp.lastIndex; - return result; + function createRecurry(func, bitmask, wrapFunc, placeholder, thisArg, partials, holders, argPos, ary, arity) { + var isCurry = bitmask & WRAP_CURRY_FLAG, + newHolders = isCurry ? holders : undefined, + newHoldersRight = isCurry ? undefined : holders, + newPartials = isCurry ? partials : undefined, + newPartialsRight = isCurry ? undefined : partials; + + bitmask |= (isCurry ? WRAP_PARTIAL_FLAG : WRAP_PARTIAL_RIGHT_FLAG); + bitmask &= ~(isCurry ? WRAP_PARTIAL_RIGHT_FLAG : WRAP_PARTIAL_FLAG); + + if (!(bitmask & WRAP_CURRY_BOUND_FLAG)) { + bitmask &= ~(WRAP_BIND_FLAG | WRAP_BIND_KEY_FLAG); + } + var newData = [ + func, bitmask, thisArg, newPartials, newHolders, newPartialsRight, + newHoldersRight, argPos, ary, arity + ]; + + var result = wrapFunc.apply(undefined, newData); + if (isLaziable(func)) { + setData(result, newData); + } + result.placeholder = placeholder; + return setWrapToString(result, func, bitmask); } /** - * Creates a clone of the `symbol` object. + * Creates a function like `_.round`. * * @private - * @param {Object} symbol The symbol object to clone. - * @returns {Object} Returns the cloned symbol object. + * @param {string} methodName The name of the `Math` method to use when rounding. + * @returns {Function} Returns the new round function. */ - function cloneSymbol(symbol) { - return symbolValueOf ? Object(symbolValueOf.call(symbol)) : {}; + function createRound(methodName) { + var func = Math[methodName]; + return function(number, precision) { + number = toNumber(number); + precision = precision == null ? 0 : nativeMin(toInteger(precision), 292); + if (precision && nativeIsFinite(number)) { + // Shift with exponential notation to avoid floating-point issues. + // See [MDN](https://mdn.io/round#Examples) for more details. + var pair = (toString(number) + 'e').split('e'), + value = func(pair[0] + 'e' + (+pair[1] + precision)); + + pair = (toString(value) + 'e').split('e'); + return +(pair[0] + 'e' + (+pair[1] - precision)); + } + return func(number); + }; } /** - * Creates a clone of `typedArray`. + * Creates a set object of `values`. * * @private - * @param {Object} typedArray The typed array to clone. - * @param {boolean} [isDeep] Specify a deep clone. - * @returns {Object} Returns the cloned typed array. + * @param {Array} values The values to add to the set. + * @returns {Object} Returns the new set. */ - function cloneTypedArray(typedArray, isDeep) { - var buffer = isDeep ? cloneArrayBuffer(typedArray.buffer) : typedArray.buffer; - return new typedArray.constructor(buffer, typedArray.byteOffset, typedArray.length); - } + var createSet = !(Set && (1 / setToArray(new Set([,-0]))[1]) == INFINITY) ? noop : function(values) { + return new Set(values); + }; /** - * Compares values to sort them in ascending order. + * Creates a `_.toPairs` or `_.toPairsIn` function. * * @private - * @param {*} value The value to compare. - * @param {*} other The other value to compare. - * @returns {number} Returns the sort order indicator for `value`. + * @param {Function} keysFunc The function to get the keys of a given object. + * @returns {Function} Returns the new pairs function. */ - function compareAscending(value, other) { - if (value !== other) { - var valIsDefined = value !== undefined, - valIsNull = value === null, - valIsReflexive = value === value, - valIsSymbol = isSymbol(value); - - var othIsDefined = other !== undefined, - othIsNull = other === null, - othIsReflexive = other === other, - othIsSymbol = isSymbol(other); - - if ((!othIsNull && !othIsSymbol && !valIsSymbol && value > other) || - (valIsSymbol && othIsDefined && othIsReflexive && !othIsNull && !othIsSymbol) || - (valIsNull && othIsDefined && othIsReflexive) || - (!valIsDefined && othIsReflexive) || - !valIsReflexive) { - return 1; + function createToPairs(keysFunc) { + return function(object) { + var tag = getTag(object); + if (tag == mapTag) { + return mapToArray(object); } - if ((!valIsNull && !valIsSymbol && !othIsSymbol && value < other) || - (othIsSymbol && valIsDefined && valIsReflexive && !valIsNull && !valIsSymbol) || - (othIsNull && valIsDefined && valIsReflexive) || - (!othIsDefined && valIsReflexive) || - !othIsReflexive) { - return -1; + if (tag == setTag) { + return setToPairs(object); } - } - return 0; + return baseToPairs(object, keysFunc(object)); + }; } /** - * Used by `_.orderBy` to compare multiple properties of a value to another - * and stable sort them. - * - * If `orders` is unspecified, all values are sorted in ascending order. Otherwise, - * specify an order of "desc" for descending or "asc" for ascending sort order - * of corresponding values. + * Creates a function that either curries or invokes `func` with optional + * `this` binding and partially applied arguments. * * @private - * @param {Object} object The object to compare. - * @param {Object} other The other object to compare. - * @param {boolean[]|string[]} orders The order to sort by for each property. - * @returns {number} Returns the sort order indicator for `object`. + * @param {Function|string} func The function or method name to wrap. + * @param {number} bitmask The bitmask flags. + * 1 - `_.bind` + * 2 - `_.bindKey` + * 4 - `_.curry` or `_.curryRight` of a bound function + * 8 - `_.curry` + * 16 - `_.curryRight` + * 32 - `_.partial` + * 64 - `_.partialRight` + * 128 - `_.rearg` + * 256 - `_.ary` + * 512 - `_.flip` + * @param {*} [thisArg] The `this` binding of `func`. + * @param {Array} [partials] The arguments to be partially applied. + * @param {Array} [holders] The `partials` placeholder indexes. + * @param {Array} [argPos] The argument positions of the new function. + * @param {number} [ary] The arity cap of `func`. + * @param {number} [arity] The arity of `func`. + * @returns {Function} Returns the new wrapped function. */ - function compareMultiple(object, other, orders) { - var index = -1, - objCriteria = object.criteria, - othCriteria = other.criteria, - length = objCriteria.length, - ordersLength = orders.length; + function createWrap(func, bitmask, thisArg, partials, holders, argPos, ary, arity) { + var isBindKey = bitmask & WRAP_BIND_KEY_FLAG; + if (!isBindKey && typeof func != 'function') { + throw new TypeError(FUNC_ERROR_TEXT); + } + var length = partials ? partials.length : 0; + if (!length) { + bitmask &= ~(WRAP_PARTIAL_FLAG | WRAP_PARTIAL_RIGHT_FLAG); + partials = holders = undefined; + } + ary = ary === undefined ? ary : nativeMax(toInteger(ary), 0); + arity = arity === undefined ? arity : toInteger(arity); + length -= holders ? holders.length : 0; - while (++index < length) { - var result = compareAscending(objCriteria[index], othCriteria[index]); - if (result) { - if (index >= ordersLength) { - return result; - } - var order = orders[index]; - return result * (order == 'desc' ? -1 : 1); - } + if (bitmask & WRAP_PARTIAL_RIGHT_FLAG) { + var partialsRight = partials, + holdersRight = holders; + + partials = holders = undefined; } - // Fixes an `Array#sort` bug in the JS engine embedded in Adobe applications - // that causes it, under certain circumstances, to provide the same value for - // `object` and `other`. See https://github.com/jashkenas/underscore/pull/1247 - // for more details. - // - // This also ensures a stable sort in V8 and other engines. - // See https://bugs.chromium.org/p/v8/issues/detail?id=90 for more details. - return object.index - other.index; - } + var data = isBindKey ? undefined : getData(func); - /** - * Creates an array that is the composition of partially applied arguments, - * placeholders, and provided arguments into a single array of arguments. - * - * @private - * @param {Array} args The provided arguments. - * @param {Array} partials The arguments to prepend to those provided. - * @param {Array} holders The `partials` placeholder indexes. - * @params {boolean} [isCurried] Specify composing for a curried function. - * @returns {Array} Returns the new array of composed arguments. - */ - function composeArgs(args, partials, holders, isCurried) { - var argsIndex = -1, - argsLength = args.length, - holdersLength = holders.length, - leftIndex = -1, - leftLength = partials.length, - rangeLength = nativeMax(argsLength - holdersLength, 0), - result = Array(leftLength + rangeLength), - isUncurried = !isCurried; + var newData = [ + func, bitmask, thisArg, partials, holders, partialsRight, holdersRight, + argPos, ary, arity + ]; - while (++leftIndex < leftLength) { - result[leftIndex] = partials[leftIndex]; + if (data) { + mergeData(newData, data); } - while (++argsIndex < holdersLength) { - if (isUncurried || argsIndex < argsLength) { - result[holders[argsIndex]] = args[argsIndex]; - } + func = newData[0]; + bitmask = newData[1]; + thisArg = newData[2]; + partials = newData[3]; + holders = newData[4]; + arity = newData[9] = newData[9] === undefined + ? (isBindKey ? 0 : func.length) + : nativeMax(newData[9] - length, 0); + + if (!arity && bitmask & (WRAP_CURRY_FLAG | WRAP_CURRY_RIGHT_FLAG)) { + bitmask &= ~(WRAP_CURRY_FLAG | WRAP_CURRY_RIGHT_FLAG); } - while (rangeLength--) { - result[leftIndex++] = args[argsIndex++]; + if (!bitmask || bitmask == WRAP_BIND_FLAG) { + var result = createBind(func, bitmask, thisArg); + } else if (bitmask == WRAP_CURRY_FLAG || bitmask == WRAP_CURRY_RIGHT_FLAG) { + result = createCurry(func, bitmask, arity); + } else if ((bitmask == WRAP_PARTIAL_FLAG || bitmask == (WRAP_BIND_FLAG | WRAP_PARTIAL_FLAG)) && !holders.length) { + result = createPartial(func, bitmask, thisArg, partials); + } else { + result = createHybrid.apply(undefined, newData); } - return result; + var setter = data ? baseSetData : setData; + return setWrapToString(setter(result, newData), func, bitmask); } /** - * This function is like `composeArgs` except that the arguments composition - * is tailored for `_.partialRight`. + * Used by `_.defaults` to customize its `_.assignIn` use to assign properties + * of source objects to the destination object for all destination properties + * that resolve to `undefined`. * * @private - * @param {Array} args The provided arguments. - * @param {Array} partials The arguments to append to those provided. - * @param {Array} holders The `partials` placeholder indexes. - * @params {boolean} [isCurried] Specify composing for a curried function. - * @returns {Array} Returns the new array of composed arguments. + * @param {*} objValue The destination value. + * @param {*} srcValue The source value. + * @param {string} key The key of the property to assign. + * @param {Object} object The parent object of `objValue`. + * @returns {*} Returns the value to assign. */ - function composeArgsRight(args, partials, holders, isCurried) { - var argsIndex = -1, - argsLength = args.length, - holdersIndex = -1, - holdersLength = holders.length, - rightIndex = -1, - rightLength = partials.length, - rangeLength = nativeMax(argsLength - holdersLength, 0), - result = Array(rangeLength + rightLength), - isUncurried = !isCurried; - - while (++argsIndex < rangeLength) { - result[argsIndex] = args[argsIndex]; - } - var offset = argsIndex; - while (++rightIndex < rightLength) { - result[offset + rightIndex] = partials[rightIndex]; + function customDefaultsAssignIn(objValue, srcValue, key, object) { + if (objValue === undefined || + (eq(objValue, objectProto[key]) && !hasOwnProperty.call(object, key))) { + return srcValue; } - while (++holdersIndex < holdersLength) { - if (isUncurried || argsIndex < argsLength) { - result[offset + holders[holdersIndex]] = args[argsIndex++]; - } + return objValue; + } + + /** + * Used by `_.defaultsDeep` to customize its `_.merge` use to merge source + * objects into destination objects that are passed thru. + * + * @private + * @param {*} objValue The destination value. + * @param {*} srcValue The source value. + * @param {string} key The key of the property to merge. + * @param {Object} object The parent object of `objValue`. + * @param {Object} source The parent object of `srcValue`. + * @param {Object} [stack] Tracks traversed source values and their merged + * counterparts. + * @returns {*} Returns the value to assign. + */ + function customDefaultsMerge(objValue, srcValue, key, object, source, stack) { + if (isObject(objValue) && isObject(srcValue)) { + // Recursively merge objects and arrays (susceptible to call stack limits). + stack.set(srcValue, objValue); + baseMerge(objValue, srcValue, undefined, customDefaultsMerge, stack); + stack['delete'](srcValue); } - return result; + return objValue; } /** - * Copies the values of `source` to `array`. + * Used by `_.omit` to customize its `_.cloneDeep` use to only clone plain + * objects. * * @private - * @param {Array} source The array to copy values from. - * @param {Array} [array=[]] The array to copy values to. - * @returns {Array} Returns `array`. + * @param {*} value The value to inspect. + * @param {string} key The key of the property to inspect. + * @returns {*} Returns the uncloned value or `undefined` to defer cloning to `_.cloneDeep`. */ - function copyArray(source, array) { - var index = -1, - length = source.length; - - array || (array = Array(length)); - while (++index < length) { - array[index] = source[index]; - } - return array; + function customOmitClone(value) { + return isPlainObject(value) ? undefined : value; } /** - * Copies properties of `source` to `object`. + * A specialized version of `baseIsEqualDeep` for arrays with support for + * partial deep comparisons. * * @private - * @param {Object} source The object to copy properties from. - * @param {Array} props The property identifiers to copy. - * @param {Object} [object={}] The object to copy properties to. - * @param {Function} [customizer] The function to customize copied values. - * @returns {Object} Returns `object`. + * @param {Array} array The array to compare. + * @param {Array} other The other array to compare. + * @param {number} bitmask The bitmask flags. See `baseIsEqual` for more details. + * @param {Function} customizer The function to customize comparisons. + * @param {Function} equalFunc The function to determine equivalents of values. + * @param {Object} stack Tracks traversed `array` and `other` objects. + * @returns {boolean} Returns `true` if the arrays are equivalent, else `false`. */ - function copyObject(source, props, object, customizer) { - var isNew = !object; - object || (object = {}); + function equalArrays(array, other, bitmask, customizer, equalFunc, stack) { + var isPartial = bitmask & COMPARE_PARTIAL_FLAG, + arrLength = array.length, + othLength = other.length; + if (arrLength != othLength && !(isPartial && othLength > arrLength)) { + return false; + } + // Check that cyclic values are equal. + var arrStacked = stack.get(array); + var othStacked = stack.get(other); + if (arrStacked && othStacked) { + return arrStacked == other && othStacked == array; + } var index = -1, - length = props.length; + result = true, + seen = (bitmask & COMPARE_UNORDERED_FLAG) ? new SetCache : undefined; - while (++index < length) { - var key = props[index]; + stack.set(array, other); + stack.set(other, array); - var newValue = customizer - ? customizer(object[key], source[key], key, object, source) - : undefined; + // Ignore non-index properties. + while (++index < arrLength) { + var arrValue = array[index], + othValue = other[index]; - if (newValue === undefined) { - newValue = source[key]; + if (customizer) { + var compared = isPartial + ? customizer(othValue, arrValue, index, other, array, stack) + : customizer(arrValue, othValue, index, array, other, stack); } - if (isNew) { - baseAssignValue(object, key, newValue); - } else { - assignValue(object, key, newValue); + if (compared !== undefined) { + if (compared) { + continue; + } + result = false; + break; + } + // Recursively compare arrays (susceptible to call stack limits). + if (seen) { + if (!arraySome(other, function(othValue, othIndex) { + if (!cacheHas(seen, othIndex) && + (arrValue === othValue || equalFunc(arrValue, othValue, bitmask, customizer, stack))) { + return seen.push(othIndex); + } + })) { + result = false; + break; + } + } else if (!( + arrValue === othValue || + equalFunc(arrValue, othValue, bitmask, customizer, stack) + )) { + result = false; + break; } } - return object; + stack['delete'](array); + stack['delete'](other); + return result; } /** - * Copies own symbols of `source` to `object`. + * A specialized version of `baseIsEqualDeep` for comparing objects of + * the same `toStringTag`. + * + * **Note:** This function only supports comparing values with tags of + * `Boolean`, `Date`, `Error`, `Number`, `RegExp`, or `String`. * * @private - * @param {Object} source The object to copy symbols from. - * @param {Object} [object={}] The object to copy symbols to. - * @returns {Object} Returns `object`. + * @param {Object} object The object to compare. + * @param {Object} other The other object to compare. + * @param {string} tag The `toStringTag` of the objects to compare. + * @param {number} bitmask The bitmask flags. See `baseIsEqual` for more details. + * @param {Function} customizer The function to customize comparisons. + * @param {Function} equalFunc The function to determine equivalents of values. + * @param {Object} stack Tracks traversed `object` and `other` objects. + * @returns {boolean} Returns `true` if the objects are equivalent, else `false`. */ - function copySymbols(source, object) { - return copyObject(source, getSymbols(source), object); + function equalByTag(object, other, tag, bitmask, customizer, equalFunc, stack) { + switch (tag) { + case dataViewTag: + if ((object.byteLength != other.byteLength) || + (object.byteOffset != other.byteOffset)) { + return false; + } + object = object.buffer; + other = other.buffer; + + case arrayBufferTag: + if ((object.byteLength != other.byteLength) || + !equalFunc(new Uint8Array(object), new Uint8Array(other))) { + return false; + } + return true; + + case boolTag: + case dateTag: + case numberTag: + // Coerce booleans to `1` or `0` and dates to milliseconds. + // Invalid dates are coerced to `NaN`. + return eq(+object, +other); + + case errorTag: + return object.name == other.name && object.message == other.message; + + case regexpTag: + case stringTag: + // Coerce regexes to strings and treat strings, primitives and objects, + // as equal. See http://www.ecma-international.org/ecma-262/7.0/#sec-regexp.prototype.tostring + // for more details. + return object == (other + ''); + + case mapTag: + var convert = mapToArray; + + case setTag: + var isPartial = bitmask & COMPARE_PARTIAL_FLAG; + convert || (convert = setToArray); + + if (object.size != other.size && !isPartial) { + return false; + } + // Assume cyclic values are equal. + var stacked = stack.get(object); + if (stacked) { + return stacked == other; + } + bitmask |= COMPARE_UNORDERED_FLAG; + + // Recursively compare objects (susceptible to call stack limits). + stack.set(object, other); + var result = equalArrays(convert(object), convert(other), bitmask, customizer, equalFunc, stack); + stack['delete'](object); + return result; + + case symbolTag: + if (symbolValueOf) { + return symbolValueOf.call(object) == symbolValueOf.call(other); + } + } + return false; } /** - * Copies own and inherited symbols of `source` to `object`. + * A specialized version of `baseIsEqualDeep` for objects with support for + * partial deep comparisons. * * @private - * @param {Object} source The object to copy symbols from. - * @param {Object} [object={}] The object to copy symbols to. - * @returns {Object} Returns `object`. + * @param {Object} object The object to compare. + * @param {Object} other The other object to compare. + * @param {number} bitmask The bitmask flags. See `baseIsEqual` for more details. + * @param {Function} customizer The function to customize comparisons. + * @param {Function} equalFunc The function to determine equivalents of values. + * @param {Object} stack Tracks traversed `object` and `other` objects. + * @returns {boolean} Returns `true` if the objects are equivalent, else `false`. */ - function copySymbolsIn(source, object) { - return copyObject(source, getSymbolsIn(source), object); + function equalObjects(object, other, bitmask, customizer, equalFunc, stack) { + var isPartial = bitmask & COMPARE_PARTIAL_FLAG, + objProps = getAllKeys(object), + objLength = objProps.length, + othProps = getAllKeys(other), + othLength = othProps.length; + + if (objLength != othLength && !isPartial) { + return false; + } + var index = objLength; + while (index--) { + var key = objProps[index]; + if (!(isPartial ? key in other : hasOwnProperty.call(other, key))) { + return false; + } + } + // Check that cyclic values are equal. + var objStacked = stack.get(object); + var othStacked = stack.get(other); + if (objStacked && othStacked) { + return objStacked == other && othStacked == object; + } + var result = true; + stack.set(object, other); + stack.set(other, object); + + var skipCtor = isPartial; + while (++index < objLength) { + key = objProps[index]; + var objValue = object[key], + othValue = other[key]; + + if (customizer) { + var compared = isPartial + ? customizer(othValue, objValue, key, other, object, stack) + : customizer(objValue, othValue, key, object, other, stack); + } + // Recursively compare objects (susceptible to call stack limits). + if (!(compared === undefined + ? (objValue === othValue || equalFunc(objValue, othValue, bitmask, customizer, stack)) + : compared + )) { + result = false; + break; + } + skipCtor || (skipCtor = key == 'constructor'); + } + if (result && !skipCtor) { + var objCtor = object.constructor, + othCtor = other.constructor; + + // Non `Object` object instances with different constructors are not equal. + if (objCtor != othCtor && + ('constructor' in object && 'constructor' in other) && + !(typeof objCtor == 'function' && objCtor instanceof objCtor && + typeof othCtor == 'function' && othCtor instanceof othCtor)) { + result = false; + } + } + stack['delete'](object); + stack['delete'](other); + return result; } /** - * Creates a function like `_.groupBy`. + * A specialized version of `baseRest` which flattens the rest array. * * @private - * @param {Function} setter The function to set accumulator values. - * @param {Function} [initializer] The accumulator object initializer. - * @returns {Function} Returns the new aggregator function. + * @param {Function} func The function to apply a rest parameter to. + * @returns {Function} Returns the new function. */ - function createAggregator(setter, initializer) { - return function(collection, iteratee) { - var func = isArray(collection) ? arrayAggregator : baseAggregator, - accumulator = initializer ? initializer() : {}; - - return func(collection, setter, getIteratee(iteratee, 2), accumulator); - }; + function flatRest(func) { + return setToString(overRest(func, undefined, flatten), func + ''); } /** - * Creates a function like `_.assign`. + * Creates an array of own enumerable property names and symbols of `object`. * * @private - * @param {Function} assigner The function to assign values. - * @returns {Function} Returns the new assigner function. + * @param {Object} object The object to query. + * @returns {Array} Returns the array of property names and symbols. */ - function createAssigner(assigner) { - return baseRest(function(object, sources) { - var index = -1, - length = sources.length, - customizer = length > 1 ? sources[length - 1] : undefined, - guard = length > 2 ? sources[2] : undefined; - - customizer = (assigner.length > 3 && typeof customizer == 'function') - ? (length--, customizer) - : undefined; - - if (guard && isIterateeCall(sources[0], sources[1], guard)) { - customizer = length < 3 ? undefined : customizer; - length = 1; - } - object = Object(object); - while (++index < length) { - var source = sources[index]; - if (source) { - assigner(object, source, index, customizer); - } - } - return object; - }); + function getAllKeys(object) { + return baseGetAllKeys(object, keys, getSymbols); } /** - * Creates a `baseEach` or `baseEachRight` function. + * Creates an array of own and inherited enumerable property names and + * symbols of `object`. * * @private - * @param {Function} eachFunc The function to iterate over a collection. - * @param {boolean} [fromRight] Specify iterating from right to left. - * @returns {Function} Returns the new base function. + * @param {Object} object The object to query. + * @returns {Array} Returns the array of property names and symbols. */ - function createBaseEach(eachFunc, fromRight) { - return function(collection, iteratee) { - if (collection == null) { - return collection; - } - if (!isArrayLike(collection)) { - return eachFunc(collection, iteratee); - } - var length = collection.length, - index = fromRight ? length : -1, - iterable = Object(collection); - - while ((fromRight ? index-- : ++index < length)) { - if (iteratee(iterable[index], index, iterable) === false) { - break; - } - } - return collection; - }; + function getAllKeysIn(object) { + return baseGetAllKeys(object, keysIn, getSymbolsIn); } /** - * Creates a base function for methods like `_.forIn` and `_.forOwn`. + * Gets metadata for `func`. * * @private - * @param {boolean} [fromRight] Specify iterating from right to left. - * @returns {Function} Returns the new base function. + * @param {Function} func The function to query. + * @returns {*} Returns the metadata for `func`. */ - function createBaseFor(fromRight) { - return function(object, iteratee, keysFunc) { - var index = -1, - iterable = Object(object), - props = keysFunc(object), - length = props.length; - - while (length--) { - var key = props[fromRight ? length : ++index]; - if (iteratee(iterable[key], key, iterable) === false) { - break; - } - } - return object; - }; - } + var getData = !metaMap ? noop : function(func) { + return metaMap.get(func); + }; /** - * Creates a function that wraps `func` to invoke it with the optional `this` - * binding of `thisArg`. + * Gets the name of `func`. * * @private - * @param {Function} func The function to wrap. - * @param {number} bitmask The bitmask flags. See `createWrap` for more details. - * @param {*} [thisArg] The `this` binding of `func`. - * @returns {Function} Returns the new wrapped function. + * @param {Function} func The function to query. + * @returns {string} Returns the function name. */ - function createBind(func, bitmask, thisArg) { - var isBind = bitmask & WRAP_BIND_FLAG, - Ctor = createCtor(func); + function getFuncName(func) { + var result = (func.name + ''), + array = realNames[result], + length = hasOwnProperty.call(realNames, result) ? array.length : 0; - function wrapper() { - var fn = (this && this !== root && this instanceof wrapper) ? Ctor : func; - return fn.apply(isBind ? thisArg : this, arguments); + while (length--) { + var data = array[length], + otherFunc = data.func; + if (otherFunc == null || otherFunc == func) { + return data.name; + } } - return wrapper; + return result; } /** - * Creates a function like `_.lowerFirst`. + * Gets the argument placeholder value for `func`. * * @private - * @param {string} methodName The name of the `String` case method to use. - * @returns {Function} Returns the new case function. + * @param {Function} func The function to inspect. + * @returns {*} Returns the placeholder value. */ - function createCaseFirst(methodName) { - return function(string) { - string = toString(string); - - var strSymbols = hasUnicode(string) - ? stringToArray(string) - : undefined; - - var chr = strSymbols - ? strSymbols[0] - : string.charAt(0); - - var trailing = strSymbols - ? castSlice(strSymbols, 1).join('') - : string.slice(1); - - return chr[methodName]() + trailing; - }; + function getHolder(func) { + var object = hasOwnProperty.call(lodash, 'placeholder') ? lodash : func; + return object.placeholder; } /** - * Creates a function like `_.camelCase`. + * Gets the appropriate "iteratee" function. If `_.iteratee` is customized, + * this function returns the custom method, otherwise it returns `baseIteratee`. + * If arguments are provided, the chosen function is invoked with them and + * its result is returned. * * @private - * @param {Function} callback The function to combine each word. - * @returns {Function} Returns the new compounder function. + * @param {*} [value] The value to convert to an iteratee. + * @param {number} [arity] The arity of the created iteratee. + * @returns {Function} Returns the chosen function or its result. */ - function createCompounder(callback) { - return function(string) { - return arrayReduce(words(deburr(string).replace(reApos, '')), callback, ''); - }; + function getIteratee() { + var result = lodash.iteratee || iteratee; + result = result === iteratee ? baseIteratee : result; + return arguments.length ? result(arguments[0], arguments[1]) : result; } /** - * Creates a function that produces an instance of `Ctor` regardless of - * whether it was invoked as part of a `new` expression or by `call` or `apply`. + * Gets the data for `map`. * * @private - * @param {Function} Ctor The constructor to wrap. - * @returns {Function} Returns the new wrapped function. + * @param {Object} map The map to query. + * @param {string} key The reference key. + * @returns {*} Returns the map data. */ - function createCtor(Ctor) { - return function() { - // Use a `switch` statement to work with class constructors. See - // http://ecma-international.org/ecma-262/7.0/#sec-ecmascript-function-objects-call-thisargument-argumentslist - // for more details. - var args = arguments; - switch (args.length) { - case 0: return new Ctor; - case 1: return new Ctor(args[0]); - case 2: return new Ctor(args[0], args[1]); - case 3: return new Ctor(args[0], args[1], args[2]); - case 4: return new Ctor(args[0], args[1], args[2], args[3]); - case 5: return new Ctor(args[0], args[1], args[2], args[3], args[4]); - case 6: return new Ctor(args[0], args[1], args[2], args[3], args[4], args[5]); - case 7: return new Ctor(args[0], args[1], args[2], args[3], args[4], args[5], args[6]); - } - var thisBinding = baseCreate(Ctor.prototype), - result = Ctor.apply(thisBinding, args); - - // Mimic the constructor's `return` behavior. - // See https://es5.github.io/#x13.2.2 for more details. - return isObject(result) ? result : thisBinding; - }; + function getMapData(map, key) { + var data = map.__data__; + return isKeyable(key) + ? data[typeof key == 'string' ? 'string' : 'hash'] + : data.map; } /** - * Creates a function that wraps `func` to enable currying. + * Gets the property names, values, and compare flags of `object`. * * @private - * @param {Function} func The function to wrap. - * @param {number} bitmask The bitmask flags. See `createWrap` for more details. - * @param {number} arity The arity of `func`. - * @returns {Function} Returns the new wrapped function. + * @param {Object} object The object to query. + * @returns {Array} Returns the match data of `object`. */ - function createCurry(func, bitmask, arity) { - var Ctor = createCtor(func); - - function wrapper() { - var length = arguments.length, - args = Array(length), - index = length, - placeholder = getHolder(wrapper); + function getMatchData(object) { + var result = keys(object), + length = result.length; - while (index--) { - args[index] = arguments[index]; - } - var holders = (length < 3 && args[0] !== placeholder && args[length - 1] !== placeholder) - ? [] - : replaceHolders(args, placeholder); + while (length--) { + var key = result[length], + value = object[key]; - length -= holders.length; - if (length < arity) { - return createRecurry( - func, bitmask, createHybrid, wrapper.placeholder, undefined, - args, holders, undefined, undefined, arity - length); - } - var fn = (this && this !== root && this instanceof wrapper) ? Ctor : func; - return apply(fn, this, args); + result[length] = [key, value, isStrictComparable(value)]; } - return wrapper; + return result; } /** - * Creates a `_.find` or `_.findLast` function. + * Gets the native function at `key` of `object`. * * @private - * @param {Function} findIndexFunc The function to find the collection index. - * @returns {Function} Returns the new find function. + * @param {Object} object The object to query. + * @param {string} key The key of the method to get. + * @returns {*} Returns the function if it's native, else `undefined`. */ - function createFind(findIndexFunc) { - return function(collection, predicate, fromIndex) { - var iterable = Object(collection); - if (!isArrayLike(collection)) { - var iteratee = getIteratee(predicate, 3); - collection = keys(collection); - predicate = function(key) { return iteratee(iterable[key], key, iterable); }; - } - var index = findIndexFunc(collection, predicate, fromIndex); - return index > -1 ? iterable[iteratee ? collection[index] : index] : undefined; - }; + function getNative(object, key) { + var value = getValue(object, key); + return baseIsNative(value) ? value : undefined; } /** - * Creates a `_.flow` or `_.flowRight` function. + * A specialized version of `baseGetTag` which ignores `Symbol.toStringTag` values. * * @private - * @param {boolean} [fromRight] Specify iterating from right to left. - * @returns {Function} Returns the new flow function. + * @param {*} value The value to query. + * @returns {string} Returns the raw `toStringTag`. */ - function createFlow(fromRight) { - return flatRest(function(funcs) { - var length = funcs.length, - index = length, - prereq = LodashWrapper.prototype.thru; - - if (fromRight) { - funcs.reverse(); - } - while (index--) { - var func = funcs[index]; - if (typeof func != 'function') { - throw new TypeError(FUNC_ERROR_TEXT); - } - if (prereq && !wrapper && getFuncName(func) == 'wrapper') { - var wrapper = new LodashWrapper([], true); - } - } - index = wrapper ? index : length; - while (++index < length) { - func = funcs[index]; + function getRawTag(value) { + var isOwn = hasOwnProperty.call(value, symToStringTag), + tag = value[symToStringTag]; - var funcName = getFuncName(func), - data = funcName == 'wrapper' ? getData(func) : undefined; + try { + value[symToStringTag] = undefined; + var unmasked = true; + } catch (e) {} - if (data && isLaziable(data[0]) && - data[1] == (WRAP_ARY_FLAG | WRAP_CURRY_FLAG | WRAP_PARTIAL_FLAG | WRAP_REARG_FLAG) && - !data[4].length && data[9] == 1 - ) { - wrapper = wrapper[getFuncName(data[0])].apply(wrapper, data[3]); - } else { - wrapper = (func.length == 1 && isLaziable(func)) - ? wrapper[funcName]() - : wrapper.thru(func); - } + var result = nativeObjectToString.call(value); + if (unmasked) { + if (isOwn) { + value[symToStringTag] = tag; + } else { + delete value[symToStringTag]; } - return function() { - var args = arguments, - value = args[0]; - - if (wrapper && args.length == 1 && isArray(value)) { - return wrapper.plant(value).value(); - } - var index = 0, - result = length ? funcs[index].apply(this, args) : value; - - while (++index < length) { - result = funcs[index].call(this, result); - } - return result; - }; - }); + } + return result; } /** - * Creates a function that wraps `func` to invoke it with optional `this` - * binding of `thisArg`, partial application, and currying. + * Creates an array of the own enumerable symbols of `object`. * * @private - * @param {Function|string} func The function or method name to wrap. - * @param {number} bitmask The bitmask flags. See `createWrap` for more details. - * @param {*} [thisArg] The `this` binding of `func`. - * @param {Array} [partials] The arguments to prepend to those provided to - * the new function. - * @param {Array} [holders] The `partials` placeholder indexes. - * @param {Array} [partialsRight] The arguments to append to those provided - * to the new function. - * @param {Array} [holdersRight] The `partialsRight` placeholder indexes. - * @param {Array} [argPos] The argument positions of the new function. - * @param {number} [ary] The arity cap of `func`. - * @param {number} [arity] The arity of `func`. - * @returns {Function} Returns the new wrapped function. + * @param {Object} object The object to query. + * @returns {Array} Returns the array of symbols. */ - function createHybrid(func, bitmask, thisArg, partials, holders, partialsRight, holdersRight, argPos, ary, arity) { - var isAry = bitmask & WRAP_ARY_FLAG, - isBind = bitmask & WRAP_BIND_FLAG, - isBindKey = bitmask & WRAP_BIND_KEY_FLAG, - isCurried = bitmask & (WRAP_CURRY_FLAG | WRAP_CURRY_RIGHT_FLAG), - isFlip = bitmask & WRAP_FLIP_FLAG, - Ctor = isBindKey ? undefined : createCtor(func); - - function wrapper() { - var length = arguments.length, - args = Array(length), - index = length; - - while (index--) { - args[index] = arguments[index]; - } - if (isCurried) { - var placeholder = getHolder(wrapper), - holdersCount = countHolders(args, placeholder); - } - if (partials) { - args = composeArgs(args, partials, holders, isCurried); - } - if (partialsRight) { - args = composeArgsRight(args, partialsRight, holdersRight, isCurried); - } - length -= holdersCount; - if (isCurried && length < arity) { - var newHolders = replaceHolders(args, placeholder); - return createRecurry( - func, bitmask, createHybrid, wrapper.placeholder, thisArg, - args, newHolders, argPos, ary, arity - length - ); - } - var thisBinding = isBind ? thisArg : this, - fn = isBindKey ? thisBinding[func] : func; - - length = args.length; - if (argPos) { - args = reorder(args, argPos); - } else if (isFlip && length > 1) { - args.reverse(); - } - if (isAry && ary < length) { - args.length = ary; - } - if (this && this !== root && this instanceof wrapper) { - fn = Ctor || createCtor(fn); - } - return fn.apply(thisBinding, args); + var getSymbols = !nativeGetSymbols ? stubArray : function(object) { + if (object == null) { + return []; } - return wrapper; - } + object = Object(object); + return arrayFilter(nativeGetSymbols(object), function(symbol) { + return propertyIsEnumerable.call(object, symbol); + }); + }; /** - * Creates a function like `_.invertBy`. + * Creates an array of the own and inherited enumerable symbols of `object`. * * @private - * @param {Function} setter The function to set accumulator values. - * @param {Function} toIteratee The function to resolve iteratees. - * @returns {Function} Returns the new inverter function. + * @param {Object} object The object to query. + * @returns {Array} Returns the array of symbols. */ - function createInverter(setter, toIteratee) { - return function(object, iteratee) { - return baseInverter(object, setter, toIteratee(iteratee), {}); - }; - } + var getSymbolsIn = !nativeGetSymbols ? stubArray : function(object) { + var result = []; + while (object) { + arrayPush(result, getSymbols(object)); + object = getPrototype(object); + } + return result; + }; /** - * Creates a function that performs a mathematical operation on two values. + * Gets the `toStringTag` of `value`. * * @private - * @param {Function} operator The function to perform the operation. - * @param {number} [defaultValue] The value used for `undefined` arguments. - * @returns {Function} Returns the new mathematical operation function. + * @param {*} value The value to query. + * @returns {string} Returns the `toStringTag`. */ - function createMathOperation(operator, defaultValue) { - return function(value, other) { - var result; - if (value === undefined && other === undefined) { - return defaultValue; - } - if (value !== undefined) { - result = value; - } - if (other !== undefined) { - if (result === undefined) { - return other; - } - if (typeof value == 'string' || typeof other == 'string') { - value = baseToString(value); - other = baseToString(other); - } else { - value = baseToNumber(value); - other = baseToNumber(other); + var getTag = baseGetTag; + + // Fallback for data views, maps, sets, and weak maps in IE 11 and promises in Node.js < 6. + if ((DataView && getTag(new DataView(new ArrayBuffer(1))) != dataViewTag) || + (Map && getTag(new Map) != mapTag) || + (Promise && getTag(Promise.resolve()) != promiseTag) || + (Set && getTag(new Set) != setTag) || + (WeakMap && getTag(new WeakMap) != weakMapTag)) { + getTag = function(value) { + var result = baseGetTag(value), + Ctor = result == objectTag ? value.constructor : undefined, + ctorString = Ctor ? toSource(Ctor) : ''; + + if (ctorString) { + switch (ctorString) { + case dataViewCtorString: return dataViewTag; + case mapCtorString: return mapTag; + case promiseCtorString: return promiseTag; + case setCtorString: return setTag; + case weakMapCtorString: return weakMapTag; } - result = operator(value, other); } return result; }; } /** - * Creates a function like `_.over`. + * Gets the view, applying any `transforms` to the `start` and `end` positions. * * @private - * @param {Function} arrayFunc The function to iterate over iteratees. - * @returns {Function} Returns the new over function. + * @param {number} start The start of the view. + * @param {number} end The end of the view. + * @param {Array} transforms The transformations to apply to the view. + * @returns {Object} Returns an object containing the `start` and `end` + * positions of the view. */ - function createOver(arrayFunc) { - return flatRest(function(iteratees) { - iteratees = arrayMap(iteratees, baseUnary(getIteratee())); - return baseRest(function(args) { - var thisArg = this; - return arrayFunc(iteratees, function(iteratee) { - return apply(iteratee, thisArg, args); - }); - }); - }); + function getView(start, end, transforms) { + var index = -1, + length = transforms.length; + + while (++index < length) { + var data = transforms[index], + size = data.size; + + switch (data.type) { + case 'drop': start += size; break; + case 'dropRight': end -= size; break; + case 'take': end = nativeMin(end, start + size); break; + case 'takeRight': start = nativeMax(start, end - size); break; + } + } + return { 'start': start, 'end': end }; } /** - * Creates the padding for `string` based on `length`. The `chars` string - * is truncated if the number of characters exceeds `length`. + * Extracts wrapper details from the `source` body comment. * * @private - * @param {number} length The padding length. - * @param {string} [chars=' '] The string used as padding. - * @returns {string} Returns the padding for `string`. + * @param {string} source The source to inspect. + * @returns {Array} Returns the wrapper details. */ - function createPadding(length, chars) { - chars = chars === undefined ? ' ' : baseToString(chars); - - var charsLength = chars.length; - if (charsLength < 2) { - return charsLength ? baseRepeat(chars, length) : chars; - } - var result = baseRepeat(chars, nativeCeil(length / stringSize(chars))); - return hasUnicode(chars) - ? castSlice(stringToArray(result), 0, length).join('') - : result.slice(0, length); + function getWrapDetails(source) { + var match = source.match(reWrapDetails); + return match ? match[1].split(reSplitDetails) : []; } /** - * Creates a function that wraps `func` to invoke it with the `this` binding - * of `thisArg` and `partials` prepended to the arguments it receives. + * Checks if `path` exists on `object`. * * @private - * @param {Function} func The function to wrap. - * @param {number} bitmask The bitmask flags. See `createWrap` for more details. - * @param {*} thisArg The `this` binding of `func`. - * @param {Array} partials The arguments to prepend to those provided to - * the new function. - * @returns {Function} Returns the new wrapped function. + * @param {Object} object The object to query. + * @param {Array|string} path The path to check. + * @param {Function} hasFunc The function to check properties. + * @returns {boolean} Returns `true` if `path` exists, else `false`. */ - function createPartial(func, bitmask, thisArg, partials) { - var isBind = bitmask & WRAP_BIND_FLAG, - Ctor = createCtor(func); + function hasPath(object, path, hasFunc) { + path = castPath(path, object); - function wrapper() { - var argsIndex = -1, - argsLength = arguments.length, - leftIndex = -1, - leftLength = partials.length, - args = Array(leftLength + argsLength), - fn = (this && this !== root && this instanceof wrapper) ? Ctor : func; + var index = -1, + length = path.length, + result = false; - while (++leftIndex < leftLength) { - args[leftIndex] = partials[leftIndex]; - } - while (argsLength--) { - args[leftIndex++] = arguments[++argsIndex]; + while (++index < length) { + var key = toKey(path[index]); + if (!(result = object != null && hasFunc(object, key))) { + break; } - return apply(fn, isBind ? thisArg : this, args); + object = object[key]; } - return wrapper; + if (result || ++index != length) { + return result; + } + length = object == null ? 0 : object.length; + return !!length && isLength(length) && isIndex(key, length) && + (isArray(object) || isArguments(object)); } /** - * Creates a `_.range` or `_.rangeRight` function. + * Initializes an array clone. * * @private - * @param {boolean} [fromRight] Specify iterating from right to left. - * @returns {Function} Returns the new range function. + * @param {Array} array The array to clone. + * @returns {Array} Returns the initialized clone. */ - function createRange(fromRight) { - return function(start, end, step) { - if (step && typeof step != 'number' && isIterateeCall(start, end, step)) { - end = step = undefined; - } - // Ensure the sign of `-0` is preserved. - start = toFinite(start); - if (end === undefined) { - end = start; - start = 0; - } else { - end = toFinite(end); - } - step = step === undefined ? (start < end ? 1 : -1) : toFinite(step); - return baseRange(start, end, step, fromRight); - }; + function initCloneArray(array) { + var length = array.length, + result = new array.constructor(length); + + // Add properties assigned by `RegExp#exec`. + if (length && typeof array[0] == 'string' && hasOwnProperty.call(array, 'index')) { + result.index = array.index; + result.input = array.input; + } + return result; } /** - * Creates a function that performs a relational operation on two values. + * Initializes an object clone. * * @private - * @param {Function} operator The function to perform the operation. - * @returns {Function} Returns the new relational operation function. + * @param {Object} object The object to clone. + * @returns {Object} Returns the initialized clone. */ - function createRelationalOperation(operator) { - return function(value, other) { - if (!(typeof value == 'string' && typeof other == 'string')) { - value = toNumber(value); - other = toNumber(other); - } - return operator(value, other); - }; + function initCloneObject(object) { + return (typeof object.constructor == 'function' && !isPrototype(object)) + ? baseCreate(getPrototype(object)) + : {}; } /** - * Creates a function that wraps `func` to continue currying. + * Initializes an object clone based on its `toStringTag`. + * + * **Note:** This function only supports cloning values with tags of + * `Boolean`, `Date`, `Error`, `Map`, `Number`, `RegExp`, `Set`, or `String`. * * @private - * @param {Function} func The function to wrap. - * @param {number} bitmask The bitmask flags. See `createWrap` for more details. - * @param {Function} wrapFunc The function to create the `func` wrapper. - * @param {*} placeholder The placeholder value. - * @param {*} [thisArg] The `this` binding of `func`. - * @param {Array} [partials] The arguments to prepend to those provided to - * the new function. - * @param {Array} [holders] The `partials` placeholder indexes. - * @param {Array} [argPos] The argument positions of the new function. - * @param {number} [ary] The arity cap of `func`. - * @param {number} [arity] The arity of `func`. - * @returns {Function} Returns the new wrapped function. + * @param {Object} object The object to clone. + * @param {string} tag The `toStringTag` of the object to clone. + * @param {boolean} [isDeep] Specify a deep clone. + * @returns {Object} Returns the initialized clone. */ - function createRecurry(func, bitmask, wrapFunc, placeholder, thisArg, partials, holders, argPos, ary, arity) { - var isCurry = bitmask & WRAP_CURRY_FLAG, - newHolders = isCurry ? holders : undefined, - newHoldersRight = isCurry ? undefined : holders, - newPartials = isCurry ? partials : undefined, - newPartialsRight = isCurry ? undefined : partials; + function initCloneByTag(object, tag, isDeep) { + var Ctor = object.constructor; + switch (tag) { + case arrayBufferTag: + return cloneArrayBuffer(object); - bitmask |= (isCurry ? WRAP_PARTIAL_FLAG : WRAP_PARTIAL_RIGHT_FLAG); - bitmask &= ~(isCurry ? WRAP_PARTIAL_RIGHT_FLAG : WRAP_PARTIAL_FLAG); + case boolTag: + case dateTag: + return new Ctor(+object); - if (!(bitmask & WRAP_CURRY_BOUND_FLAG)) { - bitmask &= ~(WRAP_BIND_FLAG | WRAP_BIND_KEY_FLAG); - } - var newData = [ - func, bitmask, thisArg, newPartials, newHolders, newPartialsRight, - newHoldersRight, argPos, ary, arity - ]; + case dataViewTag: + return cloneDataView(object, isDeep); - var result = wrapFunc.apply(undefined, newData); - if (isLaziable(func)) { - setData(result, newData); - } - result.placeholder = placeholder; - return setWrapToString(result, func, bitmask); - } + case float32Tag: case float64Tag: + case int8Tag: case int16Tag: case int32Tag: + case uint8Tag: case uint8ClampedTag: case uint16Tag: case uint32Tag: + return cloneTypedArray(object, isDeep); - /** - * Creates a function like `_.round`. - * - * @private - * @param {string} methodName The name of the `Math` method to use when rounding. - * @returns {Function} Returns the new round function. - */ - function createRound(methodName) { - var func = Math[methodName]; - return function(number, precision) { - number = toNumber(number); - precision = precision == null ? 0 : nativeMin(toInteger(precision), 292); - if (precision && nativeIsFinite(number)) { - // Shift with exponential notation to avoid floating-point issues. - // See [MDN](https://mdn.io/round#Examples) for more details. - var pair = (toString(number) + 'e').split('e'), - value = func(pair[0] + 'e' + (+pair[1] + precision)); + case mapTag: + return new Ctor; - pair = (toString(value) + 'e').split('e'); - return +(pair[0] + 'e' + (+pair[1] - precision)); - } - return func(number); - }; + case numberTag: + case stringTag: + return new Ctor(object); + + case regexpTag: + return cloneRegExp(object); + + case setTag: + return new Ctor; + + case symbolTag: + return cloneSymbol(object); + } } /** - * Creates a set object of `values`. + * Inserts wrapper `details` in a comment at the top of the `source` body. * * @private - * @param {Array} values The values to add to the set. - * @returns {Object} Returns the new set. + * @param {string} source The source to modify. + * @returns {Array} details The details to insert. + * @returns {string} Returns the modified source. */ - var createSet = !(Set && (1 / setToArray(new Set([,-0]))[1]) == INFINITY) ? noop : function(values) { - return new Set(values); - }; + function insertWrapDetails(source, details) { + var length = details.length; + if (!length) { + return source; + } + var lastIndex = length - 1; + details[lastIndex] = (length > 1 ? '& ' : '') + details[lastIndex]; + details = details.join(length > 2 ? ', ' : ' '); + return source.replace(reWrapComment, '{\n/* [wrapped with ' + details + '] */\n'); + } /** - * Creates a `_.toPairs` or `_.toPairsIn` function. + * Checks if `value` is a flattenable `arguments` object or array. * * @private - * @param {Function} keysFunc The function to get the keys of a given object. - * @returns {Function} Returns the new pairs function. + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is flattenable, else `false`. */ - function createToPairs(keysFunc) { - return function(object) { - var tag = getTag(object); - if (tag == mapTag) { - return mapToArray(object); - } - if (tag == setTag) { - return setToPairs(object); - } - return baseToPairs(object, keysFunc(object)); - }; + function isFlattenable(value) { + return isArray(value) || isArguments(value) || + !!(spreadableSymbol && value && value[spreadableSymbol]); } /** - * Creates a function that either curries or invokes `func` with optional - * `this` binding and partially applied arguments. + * Checks if `value` is a valid array-like index. * * @private - * @param {Function|string} func The function or method name to wrap. - * @param {number} bitmask The bitmask flags. - * 1 - `_.bind` - * 2 - `_.bindKey` - * 4 - `_.curry` or `_.curryRight` of a bound function - * 8 - `_.curry` - * 16 - `_.curryRight` - * 32 - `_.partial` - * 64 - `_.partialRight` - * 128 - `_.rearg` - * 256 - `_.ary` - * 512 - `_.flip` - * @param {*} [thisArg] The `this` binding of `func`. - * @param {Array} [partials] The arguments to be partially applied. - * @param {Array} [holders] The `partials` placeholder indexes. - * @param {Array} [argPos] The argument positions of the new function. - * @param {number} [ary] The arity cap of `func`. - * @param {number} [arity] The arity of `func`. - * @returns {Function} Returns the new wrapped function. + * @param {*} value The value to check. + * @param {number} [length=MAX_SAFE_INTEGER] The upper bounds of a valid index. + * @returns {boolean} Returns `true` if `value` is a valid index, else `false`. */ - function createWrap(func, bitmask, thisArg, partials, holders, argPos, ary, arity) { - var isBindKey = bitmask & WRAP_BIND_KEY_FLAG; - if (!isBindKey && typeof func != 'function') { - throw new TypeError(FUNC_ERROR_TEXT); - } - var length = partials ? partials.length : 0; - if (!length) { - bitmask &= ~(WRAP_PARTIAL_FLAG | WRAP_PARTIAL_RIGHT_FLAG); - partials = holders = undefined; - } - ary = ary === undefined ? ary : nativeMax(toInteger(ary), 0); - arity = arity === undefined ? arity : toInteger(arity); - length -= holders ? holders.length : 0; - - if (bitmask & WRAP_PARTIAL_RIGHT_FLAG) { - var partialsRight = partials, - holdersRight = holders; - - partials = holders = undefined; - } - var data = isBindKey ? undefined : getData(func); - - var newData = [ - func, bitmask, thisArg, partials, holders, partialsRight, holdersRight, - argPos, ary, arity - ]; - - if (data) { - mergeData(newData, data); - } - func = newData[0]; - bitmask = newData[1]; - thisArg = newData[2]; - partials = newData[3]; - holders = newData[4]; - arity = newData[9] = newData[9] === undefined - ? (isBindKey ? 0 : func.length) - : nativeMax(newData[9] - length, 0); + function isIndex(value, length) { + var type = typeof value; + length = length == null ? MAX_SAFE_INTEGER : length; - if (!arity && bitmask & (WRAP_CURRY_FLAG | WRAP_CURRY_RIGHT_FLAG)) { - bitmask &= ~(WRAP_CURRY_FLAG | WRAP_CURRY_RIGHT_FLAG); - } - if (!bitmask || bitmask == WRAP_BIND_FLAG) { - var result = createBind(func, bitmask, thisArg); - } else if (bitmask == WRAP_CURRY_FLAG || bitmask == WRAP_CURRY_RIGHT_FLAG) { - result = createCurry(func, bitmask, arity); - } else if ((bitmask == WRAP_PARTIAL_FLAG || bitmask == (WRAP_BIND_FLAG | WRAP_PARTIAL_FLAG)) && !holders.length) { - result = createPartial(func, bitmask, thisArg, partials); - } else { - result = createHybrid.apply(undefined, newData); - } - var setter = data ? baseSetData : setData; - return setWrapToString(setter(result, newData), func, bitmask); + return !!length && + (type == 'number' || + (type != 'symbol' && reIsUint.test(value))) && + (value > -1 && value % 1 == 0 && value < length); } /** - * Used by `_.defaults` to customize its `_.assignIn` use to assign properties - * of source objects to the destination object for all destination properties - * that resolve to `undefined`. + * Checks if the given arguments are from an iteratee call. * * @private - * @param {*} objValue The destination value. - * @param {*} srcValue The source value. - * @param {string} key The key of the property to assign. - * @param {Object} object The parent object of `objValue`. - * @returns {*} Returns the value to assign. + * @param {*} value The potential iteratee value argument. + * @param {*} index The potential iteratee index or key argument. + * @param {*} object The potential iteratee object argument. + * @returns {boolean} Returns `true` if the arguments are from an iteratee call, + * else `false`. */ - function customDefaultsAssignIn(objValue, srcValue, key, object) { - if (objValue === undefined || - (eq(objValue, objectProto[key]) && !hasOwnProperty.call(object, key))) { - return srcValue; + function isIterateeCall(value, index, object) { + if (!isObject(object)) { + return false; } - return objValue; + var type = typeof index; + if (type == 'number' + ? (isArrayLike(object) && isIndex(index, object.length)) + : (type == 'string' && index in object) + ) { + return eq(object[index], value); + } + return false; } /** - * Used by `_.defaultsDeep` to customize its `_.merge` use to merge source - * objects into destination objects that are passed thru. + * Checks if `value` is a property name and not a property path. * * @private - * @param {*} objValue The destination value. - * @param {*} srcValue The source value. - * @param {string} key The key of the property to merge. - * @param {Object} object The parent object of `objValue`. - * @param {Object} source The parent object of `srcValue`. - * @param {Object} [stack] Tracks traversed source values and their merged - * counterparts. - * @returns {*} Returns the value to assign. + * @param {*} value The value to check. + * @param {Object} [object] The object to query keys on. + * @returns {boolean} Returns `true` if `value` is a property name, else `false`. */ - function customDefaultsMerge(objValue, srcValue, key, object, source, stack) { - if (isObject(objValue) && isObject(srcValue)) { - // Recursively merge objects and arrays (susceptible to call stack limits). - stack.set(srcValue, objValue); - baseMerge(objValue, srcValue, undefined, customDefaultsMerge, stack); - stack['delete'](srcValue); + function isKey(value, object) { + if (isArray(value)) { + return false; } - return objValue; + var type = typeof value; + if (type == 'number' || type == 'symbol' || type == 'boolean' || + value == null || isSymbol(value)) { + return true; + } + return reIsPlainProp.test(value) || !reIsDeepProp.test(value) || + (object != null && value in Object(object)); } /** - * Used by `_.omit` to customize its `_.cloneDeep` use to only clone plain - * objects. + * Checks if `value` is suitable for use as unique object key. * * @private - * @param {*} value The value to inspect. - * @param {string} key The key of the property to inspect. - * @returns {*} Returns the uncloned value or `undefined` to defer cloning to `_.cloneDeep`. + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is suitable, else `false`. */ - function customOmitClone(value) { - return isPlainObject(value) ? undefined : value; + function isKeyable(value) { + var type = typeof value; + return (type == 'string' || type == 'number' || type == 'symbol' || type == 'boolean') + ? (value !== '__proto__') + : (value === null); } /** - * A specialized version of `baseIsEqualDeep` for arrays with support for - * partial deep comparisons. + * Checks if `func` has a lazy counterpart. * * @private - * @param {Array} array The array to compare. - * @param {Array} other The other array to compare. - * @param {number} bitmask The bitmask flags. See `baseIsEqual` for more details. - * @param {Function} customizer The function to customize comparisons. - * @param {Function} equalFunc The function to determine equivalents of values. - * @param {Object} stack Tracks traversed `array` and `other` objects. - * @returns {boolean} Returns `true` if the arrays are equivalent, else `false`. + * @param {Function} func The function to check. + * @returns {boolean} Returns `true` if `func` has a lazy counterpart, + * else `false`. */ - function equalArrays(array, other, bitmask, customizer, equalFunc, stack) { - var isPartial = bitmask & COMPARE_PARTIAL_FLAG, - arrLength = array.length, - othLength = other.length; + function isLaziable(func) { + var funcName = getFuncName(func), + other = lodash[funcName]; - if (arrLength != othLength && !(isPartial && othLength > arrLength)) { + if (typeof other != 'function' || !(funcName in LazyWrapper.prototype)) { return false; } - // Check that cyclic values are equal. - var arrStacked = stack.get(array); - var othStacked = stack.get(other); - if (arrStacked && othStacked) { - return arrStacked == other && othStacked == array; - } - var index = -1, - result = true, - seen = (bitmask & COMPARE_UNORDERED_FLAG) ? new SetCache : undefined; - - stack.set(array, other); - stack.set(other, array); - - // Ignore non-index properties. - while (++index < arrLength) { - var arrValue = array[index], - othValue = other[index]; - - if (customizer) { - var compared = isPartial - ? customizer(othValue, arrValue, index, other, array, stack) - : customizer(arrValue, othValue, index, array, other, stack); - } - if (compared !== undefined) { - if (compared) { - continue; - } - result = false; - break; - } - // Recursively compare arrays (susceptible to call stack limits). - if (seen) { - if (!arraySome(other, function(othValue, othIndex) { - if (!cacheHas(seen, othIndex) && - (arrValue === othValue || equalFunc(arrValue, othValue, bitmask, customizer, stack))) { - return seen.push(othIndex); - } - })) { - result = false; - break; - } - } else if (!( - arrValue === othValue || - equalFunc(arrValue, othValue, bitmask, customizer, stack) - )) { - result = false; - break; - } + if (func === other) { + return true; } - stack['delete'](array); - stack['delete'](other); - return result; + var data = getData(other); + return !!data && func === data[0]; } /** - * A specialized version of `baseIsEqualDeep` for comparing objects of - * the same `toStringTag`. - * - * **Note:** This function only supports comparing values with tags of - * `Boolean`, `Date`, `Error`, `Number`, `RegExp`, or `String`. + * Checks if `func` has its source masked. * * @private - * @param {Object} object The object to compare. - * @param {Object} other The other object to compare. - * @param {string} tag The `toStringTag` of the objects to compare. - * @param {number} bitmask The bitmask flags. See `baseIsEqual` for more details. - * @param {Function} customizer The function to customize comparisons. - * @param {Function} equalFunc The function to determine equivalents of values. - * @param {Object} stack Tracks traversed `object` and `other` objects. - * @returns {boolean} Returns `true` if the objects are equivalent, else `false`. + * @param {Function} func The function to check. + * @returns {boolean} Returns `true` if `func` is masked, else `false`. */ - function equalByTag(object, other, tag, bitmask, customizer, equalFunc, stack) { - switch (tag) { - case dataViewTag: - if ((object.byteLength != other.byteLength) || - (object.byteOffset != other.byteOffset)) { - return false; - } - object = object.buffer; - other = other.buffer; - - case arrayBufferTag: - if ((object.byteLength != other.byteLength) || - !equalFunc(new Uint8Array(object), new Uint8Array(other))) { - return false; - } - return true; - - case boolTag: - case dateTag: - case numberTag: - // Coerce booleans to `1` or `0` and dates to milliseconds. - // Invalid dates are coerced to `NaN`. - return eq(+object, +other); - - case errorTag: - return object.name == other.name && object.message == other.message; - - case regexpTag: - case stringTag: - // Coerce regexes to strings and treat strings, primitives and objects, - // as equal. See http://www.ecma-international.org/ecma-262/7.0/#sec-regexp.prototype.tostring - // for more details. - return object == (other + ''); - - case mapTag: - var convert = mapToArray; - - case setTag: - var isPartial = bitmask & COMPARE_PARTIAL_FLAG; - convert || (convert = setToArray); + function isMasked(func) { + return !!maskSrcKey && (maskSrcKey in func); + } - if (object.size != other.size && !isPartial) { - return false; - } - // Assume cyclic values are equal. - var stacked = stack.get(object); - if (stacked) { - return stacked == other; - } - bitmask |= COMPARE_UNORDERED_FLAG; + /** + * Checks if `func` is capable of being masked. + * + * @private + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `func` is maskable, else `false`. + */ + var isMaskable = coreJsData ? isFunction : stubFalse; - // Recursively compare objects (susceptible to call stack limits). - stack.set(object, other); - var result = equalArrays(convert(object), convert(other), bitmask, customizer, equalFunc, stack); - stack['delete'](object); - return result; + /** + * Checks if `value` is likely a prototype object. + * + * @private + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is a prototype, else `false`. + */ + function isPrototype(value) { + var Ctor = value && value.constructor, + proto = (typeof Ctor == 'function' && Ctor.prototype) || objectProto; - case symbolTag: - if (symbolValueOf) { - return symbolValueOf.call(object) == symbolValueOf.call(other); - } - } - return false; + return value === proto; } /** - * A specialized version of `baseIsEqualDeep` for objects with support for - * partial deep comparisons. + * Checks if `value` is suitable for strict equality comparisons, i.e. `===`. * * @private - * @param {Object} object The object to compare. - * @param {Object} other The other object to compare. - * @param {number} bitmask The bitmask flags. See `baseIsEqual` for more details. - * @param {Function} customizer The function to customize comparisons. - * @param {Function} equalFunc The function to determine equivalents of values. - * @param {Object} stack Tracks traversed `object` and `other` objects. - * @returns {boolean} Returns `true` if the objects are equivalent, else `false`. + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` if suitable for strict + * equality comparisons, else `false`. */ - function equalObjects(object, other, bitmask, customizer, equalFunc, stack) { - var isPartial = bitmask & COMPARE_PARTIAL_FLAG, - objProps = getAllKeys(object), - objLength = objProps.length, - othProps = getAllKeys(other), - othLength = othProps.length; + function isStrictComparable(value) { + return value === value && !isObject(value); + } - if (objLength != othLength && !isPartial) { - return false; - } - var index = objLength; - while (index--) { - var key = objProps[index]; - if (!(isPartial ? key in other : hasOwnProperty.call(other, key))) { + /** + * A specialized version of `matchesProperty` for source values suitable + * for strict equality comparisons, i.e. `===`. + * + * @private + * @param {string} key The key of the property to get. + * @param {*} srcValue The value to match. + * @returns {Function} Returns the new spec function. + */ + function matchesStrictComparable(key, srcValue) { + return function(object) { + if (object == null) { return false; } - } - // Check that cyclic values are equal. - var objStacked = stack.get(object); - var othStacked = stack.get(other); - if (objStacked && othStacked) { - return objStacked == other && othStacked == object; - } - var result = true; - stack.set(object, other); - stack.set(other, object); - - var skipCtor = isPartial; - while (++index < objLength) { - key = objProps[index]; - var objValue = object[key], - othValue = other[key]; + return object[key] === srcValue && + (srcValue !== undefined || (key in Object(object))); + }; + } - if (customizer) { - var compared = isPartial - ? customizer(othValue, objValue, key, other, object, stack) - : customizer(objValue, othValue, key, object, other, stack); - } - // Recursively compare objects (susceptible to call stack limits). - if (!(compared === undefined - ? (objValue === othValue || equalFunc(objValue, othValue, bitmask, customizer, stack)) - : compared - )) { - result = false; - break; + /** + * A specialized version of `_.memoize` which clears the memoized function's + * cache when it exceeds `MAX_MEMOIZE_SIZE`. + * + * @private + * @param {Function} func The function to have its output memoized. + * @returns {Function} Returns the new memoized function. + */ + function memoizeCapped(func) { + var result = memoize(func, function(key) { + if (cache.size === MAX_MEMOIZE_SIZE) { + cache.clear(); } - skipCtor || (skipCtor = key == 'constructor'); - } - if (result && !skipCtor) { - var objCtor = object.constructor, - othCtor = other.constructor; + return key; + }); - // Non `Object` object instances with different constructors are not equal. - if (objCtor != othCtor && - ('constructor' in object && 'constructor' in other) && - !(typeof objCtor == 'function' && objCtor instanceof objCtor && - typeof othCtor == 'function' && othCtor instanceof othCtor)) { - result = false; - } - } - stack['delete'](object); - stack['delete'](other); + var cache = result.cache; return result; } /** - * A specialized version of `baseRest` which flattens the rest array. + * Merges the function metadata of `source` into `data`. + * + * Merging metadata reduces the number of wrappers used to invoke a function. + * This is possible because methods like `_.bind`, `_.curry`, and `_.partial` + * may be applied regardless of execution order. Methods like `_.ary` and + * `_.rearg` modify function arguments, making the order in which they are + * executed important, preventing the merging of metadata. However, we make + * an exception for a safe combined case where curried functions have `_.ary` + * and or `_.rearg` applied. * * @private - * @param {Function} func The function to apply a rest parameter to. - * @returns {Function} Returns the new function. + * @param {Array} data The destination metadata. + * @param {Array} source The source metadata. + * @returns {Array} Returns `data`. */ - function flatRest(func) { - return setToString(overRest(func, undefined, flatten), func + ''); - } + function mergeData(data, source) { + var bitmask = data[1], + srcBitmask = source[1], + newBitmask = bitmask | srcBitmask, + isCommon = newBitmask < (WRAP_BIND_FLAG | WRAP_BIND_KEY_FLAG | WRAP_ARY_FLAG); + + var isCombo = + ((srcBitmask == WRAP_ARY_FLAG) && (bitmask == WRAP_CURRY_FLAG)) || + ((srcBitmask == WRAP_ARY_FLAG) && (bitmask == WRAP_REARG_FLAG) && (data[7].length <= source[8])) || + ((srcBitmask == (WRAP_ARY_FLAG | WRAP_REARG_FLAG)) && (source[7].length <= source[8]) && (bitmask == WRAP_CURRY_FLAG)); + + // Exit early if metadata can't be merged. + if (!(isCommon || isCombo)) { + return data; + } + // Use source `thisArg` if available. + if (srcBitmask & WRAP_BIND_FLAG) { + data[2] = source[2]; + // Set when currying a bound function. + newBitmask |= bitmask & WRAP_BIND_FLAG ? 0 : WRAP_CURRY_BOUND_FLAG; + } + // Compose partial arguments. + var value = source[3]; + if (value) { + var partials = data[3]; + data[3] = partials ? composeArgs(partials, value, source[4]) : value; + data[4] = partials ? replaceHolders(data[3], PLACEHOLDER) : source[4]; + } + // Compose partial right arguments. + value = source[5]; + if (value) { + partials = data[5]; + data[5] = partials ? composeArgsRight(partials, value, source[6]) : value; + data[6] = partials ? replaceHolders(data[5], PLACEHOLDER) : source[6]; + } + // Use source `argPos` if available. + value = source[7]; + if (value) { + data[7] = value; + } + // Use source `ary` if it's smaller. + if (srcBitmask & WRAP_ARY_FLAG) { + data[8] = data[8] == null ? source[8] : nativeMin(data[8], source[8]); + } + // Use source `arity` if one is not provided. + if (data[9] == null) { + data[9] = source[9]; + } + // Use source `func` and merge bitmasks. + data[0] = source[0]; + data[1] = newBitmask; - /** - * Creates an array of own enumerable property names and symbols of `object`. - * - * @private - * @param {Object} object The object to query. - * @returns {Array} Returns the array of property names and symbols. - */ - function getAllKeys(object) { - return baseGetAllKeys(object, keys, getSymbols); + return data; } /** - * Creates an array of own and inherited enumerable property names and - * symbols of `object`. + * This function is like + * [`Object.keys`](http://ecma-international.org/ecma-262/7.0/#sec-object.keys) + * except that it includes inherited enumerable properties. * * @private * @param {Object} object The object to query. - * @returns {Array} Returns the array of property names and symbols. - */ - function getAllKeysIn(object) { - return baseGetAllKeys(object, keysIn, getSymbolsIn); - } - - /** - * Gets metadata for `func`. - * - * @private - * @param {Function} func The function to query. - * @returns {*} Returns the metadata for `func`. - */ - var getData = !metaMap ? noop : function(func) { - return metaMap.get(func); - }; - - /** - * Gets the name of `func`. - * - * @private - * @param {Function} func The function to query. - * @returns {string} Returns the function name. + * @returns {Array} Returns the array of property names. */ - function getFuncName(func) { - var result = (func.name + ''), - array = realNames[result], - length = hasOwnProperty.call(realNames, result) ? array.length : 0; - - while (length--) { - var data = array[length], - otherFunc = data.func; - if (otherFunc == null || otherFunc == func) { - return data.name; + function nativeKeysIn(object) { + var result = []; + if (object != null) { + for (var key in Object(object)) { + result.push(key); } } return result; } /** - * Gets the argument placeholder value for `func`. + * Converts `value` to a string using `Object.prototype.toString`. * * @private - * @param {Function} func The function to inspect. - * @returns {*} Returns the placeholder value. + * @param {*} value The value to convert. + * @returns {string} Returns the converted string. */ - function getHolder(func) { - var object = hasOwnProperty.call(lodash, 'placeholder') ? lodash : func; - return object.placeholder; + function objectToString(value) { + return nativeObjectToString.call(value); } /** - * Gets the appropriate "iteratee" function. If `_.iteratee` is customized, - * this function returns the custom method, otherwise it returns `baseIteratee`. - * If arguments are provided, the chosen function is invoked with them and - * its result is returned. + * A specialized version of `baseRest` which transforms the rest array. * * @private - * @param {*} [value] The value to convert to an iteratee. - * @param {number} [arity] The arity of the created iteratee. - * @returns {Function} Returns the chosen function or its result. + * @param {Function} func The function to apply a rest parameter to. + * @param {number} [start=func.length-1] The start position of the rest parameter. + * @param {Function} transform The rest array transform. + * @returns {Function} Returns the new function. */ - function getIteratee() { - var result = lodash.iteratee || iteratee; - result = result === iteratee ? baseIteratee : result; - return arguments.length ? result(arguments[0], arguments[1]) : result; + function overRest(func, start, transform) { + start = nativeMax(start === undefined ? (func.length - 1) : start, 0); + return function() { + var args = arguments, + index = -1, + length = nativeMax(args.length - start, 0), + array = Array(length); + + while (++index < length) { + array[index] = args[start + index]; + } + index = -1; + var otherArgs = Array(start + 1); + while (++index < start) { + otherArgs[index] = args[index]; + } + otherArgs[start] = transform(array); + return apply(func, this, otherArgs); + }; } /** - * Gets the data for `map`. + * Gets the parent value at `path` of `object`. * * @private - * @param {Object} map The map to query. - * @param {string} key The reference key. - * @returns {*} Returns the map data. + * @param {Object} object The object to query. + * @param {Array} path The path to get the parent value of. + * @returns {*} Returns the parent value. */ - function getMapData(map, key) { - var data = map.__data__; - return isKeyable(key) - ? data[typeof key == 'string' ? 'string' : 'hash'] - : data.map; + function parent(object, path) { + return path.length < 2 ? object : baseGet(object, baseSlice(path, 0, -1)); } /** - * Gets the property names, values, and compare flags of `object`. + * Reorder `array` according to the specified indexes where the element at + * the first index is assigned as the first element, the element at + * the second index is assigned as the second element, and so on. * * @private - * @param {Object} object The object to query. - * @returns {Array} Returns the match data of `object`. + * @param {Array} array The array to reorder. + * @param {Array} indexes The arranged array indexes. + * @returns {Array} Returns `array`. */ - function getMatchData(object) { - var result = keys(object), - length = result.length; + function reorder(array, indexes) { + var arrLength = array.length, + length = nativeMin(indexes.length, arrLength), + oldArray = copyArray(array); while (length--) { - var key = result[length], - value = object[key]; - - result[length] = [key, value, isStrictComparable(value)]; + var index = indexes[length]; + array[length] = isIndex(index, arrLength) ? oldArray[index] : undefined; } - return result; + return array; } /** - * Gets the native function at `key` of `object`. + * Gets the value at `key`, unless `key` is "__proto__" or "constructor". * * @private * @param {Object} object The object to query. - * @param {string} key The key of the method to get. - * @returns {*} Returns the function if it's native, else `undefined`. + * @param {string} key The key of the property to get. + * @returns {*} Returns the property value. */ - function getNative(object, key) { - var value = getValue(object, key); - return baseIsNative(value) ? value : undefined; + function safeGet(object, key) { + if (key === 'constructor' && typeof object[key] === 'function') { + return; + } + + if (key == '__proto__') { + return; + } + + return object[key]; } /** - * A specialized version of `baseGetTag` which ignores `Symbol.toStringTag` values. + * Sets metadata for `func`. + * + * **Note:** If this function becomes hot, i.e. is invoked a lot in a short + * period of time, it will trip its breaker and transition to an identity + * function to avoid garbage collection pauses in V8. See + * [V8 issue 2070](https://bugs.chromium.org/p/v8/issues/detail?id=2070) + * for more details. * * @private - * @param {*} value The value to query. - * @returns {string} Returns the raw `toStringTag`. + * @param {Function} func The function to associate metadata with. + * @param {*} data The metadata. + * @returns {Function} Returns `func`. */ - function getRawTag(value) { - var isOwn = hasOwnProperty.call(value, symToStringTag), - tag = value[symToStringTag]; - - try { - value[symToStringTag] = undefined; - var unmasked = true; - } catch (e) {} - - var result = nativeObjectToString.call(value); - if (unmasked) { - if (isOwn) { - value[symToStringTag] = tag; - } else { - delete value[symToStringTag]; - } - } - return result; - } + var setData = shortOut(baseSetData); /** - * Creates an array of the own enumerable symbols of `object`. + * A simple wrapper around the global [`setTimeout`](https://mdn.io/setTimeout). * * @private - * @param {Object} object The object to query. - * @returns {Array} Returns the array of symbols. + * @param {Function} func The function to delay. + * @param {number} wait The number of milliseconds to delay invocation. + * @returns {number|Object} Returns the timer id or timeout object. */ - var getSymbols = !nativeGetSymbols ? stubArray : function(object) { - if (object == null) { - return []; - } - object = Object(object); - return arrayFilter(nativeGetSymbols(object), function(symbol) { - return propertyIsEnumerable.call(object, symbol); - }); + var setTimeout = ctxSetTimeout || function(func, wait) { + return root.setTimeout(func, wait); }; /** - * Creates an array of the own and inherited enumerable symbols of `object`. + * Sets the `toString` method of `func` to return `string`. * * @private - * @param {Object} object The object to query. - * @returns {Array} Returns the array of symbols. + * @param {Function} func The function to modify. + * @param {Function} string The `toString` result. + * @returns {Function} Returns `func`. */ - var getSymbolsIn = !nativeGetSymbols ? stubArray : function(object) { - var result = []; - while (object) { - arrayPush(result, getSymbols(object)); - object = getPrototype(object); - } - return result; - }; + var setToString = shortOut(baseSetToString); /** - * Gets the `toStringTag` of `value`. + * Sets the `toString` method of `wrapper` to mimic the source of `reference` + * with wrapper details in a comment at the top of the source body. * * @private - * @param {*} value The value to query. - * @returns {string} Returns the `toStringTag`. + * @param {Function} wrapper The function to modify. + * @param {Function} reference The reference function. + * @param {number} bitmask The bitmask flags. See `createWrap` for more details. + * @returns {Function} Returns `wrapper`. */ - var getTag = baseGetTag; - - // Fallback for data views, maps, sets, and weak maps in IE 11 and promises in Node.js < 6. - if ((DataView && getTag(new DataView(new ArrayBuffer(1))) != dataViewTag) || - (Map && getTag(new Map) != mapTag) || - (Promise && getTag(Promise.resolve()) != promiseTag) || - (Set && getTag(new Set) != setTag) || - (WeakMap && getTag(new WeakMap) != weakMapTag)) { - getTag = function(value) { - var result = baseGetTag(value), - Ctor = result == objectTag ? value.constructor : undefined, - ctorString = Ctor ? toSource(Ctor) : ''; - - if (ctorString) { - switch (ctorString) { - case dataViewCtorString: return dataViewTag; - case mapCtorString: return mapTag; - case promiseCtorString: return promiseTag; - case setCtorString: return setTag; - case weakMapCtorString: return weakMapTag; - } - } - return result; - }; + function setWrapToString(wrapper, reference, bitmask) { + var source = (reference + ''); + return setToString(wrapper, insertWrapDetails(source, updateWrapDetails(getWrapDetails(source), bitmask))); } /** - * Gets the view, applying any `transforms` to the `start` and `end` positions. + * Creates a function that'll short out and invoke `identity` instead + * of `func` when it's called `HOT_COUNT` or more times in `HOT_SPAN` + * milliseconds. * * @private - * @param {number} start The start of the view. - * @param {number} end The end of the view. - * @param {Array} transforms The transformations to apply to the view. - * @returns {Object} Returns an object containing the `start` and `end` - * positions of the view. + * @param {Function} func The function to restrict. + * @returns {Function} Returns the new shortable function. */ - function getView(start, end, transforms) { - var index = -1, - length = transforms.length; + function shortOut(func) { + var count = 0, + lastCalled = 0; - while (++index < length) { - var data = transforms[index], - size = data.size; + return function() { + var stamp = nativeNow(), + remaining = HOT_SPAN - (stamp - lastCalled); - switch (data.type) { - case 'drop': start += size; break; - case 'dropRight': end -= size; break; - case 'take': end = nativeMin(end, start + size); break; - case 'takeRight': start = nativeMax(start, end - size); break; + lastCalled = stamp; + if (remaining > 0) { + if (++count >= HOT_COUNT) { + return arguments[0]; + } + } else { + count = 0; } - } - return { 'start': start, 'end': end }; - } - - /** - * Extracts wrapper details from the `source` body comment. - * - * @private - * @param {string} source The source to inspect. - * @returns {Array} Returns the wrapper details. - */ - function getWrapDetails(source) { - var match = source.match(reWrapDetails); - return match ? match[1].split(reSplitDetails) : []; + return func.apply(undefined, arguments); + }; } /** - * Checks if `path` exists on `object`. + * A specialized version of `_.shuffle` which mutates and sets the size of `array`. * * @private - * @param {Object} object The object to query. - * @param {Array|string} path The path to check. - * @param {Function} hasFunc The function to check properties. - * @returns {boolean} Returns `true` if `path` exists, else `false`. + * @param {Array} array The array to shuffle. + * @param {number} [size=array.length] The size of `array`. + * @returns {Array} Returns `array`. */ - function hasPath(object, path, hasFunc) { - path = castPath(path, object); - + function shuffleSelf(array, size) { var index = -1, - length = path.length, - result = false; + length = array.length, + lastIndex = length - 1; - while (++index < length) { - var key = toKey(path[index]); - if (!(result = object != null && hasFunc(object, key))) { - break; - } - object = object[key]; - } - if (result || ++index != length) { - return result; + size = size === undefined ? length : size; + while (++index < size) { + var rand = baseRandom(index, lastIndex), + value = array[rand]; + + array[rand] = array[index]; + array[index] = value; } - length = object == null ? 0 : object.length; - return !!length && isLength(length) && isIndex(key, length) && - (isArray(object) || isArguments(object)); + array.length = size; + return array; } /** - * Initializes an array clone. + * Converts `string` to a property path array. * * @private - * @param {Array} array The array to clone. - * @returns {Array} Returns the initialized clone. + * @param {string} string The string to convert. + * @returns {Array} Returns the property path array. */ - function initCloneArray(array) { - var length = array.length, - result = new array.constructor(length); - - // Add properties assigned by `RegExp#exec`. - if (length && typeof array[0] == 'string' && hasOwnProperty.call(array, 'index')) { - result.index = array.index; - result.input = array.input; + var stringToPath = memoizeCapped(function(string) { + var result = []; + if (string.charCodeAt(0) === 46 /* . */) { + result.push(''); } + string.replace(rePropName, function(match, number, quote, subString) { + result.push(quote ? subString.replace(reEscapeChar, '$1') : (number || match)); + }); return result; - } - - /** - * Initializes an object clone. - * - * @private - * @param {Object} object The object to clone. - * @returns {Object} Returns the initialized clone. - */ - function initCloneObject(object) { - return (typeof object.constructor == 'function' && !isPrototype(object)) - ? baseCreate(getPrototype(object)) - : {}; - } + }); /** - * Initializes an object clone based on its `toStringTag`. - * - * **Note:** This function only supports cloning values with tags of - * `Boolean`, `Date`, `Error`, `Map`, `Number`, `RegExp`, `Set`, or `String`. + * Converts `value` to a string key if it's not a string or symbol. * * @private - * @param {Object} object The object to clone. - * @param {string} tag The `toStringTag` of the object to clone. - * @param {boolean} [isDeep] Specify a deep clone. - * @returns {Object} Returns the initialized clone. + * @param {*} value The value to inspect. + * @returns {string|symbol} Returns the key. */ - function initCloneByTag(object, tag, isDeep) { - var Ctor = object.constructor; - switch (tag) { - case arrayBufferTag: - return cloneArrayBuffer(object); - - case boolTag: - case dateTag: - return new Ctor(+object); - - case dataViewTag: - return cloneDataView(object, isDeep); - - case float32Tag: case float64Tag: - case int8Tag: case int16Tag: case int32Tag: - case uint8Tag: case uint8ClampedTag: case uint16Tag: case uint32Tag: - return cloneTypedArray(object, isDeep); - - case mapTag: - return new Ctor; - - case numberTag: - case stringTag: - return new Ctor(object); - - case regexpTag: - return cloneRegExp(object); - - case setTag: - return new Ctor; - - case symbolTag: - return cloneSymbol(object); + function toKey(value) { + if (typeof value == 'string' || isSymbol(value)) { + return value; } + var result = (value + ''); + return (result == '0' && (1 / value) == -INFINITY) ? '-0' : result; } /** - * Inserts wrapper `details` in a comment at the top of the `source` body. + * Converts `func` to its source code. * * @private - * @param {string} source The source to modify. - * @returns {Array} details The details to insert. - * @returns {string} Returns the modified source. + * @param {Function} func The function to convert. + * @returns {string} Returns the source code. */ - function insertWrapDetails(source, details) { - var length = details.length; - if (!length) { - return source; + function toSource(func) { + if (func != null) { + try { + return funcToString.call(func); + } catch (e) {} + try { + return (func + ''); + } catch (e) {} } - var lastIndex = length - 1; - details[lastIndex] = (length > 1 ? '& ' : '') + details[lastIndex]; - details = details.join(length > 2 ? ', ' : ' '); - return source.replace(reWrapComment, '{\n/* [wrapped with ' + details + '] */\n'); - } - - /** - * Checks if `value` is a flattenable `arguments` object or array. - * - * @private - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is flattenable, else `false`. - */ - function isFlattenable(value) { - return isArray(value) || isArguments(value) || - !!(spreadableSymbol && value && value[spreadableSymbol]); + return ''; } /** - * Checks if `value` is a valid array-like index. + * Updates wrapper `details` based on `bitmask` flags. * * @private - * @param {*} value The value to check. - * @param {number} [length=MAX_SAFE_INTEGER] The upper bounds of a valid index. - * @returns {boolean} Returns `true` if `value` is a valid index, else `false`. + * @returns {Array} details The details to modify. + * @param {number} bitmask The bitmask flags. See `createWrap` for more details. + * @returns {Array} Returns `details`. */ - function isIndex(value, length) { - var type = typeof value; - length = length == null ? MAX_SAFE_INTEGER : length; - - return !!length && - (type == 'number' || - (type != 'symbol' && reIsUint.test(value))) && - (value > -1 && value % 1 == 0 && value < length); + function updateWrapDetails(details, bitmask) { + arrayEach(wrapFlags, function(pair) { + var value = '_.' + pair[0]; + if ((bitmask & pair[1]) && !arrayIncludes(details, value)) { + details.push(value); + } + }); + return details.sort(); } /** - * Checks if the given arguments are from an iteratee call. + * Creates a clone of `wrapper`. * * @private - * @param {*} value The potential iteratee value argument. - * @param {*} index The potential iteratee index or key argument. - * @param {*} object The potential iteratee object argument. - * @returns {boolean} Returns `true` if the arguments are from an iteratee call, - * else `false`. + * @param {Object} wrapper The wrapper to clone. + * @returns {Object} Returns the cloned wrapper. */ - function isIterateeCall(value, index, object) { - if (!isObject(object)) { - return false; - } - var type = typeof index; - if (type == 'number' - ? (isArrayLike(object) && isIndex(index, object.length)) - : (type == 'string' && index in object) - ) { - return eq(object[index], value); + function wrapperClone(wrapper) { + if (wrapper instanceof LazyWrapper) { + return wrapper.clone(); } - return false; + var result = new LodashWrapper(wrapper.__wrapped__, wrapper.__chain__); + result.__actions__ = copyArray(wrapper.__actions__); + result.__index__ = wrapper.__index__; + result.__values__ = wrapper.__values__; + return result; } + /*------------------------------------------------------------------------*/ + /** - * Checks if `value` is a property name and not a property path. + * Creates an array of elements split into groups the length of `size`. + * If `array` can't be split evenly, the final chunk will be the remaining + * elements. * - * @private - * @param {*} value The value to check. - * @param {Object} [object] The object to query keys on. - * @returns {boolean} Returns `true` if `value` is a property name, else `false`. + * @static + * @memberOf _ + * @since 3.0.0 + * @category Array + * @param {Array} array The array to process. + * @param {number} [size=1] The length of each chunk + * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. + * @returns {Array} Returns the new array of chunks. + * @example + * + * _.chunk(['a', 'b', 'c', 'd'], 2); + * // => [['a', 'b'], ['c', 'd']] + * + * _.chunk(['a', 'b', 'c', 'd'], 3); + * // => [['a', 'b', 'c'], ['d']] */ - function isKey(value, object) { - if (isArray(value)) { - return false; + function chunk(array, size, guard) { + if ((guard ? isIterateeCall(array, size, guard) : size === undefined)) { + size = 1; + } else { + size = nativeMax(toInteger(size), 0); } - var type = typeof value; - if (type == 'number' || type == 'symbol' || type == 'boolean' || - value == null || isSymbol(value)) { - return true; + var length = array == null ? 0 : array.length; + if (!length || size < 1) { + return []; } - return reIsPlainProp.test(value) || !reIsDeepProp.test(value) || - (object != null && value in Object(object)); + var index = 0, + resIndex = 0, + result = Array(nativeCeil(length / size)); + + while (index < length) { + result[resIndex++] = baseSlice(array, index, (index += size)); + } + return result; } /** - * Checks if `value` is suitable for use as unique object key. + * Creates an array with all falsey values removed. The values `false`, `null`, + * `0`, `""`, `undefined`, and `NaN` are falsey. * - * @private - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is suitable, else `false`. + * @static + * @memberOf _ + * @since 0.1.0 + * @category Array + * @param {Array} array The array to compact. + * @returns {Array} Returns the new array of filtered values. + * @example + * + * _.compact([0, 1, false, 2, '', 3]); + * // => [1, 2, 3] */ - function isKeyable(value) { - var type = typeof value; - return (type == 'string' || type == 'number' || type == 'symbol' || type == 'boolean') - ? (value !== '__proto__') - : (value === null); + function compact(array) { + var index = -1, + length = array == null ? 0 : array.length, + resIndex = 0, + result = []; + + while (++index < length) { + var value = array[index]; + if (value) { + result[resIndex++] = value; + } + } + return result; } /** - * Checks if `func` has a lazy counterpart. + * Creates a new array concatenating `array` with any additional arrays + * and/or values. * - * @private - * @param {Function} func The function to check. - * @returns {boolean} Returns `true` if `func` has a lazy counterpart, - * else `false`. + * @static + * @memberOf _ + * @since 4.0.0 + * @category Array + * @param {Array} array The array to concatenate. + * @param {...*} [values] The values to concatenate. + * @returns {Array} Returns the new concatenated array. + * @example + * + * var array = [1]; + * var other = _.concat(array, 2, [3], [[4]]); + * + * console.log(other); + * // => [1, 2, 3, [4]] + * + * console.log(array); + * // => [1] */ - function isLaziable(func) { - var funcName = getFuncName(func), - other = lodash[funcName]; - - if (typeof other != 'function' || !(funcName in LazyWrapper.prototype)) { - return false; + function concat() { + var length = arguments.length; + if (!length) { + return []; } - if (func === other) { - return true; + var args = Array(length - 1), + array = arguments[0], + index = length; + + while (index--) { + args[index - 1] = arguments[index]; } - var data = getData(other); - return !!data && func === data[0]; + return arrayPush(isArray(array) ? copyArray(array) : [array], baseFlatten(args, 1)); } /** - * Checks if `func` has its source masked. + * Creates an array of `array` values not included in the other given arrays + * using [`SameValueZero`](http://ecma-international.org/ecma-262/7.0/#sec-samevaluezero) + * for equality comparisons. The order and references of result values are + * determined by the first array. * - * @private - * @param {Function} func The function to check. - * @returns {boolean} Returns `true` if `func` is masked, else `false`. + * **Note:** Unlike `_.pullAll`, this method returns a new array. + * + * @static + * @memberOf _ + * @since 0.1.0 + * @category Array + * @param {Array} array The array to inspect. + * @param {...Array} [values] The values to exclude. + * @returns {Array} Returns the new array of filtered values. + * @see _.without, _.xor + * @example + * + * _.difference([2, 1], [2, 3]); + * // => [1] */ - function isMasked(func) { - return !!maskSrcKey && (maskSrcKey in func); - } + var difference = baseRest(function(array, values) { + return isArrayLikeObject(array) + ? baseDifference(array, baseFlatten(values, 1, isArrayLikeObject, true)) + : []; + }); /** - * Checks if `func` is capable of being masked. + * This method is like `_.difference` except that it accepts `iteratee` which + * is invoked for each element of `array` and `values` to generate the criterion + * by which they're compared. The order and references of result values are + * determined by the first array. The iteratee is invoked with one argument: + * (value). * - * @private - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `func` is maskable, else `false`. + * **Note:** Unlike `_.pullAllBy`, this method returns a new array. + * + * @static + * @memberOf _ + * @since 4.0.0 + * @category Array + * @param {Array} array The array to inspect. + * @param {...Array} [values] The values to exclude. + * @param {Function} [iteratee=_.identity] The iteratee invoked per element. + * @returns {Array} Returns the new array of filtered values. + * @example + * + * _.differenceBy([2.1, 1.2], [2.3, 3.4], Math.floor); + * // => [1.2] + * + * // The `_.property` iteratee shorthand. + * _.differenceBy([{ 'x': 2 }, { 'x': 1 }], [{ 'x': 1 }], 'x'); + * // => [{ 'x': 2 }] */ - var isMaskable = coreJsData ? isFunction : stubFalse; + var differenceBy = baseRest(function(array, values) { + var iteratee = last(values); + if (isArrayLikeObject(iteratee)) { + iteratee = undefined; + } + return isArrayLikeObject(array) + ? baseDifference(array, baseFlatten(values, 1, isArrayLikeObject, true), getIteratee(iteratee, 2)) + : []; + }); /** - * Checks if `value` is likely a prototype object. + * This method is like `_.difference` except that it accepts `comparator` + * which is invoked to compare elements of `array` to `values`. The order and + * references of result values are determined by the first array. The comparator + * is invoked with two arguments: (arrVal, othVal). * - * @private - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a prototype, else `false`. + * **Note:** Unlike `_.pullAllWith`, this method returns a new array. + * + * @static + * @memberOf _ + * @since 4.0.0 + * @category Array + * @param {Array} array The array to inspect. + * @param {...Array} [values] The values to exclude. + * @param {Function} [comparator] The comparator invoked per element. + * @returns {Array} Returns the new array of filtered values. + * @example + * + * var objects = [{ 'x': 1, 'y': 2 }, { 'x': 2, 'y': 1 }]; + * + * _.differenceWith(objects, [{ 'x': 1, 'y': 2 }], _.isEqual); + * // => [{ 'x': 2, 'y': 1 }] */ - function isPrototype(value) { - var Ctor = value && value.constructor, - proto = (typeof Ctor == 'function' && Ctor.prototype) || objectProto; + var differenceWith = baseRest(function(array, values) { + var comparator = last(values); + if (isArrayLikeObject(comparator)) { + comparator = undefined; + } + return isArrayLikeObject(array) + ? baseDifference(array, baseFlatten(values, 1, isArrayLikeObject, true), undefined, comparator) + : []; + }); - return value === proto; + /** + * Creates a slice of `array` with `n` elements dropped from the beginning. + * + * @static + * @memberOf _ + * @since 0.5.0 + * @category Array + * @param {Array} array The array to query. + * @param {number} [n=1] The number of elements to drop. + * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. + * @returns {Array} Returns the slice of `array`. + * @example + * + * _.drop([1, 2, 3]); + * // => [2, 3] + * + * _.drop([1, 2, 3], 2); + * // => [3] + * + * _.drop([1, 2, 3], 5); + * // => [] + * + * _.drop([1, 2, 3], 0); + * // => [1, 2, 3] + */ + function drop(array, n, guard) { + var length = array == null ? 0 : array.length; + if (!length) { + return []; + } + n = (guard || n === undefined) ? 1 : toInteger(n); + return baseSlice(array, n < 0 ? 0 : n, length); } /** - * Checks if `value` is suitable for strict equality comparisons, i.e. `===`. + * Creates a slice of `array` with `n` elements dropped from the end. * - * @private - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` if suitable for strict - * equality comparisons, else `false`. + * @static + * @memberOf _ + * @since 3.0.0 + * @category Array + * @param {Array} array The array to query. + * @param {number} [n=1] The number of elements to drop. + * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. + * @returns {Array} Returns the slice of `array`. + * @example + * + * _.dropRight([1, 2, 3]); + * // => [1, 2] + * + * _.dropRight([1, 2, 3], 2); + * // => [1] + * + * _.dropRight([1, 2, 3], 5); + * // => [] + * + * _.dropRight([1, 2, 3], 0); + * // => [1, 2, 3] */ - function isStrictComparable(value) { - return value === value && !isObject(value); + function dropRight(array, n, guard) { + var length = array == null ? 0 : array.length; + if (!length) { + return []; + } + n = (guard || n === undefined) ? 1 : toInteger(n); + n = length - n; + return baseSlice(array, 0, n < 0 ? 0 : n); } /** - * A specialized version of `matchesProperty` for source values suitable - * for strict equality comparisons, i.e. `===`. + * Creates a slice of `array` excluding elements dropped from the end. + * Elements are dropped until `predicate` returns falsey. The predicate is + * invoked with three arguments: (value, index, array). * - * @private - * @param {string} key The key of the property to get. - * @param {*} srcValue The value to match. - * @returns {Function} Returns the new spec function. + * @static + * @memberOf _ + * @since 3.0.0 + * @category Array + * @param {Array} array The array to query. + * @param {Function} [predicate=_.identity] The function invoked per iteration. + * @returns {Array} Returns the slice of `array`. + * @example + * + * var users = [ + * { 'user': 'barney', 'active': true }, + * { 'user': 'fred', 'active': false }, + * { 'user': 'pebbles', 'active': false } + * ]; + * + * _.dropRightWhile(users, function(o) { return !o.active; }); + * // => objects for ['barney'] + * + * // The `_.matches` iteratee shorthand. + * _.dropRightWhile(users, { 'user': 'pebbles', 'active': false }); + * // => objects for ['barney', 'fred'] + * + * // The `_.matchesProperty` iteratee shorthand. + * _.dropRightWhile(users, ['active', false]); + * // => objects for ['barney'] + * + * // The `_.property` iteratee shorthand. + * _.dropRightWhile(users, 'active'); + * // => objects for ['barney', 'fred', 'pebbles'] */ - function matchesStrictComparable(key, srcValue) { - return function(object) { - if (object == null) { - return false; - } - return object[key] === srcValue && - (srcValue !== undefined || (key in Object(object))); - }; + function dropRightWhile(array, predicate) { + return (array && array.length) + ? baseWhile(array, getIteratee(predicate, 3), true, true) + : []; } /** - * A specialized version of `_.memoize` which clears the memoized function's - * cache when it exceeds `MAX_MEMOIZE_SIZE`. + * Creates a slice of `array` excluding elements dropped from the beginning. + * Elements are dropped until `predicate` returns falsey. The predicate is + * invoked with three arguments: (value, index, array). * - * @private - * @param {Function} func The function to have its output memoized. - * @returns {Function} Returns the new memoized function. + * @static + * @memberOf _ + * @since 3.0.0 + * @category Array + * @param {Array} array The array to query. + * @param {Function} [predicate=_.identity] The function invoked per iteration. + * @returns {Array} Returns the slice of `array`. + * @example + * + * var users = [ + * { 'user': 'barney', 'active': false }, + * { 'user': 'fred', 'active': false }, + * { 'user': 'pebbles', 'active': true } + * ]; + * + * _.dropWhile(users, function(o) { return !o.active; }); + * // => objects for ['pebbles'] + * + * // The `_.matches` iteratee shorthand. + * _.dropWhile(users, { 'user': 'barney', 'active': false }); + * // => objects for ['fred', 'pebbles'] + * + * // The `_.matchesProperty` iteratee shorthand. + * _.dropWhile(users, ['active', false]); + * // => objects for ['pebbles'] + * + * // The `_.property` iteratee shorthand. + * _.dropWhile(users, 'active'); + * // => objects for ['barney', 'fred', 'pebbles'] */ - function memoizeCapped(func) { - var result = memoize(func, function(key) { - if (cache.size === MAX_MEMOIZE_SIZE) { - cache.clear(); - } - return key; - }); - - var cache = result.cache; - return result; + function dropWhile(array, predicate) { + return (array && array.length) + ? baseWhile(array, getIteratee(predicate, 3), true) + : []; } /** - * Merges the function metadata of `source` into `data`. + * Fills elements of `array` with `value` from `start` up to, but not + * including, `end`. * - * Merging metadata reduces the number of wrappers used to invoke a function. - * This is possible because methods like `_.bind`, `_.curry`, and `_.partial` - * may be applied regardless of execution order. Methods like `_.ary` and - * `_.rearg` modify function arguments, making the order in which they are - * executed important, preventing the merging of metadata. However, we make - * an exception for a safe combined case where curried functions have `_.ary` - * and or `_.rearg` applied. + * **Note:** This method mutates `array`. * - * @private - * @param {Array} data The destination metadata. - * @param {Array} source The source metadata. - * @returns {Array} Returns `data`. + * @static + * @memberOf _ + * @since 3.2.0 + * @category Array + * @param {Array} array The array to fill. + * @param {*} value The value to fill `array` with. + * @param {number} [start=0] The start position. + * @param {number} [end=array.length] The end position. + * @returns {Array} Returns `array`. + * @example + * + * var array = [1, 2, 3]; + * + * _.fill(array, 'a'); + * console.log(array); + * // => ['a', 'a', 'a'] + * + * _.fill(Array(3), 2); + * // => [2, 2, 2] + * + * _.fill([4, 6, 8, 10], '*', 1, 3); + * // => [4, '*', '*', 10] */ - function mergeData(data, source) { - var bitmask = data[1], - srcBitmask = source[1], - newBitmask = bitmask | srcBitmask, - isCommon = newBitmask < (WRAP_BIND_FLAG | WRAP_BIND_KEY_FLAG | WRAP_ARY_FLAG); - - var isCombo = - ((srcBitmask == WRAP_ARY_FLAG) && (bitmask == WRAP_CURRY_FLAG)) || - ((srcBitmask == WRAP_ARY_FLAG) && (bitmask == WRAP_REARG_FLAG) && (data[7].length <= source[8])) || - ((srcBitmask == (WRAP_ARY_FLAG | WRAP_REARG_FLAG)) && (source[7].length <= source[8]) && (bitmask == WRAP_CURRY_FLAG)); - - // Exit early if metadata can't be merged. - if (!(isCommon || isCombo)) { - return data; - } - // Use source `thisArg` if available. - if (srcBitmask & WRAP_BIND_FLAG) { - data[2] = source[2]; - // Set when currying a bound function. - newBitmask |= bitmask & WRAP_BIND_FLAG ? 0 : WRAP_CURRY_BOUND_FLAG; - } - // Compose partial arguments. - var value = source[3]; - if (value) { - var partials = data[3]; - data[3] = partials ? composeArgs(partials, value, source[4]) : value; - data[4] = partials ? replaceHolders(data[3], PLACEHOLDER) : source[4]; - } - // Compose partial right arguments. - value = source[5]; - if (value) { - partials = data[5]; - data[5] = partials ? composeArgsRight(partials, value, source[6]) : value; - data[6] = partials ? replaceHolders(data[5], PLACEHOLDER) : source[6]; - } - // Use source `argPos` if available. - value = source[7]; - if (value) { - data[7] = value; - } - // Use source `ary` if it's smaller. - if (srcBitmask & WRAP_ARY_FLAG) { - data[8] = data[8] == null ? source[8] : nativeMin(data[8], source[8]); + function fill(array, value, start, end) { + var length = array == null ? 0 : array.length; + if (!length) { + return []; } - // Use source `arity` if one is not provided. - if (data[9] == null) { - data[9] = source[9]; + if (start && typeof start != 'number' && isIterateeCall(array, value, start)) { + start = 0; + end = length; } - // Use source `func` and merge bitmasks. - data[0] = source[0]; - data[1] = newBitmask; - - return data; + return baseFill(array, value, start, end); } /** - * This function is like - * [`Object.keys`](http://ecma-international.org/ecma-262/7.0/#sec-object.keys) - * except that it includes inherited enumerable properties. + * This method is like `_.find` except that it returns the index of the first + * element `predicate` returns truthy for instead of the element itself. * - * @private - * @param {Object} object The object to query. - * @returns {Array} Returns the array of property names. + * @static + * @memberOf _ + * @since 1.1.0 + * @category Array + * @param {Array} array The array to inspect. + * @param {Function} [predicate=_.identity] The function invoked per iteration. + * @param {number} [fromIndex=0] The index to search from. + * @returns {number} Returns the index of the found element, else `-1`. + * @example + * + * var users = [ + * { 'user': 'barney', 'active': false }, + * { 'user': 'fred', 'active': false }, + * { 'user': 'pebbles', 'active': true } + * ]; + * + * _.findIndex(users, function(o) { return o.user == 'barney'; }); + * // => 0 + * + * // The `_.matches` iteratee shorthand. + * _.findIndex(users, { 'user': 'fred', 'active': false }); + * // => 1 + * + * // The `_.matchesProperty` iteratee shorthand. + * _.findIndex(users, ['active', false]); + * // => 0 + * + * // The `_.property` iteratee shorthand. + * _.findIndex(users, 'active'); + * // => 2 */ - function nativeKeysIn(object) { - var result = []; - if (object != null) { - for (var key in Object(object)) { - result.push(key); - } + function findIndex(array, predicate, fromIndex) { + var length = array == null ? 0 : array.length; + if (!length) { + return -1; } - return result; + var index = fromIndex == null ? 0 : toInteger(fromIndex); + if (index < 0) { + index = nativeMax(length + index, 0); + } + return baseFindIndex(array, getIteratee(predicate, 3), index); } /** - * Converts `value` to a string using `Object.prototype.toString`. + * This method is like `_.findIndex` except that it iterates over elements + * of `collection` from right to left. * - * @private - * @param {*} value The value to convert. - * @returns {string} Returns the converted string. + * @static + * @memberOf _ + * @since 2.0.0 + * @category Array + * @param {Array} array The array to inspect. + * @param {Function} [predicate=_.identity] The function invoked per iteration. + * @param {number} [fromIndex=array.length-1] The index to search from. + * @returns {number} Returns the index of the found element, else `-1`. + * @example + * + * var users = [ + * { 'user': 'barney', 'active': true }, + * { 'user': 'fred', 'active': false }, + * { 'user': 'pebbles', 'active': false } + * ]; + * + * _.findLastIndex(users, function(o) { return o.user == 'pebbles'; }); + * // => 2 + * + * // The `_.matches` iteratee shorthand. + * _.findLastIndex(users, { 'user': 'barney', 'active': true }); + * // => 0 + * + * // The `_.matchesProperty` iteratee shorthand. + * _.findLastIndex(users, ['active', false]); + * // => 2 + * + * // The `_.property` iteratee shorthand. + * _.findLastIndex(users, 'active'); + * // => 0 */ - function objectToString(value) { - return nativeObjectToString.call(value); + function findLastIndex(array, predicate, fromIndex) { + var length = array == null ? 0 : array.length; + if (!length) { + return -1; + } + var index = length - 1; + if (fromIndex !== undefined) { + index = toInteger(fromIndex); + index = fromIndex < 0 + ? nativeMax(length + index, 0) + : nativeMin(index, length - 1); + } + return baseFindIndex(array, getIteratee(predicate, 3), index, true); } /** - * A specialized version of `baseRest` which transforms the rest array. + * Flattens `array` a single level deep. * - * @private - * @param {Function} func The function to apply a rest parameter to. - * @param {number} [start=func.length-1] The start position of the rest parameter. - * @param {Function} transform The rest array transform. - * @returns {Function} Returns the new function. + * @static + * @memberOf _ + * @since 0.1.0 + * @category Array + * @param {Array} array The array to flatten. + * @returns {Array} Returns the new flattened array. + * @example + * + * _.flatten([1, [2, [3, [4]], 5]]); + * // => [1, 2, [3, [4]], 5] */ - function overRest(func, start, transform) { - start = nativeMax(start === undefined ? (func.length - 1) : start, 0); - return function() { - var args = arguments, - index = -1, - length = nativeMax(args.length - start, 0), - array = Array(length); - - while (++index < length) { - array[index] = args[start + index]; - } - index = -1; - var otherArgs = Array(start + 1); - while (++index < start) { - otherArgs[index] = args[index]; - } - otherArgs[start] = transform(array); - return apply(func, this, otherArgs); - }; + function flatten(array) { + var length = array == null ? 0 : array.length; + return length ? baseFlatten(array, 1) : []; } /** - * Gets the parent value at `path` of `object`. + * Recursively flattens `array`. * - * @private - * @param {Object} object The object to query. - * @param {Array} path The path to get the parent value of. - * @returns {*} Returns the parent value. + * @static + * @memberOf _ + * @since 3.0.0 + * @category Array + * @param {Array} array The array to flatten. + * @returns {Array} Returns the new flattened array. + * @example + * + * _.flattenDeep([1, [2, [3, [4]], 5]]); + * // => [1, 2, 3, 4, 5] */ - function parent(object, path) { - return path.length < 2 ? object : baseGet(object, baseSlice(path, 0, -1)); + function flattenDeep(array) { + var length = array == null ? 0 : array.length; + return length ? baseFlatten(array, INFINITY) : []; } /** - * Reorder `array` according to the specified indexes where the element at - * the first index is assigned as the first element, the element at - * the second index is assigned as the second element, and so on. + * Recursively flatten `array` up to `depth` times. * - * @private - * @param {Array} array The array to reorder. - * @param {Array} indexes The arranged array indexes. - * @returns {Array} Returns `array`. + * @static + * @memberOf _ + * @since 4.4.0 + * @category Array + * @param {Array} array The array to flatten. + * @param {number} [depth=1] The maximum recursion depth. + * @returns {Array} Returns the new flattened array. + * @example + * + * var array = [1, [2, [3, [4]], 5]]; + * + * _.flattenDepth(array, 1); + * // => [1, 2, [3, [4]], 5] + * + * _.flattenDepth(array, 2); + * // => [1, 2, 3, [4], 5] */ - function reorder(array, indexes) { - var arrLength = array.length, - length = nativeMin(indexes.length, arrLength), - oldArray = copyArray(array); - - while (length--) { - var index = indexes[length]; - array[length] = isIndex(index, arrLength) ? oldArray[index] : undefined; + function flattenDepth(array, depth) { + var length = array == null ? 0 : array.length; + if (!length) { + return []; } - return array; + depth = depth === undefined ? 1 : toInteger(depth); + return baseFlatten(array, depth); } /** - * Gets the value at `key`, unless `key` is "__proto__" or "constructor". + * The inverse of `_.toPairs`; this method returns an object composed + * from key-value `pairs`. * - * @private - * @param {Object} object The object to query. - * @param {string} key The key of the property to get. - * @returns {*} Returns the property value. + * @static + * @memberOf _ + * @since 4.0.0 + * @category Array + * @param {Array} pairs The key-value pairs. + * @returns {Object} Returns the new object. + * @example + * + * _.fromPairs([['a', 1], ['b', 2]]); + * // => { 'a': 1, 'b': 2 } */ - function safeGet(object, key) { - if (key === 'constructor' && typeof object[key] === 'function') { - return; - } + function fromPairs(pairs) { + var index = -1, + length = pairs == null ? 0 : pairs.length, + result = {}; - if (key == '__proto__') { - return; + while (++index < length) { + var pair = pairs[index]; + result[pair[0]] = pair[1]; } - - return object[key]; + return result; } /** - * Sets metadata for `func`. + * Gets the first element of `array`. * - * **Note:** If this function becomes hot, i.e. is invoked a lot in a short - * period of time, it will trip its breaker and transition to an identity - * function to avoid garbage collection pauses in V8. See - * [V8 issue 2070](https://bugs.chromium.org/p/v8/issues/detail?id=2070) - * for more details. + * @static + * @memberOf _ + * @since 0.1.0 + * @alias first + * @category Array + * @param {Array} array The array to query. + * @returns {*} Returns the first element of `array`. + * @example * - * @private - * @param {Function} func The function to associate metadata with. - * @param {*} data The metadata. - * @returns {Function} Returns `func`. - */ - var setData = shortOut(baseSetData); - - /** - * A simple wrapper around the global [`setTimeout`](https://mdn.io/setTimeout). + * _.head([1, 2, 3]); + * // => 1 * - * @private - * @param {Function} func The function to delay. - * @param {number} wait The number of milliseconds to delay invocation. - * @returns {number|Object} Returns the timer id or timeout object. + * _.head([]); + * // => undefined */ - var setTimeout = ctxSetTimeout || function(func, wait) { - return root.setTimeout(func, wait); - }; + function head(array) { + return (array && array.length) ? array[0] : undefined; + } /** - * Sets the `toString` method of `func` to return `string`. + * Gets the index at which the first occurrence of `value` is found in `array` + * using [`SameValueZero`](http://ecma-international.org/ecma-262/7.0/#sec-samevaluezero) + * for equality comparisons. If `fromIndex` is negative, it's used as the + * offset from the end of `array`. * - * @private - * @param {Function} func The function to modify. - * @param {Function} string The `toString` result. - * @returns {Function} Returns `func`. - */ - var setToString = shortOut(baseSetToString); - - /** - * Sets the `toString` method of `wrapper` to mimic the source of `reference` - * with wrapper details in a comment at the top of the source body. + * @static + * @memberOf _ + * @since 0.1.0 + * @category Array + * @param {Array} array The array to inspect. + * @param {*} value The value to search for. + * @param {number} [fromIndex=0] The index to search from. + * @returns {number} Returns the index of the matched value, else `-1`. + * @example * - * @private - * @param {Function} wrapper The function to modify. - * @param {Function} reference The reference function. - * @param {number} bitmask The bitmask flags. See `createWrap` for more details. - * @returns {Function} Returns `wrapper`. - */ - function setWrapToString(wrapper, reference, bitmask) { - var source = (reference + ''); - return setToString(wrapper, insertWrapDetails(source, updateWrapDetails(getWrapDetails(source), bitmask))); - } - - /** - * Creates a function that'll short out and invoke `identity` instead - * of `func` when it's called `HOT_COUNT` or more times in `HOT_SPAN` - * milliseconds. + * _.indexOf([1, 2, 1, 2], 2); + * // => 1 * - * @private - * @param {Function} func The function to restrict. - * @returns {Function} Returns the new shortable function. + * // Search from the `fromIndex`. + * _.indexOf([1, 2, 1, 2], 2, 2); + * // => 3 */ - function shortOut(func) { - var count = 0, - lastCalled = 0; - - return function() { - var stamp = nativeNow(), - remaining = HOT_SPAN - (stamp - lastCalled); - - lastCalled = stamp; - if (remaining > 0) { - if (++count >= HOT_COUNT) { - return arguments[0]; - } - } else { - count = 0; - } - return func.apply(undefined, arguments); - }; + function indexOf(array, value, fromIndex) { + var length = array == null ? 0 : array.length; + if (!length) { + return -1; + } + var index = fromIndex == null ? 0 : toInteger(fromIndex); + if (index < 0) { + index = nativeMax(length + index, 0); + } + return baseIndexOf(array, value, index); } /** - * A specialized version of `_.shuffle` which mutates and sets the size of `array`. + * Gets all but the last element of `array`. * - * @private - * @param {Array} array The array to shuffle. - * @param {number} [size=array.length] The size of `array`. - * @returns {Array} Returns `array`. + * @static + * @memberOf _ + * @since 0.1.0 + * @category Array + * @param {Array} array The array to query. + * @returns {Array} Returns the slice of `array`. + * @example + * + * _.initial([1, 2, 3]); + * // => [1, 2] */ - function shuffleSelf(array, size) { - var index = -1, - length = array.length, - lastIndex = length - 1; - - size = size === undefined ? length : size; - while (++index < size) { - var rand = baseRandom(index, lastIndex), - value = array[rand]; - - array[rand] = array[index]; - array[index] = value; - } - array.length = size; - return array; + function initial(array) { + var length = array == null ? 0 : array.length; + return length ? baseSlice(array, 0, -1) : []; } /** - * Converts `string` to a property path array. + * Creates an array of unique values that are included in all given arrays + * using [`SameValueZero`](http://ecma-international.org/ecma-262/7.0/#sec-samevaluezero) + * for equality comparisons. The order and references of result values are + * determined by the first array. * - * @private - * @param {string} string The string to convert. - * @returns {Array} Returns the property path array. + * @static + * @memberOf _ + * @since 0.1.0 + * @category Array + * @param {...Array} [arrays] The arrays to inspect. + * @returns {Array} Returns the new array of intersecting values. + * @example + * + * _.intersection([2, 1], [2, 3]); + * // => [2] */ - var stringToPath = memoizeCapped(function(string) { - var result = []; - if (string.charCodeAt(0) === 46 /* . */) { - result.push(''); - } - string.replace(rePropName, function(match, number, quote, subString) { - result.push(quote ? subString.replace(reEscapeChar, '$1') : (number || match)); - }); - return result; + var intersection = baseRest(function(arrays) { + var mapped = arrayMap(arrays, castArrayLikeObject); + return (mapped.length && mapped[0] === arrays[0]) + ? baseIntersection(mapped) + : []; }); /** - * Converts `value` to a string key if it's not a string or symbol. + * This method is like `_.intersection` except that it accepts `iteratee` + * which is invoked for each element of each `arrays` to generate the criterion + * by which they're compared. The order and references of result values are + * determined by the first array. The iteratee is invoked with one argument: + * (value). * - * @private - * @param {*} value The value to inspect. - * @returns {string|symbol} Returns the key. + * @static + * @memberOf _ + * @since 4.0.0 + * @category Array + * @param {...Array} [arrays] The arrays to inspect. + * @param {Function} [iteratee=_.identity] The iteratee invoked per element. + * @returns {Array} Returns the new array of intersecting values. + * @example + * + * _.intersectionBy([2.1, 1.2], [2.3, 3.4], Math.floor); + * // => [2.1] + * + * // The `_.property` iteratee shorthand. + * _.intersectionBy([{ 'x': 1 }], [{ 'x': 2 }, { 'x': 1 }], 'x'); + * // => [{ 'x': 1 }] */ - function toKey(value) { - if (typeof value == 'string' || isSymbol(value)) { - return value; + var intersectionBy = baseRest(function(arrays) { + var iteratee = last(arrays), + mapped = arrayMap(arrays, castArrayLikeObject); + + if (iteratee === last(mapped)) { + iteratee = undefined; + } else { + mapped.pop(); } - var result = (value + ''); - return (result == '0' && (1 / value) == -INFINITY) ? '-0' : result; - } + return (mapped.length && mapped[0] === arrays[0]) + ? baseIntersection(mapped, getIteratee(iteratee, 2)) + : []; + }); /** - * Converts `func` to its source code. + * This method is like `_.intersection` except that it accepts `comparator` + * which is invoked to compare elements of `arrays`. The order and references + * of result values are determined by the first array. The comparator is + * invoked with two arguments: (arrVal, othVal). * - * @private - * @param {Function} func The function to convert. - * @returns {string} Returns the source code. + * @static + * @memberOf _ + * @since 4.0.0 + * @category Array + * @param {...Array} [arrays] The arrays to inspect. + * @param {Function} [comparator] The comparator invoked per element. + * @returns {Array} Returns the new array of intersecting values. + * @example + * + * var objects = [{ 'x': 1, 'y': 2 }, { 'x': 2, 'y': 1 }]; + * var others = [{ 'x': 1, 'y': 1 }, { 'x': 1, 'y': 2 }]; + * + * _.intersectionWith(objects, others, _.isEqual); + * // => [{ 'x': 1, 'y': 2 }] */ - function toSource(func) { - if (func != null) { - try { - return funcToString.call(func); - } catch (e) {} - try { - return (func + ''); - } catch (e) {} + var intersectionWith = baseRest(function(arrays) { + var comparator = last(arrays), + mapped = arrayMap(arrays, castArrayLikeObject); + + comparator = typeof comparator == 'function' ? comparator : undefined; + if (comparator) { + mapped.pop(); } - return ''; - } + return (mapped.length && mapped[0] === arrays[0]) + ? baseIntersection(mapped, undefined, comparator) + : []; + }); /** - * Updates wrapper `details` based on `bitmask` flags. + * Converts all elements in `array` into a string separated by `separator`. * - * @private - * @returns {Array} details The details to modify. - * @param {number} bitmask The bitmask flags. See `createWrap` for more details. - * @returns {Array} Returns `details`. + * @static + * @memberOf _ + * @since 4.0.0 + * @category Array + * @param {Array} array The array to convert. + * @param {string} [separator=','] The element separator. + * @returns {string} Returns the joined string. + * @example + * + * _.join(['a', 'b', 'c'], '~'); + * // => 'a~b~c' */ - function updateWrapDetails(details, bitmask) { - arrayEach(wrapFlags, function(pair) { - var value = '_.' + pair[0]; - if ((bitmask & pair[1]) && !arrayIncludes(details, value)) { - details.push(value); - } - }); - return details.sort(); + function join(array, separator) { + return array == null ? '' : nativeJoin.call(array, separator); } /** - * Creates a clone of `wrapper`. + * Gets the last element of `array`. * - * @private - * @param {Object} wrapper The wrapper to clone. - * @returns {Object} Returns the cloned wrapper. + * @static + * @memberOf _ + * @since 0.1.0 + * @category Array + * @param {Array} array The array to query. + * @returns {*} Returns the last element of `array`. + * @example + * + * _.last([1, 2, 3]); + * // => 3 */ - function wrapperClone(wrapper) { - if (wrapper instanceof LazyWrapper) { - return wrapper.clone(); - } - var result = new LodashWrapper(wrapper.__wrapped__, wrapper.__chain__); - result.__actions__ = copyArray(wrapper.__actions__); - result.__index__ = wrapper.__index__; - result.__values__ = wrapper.__values__; - return result; + function last(array) { + var length = array == null ? 0 : array.length; + return length ? array[length - 1] : undefined; } - /*------------------------------------------------------------------------*/ - /** - * Creates an array of elements split into groups the length of `size`. - * If `array` can't be split evenly, the final chunk will be the remaining - * elements. + * This method is like `_.indexOf` except that it iterates over elements of + * `array` from right to left. * * @static * @memberOf _ - * @since 3.0.0 + * @since 0.1.0 * @category Array - * @param {Array} array The array to process. - * @param {number} [size=1] The length of each chunk - * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. - * @returns {Array} Returns the new array of chunks. + * @param {Array} array The array to inspect. + * @param {*} value The value to search for. + * @param {number} [fromIndex=array.length-1] The index to search from. + * @returns {number} Returns the index of the matched value, else `-1`. * @example * - * _.chunk(['a', 'b', 'c', 'd'], 2); - * // => [['a', 'b'], ['c', 'd']] + * _.lastIndexOf([1, 2, 1, 2], 2); + * // => 3 * - * _.chunk(['a', 'b', 'c', 'd'], 3); - * // => [['a', 'b', 'c'], ['d']] + * // Search from the `fromIndex`. + * _.lastIndexOf([1, 2, 1, 2], 2, 2); + * // => 1 */ - function chunk(array, size, guard) { - if ((guard ? isIterateeCall(array, size, guard) : size === undefined)) { - size = 1; - } else { - size = nativeMax(toInteger(size), 0); - } + function lastIndexOf(array, value, fromIndex) { var length = array == null ? 0 : array.length; - if (!length || size < 1) { - return []; + if (!length) { + return -1; } - var index = 0, - resIndex = 0, - result = Array(nativeCeil(length / size)); - - while (index < length) { - result[resIndex++] = baseSlice(array, index, (index += size)); + var index = length; + if (fromIndex !== undefined) { + index = toInteger(fromIndex); + index = index < 0 ? nativeMax(length + index, 0) : nativeMin(index, length - 1); } - return result; + return value === value + ? strictLastIndexOf(array, value, index) + : baseFindIndex(array, baseIsNaN, index, true); } /** - * Creates an array with all falsey values removed. The values `false`, `null`, - * `0`, `""`, `undefined`, and `NaN` are falsey. + * Gets the element at index `n` of `array`. If `n` is negative, the nth + * element from the end is returned. * * @static * @memberOf _ - * @since 0.1.0 + * @since 4.11.0 * @category Array - * @param {Array} array The array to compact. - * @returns {Array} Returns the new array of filtered values. + * @param {Array} array The array to query. + * @param {number} [n=0] The index of the element to return. + * @returns {*} Returns the nth element of `array`. * @example * - * _.compact([0, 1, false, 2, '', 3]); - * // => [1, 2, 3] + * var array = ['a', 'b', 'c', 'd']; + * + * _.nth(array, 1); + * // => 'b' + * + * _.nth(array, -2); + * // => 'c'; */ - function compact(array) { - var index = -1, - length = array == null ? 0 : array.length, - resIndex = 0, - result = []; - - while (++index < length) { - var value = array[index]; - if (value) { - result[resIndex++] = value; - } - } - return result; + function nth(array, n) { + return (array && array.length) ? baseNth(array, toInteger(n)) : undefined; } /** - * Creates a new array concatenating `array` with any additional arrays - * and/or values. + * Removes all given values from `array` using + * [`SameValueZero`](http://ecma-international.org/ecma-262/7.0/#sec-samevaluezero) + * for equality comparisons. + * + * **Note:** Unlike `_.without`, this method mutates `array`. Use `_.remove` + * to remove elements from an array by predicate. * * @static * @memberOf _ - * @since 4.0.0 + * @since 2.0.0 * @category Array - * @param {Array} array The array to concatenate. - * @param {...*} [values] The values to concatenate. - * @returns {Array} Returns the new concatenated array. + * @param {Array} array The array to modify. + * @param {...*} [values] The values to remove. + * @returns {Array} Returns `array`. * @example * - * var array = [1]; - * var other = _.concat(array, 2, [3], [[4]]); - * - * console.log(other); - * // => [1, 2, 3, [4]] + * var array = ['a', 'b', 'c', 'a', 'b', 'c']; * + * _.pull(array, 'a', 'c'); * console.log(array); - * // => [1] + * // => ['b', 'b'] */ - function concat() { - var length = arguments.length; - if (!length) { - return []; - } - var args = Array(length - 1), - array = arguments[0], - index = length; - - while (index--) { - args[index - 1] = arguments[index]; - } - return arrayPush(isArray(array) ? copyArray(array) : [array], baseFlatten(args, 1)); - } + var pull = baseRest(pullAll); /** - * Creates an array of `array` values not included in the other given arrays - * using [`SameValueZero`](http://ecma-international.org/ecma-262/7.0/#sec-samevaluezero) - * for equality comparisons. The order and references of result values are - * determined by the first array. + * This method is like `_.pull` except that it accepts an array of values to remove. * - * **Note:** Unlike `_.pullAll`, this method returns a new array. + * **Note:** Unlike `_.difference`, this method mutates `array`. * * @static * @memberOf _ - * @since 0.1.0 + * @since 4.0.0 * @category Array - * @param {Array} array The array to inspect. - * @param {...Array} [values] The values to exclude. - * @returns {Array} Returns the new array of filtered values. - * @see _.without, _.xor + * @param {Array} array The array to modify. + * @param {Array} values The values to remove. + * @returns {Array} Returns `array`. * @example * - * _.difference([2, 1], [2, 3]); - * // => [1] + * var array = ['a', 'b', 'c', 'a', 'b', 'c']; + * + * _.pullAll(array, ['a', 'c']); + * console.log(array); + * // => ['b', 'b'] */ - var difference = baseRest(function(array, values) { - return isArrayLikeObject(array) - ? baseDifference(array, baseFlatten(values, 1, isArrayLikeObject, true)) - : []; - }); + function pullAll(array, values) { + return (array && array.length && values && values.length) + ? basePullAll(array, values) + : array; + } /** - * This method is like `_.difference` except that it accepts `iteratee` which - * is invoked for each element of `array` and `values` to generate the criterion - * by which they're compared. The order and references of result values are - * determined by the first array. The iteratee is invoked with one argument: - * (value). + * This method is like `_.pullAll` except that it accepts `iteratee` which is + * invoked for each element of `array` and `values` to generate the criterion + * by which they're compared. The iteratee is invoked with one argument: (value). * - * **Note:** Unlike `_.pullAllBy`, this method returns a new array. + * **Note:** Unlike `_.differenceBy`, this method mutates `array`. * * @static * @memberOf _ * @since 4.0.0 * @category Array - * @param {Array} array The array to inspect. - * @param {...Array} [values] The values to exclude. + * @param {Array} array The array to modify. + * @param {Array} values The values to remove. * @param {Function} [iteratee=_.identity] The iteratee invoked per element. - * @returns {Array} Returns the new array of filtered values. + * @returns {Array} Returns `array`. * @example * - * _.differenceBy([2.1, 1.2], [2.3, 3.4], Math.floor); - * // => [1.2] + * var array = [{ 'x': 1 }, { 'x': 2 }, { 'x': 3 }, { 'x': 1 }]; * - * // The `_.property` iteratee shorthand. - * _.differenceBy([{ 'x': 2 }, { 'x': 1 }], [{ 'x': 1 }], 'x'); + * _.pullAllBy(array, [{ 'x': 1 }, { 'x': 3 }], 'x'); + * console.log(array); * // => [{ 'x': 2 }] */ - var differenceBy = baseRest(function(array, values) { - var iteratee = last(values); - if (isArrayLikeObject(iteratee)) { - iteratee = undefined; - } - return isArrayLikeObject(array) - ? baseDifference(array, baseFlatten(values, 1, isArrayLikeObject, true), getIteratee(iteratee, 2)) - : []; - }); + function pullAllBy(array, values, iteratee) { + return (array && array.length && values && values.length) + ? basePullAll(array, values, getIteratee(iteratee, 2)) + : array; + } /** - * This method is like `_.difference` except that it accepts `comparator` - * which is invoked to compare elements of `array` to `values`. The order and - * references of result values are determined by the first array. The comparator - * is invoked with two arguments: (arrVal, othVal). + * This method is like `_.pullAll` except that it accepts `comparator` which + * is invoked to compare elements of `array` to `values`. The comparator is + * invoked with two arguments: (arrVal, othVal). * - * **Note:** Unlike `_.pullAllWith`, this method returns a new array. + * **Note:** Unlike `_.differenceWith`, this method mutates `array`. * * @static * @memberOf _ - * @since 4.0.0 + * @since 4.6.0 * @category Array - * @param {Array} array The array to inspect. - * @param {...Array} [values] The values to exclude. + * @param {Array} array The array to modify. + * @param {Array} values The values to remove. * @param {Function} [comparator] The comparator invoked per element. - * @returns {Array} Returns the new array of filtered values. + * @returns {Array} Returns `array`. * @example * - * var objects = [{ 'x': 1, 'y': 2 }, { 'x': 2, 'y': 1 }]; + * var array = [{ 'x': 1, 'y': 2 }, { 'x': 3, 'y': 4 }, { 'x': 5, 'y': 6 }]; * - * _.differenceWith(objects, [{ 'x': 1, 'y': 2 }], _.isEqual); - * // => [{ 'x': 2, 'y': 1 }] + * _.pullAllWith(array, [{ 'x': 3, 'y': 4 }], _.isEqual); + * console.log(array); + * // => [{ 'x': 1, 'y': 2 }, { 'x': 5, 'y': 6 }] */ - var differenceWith = baseRest(function(array, values) { - var comparator = last(values); - if (isArrayLikeObject(comparator)) { - comparator = undefined; - } - return isArrayLikeObject(array) - ? baseDifference(array, baseFlatten(values, 1, isArrayLikeObject, true), undefined, comparator) - : []; - }); + function pullAllWith(array, values, comparator) { + return (array && array.length && values && values.length) + ? basePullAll(array, values, undefined, comparator) + : array; + } /** - * Creates a slice of `array` with `n` elements dropped from the beginning. + * Removes elements from `array` corresponding to `indexes` and returns an + * array of removed elements. + * + * **Note:** Unlike `_.at`, this method mutates `array`. * * @static * @memberOf _ - * @since 0.5.0 + * @since 3.0.0 * @category Array - * @param {Array} array The array to query. - * @param {number} [n=1] The number of elements to drop. - * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. - * @returns {Array} Returns the slice of `array`. + * @param {Array} array The array to modify. + * @param {...(number|number[])} [indexes] The indexes of elements to remove. + * @returns {Array} Returns the new array of removed elements. * @example * - * _.drop([1, 2, 3]); - * // => [2, 3] + * var array = ['a', 'b', 'c', 'd']; + * var pulled = _.pullAt(array, [1, 3]); * - * _.drop([1, 2, 3], 2); - * // => [3] + * console.log(array); + * // => ['a', 'c'] * - * _.drop([1, 2, 3], 5); - * // => [] + * console.log(pulled); + * // => ['b', 'd'] + */ + var pullAt = flatRest(function(array, indexes) { + var length = array == null ? 0 : array.length, + result = baseAt(array, indexes); + + basePullAt(array, arrayMap(indexes, function(index) { + return isIndex(index, length) ? +index : index; + }).sort(compareAscending)); + + return result; + }); + + /** + * Removes all elements from `array` that `predicate` returns truthy for + * and returns an array of the removed elements. The predicate is invoked + * with three arguments: (value, index, array). * - * _.drop([1, 2, 3], 0); - * // => [1, 2, 3] + * **Note:** Unlike `_.filter`, this method mutates `array`. Use `_.pull` + * to pull elements from an array by value. + * + * @static + * @memberOf _ + * @since 2.0.0 + * @category Array + * @param {Array} array The array to modify. + * @param {Function} [predicate=_.identity] The function invoked per iteration. + * @returns {Array} Returns the new array of removed elements. + * @example + * + * var array = [1, 2, 3, 4]; + * var evens = _.remove(array, function(n) { + * return n % 2 == 0; + * }); + * + * console.log(array); + * // => [1, 3] + * + * console.log(evens); + * // => [2, 4] */ - function drop(array, n, guard) { - var length = array == null ? 0 : array.length; - if (!length) { - return []; + function remove(array, predicate) { + var result = []; + if (!(array && array.length)) { + return result; } - n = (guard || n === undefined) ? 1 : toInteger(n); - return baseSlice(array, n < 0 ? 0 : n, length); + var index = -1, + indexes = [], + length = array.length; + + predicate = getIteratee(predicate, 3); + while (++index < length) { + var value = array[index]; + if (predicate(value, index, array)) { + result.push(value); + indexes.push(index); + } + } + basePullAt(array, indexes); + return result; } /** - * Creates a slice of `array` with `n` elements dropped from the end. + * Reverses `array` so that the first element becomes the last, the second + * element becomes the second to last, and so on. + * + * **Note:** This method mutates `array` and is based on + * [`Array#reverse`](https://mdn.io/Array/reverse). * * @static * @memberOf _ - * @since 3.0.0 + * @since 4.0.0 * @category Array - * @param {Array} array The array to query. - * @param {number} [n=1] The number of elements to drop. - * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. - * @returns {Array} Returns the slice of `array`. + * @param {Array} array The array to modify. + * @returns {Array} Returns `array`. * @example * - * _.dropRight([1, 2, 3]); - * // => [1, 2] + * var array = [1, 2, 3]; * - * _.dropRight([1, 2, 3], 2); - * // => [1] + * _.reverse(array); + * // => [3, 2, 1] * - * _.dropRight([1, 2, 3], 5); - * // => [] + * console.log(array); + * // => [3, 2, 1] + */ + function reverse(array) { + return array == null ? array : nativeReverse.call(array); + } + + /** + * Creates a slice of `array` from `start` up to, but not including, `end`. * - * _.dropRight([1, 2, 3], 0); - * // => [1, 2, 3] + * **Note:** This method is used instead of + * [`Array#slice`](https://mdn.io/Array/slice) to ensure dense arrays are + * returned. + * + * @static + * @memberOf _ + * @since 3.0.0 + * @category Array + * @param {Array} array The array to slice. + * @param {number} [start=0] The start position. + * @param {number} [end=array.length] The end position. + * @returns {Array} Returns the slice of `array`. */ - function dropRight(array, n, guard) { + function slice(array, start, end) { var length = array == null ? 0 : array.length; if (!length) { return []; } - n = (guard || n === undefined) ? 1 : toInteger(n); - n = length - n; - return baseSlice(array, 0, n < 0 ? 0 : n); + if (end && typeof end != 'number' && isIterateeCall(array, start, end)) { + start = 0; + end = length; + } + else { + start = start == null ? 0 : toInteger(start); + end = end === undefined ? length : toInteger(end); + } + return baseSlice(array, start, end); } /** - * Creates a slice of `array` excluding elements dropped from the end. - * Elements are dropped until `predicate` returns falsey. The predicate is - * invoked with three arguments: (value, index, array). + * Uses a binary search to determine the lowest index at which `value` + * should be inserted into `array` in order to maintain its sort order. * * @static * @memberOf _ - * @since 3.0.0 + * @since 0.1.0 * @category Array - * @param {Array} array The array to query. - * @param {Function} [predicate=_.identity] The function invoked per iteration. - * @returns {Array} Returns the slice of `array`. + * @param {Array} array The sorted array to inspect. + * @param {*} value The value to evaluate. + * @returns {number} Returns the index at which `value` should be inserted + * into `array`. * @example * - * var users = [ - * { 'user': 'barney', 'active': true }, - * { 'user': 'fred', 'active': false }, - * { 'user': 'pebbles', 'active': false } - * ]; - * - * _.dropRightWhile(users, function(o) { return !o.active; }); - * // => objects for ['barney'] - * - * // The `_.matches` iteratee shorthand. - * _.dropRightWhile(users, { 'user': 'pebbles', 'active': false }); - * // => objects for ['barney', 'fred'] - * - * // The `_.matchesProperty` iteratee shorthand. - * _.dropRightWhile(users, ['active', false]); - * // => objects for ['barney'] - * - * // The `_.property` iteratee shorthand. - * _.dropRightWhile(users, 'active'); - * // => objects for ['barney', 'fred', 'pebbles'] + * _.sortedIndex([30, 50], 40); + * // => 1 */ - function dropRightWhile(array, predicate) { - return (array && array.length) - ? baseWhile(array, getIteratee(predicate, 3), true, true) - : []; + function sortedIndex(array, value) { + return baseSortedIndex(array, value); } /** - * Creates a slice of `array` excluding elements dropped from the beginning. - * Elements are dropped until `predicate` returns falsey. The predicate is - * invoked with three arguments: (value, index, array). + * This method is like `_.sortedIndex` except that it accepts `iteratee` + * which is invoked for `value` and each element of `array` to compute their + * sort ranking. The iteratee is invoked with one argument: (value). * * @static * @memberOf _ - * @since 3.0.0 + * @since 4.0.0 * @category Array - * @param {Array} array The array to query. - * @param {Function} [predicate=_.identity] The function invoked per iteration. - * @returns {Array} Returns the slice of `array`. + * @param {Array} array The sorted array to inspect. + * @param {*} value The value to evaluate. + * @param {Function} [iteratee=_.identity] The iteratee invoked per element. + * @returns {number} Returns the index at which `value` should be inserted + * into `array`. * @example * - * var users = [ - * { 'user': 'barney', 'active': false }, - * { 'user': 'fred', 'active': false }, - * { 'user': 'pebbles', 'active': true } - * ]; - * - * _.dropWhile(users, function(o) { return !o.active; }); - * // => objects for ['pebbles'] - * - * // The `_.matches` iteratee shorthand. - * _.dropWhile(users, { 'user': 'barney', 'active': false }); - * // => objects for ['fred', 'pebbles'] - * - * // The `_.matchesProperty` iteratee shorthand. - * _.dropWhile(users, ['active', false]); - * // => objects for ['pebbles'] + * var objects = [{ 'x': 4 }, { 'x': 5 }]; + * + * _.sortedIndexBy(objects, { 'x': 4 }, function(o) { return o.x; }); + * // => 0 * * // The `_.property` iteratee shorthand. - * _.dropWhile(users, 'active'); - * // => objects for ['barney', 'fred', 'pebbles'] + * _.sortedIndexBy(objects, { 'x': 4 }, 'x'); + * // => 0 */ - function dropWhile(array, predicate) { - return (array && array.length) - ? baseWhile(array, getIteratee(predicate, 3), true) - : []; + function sortedIndexBy(array, value, iteratee) { + return baseSortedIndexBy(array, value, getIteratee(iteratee, 2)); } /** - * Fills elements of `array` with `value` from `start` up to, but not - * including, `end`. - * - * **Note:** This method mutates `array`. + * This method is like `_.indexOf` except that it performs a binary + * search on a sorted `array`. * * @static * @memberOf _ - * @since 3.2.0 + * @since 4.0.0 * @category Array - * @param {Array} array The array to fill. - * @param {*} value The value to fill `array` with. - * @param {number} [start=0] The start position. - * @param {number} [end=array.length] The end position. - * @returns {Array} Returns `array`. + * @param {Array} array The array to inspect. + * @param {*} value The value to search for. + * @returns {number} Returns the index of the matched value, else `-1`. * @example * - * var array = [1, 2, 3]; - * - * _.fill(array, 'a'); - * console.log(array); - * // => ['a', 'a', 'a'] - * - * _.fill(Array(3), 2); - * // => [2, 2, 2] - * - * _.fill([4, 6, 8, 10], '*', 1, 3); - * // => [4, '*', '*', 10] + * _.sortedIndexOf([4, 5, 5, 5, 6], 5); + * // => 1 */ - function fill(array, value, start, end) { + function sortedIndexOf(array, value) { var length = array == null ? 0 : array.length; - if (!length) { - return []; - } - if (start && typeof start != 'number' && isIterateeCall(array, value, start)) { - start = 0; - end = length; + if (length) { + var index = baseSortedIndex(array, value); + if (index < length && eq(array[index], value)) { + return index; + } } - return baseFill(array, value, start, end); + return -1; } /** - * This method is like `_.find` except that it returns the index of the first - * element `predicate` returns truthy for instead of the element itself. + * This method is like `_.sortedIndex` except that it returns the highest + * index at which `value` should be inserted into `array` in order to + * maintain its sort order. * * @static * @memberOf _ - * @since 1.1.0 + * @since 3.0.0 * @category Array - * @param {Array} array The array to inspect. - * @param {Function} [predicate=_.identity] The function invoked per iteration. - * @param {number} [fromIndex=0] The index to search from. - * @returns {number} Returns the index of the found element, else `-1`. + * @param {Array} array The sorted array to inspect. + * @param {*} value The value to evaluate. + * @returns {number} Returns the index at which `value` should be inserted + * into `array`. * @example * - * var users = [ - * { 'user': 'barney', 'active': false }, - * { 'user': 'fred', 'active': false }, - * { 'user': 'pebbles', 'active': true } - * ]; + * _.sortedLastIndex([4, 5, 5, 5, 6], 5); + * // => 4 + */ + function sortedLastIndex(array, value) { + return baseSortedIndex(array, value, true); + } + + /** + * This method is like `_.sortedLastIndex` except that it accepts `iteratee` + * which is invoked for `value` and each element of `array` to compute their + * sort ranking. The iteratee is invoked with one argument: (value). * - * _.findIndex(users, function(o) { return o.user == 'barney'; }); - * // => 0 + * @static + * @memberOf _ + * @since 4.0.0 + * @category Array + * @param {Array} array The sorted array to inspect. + * @param {*} value The value to evaluate. + * @param {Function} [iteratee=_.identity] The iteratee invoked per element. + * @returns {number} Returns the index at which `value` should be inserted + * into `array`. + * @example * - * // The `_.matches` iteratee shorthand. - * _.findIndex(users, { 'user': 'fred', 'active': false }); - * // => 1 + * var objects = [{ 'x': 4 }, { 'x': 5 }]; * - * // The `_.matchesProperty` iteratee shorthand. - * _.findIndex(users, ['active', false]); - * // => 0 + * _.sortedLastIndexBy(objects, { 'x': 4 }, function(o) { return o.x; }); + * // => 1 * * // The `_.property` iteratee shorthand. - * _.findIndex(users, 'active'); - * // => 2 + * _.sortedLastIndexBy(objects, { 'x': 4 }, 'x'); + * // => 1 */ - function findIndex(array, predicate, fromIndex) { - var length = array == null ? 0 : array.length; - if (!length) { - return -1; - } - var index = fromIndex == null ? 0 : toInteger(fromIndex); - if (index < 0) { - index = nativeMax(length + index, 0); - } - return baseFindIndex(array, getIteratee(predicate, 3), index); + function sortedLastIndexBy(array, value, iteratee) { + return baseSortedIndexBy(array, value, getIteratee(iteratee, 2), true); } /** - * This method is like `_.findIndex` except that it iterates over elements - * of `collection` from right to left. + * This method is like `_.lastIndexOf` except that it performs a binary + * search on a sorted `array`. * * @static * @memberOf _ - * @since 2.0.0 + * @since 4.0.0 * @category Array * @param {Array} array The array to inspect. - * @param {Function} [predicate=_.identity] The function invoked per iteration. - * @param {number} [fromIndex=array.length-1] The index to search from. - * @returns {number} Returns the index of the found element, else `-1`. + * @param {*} value The value to search for. + * @returns {number} Returns the index of the matched value, else `-1`. * @example * - * var users = [ - * { 'user': 'barney', 'active': true }, - * { 'user': 'fred', 'active': false }, - * { 'user': 'pebbles', 'active': false } - * ]; - * - * _.findLastIndex(users, function(o) { return o.user == 'pebbles'; }); - * // => 2 - * - * // The `_.matches` iteratee shorthand. - * _.findLastIndex(users, { 'user': 'barney', 'active': true }); - * // => 0 - * - * // The `_.matchesProperty` iteratee shorthand. - * _.findLastIndex(users, ['active', false]); - * // => 2 - * - * // The `_.property` iteratee shorthand. - * _.findLastIndex(users, 'active'); - * // => 0 + * _.sortedLastIndexOf([4, 5, 5, 5, 6], 5); + * // => 3 */ - function findLastIndex(array, predicate, fromIndex) { + function sortedLastIndexOf(array, value) { var length = array == null ? 0 : array.length; - if (!length) { - return -1; - } - var index = length - 1; - if (fromIndex !== undefined) { - index = toInteger(fromIndex); - index = fromIndex < 0 - ? nativeMax(length + index, 0) - : nativeMin(index, length - 1); + if (length) { + var index = baseSortedIndex(array, value, true) - 1; + if (eq(array[index], value)) { + return index; + } } - return baseFindIndex(array, getIteratee(predicate, 3), index, true); + return -1; } /** - * Flattens `array` a single level deep. + * This method is like `_.uniq` except that it's designed and optimized + * for sorted arrays. * * @static * @memberOf _ - * @since 0.1.0 + * @since 4.0.0 * @category Array - * @param {Array} array The array to flatten. - * @returns {Array} Returns the new flattened array. + * @param {Array} array The array to inspect. + * @returns {Array} Returns the new duplicate free array. * @example * - * _.flatten([1, [2, [3, [4]], 5]]); - * // => [1, 2, [3, [4]], 5] + * _.sortedUniq([1, 1, 2]); + * // => [1, 2] */ - function flatten(array) { - var length = array == null ? 0 : array.length; - return length ? baseFlatten(array, 1) : []; + function sortedUniq(array) { + return (array && array.length) + ? baseSortedUniq(array) + : []; } /** - * Recursively flattens `array`. + * This method is like `_.uniqBy` except that it's designed and optimized + * for sorted arrays. * * @static * @memberOf _ - * @since 3.0.0 + * @since 4.0.0 * @category Array - * @param {Array} array The array to flatten. - * @returns {Array} Returns the new flattened array. + * @param {Array} array The array to inspect. + * @param {Function} [iteratee] The iteratee invoked per element. + * @returns {Array} Returns the new duplicate free array. * @example * - * _.flattenDeep([1, [2, [3, [4]], 5]]); - * // => [1, 2, 3, 4, 5] + * _.sortedUniqBy([1.1, 1.2, 2.3, 2.4], Math.floor); + * // => [1.1, 2.3] */ - function flattenDeep(array) { - var length = array == null ? 0 : array.length; - return length ? baseFlatten(array, INFINITY) : []; + function sortedUniqBy(array, iteratee) { + return (array && array.length) + ? baseSortedUniq(array, getIteratee(iteratee, 2)) + : []; } /** - * Recursively flatten `array` up to `depth` times. + * Gets all but the first element of `array`. * * @static * @memberOf _ - * @since 4.4.0 + * @since 4.0.0 * @category Array - * @param {Array} array The array to flatten. - * @param {number} [depth=1] The maximum recursion depth. - * @returns {Array} Returns the new flattened array. + * @param {Array} array The array to query. + * @returns {Array} Returns the slice of `array`. * @example * - * var array = [1, [2, [3, [4]], 5]]; - * - * _.flattenDepth(array, 1); - * // => [1, 2, [3, [4]], 5] - * - * _.flattenDepth(array, 2); - * // => [1, 2, 3, [4], 5] + * _.tail([1, 2, 3]); + * // => [2, 3] */ - function flattenDepth(array, depth) { + function tail(array) { var length = array == null ? 0 : array.length; - if (!length) { - return []; - } - depth = depth === undefined ? 1 : toInteger(depth); - return baseFlatten(array, depth); + return length ? baseSlice(array, 1, length) : []; } /** - * The inverse of `_.toPairs`; this method returns an object composed - * from key-value `pairs`. + * Creates a slice of `array` with `n` elements taken from the beginning. * * @static * @memberOf _ - * @since 4.0.0 + * @since 0.1.0 * @category Array - * @param {Array} pairs The key-value pairs. - * @returns {Object} Returns the new object. + * @param {Array} array The array to query. + * @param {number} [n=1] The number of elements to take. + * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. + * @returns {Array} Returns the slice of `array`. * @example * - * _.fromPairs([['a', 1], ['b', 2]]); - * // => { 'a': 1, 'b': 2 } + * _.take([1, 2, 3]); + * // => [1] + * + * _.take([1, 2, 3], 2); + * // => [1, 2] + * + * _.take([1, 2, 3], 5); + * // => [1, 2, 3] + * + * _.take([1, 2, 3], 0); + * // => [] */ - function fromPairs(pairs) { - var index = -1, - length = pairs == null ? 0 : pairs.length, - result = {}; - - while (++index < length) { - var pair = pairs[index]; - result[pair[0]] = pair[1]; + function take(array, n, guard) { + if (!(array && array.length)) { + return []; } - return result; + n = (guard || n === undefined) ? 1 : toInteger(n); + return baseSlice(array, 0, n < 0 ? 0 : n); } /** - * Gets the first element of `array`. + * Creates a slice of `array` with `n` elements taken from the end. * * @static * @memberOf _ - * @since 0.1.0 - * @alias first + * @since 3.0.0 * @category Array * @param {Array} array The array to query. - * @returns {*} Returns the first element of `array`. + * @param {number} [n=1] The number of elements to take. + * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. + * @returns {Array} Returns the slice of `array`. * @example * - * _.head([1, 2, 3]); - * // => 1 + * _.takeRight([1, 2, 3]); + * // => [3] * - * _.head([]); - * // => undefined + * _.takeRight([1, 2, 3], 2); + * // => [2, 3] + * + * _.takeRight([1, 2, 3], 5); + * // => [1, 2, 3] + * + * _.takeRight([1, 2, 3], 0); + * // => [] */ - function head(array) { - return (array && array.length) ? array[0] : undefined; + function takeRight(array, n, guard) { + var length = array == null ? 0 : array.length; + if (!length) { + return []; + } + n = (guard || n === undefined) ? 1 : toInteger(n); + n = length - n; + return baseSlice(array, n < 0 ? 0 : n, length); } /** - * Gets the index at which the first occurrence of `value` is found in `array` - * using [`SameValueZero`](http://ecma-international.org/ecma-262/7.0/#sec-samevaluezero) - * for equality comparisons. If `fromIndex` is negative, it's used as the - * offset from the end of `array`. + * Creates a slice of `array` with elements taken from the end. Elements are + * taken until `predicate` returns falsey. The predicate is invoked with + * three arguments: (value, index, array). * * @static * @memberOf _ - * @since 0.1.0 + * @since 3.0.0 * @category Array - * @param {Array} array The array to inspect. - * @param {*} value The value to search for. - * @param {number} [fromIndex=0] The index to search from. - * @returns {number} Returns the index of the matched value, else `-1`. + * @param {Array} array The array to query. + * @param {Function} [predicate=_.identity] The function invoked per iteration. + * @returns {Array} Returns the slice of `array`. * @example * - * _.indexOf([1, 2, 1, 2], 2); - * // => 1 + * var users = [ + * { 'user': 'barney', 'active': true }, + * { 'user': 'fred', 'active': false }, + * { 'user': 'pebbles', 'active': false } + * ]; * - * // Search from the `fromIndex`. - * _.indexOf([1, 2, 1, 2], 2, 2); - * // => 3 + * _.takeRightWhile(users, function(o) { return !o.active; }); + * // => objects for ['fred', 'pebbles'] + * + * // The `_.matches` iteratee shorthand. + * _.takeRightWhile(users, { 'user': 'pebbles', 'active': false }); + * // => objects for ['pebbles'] + * + * // The `_.matchesProperty` iteratee shorthand. + * _.takeRightWhile(users, ['active', false]); + * // => objects for ['fred', 'pebbles'] + * + * // The `_.property` iteratee shorthand. + * _.takeRightWhile(users, 'active'); + * // => [] */ - function indexOf(array, value, fromIndex) { - var length = array == null ? 0 : array.length; - if (!length) { - return -1; - } - var index = fromIndex == null ? 0 : toInteger(fromIndex); - if (index < 0) { - index = nativeMax(length + index, 0); - } - return baseIndexOf(array, value, index); + function takeRightWhile(array, predicate) { + return (array && array.length) + ? baseWhile(array, getIteratee(predicate, 3), false, true) + : []; } /** - * Gets all but the last element of `array`. + * Creates a slice of `array` with elements taken from the beginning. Elements + * are taken until `predicate` returns falsey. The predicate is invoked with + * three arguments: (value, index, array). * * @static * @memberOf _ - * @since 0.1.0 + * @since 3.0.0 * @category Array * @param {Array} array The array to query. + * @param {Function} [predicate=_.identity] The function invoked per iteration. * @returns {Array} Returns the slice of `array`. * @example * - * _.initial([1, 2, 3]); - * // => [1, 2] + * var users = [ + * { 'user': 'barney', 'active': false }, + * { 'user': 'fred', 'active': false }, + * { 'user': 'pebbles', 'active': true } + * ]; + * + * _.takeWhile(users, function(o) { return !o.active; }); + * // => objects for ['barney', 'fred'] + * + * // The `_.matches` iteratee shorthand. + * _.takeWhile(users, { 'user': 'barney', 'active': false }); + * // => objects for ['barney'] + * + * // The `_.matchesProperty` iteratee shorthand. + * _.takeWhile(users, ['active', false]); + * // => objects for ['barney', 'fred'] + * + * // The `_.property` iteratee shorthand. + * _.takeWhile(users, 'active'); + * // => [] */ - function initial(array) { - var length = array == null ? 0 : array.length; - return length ? baseSlice(array, 0, -1) : []; + function takeWhile(array, predicate) { + return (array && array.length) + ? baseWhile(array, getIteratee(predicate, 3)) + : []; } /** - * Creates an array of unique values that are included in all given arrays - * using [`SameValueZero`](http://ecma-international.org/ecma-262/7.0/#sec-samevaluezero) - * for equality comparisons. The order and references of result values are - * determined by the first array. + * Creates an array of unique values, in order, from all given arrays using + * [`SameValueZero`](http://ecma-international.org/ecma-262/7.0/#sec-samevaluezero) + * for equality comparisons. * * @static * @memberOf _ * @since 0.1.0 * @category Array * @param {...Array} [arrays] The arrays to inspect. - * @returns {Array} Returns the new array of intersecting values. + * @returns {Array} Returns the new array of combined values. * @example * - * _.intersection([2, 1], [2, 3]); - * // => [2] + * _.union([2], [1, 2]); + * // => [2, 1] */ - var intersection = baseRest(function(arrays) { - var mapped = arrayMap(arrays, castArrayLikeObject); - return (mapped.length && mapped[0] === arrays[0]) - ? baseIntersection(mapped) - : []; + var union = baseRest(function(arrays) { + return baseUniq(baseFlatten(arrays, 1, isArrayLikeObject, true)); }); /** - * This method is like `_.intersection` except that it accepts `iteratee` - * which is invoked for each element of each `arrays` to generate the criterion - * by which they're compared. The order and references of result values are - * determined by the first array. The iteratee is invoked with one argument: + * This method is like `_.union` except that it accepts `iteratee` which is + * invoked for each element of each `arrays` to generate the criterion by + * which uniqueness is computed. Result values are chosen from the first + * array in which the value occurs. The iteratee is invoked with one argument: * (value). * * @static @@ -47670,35 +47928,29 @@ exports.Deprecation = Deprecation; * @category Array * @param {...Array} [arrays] The arrays to inspect. * @param {Function} [iteratee=_.identity] The iteratee invoked per element. - * @returns {Array} Returns the new array of intersecting values. + * @returns {Array} Returns the new array of combined values. * @example * - * _.intersectionBy([2.1, 1.2], [2.3, 3.4], Math.floor); - * // => [2.1] + * _.unionBy([2.1], [1.2, 2.3], Math.floor); + * // => [2.1, 1.2] * * // The `_.property` iteratee shorthand. - * _.intersectionBy([{ 'x': 1 }], [{ 'x': 2 }, { 'x': 1 }], 'x'); - * // => [{ 'x': 1 }] + * _.unionBy([{ 'x': 1 }], [{ 'x': 2 }, { 'x': 1 }], 'x'); + * // => [{ 'x': 1 }, { 'x': 2 }] */ - var intersectionBy = baseRest(function(arrays) { - var iteratee = last(arrays), - mapped = arrayMap(arrays, castArrayLikeObject); - - if (iteratee === last(mapped)) { + var unionBy = baseRest(function(arrays) { + var iteratee = last(arrays); + if (isArrayLikeObject(iteratee)) { iteratee = undefined; - } else { - mapped.pop(); } - return (mapped.length && mapped[0] === arrays[0]) - ? baseIntersection(mapped, getIteratee(iteratee, 2)) - : []; + return baseUniq(baseFlatten(arrays, 1, isArrayLikeObject, true), getIteratee(iteratee, 2)); }); /** - * This method is like `_.intersection` except that it accepts `comparator` - * which is invoked to compare elements of `arrays`. The order and references - * of result values are determined by the first array. The comparator is - * invoked with two arguments: (arrVal, othVal). + * This method is like `_.union` except that it accepts `comparator` which + * is invoked to compare elements of `arrays`. Result values are chosen from + * the first array in which the value occurs. The comparator is invoked + * with two arguments: (arrVal, othVal). * * @static * @memberOf _ @@ -47706,3646 +47958,3899 @@ exports.Deprecation = Deprecation; * @category Array * @param {...Array} [arrays] The arrays to inspect. * @param {Function} [comparator] The comparator invoked per element. - * @returns {Array} Returns the new array of intersecting values. + * @returns {Array} Returns the new array of combined values. * @example * * var objects = [{ 'x': 1, 'y': 2 }, { 'x': 2, 'y': 1 }]; * var others = [{ 'x': 1, 'y': 1 }, { 'x': 1, 'y': 2 }]; * - * _.intersectionWith(objects, others, _.isEqual); - * // => [{ 'x': 1, 'y': 2 }] + * _.unionWith(objects, others, _.isEqual); + * // => [{ 'x': 1, 'y': 2 }, { 'x': 2, 'y': 1 }, { 'x': 1, 'y': 1 }] */ - var intersectionWith = baseRest(function(arrays) { - var comparator = last(arrays), - mapped = arrayMap(arrays, castArrayLikeObject); - + var unionWith = baseRest(function(arrays) { + var comparator = last(arrays); comparator = typeof comparator == 'function' ? comparator : undefined; - if (comparator) { - mapped.pop(); - } - return (mapped.length && mapped[0] === arrays[0]) - ? baseIntersection(mapped, undefined, comparator) - : []; + return baseUniq(baseFlatten(arrays, 1, isArrayLikeObject, true), undefined, comparator); }); /** - * Converts all elements in `array` into a string separated by `separator`. + * Creates a duplicate-free version of an array, using + * [`SameValueZero`](http://ecma-international.org/ecma-262/7.0/#sec-samevaluezero) + * for equality comparisons, in which only the first occurrence of each element + * is kept. The order of result values is determined by the order they occur + * in the array. + * + * @static + * @memberOf _ + * @since 0.1.0 + * @category Array + * @param {Array} array The array to inspect. + * @returns {Array} Returns the new duplicate free array. + * @example + * + * _.uniq([2, 1, 2]); + * // => [2, 1] + */ + function uniq(array) { + return (array && array.length) ? baseUniq(array) : []; + } + + /** + * This method is like `_.uniq` except that it accepts `iteratee` which is + * invoked for each element in `array` to generate the criterion by which + * uniqueness is computed. The order of result values is determined by the + * order they occur in the array. The iteratee is invoked with one argument: + * (value). * * @static * @memberOf _ * @since 4.0.0 * @category Array - * @param {Array} array The array to convert. - * @param {string} [separator=','] The element separator. - * @returns {string} Returns the joined string. + * @param {Array} array The array to inspect. + * @param {Function} [iteratee=_.identity] The iteratee invoked per element. + * @returns {Array} Returns the new duplicate free array. * @example * - * _.join(['a', 'b', 'c'], '~'); - * // => 'a~b~c' + * _.uniqBy([2.1, 1.2, 2.3], Math.floor); + * // => [2.1, 1.2] + * + * // The `_.property` iteratee shorthand. + * _.uniqBy([{ 'x': 1 }, { 'x': 2 }, { 'x': 1 }], 'x'); + * // => [{ 'x': 1 }, { 'x': 2 }] */ - function join(array, separator) { - return array == null ? '' : nativeJoin.call(array, separator); + function uniqBy(array, iteratee) { + return (array && array.length) ? baseUniq(array, getIteratee(iteratee, 2)) : []; } /** - * Gets the last element of `array`. + * This method is like `_.uniq` except that it accepts `comparator` which + * is invoked to compare elements of `array`. The order of result values is + * determined by the order they occur in the array.The comparator is invoked + * with two arguments: (arrVal, othVal). * * @static * @memberOf _ - * @since 0.1.0 + * @since 4.0.0 * @category Array - * @param {Array} array The array to query. - * @returns {*} Returns the last element of `array`. + * @param {Array} array The array to inspect. + * @param {Function} [comparator] The comparator invoked per element. + * @returns {Array} Returns the new duplicate free array. * @example * - * _.last([1, 2, 3]); - * // => 3 + * var objects = [{ 'x': 1, 'y': 2 }, { 'x': 2, 'y': 1 }, { 'x': 1, 'y': 2 }]; + * + * _.uniqWith(objects, _.isEqual); + * // => [{ 'x': 1, 'y': 2 }, { 'x': 2, 'y': 1 }] */ - function last(array) { - var length = array == null ? 0 : array.length; - return length ? array[length - 1] : undefined; + function uniqWith(array, comparator) { + comparator = typeof comparator == 'function' ? comparator : undefined; + return (array && array.length) ? baseUniq(array, undefined, comparator) : []; } /** - * This method is like `_.indexOf` except that it iterates over elements of - * `array` from right to left. + * This method is like `_.zip` except that it accepts an array of grouped + * elements and creates an array regrouping the elements to their pre-zip + * configuration. * * @static * @memberOf _ - * @since 0.1.0 + * @since 1.2.0 * @category Array - * @param {Array} array The array to inspect. - * @param {*} value The value to search for. - * @param {number} [fromIndex=array.length-1] The index to search from. - * @returns {number} Returns the index of the matched value, else `-1`. + * @param {Array} array The array of grouped elements to process. + * @returns {Array} Returns the new array of regrouped elements. * @example * - * _.lastIndexOf([1, 2, 1, 2], 2); - * // => 3 + * var zipped = _.zip(['a', 'b'], [1, 2], [true, false]); + * // => [['a', 1, true], ['b', 2, false]] * - * // Search from the `fromIndex`. - * _.lastIndexOf([1, 2, 1, 2], 2, 2); - * // => 1 + * _.unzip(zipped); + * // => [['a', 'b'], [1, 2], [true, false]] */ - function lastIndexOf(array, value, fromIndex) { - var length = array == null ? 0 : array.length; - if (!length) { - return -1; - } - var index = length; - if (fromIndex !== undefined) { - index = toInteger(fromIndex); - index = index < 0 ? nativeMax(length + index, 0) : nativeMin(index, length - 1); + function unzip(array) { + if (!(array && array.length)) { + return []; } - return value === value - ? strictLastIndexOf(array, value, index) - : baseFindIndex(array, baseIsNaN, index, true); + var length = 0; + array = arrayFilter(array, function(group) { + if (isArrayLikeObject(group)) { + length = nativeMax(group.length, length); + return true; + } + }); + return baseTimes(length, function(index) { + return arrayMap(array, baseProperty(index)); + }); } /** - * Gets the element at index `n` of `array`. If `n` is negative, the nth - * element from the end is returned. + * This method is like `_.unzip` except that it accepts `iteratee` to specify + * how regrouped values should be combined. The iteratee is invoked with the + * elements of each group: (...group). * * @static * @memberOf _ - * @since 4.11.0 + * @since 3.8.0 * @category Array - * @param {Array} array The array to query. - * @param {number} [n=0] The index of the element to return. - * @returns {*} Returns the nth element of `array`. + * @param {Array} array The array of grouped elements to process. + * @param {Function} [iteratee=_.identity] The function to combine + * regrouped values. + * @returns {Array} Returns the new array of regrouped elements. * @example * - * var array = ['a', 'b', 'c', 'd']; - * - * _.nth(array, 1); - * // => 'b' + * var zipped = _.zip([1, 2], [10, 20], [100, 200]); + * // => [[1, 10, 100], [2, 20, 200]] * - * _.nth(array, -2); - * // => 'c'; + * _.unzipWith(zipped, _.add); + * // => [3, 30, 300] */ - function nth(array, n) { - return (array && array.length) ? baseNth(array, toInteger(n)) : undefined; + function unzipWith(array, iteratee) { + if (!(array && array.length)) { + return []; + } + var result = unzip(array); + if (iteratee == null) { + return result; + } + return arrayMap(result, function(group) { + return apply(iteratee, undefined, group); + }); } /** - * Removes all given values from `array` using + * Creates an array excluding all given values using * [`SameValueZero`](http://ecma-international.org/ecma-262/7.0/#sec-samevaluezero) * for equality comparisons. * - * **Note:** Unlike `_.without`, this method mutates `array`. Use `_.remove` - * to remove elements from an array by predicate. + * **Note:** Unlike `_.pull`, this method returns a new array. * * @static * @memberOf _ - * @since 2.0.0 + * @since 0.1.0 * @category Array - * @param {Array} array The array to modify. - * @param {...*} [values] The values to remove. - * @returns {Array} Returns `array`. + * @param {Array} array The array to inspect. + * @param {...*} [values] The values to exclude. + * @returns {Array} Returns the new array of filtered values. + * @see _.difference, _.xor * @example * - * var array = ['a', 'b', 'c', 'a', 'b', 'c']; - * - * _.pull(array, 'a', 'c'); - * console.log(array); - * // => ['b', 'b'] + * _.without([2, 1, 2, 3], 1, 2); + * // => [3] */ - var pull = baseRest(pullAll); + var without = baseRest(function(array, values) { + return isArrayLikeObject(array) + ? baseDifference(array, values) + : []; + }); /** - * This method is like `_.pull` except that it accepts an array of values to remove. - * - * **Note:** Unlike `_.difference`, this method mutates `array`. + * Creates an array of unique values that is the + * [symmetric difference](https://en.wikipedia.org/wiki/Symmetric_difference) + * of the given arrays. The order of result values is determined by the order + * they occur in the arrays. * * @static * @memberOf _ - * @since 4.0.0 + * @since 2.4.0 * @category Array - * @param {Array} array The array to modify. - * @param {Array} values The values to remove. - * @returns {Array} Returns `array`. + * @param {...Array} [arrays] The arrays to inspect. + * @returns {Array} Returns the new array of filtered values. + * @see _.difference, _.without * @example * - * var array = ['a', 'b', 'c', 'a', 'b', 'c']; - * - * _.pullAll(array, ['a', 'c']); - * console.log(array); - * // => ['b', 'b'] + * _.xor([2, 1], [2, 3]); + * // => [1, 3] */ - function pullAll(array, values) { - return (array && array.length && values && values.length) - ? basePullAll(array, values) - : array; - } + var xor = baseRest(function(arrays) { + return baseXor(arrayFilter(arrays, isArrayLikeObject)); + }); /** - * This method is like `_.pullAll` except that it accepts `iteratee` which is - * invoked for each element of `array` and `values` to generate the criterion - * by which they're compared. The iteratee is invoked with one argument: (value). - * - * **Note:** Unlike `_.differenceBy`, this method mutates `array`. + * This method is like `_.xor` except that it accepts `iteratee` which is + * invoked for each element of each `arrays` to generate the criterion by + * which by which they're compared. The order of result values is determined + * by the order they occur in the arrays. The iteratee is invoked with one + * argument: (value). * * @static * @memberOf _ * @since 4.0.0 * @category Array - * @param {Array} array The array to modify. - * @param {Array} values The values to remove. + * @param {...Array} [arrays] The arrays to inspect. * @param {Function} [iteratee=_.identity] The iteratee invoked per element. - * @returns {Array} Returns `array`. + * @returns {Array} Returns the new array of filtered values. * @example * - * var array = [{ 'x': 1 }, { 'x': 2 }, { 'x': 3 }, { 'x': 1 }]; + * _.xorBy([2.1, 1.2], [2.3, 3.4], Math.floor); + * // => [1.2, 3.4] * - * _.pullAllBy(array, [{ 'x': 1 }, { 'x': 3 }], 'x'); - * console.log(array); + * // The `_.property` iteratee shorthand. + * _.xorBy([{ 'x': 1 }], [{ 'x': 2 }, { 'x': 1 }], 'x'); * // => [{ 'x': 2 }] */ - function pullAllBy(array, values, iteratee) { - return (array && array.length && values && values.length) - ? basePullAll(array, values, getIteratee(iteratee, 2)) - : array; - } + var xorBy = baseRest(function(arrays) { + var iteratee = last(arrays); + if (isArrayLikeObject(iteratee)) { + iteratee = undefined; + } + return baseXor(arrayFilter(arrays, isArrayLikeObject), getIteratee(iteratee, 2)); + }); /** - * This method is like `_.pullAll` except that it accepts `comparator` which - * is invoked to compare elements of `array` to `values`. The comparator is - * invoked with two arguments: (arrVal, othVal). - * - * **Note:** Unlike `_.differenceWith`, this method mutates `array`. + * This method is like `_.xor` except that it accepts `comparator` which is + * invoked to compare elements of `arrays`. The order of result values is + * determined by the order they occur in the arrays. The comparator is invoked + * with two arguments: (arrVal, othVal). * * @static * @memberOf _ - * @since 4.6.0 + * @since 4.0.0 * @category Array - * @param {Array} array The array to modify. - * @param {Array} values The values to remove. + * @param {...Array} [arrays] The arrays to inspect. * @param {Function} [comparator] The comparator invoked per element. - * @returns {Array} Returns `array`. + * @returns {Array} Returns the new array of filtered values. * @example * - * var array = [{ 'x': 1, 'y': 2 }, { 'x': 3, 'y': 4 }, { 'x': 5, 'y': 6 }]; + * var objects = [{ 'x': 1, 'y': 2 }, { 'x': 2, 'y': 1 }]; + * var others = [{ 'x': 1, 'y': 1 }, { 'x': 1, 'y': 2 }]; * - * _.pullAllWith(array, [{ 'x': 3, 'y': 4 }], _.isEqual); - * console.log(array); - * // => [{ 'x': 1, 'y': 2 }, { 'x': 5, 'y': 6 }] + * _.xorWith(objects, others, _.isEqual); + * // => [{ 'x': 2, 'y': 1 }, { 'x': 1, 'y': 1 }] */ - function pullAllWith(array, values, comparator) { - return (array && array.length && values && values.length) - ? basePullAll(array, values, undefined, comparator) - : array; - } + var xorWith = baseRest(function(arrays) { + var comparator = last(arrays); + comparator = typeof comparator == 'function' ? comparator : undefined; + return baseXor(arrayFilter(arrays, isArrayLikeObject), undefined, comparator); + }); /** - * Removes elements from `array` corresponding to `indexes` and returns an - * array of removed elements. - * - * **Note:** Unlike `_.at`, this method mutates `array`. + * Creates an array of grouped elements, the first of which contains the + * first elements of the given arrays, the second of which contains the + * second elements of the given arrays, and so on. * * @static * @memberOf _ - * @since 3.0.0 + * @since 0.1.0 * @category Array - * @param {Array} array The array to modify. - * @param {...(number|number[])} [indexes] The indexes of elements to remove. - * @returns {Array} Returns the new array of removed elements. + * @param {...Array} [arrays] The arrays to process. + * @returns {Array} Returns the new array of grouped elements. * @example * - * var array = ['a', 'b', 'c', 'd']; - * var pulled = _.pullAt(array, [1, 3]); - * - * console.log(array); - * // => ['a', 'c'] - * - * console.log(pulled); - * // => ['b', 'd'] + * _.zip(['a', 'b'], [1, 2], [true, false]); + * // => [['a', 1, true], ['b', 2, false]] */ - var pullAt = flatRest(function(array, indexes) { - var length = array == null ? 0 : array.length, - result = baseAt(array, indexes); - - basePullAt(array, arrayMap(indexes, function(index) { - return isIndex(index, length) ? +index : index; - }).sort(compareAscending)); - - return result; - }); + var zip = baseRest(unzip); /** - * Removes all elements from `array` that `predicate` returns truthy for - * and returns an array of the removed elements. The predicate is invoked - * with three arguments: (value, index, array). - * - * **Note:** Unlike `_.filter`, this method mutates `array`. Use `_.pull` - * to pull elements from an array by value. + * This method is like `_.fromPairs` except that it accepts two arrays, + * one of property identifiers and one of corresponding values. * * @static * @memberOf _ - * @since 2.0.0 + * @since 0.4.0 * @category Array - * @param {Array} array The array to modify. - * @param {Function} [predicate=_.identity] The function invoked per iteration. - * @returns {Array} Returns the new array of removed elements. + * @param {Array} [props=[]] The property identifiers. + * @param {Array} [values=[]] The property values. + * @returns {Object} Returns the new object. * @example * - * var array = [1, 2, 3, 4]; - * var evens = _.remove(array, function(n) { - * return n % 2 == 0; - * }); - * - * console.log(array); - * // => [1, 3] - * - * console.log(evens); - * // => [2, 4] + * _.zipObject(['a', 'b'], [1, 2]); + * // => { 'a': 1, 'b': 2 } */ - function remove(array, predicate) { - var result = []; - if (!(array && array.length)) { - return result; - } - var index = -1, - indexes = [], - length = array.length; - - predicate = getIteratee(predicate, 3); - while (++index < length) { - var value = array[index]; - if (predicate(value, index, array)) { - result.push(value); - indexes.push(index); - } - } - basePullAt(array, indexes); - return result; + function zipObject(props, values) { + return baseZipObject(props || [], values || [], assignValue); } /** - * Reverses `array` so that the first element becomes the last, the second - * element becomes the second to last, and so on. - * - * **Note:** This method mutates `array` and is based on - * [`Array#reverse`](https://mdn.io/Array/reverse). + * This method is like `_.zipObject` except that it supports property paths. * * @static * @memberOf _ - * @since 4.0.0 + * @since 4.1.0 * @category Array - * @param {Array} array The array to modify. - * @returns {Array} Returns `array`. + * @param {Array} [props=[]] The property identifiers. + * @param {Array} [values=[]] The property values. + * @returns {Object} Returns the new object. * @example * - * var array = [1, 2, 3]; - * - * _.reverse(array); - * // => [3, 2, 1] - * - * console.log(array); - * // => [3, 2, 1] + * _.zipObjectDeep(['a.b[0].c', 'a.b[1].d'], [1, 2]); + * // => { 'a': { 'b': [{ 'c': 1 }, { 'd': 2 }] } } */ - function reverse(array) { - return array == null ? array : nativeReverse.call(array); + function zipObjectDeep(props, values) { + return baseZipObject(props || [], values || [], baseSet); } /** - * Creates a slice of `array` from `start` up to, but not including, `end`. - * - * **Note:** This method is used instead of - * [`Array#slice`](https://mdn.io/Array/slice) to ensure dense arrays are - * returned. + * This method is like `_.zip` except that it accepts `iteratee` to specify + * how grouped values should be combined. The iteratee is invoked with the + * elements of each group: (...group). * * @static * @memberOf _ - * @since 3.0.0 + * @since 3.8.0 * @category Array - * @param {Array} array The array to slice. - * @param {number} [start=0] The start position. - * @param {number} [end=array.length] The end position. - * @returns {Array} Returns the slice of `array`. + * @param {...Array} [arrays] The arrays to process. + * @param {Function} [iteratee=_.identity] The function to combine + * grouped values. + * @returns {Array} Returns the new array of grouped elements. + * @example + * + * _.zipWith([1, 2], [10, 20], [100, 200], function(a, b, c) { + * return a + b + c; + * }); + * // => [111, 222] */ - function slice(array, start, end) { - var length = array == null ? 0 : array.length; - if (!length) { - return []; - } - if (end && typeof end != 'number' && isIterateeCall(array, start, end)) { - start = 0; - end = length; - } - else { - start = start == null ? 0 : toInteger(start); - end = end === undefined ? length : toInteger(end); - } - return baseSlice(array, start, end); - } + var zipWith = baseRest(function(arrays) { + var length = arrays.length, + iteratee = length > 1 ? arrays[length - 1] : undefined; + + iteratee = typeof iteratee == 'function' ? (arrays.pop(), iteratee) : undefined; + return unzipWith(arrays, iteratee); + }); + + /*------------------------------------------------------------------------*/ /** - * Uses a binary search to determine the lowest index at which `value` - * should be inserted into `array` in order to maintain its sort order. + * Creates a `lodash` wrapper instance that wraps `value` with explicit method + * chain sequences enabled. The result of such sequences must be unwrapped + * with `_#value`. * * @static * @memberOf _ - * @since 0.1.0 - * @category Array - * @param {Array} array The sorted array to inspect. - * @param {*} value The value to evaluate. - * @returns {number} Returns the index at which `value` should be inserted - * into `array`. + * @since 1.3.0 + * @category Seq + * @param {*} value The value to wrap. + * @returns {Object} Returns the new `lodash` wrapper instance. * @example * - * _.sortedIndex([30, 50], 40); - * // => 1 + * var users = [ + * { 'user': 'barney', 'age': 36 }, + * { 'user': 'fred', 'age': 40 }, + * { 'user': 'pebbles', 'age': 1 } + * ]; + * + * var youngest = _ + * .chain(users) + * .sortBy('age') + * .map(function(o) { + * return o.user + ' is ' + o.age; + * }) + * .head() + * .value(); + * // => 'pebbles is 1' */ - function sortedIndex(array, value) { - return baseSortedIndex(array, value); + function chain(value) { + var result = lodash(value); + result.__chain__ = true; + return result; } /** - * This method is like `_.sortedIndex` except that it accepts `iteratee` - * which is invoked for `value` and each element of `array` to compute their - * sort ranking. The iteratee is invoked with one argument: (value). + * This method invokes `interceptor` and returns `value`. The interceptor + * is invoked with one argument; (value). The purpose of this method is to + * "tap into" a method chain sequence in order to modify intermediate results. * * @static * @memberOf _ - * @since 4.0.0 - * @category Array - * @param {Array} array The sorted array to inspect. - * @param {*} value The value to evaluate. - * @param {Function} [iteratee=_.identity] The iteratee invoked per element. - * @returns {number} Returns the index at which `value` should be inserted - * into `array`. + * @since 0.1.0 + * @category Seq + * @param {*} value The value to provide to `interceptor`. + * @param {Function} interceptor The function to invoke. + * @returns {*} Returns `value`. * @example * - * var objects = [{ 'x': 4 }, { 'x': 5 }]; + * _([1, 2, 3]) + * .tap(function(array) { + * // Mutate input array. + * array.pop(); + * }) + * .reverse() + * .value(); + * // => [2, 1] + */ + function tap(value, interceptor) { + interceptor(value); + return value; + } + + /** + * This method is like `_.tap` except that it returns the result of `interceptor`. + * The purpose of this method is to "pass thru" values replacing intermediate + * results in a method chain sequence. * - * _.sortedIndexBy(objects, { 'x': 4 }, function(o) { return o.x; }); - * // => 0 + * @static + * @memberOf _ + * @since 3.0.0 + * @category Seq + * @param {*} value The value to provide to `interceptor`. + * @param {Function} interceptor The function to invoke. + * @returns {*} Returns the result of `interceptor`. + * @example * - * // The `_.property` iteratee shorthand. - * _.sortedIndexBy(objects, { 'x': 4 }, 'x'); - * // => 0 + * _(' abc ') + * .chain() + * .trim() + * .thru(function(value) { + * return [value]; + * }) + * .value(); + * // => ['abc'] */ - function sortedIndexBy(array, value, iteratee) { - return baseSortedIndexBy(array, value, getIteratee(iteratee, 2)); + function thru(value, interceptor) { + return interceptor(value); } /** - * This method is like `_.indexOf` except that it performs a binary - * search on a sorted `array`. + * This method is the wrapper version of `_.at`. * - * @static + * @name at * @memberOf _ - * @since 4.0.0 - * @category Array - * @param {Array} array The array to inspect. - * @param {*} value The value to search for. - * @returns {number} Returns the index of the matched value, else `-1`. + * @since 1.0.0 + * @category Seq + * @param {...(string|string[])} [paths] The property paths to pick. + * @returns {Object} Returns the new `lodash` wrapper instance. * @example * - * _.sortedIndexOf([4, 5, 5, 5, 6], 5); - * // => 1 + * var object = { 'a': [{ 'b': { 'c': 3 } }, 4] }; + * + * _(object).at(['a[0].b.c', 'a[1]']).value(); + * // => [3, 4] */ - function sortedIndexOf(array, value) { - var length = array == null ? 0 : array.length; - if (length) { - var index = baseSortedIndex(array, value); - if (index < length && eq(array[index], value)) { - return index; - } + var wrapperAt = flatRest(function(paths) { + var length = paths.length, + start = length ? paths[0] : 0, + value = this.__wrapped__, + interceptor = function(object) { return baseAt(object, paths); }; + + if (length > 1 || this.__actions__.length || + !(value instanceof LazyWrapper) || !isIndex(start)) { + return this.thru(interceptor); } - return -1; - } + value = value.slice(start, +start + (length ? 1 : 0)); + value.__actions__.push({ + 'func': thru, + 'args': [interceptor], + 'thisArg': undefined + }); + return new LodashWrapper(value, this.__chain__).thru(function(array) { + if (length && !array.length) { + array.push(undefined); + } + return array; + }); + }); /** - * This method is like `_.sortedIndex` except that it returns the highest - * index at which `value` should be inserted into `array` in order to - * maintain its sort order. + * Creates a `lodash` wrapper instance with explicit method chain sequences enabled. * - * @static + * @name chain * @memberOf _ - * @since 3.0.0 - * @category Array - * @param {Array} array The sorted array to inspect. - * @param {*} value The value to evaluate. - * @returns {number} Returns the index at which `value` should be inserted - * into `array`. + * @since 0.1.0 + * @category Seq + * @returns {Object} Returns the new `lodash` wrapper instance. * @example * - * _.sortedLastIndex([4, 5, 5, 5, 6], 5); - * // => 4 + * var users = [ + * { 'user': 'barney', 'age': 36 }, + * { 'user': 'fred', 'age': 40 } + * ]; + * + * // A sequence without explicit chaining. + * _(users).head(); + * // => { 'user': 'barney', 'age': 36 } + * + * // A sequence with explicit chaining. + * _(users) + * .chain() + * .head() + * .pick('user') + * .value(); + * // => { 'user': 'barney' } */ - function sortedLastIndex(array, value) { - return baseSortedIndex(array, value, true); + function wrapperChain() { + return chain(this); } /** - * This method is like `_.sortedLastIndex` except that it accepts `iteratee` - * which is invoked for `value` and each element of `array` to compute their - * sort ranking. The iteratee is invoked with one argument: (value). + * Executes the chain sequence and returns the wrapped result. * - * @static + * @name commit * @memberOf _ - * @since 4.0.0 - * @category Array - * @param {Array} array The sorted array to inspect. - * @param {*} value The value to evaluate. - * @param {Function} [iteratee=_.identity] The iteratee invoked per element. - * @returns {number} Returns the index at which `value` should be inserted - * into `array`. + * @since 3.2.0 + * @category Seq + * @returns {Object} Returns the new `lodash` wrapper instance. * @example * - * var objects = [{ 'x': 4 }, { 'x': 5 }]; + * var array = [1, 2]; + * var wrapped = _(array).push(3); * - * _.sortedLastIndexBy(objects, { 'x': 4 }, function(o) { return o.x; }); - * // => 1 + * console.log(array); + * // => [1, 2] * - * // The `_.property` iteratee shorthand. - * _.sortedLastIndexBy(objects, { 'x': 4 }, 'x'); - * // => 1 + * wrapped = wrapped.commit(); + * console.log(array); + * // => [1, 2, 3] + * + * wrapped.last(); + * // => 3 + * + * console.log(array); + * // => [1, 2, 3] */ - function sortedLastIndexBy(array, value, iteratee) { - return baseSortedIndexBy(array, value, getIteratee(iteratee, 2), true); + function wrapperCommit() { + return new LodashWrapper(this.value(), this.__chain__); } /** - * This method is like `_.lastIndexOf` except that it performs a binary - * search on a sorted `array`. + * Gets the next value on a wrapped object following the + * [iterator protocol](https://mdn.io/iteration_protocols#iterator). * - * @static + * @name next * @memberOf _ * @since 4.0.0 - * @category Array - * @param {Array} array The array to inspect. - * @param {*} value The value to search for. - * @returns {number} Returns the index of the matched value, else `-1`. + * @category Seq + * @returns {Object} Returns the next iterator value. * @example * - * _.sortedLastIndexOf([4, 5, 5, 5, 6], 5); - * // => 3 + * var wrapped = _([1, 2]); + * + * wrapped.next(); + * // => { 'done': false, 'value': 1 } + * + * wrapped.next(); + * // => { 'done': false, 'value': 2 } + * + * wrapped.next(); + * // => { 'done': true, 'value': undefined } */ - function sortedLastIndexOf(array, value) { - var length = array == null ? 0 : array.length; - if (length) { - var index = baseSortedIndex(array, value, true) - 1; - if (eq(array[index], value)) { - return index; - } + function wrapperNext() { + if (this.__values__ === undefined) { + this.__values__ = toArray(this.value()); } - return -1; + var done = this.__index__ >= this.__values__.length, + value = done ? undefined : this.__values__[this.__index__++]; + + return { 'done': done, 'value': value }; } /** - * This method is like `_.uniq` except that it's designed and optimized - * for sorted arrays. + * Enables the wrapper to be iterable. * - * @static + * @name Symbol.iterator * @memberOf _ * @since 4.0.0 - * @category Array - * @param {Array} array The array to inspect. - * @returns {Array} Returns the new duplicate free array. + * @category Seq + * @returns {Object} Returns the wrapper object. * @example * - * _.sortedUniq([1, 1, 2]); + * var wrapped = _([1, 2]); + * + * wrapped[Symbol.iterator]() === wrapped; + * // => true + * + * Array.from(wrapped); * // => [1, 2] */ - function sortedUniq(array) { - return (array && array.length) - ? baseSortedUniq(array) - : []; + function wrapperToIterator() { + return this; } /** - * This method is like `_.uniqBy` except that it's designed and optimized - * for sorted arrays. + * Creates a clone of the chain sequence planting `value` as the wrapped value. * - * @static + * @name plant * @memberOf _ - * @since 4.0.0 - * @category Array - * @param {Array} array The array to inspect. - * @param {Function} [iteratee] The iteratee invoked per element. - * @returns {Array} Returns the new duplicate free array. + * @since 3.2.0 + * @category Seq + * @param {*} value The value to plant. + * @returns {Object} Returns the new `lodash` wrapper instance. * @example * - * _.sortedUniqBy([1.1, 1.2, 2.3, 2.4], Math.floor); - * // => [1.1, 2.3] + * function square(n) { + * return n * n; + * } + * + * var wrapped = _([1, 2]).map(square); + * var other = wrapped.plant([3, 4]); + * + * other.value(); + * // => [9, 16] + * + * wrapped.value(); + * // => [1, 4] */ - function sortedUniqBy(array, iteratee) { - return (array && array.length) - ? baseSortedUniq(array, getIteratee(iteratee, 2)) - : []; + function wrapperPlant(value) { + var result, + parent = this; + + while (parent instanceof baseLodash) { + var clone = wrapperClone(parent); + clone.__index__ = 0; + clone.__values__ = undefined; + if (result) { + previous.__wrapped__ = clone; + } else { + result = clone; + } + var previous = clone; + parent = parent.__wrapped__; + } + previous.__wrapped__ = value; + return result; } /** - * Gets all but the first element of `array`. + * This method is the wrapper version of `_.reverse`. * - * @static + * **Note:** This method mutates the wrapped array. + * + * @name reverse * @memberOf _ - * @since 4.0.0 - * @category Array - * @param {Array} array The array to query. - * @returns {Array} Returns the slice of `array`. + * @since 0.1.0 + * @category Seq + * @returns {Object} Returns the new `lodash` wrapper instance. * @example * - * _.tail([1, 2, 3]); - * // => [2, 3] + * var array = [1, 2, 3]; + * + * _(array).reverse().value() + * // => [3, 2, 1] + * + * console.log(array); + * // => [3, 2, 1] */ - function tail(array) { - var length = array == null ? 0 : array.length; - return length ? baseSlice(array, 1, length) : []; + function wrapperReverse() { + var value = this.__wrapped__; + if (value instanceof LazyWrapper) { + var wrapped = value; + if (this.__actions__.length) { + wrapped = new LazyWrapper(this); + } + wrapped = wrapped.reverse(); + wrapped.__actions__.push({ + 'func': thru, + 'args': [reverse], + 'thisArg': undefined + }); + return new LodashWrapper(wrapped, this.__chain__); + } + return this.thru(reverse); } /** - * Creates a slice of `array` with `n` elements taken from the beginning. + * Executes the chain sequence to resolve the unwrapped value. * - * @static + * @name value * @memberOf _ * @since 0.1.0 - * @category Array - * @param {Array} array The array to query. - * @param {number} [n=1] The number of elements to take. - * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. - * @returns {Array} Returns the slice of `array`. + * @alias toJSON, valueOf + * @category Seq + * @returns {*} Returns the resolved unwrapped value. * @example * - * _.take([1, 2, 3]); - * // => [1] - * - * _.take([1, 2, 3], 2); - * // => [1, 2] - * - * _.take([1, 2, 3], 5); + * _([1, 2, 3]).value(); * // => [1, 2, 3] - * - * _.take([1, 2, 3], 0); - * // => [] */ - function take(array, n, guard) { - if (!(array && array.length)) { - return []; - } - n = (guard || n === undefined) ? 1 : toInteger(n); - return baseSlice(array, 0, n < 0 ? 0 : n); + function wrapperValue() { + return baseWrapperValue(this.__wrapped__, this.__actions__); } + /*------------------------------------------------------------------------*/ + /** - * Creates a slice of `array` with `n` elements taken from the end. + * Creates an object composed of keys generated from the results of running + * each element of `collection` thru `iteratee`. The corresponding value of + * each key is the number of times the key was returned by `iteratee`. The + * iteratee is invoked with one argument: (value). * * @static * @memberOf _ - * @since 3.0.0 - * @category Array - * @param {Array} array The array to query. - * @param {number} [n=1] The number of elements to take. - * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. - * @returns {Array} Returns the slice of `array`. + * @since 0.5.0 + * @category Collection + * @param {Array|Object} collection The collection to iterate over. + * @param {Function} [iteratee=_.identity] The iteratee to transform keys. + * @returns {Object} Returns the composed aggregate object. * @example * - * _.takeRight([1, 2, 3]); - * // => [3] - * - * _.takeRight([1, 2, 3], 2); - * // => [2, 3] - * - * _.takeRight([1, 2, 3], 5); - * // => [1, 2, 3] + * _.countBy([6.1, 4.2, 6.3], Math.floor); + * // => { '4': 1, '6': 2 } * - * _.takeRight([1, 2, 3], 0); - * // => [] + * // The `_.property` iteratee shorthand. + * _.countBy(['one', 'two', 'three'], 'length'); + * // => { '3': 2, '5': 1 } */ - function takeRight(array, n, guard) { - var length = array == null ? 0 : array.length; - if (!length) { - return []; + var countBy = createAggregator(function(result, value, key) { + if (hasOwnProperty.call(result, key)) { + ++result[key]; + } else { + baseAssignValue(result, key, 1); } - n = (guard || n === undefined) ? 1 : toInteger(n); - n = length - n; - return baseSlice(array, n < 0 ? 0 : n, length); - } + }); /** - * Creates a slice of `array` with elements taken from the end. Elements are - * taken until `predicate` returns falsey. The predicate is invoked with - * three arguments: (value, index, array). + * Checks if `predicate` returns truthy for **all** elements of `collection`. + * Iteration is stopped once `predicate` returns falsey. The predicate is + * invoked with three arguments: (value, index|key, collection). + * + * **Note:** This method returns `true` for + * [empty collections](https://en.wikipedia.org/wiki/Empty_set) because + * [everything is true](https://en.wikipedia.org/wiki/Vacuous_truth) of + * elements of empty collections. * * @static * @memberOf _ - * @since 3.0.0 - * @category Array - * @param {Array} array The array to query. + * @since 0.1.0 + * @category Collection + * @param {Array|Object} collection The collection to iterate over. * @param {Function} [predicate=_.identity] The function invoked per iteration. - * @returns {Array} Returns the slice of `array`. + * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. + * @returns {boolean} Returns `true` if all elements pass the predicate check, + * else `false`. * @example * + * _.every([true, 1, null, 'yes'], Boolean); + * // => false + * * var users = [ - * { 'user': 'barney', 'active': true }, - * { 'user': 'fred', 'active': false }, - * { 'user': 'pebbles', 'active': false } + * { 'user': 'barney', 'age': 36, 'active': false }, + * { 'user': 'fred', 'age': 40, 'active': false } * ]; * - * _.takeRightWhile(users, function(o) { return !o.active; }); - * // => objects for ['fred', 'pebbles'] - * * // The `_.matches` iteratee shorthand. - * _.takeRightWhile(users, { 'user': 'pebbles', 'active': false }); - * // => objects for ['pebbles'] + * _.every(users, { 'user': 'barney', 'active': false }); + * // => false * * // The `_.matchesProperty` iteratee shorthand. - * _.takeRightWhile(users, ['active', false]); - * // => objects for ['fred', 'pebbles'] + * _.every(users, ['active', false]); + * // => true * * // The `_.property` iteratee shorthand. - * _.takeRightWhile(users, 'active'); - * // => [] + * _.every(users, 'active'); + * // => false */ - function takeRightWhile(array, predicate) { - return (array && array.length) - ? baseWhile(array, getIteratee(predicate, 3), false, true) - : []; + function every(collection, predicate, guard) { + var func = isArray(collection) ? arrayEvery : baseEvery; + if (guard && isIterateeCall(collection, predicate, guard)) { + predicate = undefined; + } + return func(collection, getIteratee(predicate, 3)); } /** - * Creates a slice of `array` with elements taken from the beginning. Elements - * are taken until `predicate` returns falsey. The predicate is invoked with - * three arguments: (value, index, array). + * Iterates over elements of `collection`, returning an array of all elements + * `predicate` returns truthy for. The predicate is invoked with three + * arguments: (value, index|key, collection). + * + * **Note:** Unlike `_.remove`, this method returns a new array. * * @static * @memberOf _ - * @since 3.0.0 - * @category Array - * @param {Array} array The array to query. + * @since 0.1.0 + * @category Collection + * @param {Array|Object} collection The collection to iterate over. * @param {Function} [predicate=_.identity] The function invoked per iteration. - * @returns {Array} Returns the slice of `array`. + * @returns {Array} Returns the new filtered array. + * @see _.reject * @example * * var users = [ - * { 'user': 'barney', 'active': false }, - * { 'user': 'fred', 'active': false }, - * { 'user': 'pebbles', 'active': true } + * { 'user': 'barney', 'age': 36, 'active': true }, + * { 'user': 'fred', 'age': 40, 'active': false } * ]; * - * _.takeWhile(users, function(o) { return !o.active; }); - * // => objects for ['barney', 'fred'] + * _.filter(users, function(o) { return !o.active; }); + * // => objects for ['fred'] * * // The `_.matches` iteratee shorthand. - * _.takeWhile(users, { 'user': 'barney', 'active': false }); + * _.filter(users, { 'age': 36, 'active': true }); * // => objects for ['barney'] * * // The `_.matchesProperty` iteratee shorthand. - * _.takeWhile(users, ['active', false]); - * // => objects for ['barney', 'fred'] + * _.filter(users, ['active', false]); + * // => objects for ['fred'] * * // The `_.property` iteratee shorthand. - * _.takeWhile(users, 'active'); - * // => [] + * _.filter(users, 'active'); + * // => objects for ['barney'] + * + * // Combining several predicates using `_.overEvery` or `_.overSome`. + * _.filter(users, _.overSome([{ 'age': 36 }, ['age', 40]])); + * // => objects for ['fred', 'barney'] */ - function takeWhile(array, predicate) { - return (array && array.length) - ? baseWhile(array, getIteratee(predicate, 3)) - : []; + function filter(collection, predicate) { + var func = isArray(collection) ? arrayFilter : baseFilter; + return func(collection, getIteratee(predicate, 3)); } /** - * Creates an array of unique values, in order, from all given arrays using - * [`SameValueZero`](http://ecma-international.org/ecma-262/7.0/#sec-samevaluezero) - * for equality comparisons. + * Iterates over elements of `collection`, returning the first element + * `predicate` returns truthy for. The predicate is invoked with three + * arguments: (value, index|key, collection). * * @static * @memberOf _ * @since 0.1.0 - * @category Array - * @param {...Array} [arrays] The arrays to inspect. - * @returns {Array} Returns the new array of combined values. + * @category Collection + * @param {Array|Object} collection The collection to inspect. + * @param {Function} [predicate=_.identity] The function invoked per iteration. + * @param {number} [fromIndex=0] The index to search from. + * @returns {*} Returns the matched element, else `undefined`. * @example * - * _.union([2], [1, 2]); - * // => [2, 1] + * var users = [ + * { 'user': 'barney', 'age': 36, 'active': true }, + * { 'user': 'fred', 'age': 40, 'active': false }, + * { 'user': 'pebbles', 'age': 1, 'active': true } + * ]; + * + * _.find(users, function(o) { return o.age < 40; }); + * // => object for 'barney' + * + * // The `_.matches` iteratee shorthand. + * _.find(users, { 'age': 1, 'active': true }); + * // => object for 'pebbles' + * + * // The `_.matchesProperty` iteratee shorthand. + * _.find(users, ['active', false]); + * // => object for 'fred' + * + * // The `_.property` iteratee shorthand. + * _.find(users, 'active'); + * // => object for 'barney' */ - var union = baseRest(function(arrays) { - return baseUniq(baseFlatten(arrays, 1, isArrayLikeObject, true)); - }); + var find = createFind(findIndex); /** - * This method is like `_.union` except that it accepts `iteratee` which is - * invoked for each element of each `arrays` to generate the criterion by - * which uniqueness is computed. Result values are chosen from the first - * array in which the value occurs. The iteratee is invoked with one argument: - * (value). + * This method is like `_.find` except that it iterates over elements of + * `collection` from right to left. * * @static - * @memberOf _ - * @since 4.0.0 - * @category Array - * @param {...Array} [arrays] The arrays to inspect. - * @param {Function} [iteratee=_.identity] The iteratee invoked per element. - * @returns {Array} Returns the new array of combined values. + * @memberOf _ + * @since 2.0.0 + * @category Collection + * @param {Array|Object} collection The collection to inspect. + * @param {Function} [predicate=_.identity] The function invoked per iteration. + * @param {number} [fromIndex=collection.length-1] The index to search from. + * @returns {*} Returns the matched element, else `undefined`. * @example * - * _.unionBy([2.1], [1.2, 2.3], Math.floor); - * // => [2.1, 1.2] - * - * // The `_.property` iteratee shorthand. - * _.unionBy([{ 'x': 1 }], [{ 'x': 2 }, { 'x': 1 }], 'x'); - * // => [{ 'x': 1 }, { 'x': 2 }] + * _.findLast([1, 2, 3, 4], function(n) { + * return n % 2 == 1; + * }); + * // => 3 */ - var unionBy = baseRest(function(arrays) { - var iteratee = last(arrays); - if (isArrayLikeObject(iteratee)) { - iteratee = undefined; - } - return baseUniq(baseFlatten(arrays, 1, isArrayLikeObject, true), getIteratee(iteratee, 2)); - }); + var findLast = createFind(findLastIndex); /** - * This method is like `_.union` except that it accepts `comparator` which - * is invoked to compare elements of `arrays`. Result values are chosen from - * the first array in which the value occurs. The comparator is invoked - * with two arguments: (arrVal, othVal). + * Creates a flattened array of values by running each element in `collection` + * thru `iteratee` and flattening the mapped results. The iteratee is invoked + * with three arguments: (value, index|key, collection). * * @static * @memberOf _ * @since 4.0.0 - * @category Array - * @param {...Array} [arrays] The arrays to inspect. - * @param {Function} [comparator] The comparator invoked per element. - * @returns {Array} Returns the new array of combined values. + * @category Collection + * @param {Array|Object} collection The collection to iterate over. + * @param {Function} [iteratee=_.identity] The function invoked per iteration. + * @returns {Array} Returns the new flattened array. * @example * - * var objects = [{ 'x': 1, 'y': 2 }, { 'x': 2, 'y': 1 }]; - * var others = [{ 'x': 1, 'y': 1 }, { 'x': 1, 'y': 2 }]; + * function duplicate(n) { + * return [n, n]; + * } * - * _.unionWith(objects, others, _.isEqual); - * // => [{ 'x': 1, 'y': 2 }, { 'x': 2, 'y': 1 }, { 'x': 1, 'y': 1 }] + * _.flatMap([1, 2], duplicate); + * // => [1, 1, 2, 2] */ - var unionWith = baseRest(function(arrays) { - var comparator = last(arrays); - comparator = typeof comparator == 'function' ? comparator : undefined; - return baseUniq(baseFlatten(arrays, 1, isArrayLikeObject, true), undefined, comparator); - }); + function flatMap(collection, iteratee) { + return baseFlatten(map(collection, iteratee), 1); + } /** - * Creates a duplicate-free version of an array, using - * [`SameValueZero`](http://ecma-international.org/ecma-262/7.0/#sec-samevaluezero) - * for equality comparisons, in which only the first occurrence of each element - * is kept. The order of result values is determined by the order they occur - * in the array. + * This method is like `_.flatMap` except that it recursively flattens the + * mapped results. * * @static * @memberOf _ - * @since 0.1.0 - * @category Array - * @param {Array} array The array to inspect. - * @returns {Array} Returns the new duplicate free array. + * @since 4.7.0 + * @category Collection + * @param {Array|Object} collection The collection to iterate over. + * @param {Function} [iteratee=_.identity] The function invoked per iteration. + * @returns {Array} Returns the new flattened array. * @example * - * _.uniq([2, 1, 2]); - * // => [2, 1] + * function duplicate(n) { + * return [[[n, n]]]; + * } + * + * _.flatMapDeep([1, 2], duplicate); + * // => [1, 1, 2, 2] */ - function uniq(array) { - return (array && array.length) ? baseUniq(array) : []; + function flatMapDeep(collection, iteratee) { + return baseFlatten(map(collection, iteratee), INFINITY); } /** - * This method is like `_.uniq` except that it accepts `iteratee` which is - * invoked for each element in `array` to generate the criterion by which - * uniqueness is computed. The order of result values is determined by the - * order they occur in the array. The iteratee is invoked with one argument: - * (value). + * This method is like `_.flatMap` except that it recursively flattens the + * mapped results up to `depth` times. * * @static * @memberOf _ - * @since 4.0.0 - * @category Array - * @param {Array} array The array to inspect. - * @param {Function} [iteratee=_.identity] The iteratee invoked per element. - * @returns {Array} Returns the new duplicate free array. + * @since 4.7.0 + * @category Collection + * @param {Array|Object} collection The collection to iterate over. + * @param {Function} [iteratee=_.identity] The function invoked per iteration. + * @param {number} [depth=1] The maximum recursion depth. + * @returns {Array} Returns the new flattened array. * @example * - * _.uniqBy([2.1, 1.2, 2.3], Math.floor); - * // => [2.1, 1.2] + * function duplicate(n) { + * return [[[n, n]]]; + * } * - * // The `_.property` iteratee shorthand. - * _.uniqBy([{ 'x': 1 }, { 'x': 2 }, { 'x': 1 }], 'x'); - * // => [{ 'x': 1 }, { 'x': 2 }] + * _.flatMapDepth([1, 2], duplicate, 2); + * // => [[1, 1], [2, 2]] */ - function uniqBy(array, iteratee) { - return (array && array.length) ? baseUniq(array, getIteratee(iteratee, 2)) : []; + function flatMapDepth(collection, iteratee, depth) { + depth = depth === undefined ? 1 : toInteger(depth); + return baseFlatten(map(collection, iteratee), depth); } /** - * This method is like `_.uniq` except that it accepts `comparator` which - * is invoked to compare elements of `array`. The order of result values is - * determined by the order they occur in the array.The comparator is invoked - * with two arguments: (arrVal, othVal). + * Iterates over elements of `collection` and invokes `iteratee` for each element. + * The iteratee is invoked with three arguments: (value, index|key, collection). + * Iteratee functions may exit iteration early by explicitly returning `false`. + * + * **Note:** As with other "Collections" methods, objects with a "length" + * property are iterated like arrays. To avoid this behavior use `_.forIn` + * or `_.forOwn` for object iteration. * * @static * @memberOf _ - * @since 4.0.0 - * @category Array - * @param {Array} array The array to inspect. - * @param {Function} [comparator] The comparator invoked per element. - * @returns {Array} Returns the new duplicate free array. + * @since 0.1.0 + * @alias each + * @category Collection + * @param {Array|Object} collection The collection to iterate over. + * @param {Function} [iteratee=_.identity] The function invoked per iteration. + * @returns {Array|Object} Returns `collection`. + * @see _.forEachRight * @example * - * var objects = [{ 'x': 1, 'y': 2 }, { 'x': 2, 'y': 1 }, { 'x': 1, 'y': 2 }]; + * _.forEach([1, 2], function(value) { + * console.log(value); + * }); + * // => Logs `1` then `2`. * - * _.uniqWith(objects, _.isEqual); - * // => [{ 'x': 1, 'y': 2 }, { 'x': 2, 'y': 1 }] + * _.forEach({ 'a': 1, 'b': 2 }, function(value, key) { + * console.log(key); + * }); + * // => Logs 'a' then 'b' (iteration order is not guaranteed). */ - function uniqWith(array, comparator) { - comparator = typeof comparator == 'function' ? comparator : undefined; - return (array && array.length) ? baseUniq(array, undefined, comparator) : []; + function forEach(collection, iteratee) { + var func = isArray(collection) ? arrayEach : baseEach; + return func(collection, getIteratee(iteratee, 3)); } /** - * This method is like `_.zip` except that it accepts an array of grouped - * elements and creates an array regrouping the elements to their pre-zip - * configuration. + * This method is like `_.forEach` except that it iterates over elements of + * `collection` from right to left. * * @static * @memberOf _ - * @since 1.2.0 - * @category Array - * @param {Array} array The array of grouped elements to process. - * @returns {Array} Returns the new array of regrouped elements. + * @since 2.0.0 + * @alias eachRight + * @category Collection + * @param {Array|Object} collection The collection to iterate over. + * @param {Function} [iteratee=_.identity] The function invoked per iteration. + * @returns {Array|Object} Returns `collection`. + * @see _.forEach * @example * - * var zipped = _.zip(['a', 'b'], [1, 2], [true, false]); - * // => [['a', 1, true], ['b', 2, false]] - * - * _.unzip(zipped); - * // => [['a', 'b'], [1, 2], [true, false]] + * _.forEachRight([1, 2], function(value) { + * console.log(value); + * }); + * // => Logs `2` then `1`. */ - function unzip(array) { - if (!(array && array.length)) { - return []; - } - var length = 0; - array = arrayFilter(array, function(group) { - if (isArrayLikeObject(group)) { - length = nativeMax(group.length, length); - return true; - } - }); - return baseTimes(length, function(index) { - return arrayMap(array, baseProperty(index)); - }); + function forEachRight(collection, iteratee) { + var func = isArray(collection) ? arrayEachRight : baseEachRight; + return func(collection, getIteratee(iteratee, 3)); } /** - * This method is like `_.unzip` except that it accepts `iteratee` to specify - * how regrouped values should be combined. The iteratee is invoked with the - * elements of each group: (...group). + * Creates an object composed of keys generated from the results of running + * each element of `collection` thru `iteratee`. The order of grouped values + * is determined by the order they occur in `collection`. The corresponding + * value of each key is an array of elements responsible for generating the + * key. The iteratee is invoked with one argument: (value). * * @static * @memberOf _ - * @since 3.8.0 - * @category Array - * @param {Array} array The array of grouped elements to process. - * @param {Function} [iteratee=_.identity] The function to combine - * regrouped values. - * @returns {Array} Returns the new array of regrouped elements. + * @since 0.1.0 + * @category Collection + * @param {Array|Object} collection The collection to iterate over. + * @param {Function} [iteratee=_.identity] The iteratee to transform keys. + * @returns {Object} Returns the composed aggregate object. * @example * - * var zipped = _.zip([1, 2], [10, 20], [100, 200]); - * // => [[1, 10, 100], [2, 20, 200]] + * _.groupBy([6.1, 4.2, 6.3], Math.floor); + * // => { '4': [4.2], '6': [6.1, 6.3] } * - * _.unzipWith(zipped, _.add); - * // => [3, 30, 300] + * // The `_.property` iteratee shorthand. + * _.groupBy(['one', 'two', 'three'], 'length'); + * // => { '3': ['one', 'two'], '5': ['three'] } */ - function unzipWith(array, iteratee) { - if (!(array && array.length)) { - return []; - } - var result = unzip(array); - if (iteratee == null) { - return result; + var groupBy = createAggregator(function(result, value, key) { + if (hasOwnProperty.call(result, key)) { + result[key].push(value); + } else { + baseAssignValue(result, key, [value]); } - return arrayMap(result, function(group) { - return apply(iteratee, undefined, group); - }); - } + }); /** - * Creates an array excluding all given values using + * Checks if `value` is in `collection`. If `collection` is a string, it's + * checked for a substring of `value`, otherwise * [`SameValueZero`](http://ecma-international.org/ecma-262/7.0/#sec-samevaluezero) - * for equality comparisons. - * - * **Note:** Unlike `_.pull`, this method returns a new array. + * is used for equality comparisons. If `fromIndex` is negative, it's used as + * the offset from the end of `collection`. * * @static * @memberOf _ * @since 0.1.0 - * @category Array - * @param {Array} array The array to inspect. - * @param {...*} [values] The values to exclude. - * @returns {Array} Returns the new array of filtered values. - * @see _.difference, _.xor + * @category Collection + * @param {Array|Object|string} collection The collection to inspect. + * @param {*} value The value to search for. + * @param {number} [fromIndex=0] The index to search from. + * @param- {Object} [guard] Enables use as an iteratee for methods like `_.reduce`. + * @returns {boolean} Returns `true` if `value` is found, else `false`. * @example * - * _.without([2, 1, 2, 3], 1, 2); - * // => [3] + * _.includes([1, 2, 3], 1); + * // => true + * + * _.includes([1, 2, 3], 1, 2); + * // => false + * + * _.includes({ 'a': 1, 'b': 2 }, 1); + * // => true + * + * _.includes('abcd', 'bc'); + * // => true */ - var without = baseRest(function(array, values) { - return isArrayLikeObject(array) - ? baseDifference(array, values) - : []; - }); + function includes(collection, value, fromIndex, guard) { + collection = isArrayLike(collection) ? collection : values(collection); + fromIndex = (fromIndex && !guard) ? toInteger(fromIndex) : 0; + + var length = collection.length; + if (fromIndex < 0) { + fromIndex = nativeMax(length + fromIndex, 0); + } + return isString(collection) + ? (fromIndex <= length && collection.indexOf(value, fromIndex) > -1) + : (!!length && baseIndexOf(collection, value, fromIndex) > -1); + } /** - * Creates an array of unique values that is the - * [symmetric difference](https://en.wikipedia.org/wiki/Symmetric_difference) - * of the given arrays. The order of result values is determined by the order - * they occur in the arrays. + * Invokes the method at `path` of each element in `collection`, returning + * an array of the results of each invoked method. Any additional arguments + * are provided to each invoked method. If `path` is a function, it's invoked + * for, and `this` bound to, each element in `collection`. * * @static * @memberOf _ - * @since 2.4.0 - * @category Array - * @param {...Array} [arrays] The arrays to inspect. - * @returns {Array} Returns the new array of filtered values. - * @see _.difference, _.without + * @since 4.0.0 + * @category Collection + * @param {Array|Object} collection The collection to iterate over. + * @param {Array|Function|string} path The path of the method to invoke or + * the function invoked per iteration. + * @param {...*} [args] The arguments to invoke each method with. + * @returns {Array} Returns the array of results. * @example * - * _.xor([2, 1], [2, 3]); - * // => [1, 3] + * _.invokeMap([[5, 1, 7], [3, 2, 1]], 'sort'); + * // => [[1, 5, 7], [1, 2, 3]] + * + * _.invokeMap([123, 456], String.prototype.split, ''); + * // => [['1', '2', '3'], ['4', '5', '6']] */ - var xor = baseRest(function(arrays) { - return baseXor(arrayFilter(arrays, isArrayLikeObject)); + var invokeMap = baseRest(function(collection, path, args) { + var index = -1, + isFunc = typeof path == 'function', + result = isArrayLike(collection) ? Array(collection.length) : []; + + baseEach(collection, function(value) { + result[++index] = isFunc ? apply(path, value, args) : baseInvoke(value, path, args); + }); + return result; }); /** - * This method is like `_.xor` except that it accepts `iteratee` which is - * invoked for each element of each `arrays` to generate the criterion by - * which by which they're compared. The order of result values is determined - * by the order they occur in the arrays. The iteratee is invoked with one - * argument: (value). + * Creates an object composed of keys generated from the results of running + * each element of `collection` thru `iteratee`. The corresponding value of + * each key is the last element responsible for generating the key. The + * iteratee is invoked with one argument: (value). * * @static * @memberOf _ * @since 4.0.0 - * @category Array - * @param {...Array} [arrays] The arrays to inspect. - * @param {Function} [iteratee=_.identity] The iteratee invoked per element. - * @returns {Array} Returns the new array of filtered values. + * @category Collection + * @param {Array|Object} collection The collection to iterate over. + * @param {Function} [iteratee=_.identity] The iteratee to transform keys. + * @returns {Object} Returns the composed aggregate object. * @example * - * _.xorBy([2.1, 1.2], [2.3, 3.4], Math.floor); - * // => [1.2, 3.4] + * var array = [ + * { 'dir': 'left', 'code': 97 }, + * { 'dir': 'right', 'code': 100 } + * ]; * - * // The `_.property` iteratee shorthand. - * _.xorBy([{ 'x': 1 }], [{ 'x': 2 }, { 'x': 1 }], 'x'); - * // => [{ 'x': 2 }] + * _.keyBy(array, function(o) { + * return String.fromCharCode(o.code); + * }); + * // => { 'a': { 'dir': 'left', 'code': 97 }, 'd': { 'dir': 'right', 'code': 100 } } + * + * _.keyBy(array, 'dir'); + * // => { 'left': { 'dir': 'left', 'code': 97 }, 'right': { 'dir': 'right', 'code': 100 } } */ - var xorBy = baseRest(function(arrays) { - var iteratee = last(arrays); - if (isArrayLikeObject(iteratee)) { - iteratee = undefined; - } - return baseXor(arrayFilter(arrays, isArrayLikeObject), getIteratee(iteratee, 2)); + var keyBy = createAggregator(function(result, value, key) { + baseAssignValue(result, key, value); }); /** - * This method is like `_.xor` except that it accepts `comparator` which is - * invoked to compare elements of `arrays`. The order of result values is - * determined by the order they occur in the arrays. The comparator is invoked - * with two arguments: (arrVal, othVal). + * Creates an array of values by running each element in `collection` thru + * `iteratee`. The iteratee is invoked with three arguments: + * (value, index|key, collection). + * + * Many lodash methods are guarded to work as iteratees for methods like + * `_.every`, `_.filter`, `_.map`, `_.mapValues`, `_.reject`, and `_.some`. + * + * The guarded methods are: + * `ary`, `chunk`, `curry`, `curryRight`, `drop`, `dropRight`, `every`, + * `fill`, `invert`, `parseInt`, `random`, `range`, `rangeRight`, `repeat`, + * `sampleSize`, `slice`, `some`, `sortBy`, `split`, `take`, `takeRight`, + * `template`, `trim`, `trimEnd`, `trimStart`, and `words` * * @static * @memberOf _ - * @since 4.0.0 - * @category Array - * @param {...Array} [arrays] The arrays to inspect. - * @param {Function} [comparator] The comparator invoked per element. - * @returns {Array} Returns the new array of filtered values. + * @since 0.1.0 + * @category Collection + * @param {Array|Object} collection The collection to iterate over. + * @param {Function} [iteratee=_.identity] The function invoked per iteration. + * @returns {Array} Returns the new mapped array. * @example * - * var objects = [{ 'x': 1, 'y': 2 }, { 'x': 2, 'y': 1 }]; - * var others = [{ 'x': 1, 'y': 1 }, { 'x': 1, 'y': 2 }]; + * function square(n) { + * return n * n; + * } * - * _.xorWith(objects, others, _.isEqual); - * // => [{ 'x': 2, 'y': 1 }, { 'x': 1, 'y': 1 }] + * _.map([4, 8], square); + * // => [16, 64] + * + * _.map({ 'a': 4, 'b': 8 }, square); + * // => [16, 64] (iteration order is not guaranteed) + * + * var users = [ + * { 'user': 'barney' }, + * { 'user': 'fred' } + * ]; + * + * // The `_.property` iteratee shorthand. + * _.map(users, 'user'); + * // => ['barney', 'fred'] */ - var xorWith = baseRest(function(arrays) { - var comparator = last(arrays); - comparator = typeof comparator == 'function' ? comparator : undefined; - return baseXor(arrayFilter(arrays, isArrayLikeObject), undefined, comparator); - }); + function map(collection, iteratee) { + var func = isArray(collection) ? arrayMap : baseMap; + return func(collection, getIteratee(iteratee, 3)); + } /** - * Creates an array of grouped elements, the first of which contains the - * first elements of the given arrays, the second of which contains the - * second elements of the given arrays, and so on. + * This method is like `_.sortBy` except that it allows specifying the sort + * orders of the iteratees to sort by. If `orders` is unspecified, all values + * are sorted in ascending order. Otherwise, specify an order of "desc" for + * descending or "asc" for ascending sort order of corresponding values. * * @static * @memberOf _ - * @since 0.1.0 - * @category Array - * @param {...Array} [arrays] The arrays to process. - * @returns {Array} Returns the new array of grouped elements. + * @since 4.0.0 + * @category Collection + * @param {Array|Object} collection The collection to iterate over. + * @param {Array[]|Function[]|Object[]|string[]} [iteratees=[_.identity]] + * The iteratees to sort by. + * @param {string[]} [orders] The sort orders of `iteratees`. + * @param- {Object} [guard] Enables use as an iteratee for methods like `_.reduce`. + * @returns {Array} Returns the new sorted array. * @example * - * _.zip(['a', 'b'], [1, 2], [true, false]); - * // => [['a', 1, true], ['b', 2, false]] + * var users = [ + * { 'user': 'fred', 'age': 48 }, + * { 'user': 'barney', 'age': 34 }, + * { 'user': 'fred', 'age': 40 }, + * { 'user': 'barney', 'age': 36 } + * ]; + * + * // Sort by `user` in ascending order and by `age` in descending order. + * _.orderBy(users, ['user', 'age'], ['asc', 'desc']); + * // => objects for [['barney', 36], ['barney', 34], ['fred', 48], ['fred', 40]] */ - var zip = baseRest(unzip); + function orderBy(collection, iteratees, orders, guard) { + if (collection == null) { + return []; + } + if (!isArray(iteratees)) { + iteratees = iteratees == null ? [] : [iteratees]; + } + orders = guard ? undefined : orders; + if (!isArray(orders)) { + orders = orders == null ? [] : [orders]; + } + return baseOrderBy(collection, iteratees, orders); + } /** - * This method is like `_.fromPairs` except that it accepts two arrays, - * one of property identifiers and one of corresponding values. + * Creates an array of elements split into two groups, the first of which + * contains elements `predicate` returns truthy for, the second of which + * contains elements `predicate` returns falsey for. The predicate is + * invoked with one argument: (value). * * @static * @memberOf _ - * @since 0.4.0 - * @category Array - * @param {Array} [props=[]] The property identifiers. - * @param {Array} [values=[]] The property values. - * @returns {Object} Returns the new object. + * @since 3.0.0 + * @category Collection + * @param {Array|Object} collection The collection to iterate over. + * @param {Function} [predicate=_.identity] The function invoked per iteration. + * @returns {Array} Returns the array of grouped elements. * @example * - * _.zipObject(['a', 'b'], [1, 2]); - * // => { 'a': 1, 'b': 2 } + * var users = [ + * { 'user': 'barney', 'age': 36, 'active': false }, + * { 'user': 'fred', 'age': 40, 'active': true }, + * { 'user': 'pebbles', 'age': 1, 'active': false } + * ]; + * + * _.partition(users, function(o) { return o.active; }); + * // => objects for [['fred'], ['barney', 'pebbles']] + * + * // The `_.matches` iteratee shorthand. + * _.partition(users, { 'age': 1, 'active': false }); + * // => objects for [['pebbles'], ['barney', 'fred']] + * + * // The `_.matchesProperty` iteratee shorthand. + * _.partition(users, ['active', false]); + * // => objects for [['barney', 'pebbles'], ['fred']] + * + * // The `_.property` iteratee shorthand. + * _.partition(users, 'active'); + * // => objects for [['fred'], ['barney', 'pebbles']] */ - function zipObject(props, values) { - return baseZipObject(props || [], values || [], assignValue); - } + var partition = createAggregator(function(result, value, key) { + result[key ? 0 : 1].push(value); + }, function() { return [[], []]; }); /** - * This method is like `_.zipObject` except that it supports property paths. + * Reduces `collection` to a value which is the accumulated result of running + * each element in `collection` thru `iteratee`, where each successive + * invocation is supplied the return value of the previous. If `accumulator` + * is not given, the first element of `collection` is used as the initial + * value. The iteratee is invoked with four arguments: + * (accumulator, value, index|key, collection). + * + * Many lodash methods are guarded to work as iteratees for methods like + * `_.reduce`, `_.reduceRight`, and `_.transform`. + * + * The guarded methods are: + * `assign`, `defaults`, `defaultsDeep`, `includes`, `merge`, `orderBy`, + * and `sortBy` * * @static * @memberOf _ - * @since 4.1.0 - * @category Array - * @param {Array} [props=[]] The property identifiers. - * @param {Array} [values=[]] The property values. - * @returns {Object} Returns the new object. + * @since 0.1.0 + * @category Collection + * @param {Array|Object} collection The collection to iterate over. + * @param {Function} [iteratee=_.identity] The function invoked per iteration. + * @param {*} [accumulator] The initial value. + * @returns {*} Returns the accumulated value. + * @see _.reduceRight * @example * - * _.zipObjectDeep(['a.b[0].c', 'a.b[1].d'], [1, 2]); - * // => { 'a': { 'b': [{ 'c': 1 }, { 'd': 2 }] } } + * _.reduce([1, 2], function(sum, n) { + * return sum + n; + * }, 0); + * // => 3 + * + * _.reduce({ 'a': 1, 'b': 2, 'c': 1 }, function(result, value, key) { + * (result[value] || (result[value] = [])).push(key); + * return result; + * }, {}); + * // => { '1': ['a', 'c'], '2': ['b'] } (iteration order is not guaranteed) */ - function zipObjectDeep(props, values) { - return baseZipObject(props || [], values || [], baseSet); + function reduce(collection, iteratee, accumulator) { + var func = isArray(collection) ? arrayReduce : baseReduce, + initAccum = arguments.length < 3; + + return func(collection, getIteratee(iteratee, 4), accumulator, initAccum, baseEach); } /** - * This method is like `_.zip` except that it accepts `iteratee` to specify - * how grouped values should be combined. The iteratee is invoked with the - * elements of each group: (...group). + * This method is like `_.reduce` except that it iterates over elements of + * `collection` from right to left. * * @static * @memberOf _ - * @since 3.8.0 - * @category Array - * @param {...Array} [arrays] The arrays to process. - * @param {Function} [iteratee=_.identity] The function to combine - * grouped values. - * @returns {Array} Returns the new array of grouped elements. + * @since 0.1.0 + * @category Collection + * @param {Array|Object} collection The collection to iterate over. + * @param {Function} [iteratee=_.identity] The function invoked per iteration. + * @param {*} [accumulator] The initial value. + * @returns {*} Returns the accumulated value. + * @see _.reduce * @example * - * _.zipWith([1, 2], [10, 20], [100, 200], function(a, b, c) { - * return a + b + c; - * }); - * // => [111, 222] + * var array = [[0, 1], [2, 3], [4, 5]]; + * + * _.reduceRight(array, function(flattened, other) { + * return flattened.concat(other); + * }, []); + * // => [4, 5, 2, 3, 0, 1] */ - var zipWith = baseRest(function(arrays) { - var length = arrays.length, - iteratee = length > 1 ? arrays[length - 1] : undefined; - - iteratee = typeof iteratee == 'function' ? (arrays.pop(), iteratee) : undefined; - return unzipWith(arrays, iteratee); - }); + function reduceRight(collection, iteratee, accumulator) { + var func = isArray(collection) ? arrayReduceRight : baseReduce, + initAccum = arguments.length < 3; - /*------------------------------------------------------------------------*/ + return func(collection, getIteratee(iteratee, 4), accumulator, initAccum, baseEachRight); + } /** - * Creates a `lodash` wrapper instance that wraps `value` with explicit method - * chain sequences enabled. The result of such sequences must be unwrapped - * with `_#value`. + * The opposite of `_.filter`; this method returns the elements of `collection` + * that `predicate` does **not** return truthy for. * * @static * @memberOf _ - * @since 1.3.0 - * @category Seq - * @param {*} value The value to wrap. - * @returns {Object} Returns the new `lodash` wrapper instance. + * @since 0.1.0 + * @category Collection + * @param {Array|Object} collection The collection to iterate over. + * @param {Function} [predicate=_.identity] The function invoked per iteration. + * @returns {Array} Returns the new filtered array. + * @see _.filter * @example * * var users = [ - * { 'user': 'barney', 'age': 36 }, - * { 'user': 'fred', 'age': 40 }, - * { 'user': 'pebbles', 'age': 1 } + * { 'user': 'barney', 'age': 36, 'active': false }, + * { 'user': 'fred', 'age': 40, 'active': true } * ]; * - * var youngest = _ - * .chain(users) - * .sortBy('age') - * .map(function(o) { - * return o.user + ' is ' + o.age; - * }) - * .head() - * .value(); - * // => 'pebbles is 1' + * _.reject(users, function(o) { return !o.active; }); + * // => objects for ['fred'] + * + * // The `_.matches` iteratee shorthand. + * _.reject(users, { 'age': 40, 'active': true }); + * // => objects for ['barney'] + * + * // The `_.matchesProperty` iteratee shorthand. + * _.reject(users, ['active', false]); + * // => objects for ['fred'] + * + * // The `_.property` iteratee shorthand. + * _.reject(users, 'active'); + * // => objects for ['barney'] */ - function chain(value) { - var result = lodash(value); - result.__chain__ = true; - return result; + function reject(collection, predicate) { + var func = isArray(collection) ? arrayFilter : baseFilter; + return func(collection, negate(getIteratee(predicate, 3))); } /** - * This method invokes `interceptor` and returns `value`. The interceptor - * is invoked with one argument; (value). The purpose of this method is to - * "tap into" a method chain sequence in order to modify intermediate results. + * Gets a random element from `collection`. * * @static * @memberOf _ - * @since 0.1.0 - * @category Seq - * @param {*} value The value to provide to `interceptor`. - * @param {Function} interceptor The function to invoke. - * @returns {*} Returns `value`. + * @since 2.0.0 + * @category Collection + * @param {Array|Object} collection The collection to sample. + * @returns {*} Returns the random element. * @example * - * _([1, 2, 3]) - * .tap(function(array) { - * // Mutate input array. - * array.pop(); - * }) - * .reverse() - * .value(); - * // => [2, 1] + * _.sample([1, 2, 3, 4]); + * // => 2 */ - function tap(value, interceptor) { - interceptor(value); - return value; + function sample(collection) { + var func = isArray(collection) ? arraySample : baseSample; + return func(collection); } /** - * This method is like `_.tap` except that it returns the result of `interceptor`. - * The purpose of this method is to "pass thru" values replacing intermediate - * results in a method chain sequence. + * Gets `n` random elements at unique keys from `collection` up to the + * size of `collection`. * * @static * @memberOf _ - * @since 3.0.0 - * @category Seq - * @param {*} value The value to provide to `interceptor`. - * @param {Function} interceptor The function to invoke. - * @returns {*} Returns the result of `interceptor`. + * @since 4.0.0 + * @category Collection + * @param {Array|Object} collection The collection to sample. + * @param {number} [n=1] The number of elements to sample. + * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. + * @returns {Array} Returns the random elements. * @example * - * _(' abc ') - * .chain() - * .trim() - * .thru(function(value) { - * return [value]; - * }) - * .value(); - * // => ['abc'] + * _.sampleSize([1, 2, 3], 2); + * // => [3, 1] + * + * _.sampleSize([1, 2, 3], 4); + * // => [2, 3, 1] */ - function thru(value, interceptor) { - return interceptor(value); + function sampleSize(collection, n, guard) { + if ((guard ? isIterateeCall(collection, n, guard) : n === undefined)) { + n = 1; + } else { + n = toInteger(n); + } + var func = isArray(collection) ? arraySampleSize : baseSampleSize; + return func(collection, n); } /** - * This method is the wrapper version of `_.at`. + * Creates an array of shuffled values, using a version of the + * [Fisher-Yates shuffle](https://en.wikipedia.org/wiki/Fisher-Yates_shuffle). * - * @name at + * @static * @memberOf _ - * @since 1.0.0 - * @category Seq - * @param {...(string|string[])} [paths] The property paths to pick. - * @returns {Object} Returns the new `lodash` wrapper instance. + * @since 0.1.0 + * @category Collection + * @param {Array|Object} collection The collection to shuffle. + * @returns {Array} Returns the new shuffled array. * @example * - * var object = { 'a': [{ 'b': { 'c': 3 } }, 4] }; - * - * _(object).at(['a[0].b.c', 'a[1]']).value(); - * // => [3, 4] + * _.shuffle([1, 2, 3, 4]); + * // => [4, 1, 3, 2] */ - var wrapperAt = flatRest(function(paths) { - var length = paths.length, - start = length ? paths[0] : 0, - value = this.__wrapped__, - interceptor = function(object) { return baseAt(object, paths); }; - - if (length > 1 || this.__actions__.length || - !(value instanceof LazyWrapper) || !isIndex(start)) { - return this.thru(interceptor); - } - value = value.slice(start, +start + (length ? 1 : 0)); - value.__actions__.push({ - 'func': thru, - 'args': [interceptor], - 'thisArg': undefined - }); - return new LodashWrapper(value, this.__chain__).thru(function(array) { - if (length && !array.length) { - array.push(undefined); - } - return array; - }); - }); + function shuffle(collection) { + var func = isArray(collection) ? arrayShuffle : baseShuffle; + return func(collection); + } /** - * Creates a `lodash` wrapper instance with explicit method chain sequences enabled. + * Gets the size of `collection` by returning its length for array-like + * values or the number of own enumerable string keyed properties for objects. * - * @name chain + * @static * @memberOf _ * @since 0.1.0 - * @category Seq - * @returns {Object} Returns the new `lodash` wrapper instance. + * @category Collection + * @param {Array|Object|string} collection The collection to inspect. + * @returns {number} Returns the collection size. * @example * - * var users = [ - * { 'user': 'barney', 'age': 36 }, - * { 'user': 'fred', 'age': 40 } - * ]; + * _.size([1, 2, 3]); + * // => 3 * - * // A sequence without explicit chaining. - * _(users).head(); - * // => { 'user': 'barney', 'age': 36 } + * _.size({ 'a': 1, 'b': 2 }); + * // => 2 * - * // A sequence with explicit chaining. - * _(users) - * .chain() - * .head() - * .pick('user') - * .value(); - * // => { 'user': 'barney' } + * _.size('pebbles'); + * // => 7 */ - function wrapperChain() { - return chain(this); + function size(collection) { + if (collection == null) { + return 0; + } + if (isArrayLike(collection)) { + return isString(collection) ? stringSize(collection) : collection.length; + } + var tag = getTag(collection); + if (tag == mapTag || tag == setTag) { + return collection.size; + } + return baseKeys(collection).length; } /** - * Executes the chain sequence and returns the wrapped result. + * Checks if `predicate` returns truthy for **any** element of `collection`. + * Iteration is stopped once `predicate` returns truthy. The predicate is + * invoked with three arguments: (value, index|key, collection). * - * @name commit + * @static * @memberOf _ - * @since 3.2.0 - * @category Seq - * @returns {Object} Returns the new `lodash` wrapper instance. + * @since 0.1.0 + * @category Collection + * @param {Array|Object} collection The collection to iterate over. + * @param {Function} [predicate=_.identity] The function invoked per iteration. + * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. + * @returns {boolean} Returns `true` if any element passes the predicate check, + * else `false`. * @example * - * var array = [1, 2]; - * var wrapped = _(array).push(3); + * _.some([null, 0, 'yes', false], Boolean); + * // => true * - * console.log(array); - * // => [1, 2] + * var users = [ + * { 'user': 'barney', 'active': true }, + * { 'user': 'fred', 'active': false } + * ]; * - * wrapped = wrapped.commit(); - * console.log(array); - * // => [1, 2, 3] + * // The `_.matches` iteratee shorthand. + * _.some(users, { 'user': 'barney', 'active': false }); + * // => false * - * wrapped.last(); - * // => 3 + * // The `_.matchesProperty` iteratee shorthand. + * _.some(users, ['active', false]); + * // => true * - * console.log(array); - * // => [1, 2, 3] + * // The `_.property` iteratee shorthand. + * _.some(users, 'active'); + * // => true */ - function wrapperCommit() { - return new LodashWrapper(this.value(), this.__chain__); + function some(collection, predicate, guard) { + var func = isArray(collection) ? arraySome : baseSome; + if (guard && isIterateeCall(collection, predicate, guard)) { + predicate = undefined; + } + return func(collection, getIteratee(predicate, 3)); } /** - * Gets the next value on a wrapped object following the - * [iterator protocol](https://mdn.io/iteration_protocols#iterator). + * Creates an array of elements, sorted in ascending order by the results of + * running each element in a collection thru each iteratee. This method + * performs a stable sort, that is, it preserves the original sort order of + * equal elements. The iteratees are invoked with one argument: (value). * - * @name next + * @static * @memberOf _ - * @since 4.0.0 - * @category Seq - * @returns {Object} Returns the next iterator value. + * @since 0.1.0 + * @category Collection + * @param {Array|Object} collection The collection to iterate over. + * @param {...(Function|Function[])} [iteratees=[_.identity]] + * The iteratees to sort by. + * @returns {Array} Returns the new sorted array. * @example * - * var wrapped = _([1, 2]); - * - * wrapped.next(); - * // => { 'done': false, 'value': 1 } + * var users = [ + * { 'user': 'fred', 'age': 48 }, + * { 'user': 'barney', 'age': 36 }, + * { 'user': 'fred', 'age': 30 }, + * { 'user': 'barney', 'age': 34 } + * ]; * - * wrapped.next(); - * // => { 'done': false, 'value': 2 } + * _.sortBy(users, [function(o) { return o.user; }]); + * // => objects for [['barney', 36], ['barney', 34], ['fred', 48], ['fred', 30]] * - * wrapped.next(); - * // => { 'done': true, 'value': undefined } + * _.sortBy(users, ['user', 'age']); + * // => objects for [['barney', 34], ['barney', 36], ['fred', 30], ['fred', 48]] */ - function wrapperNext() { - if (this.__values__ === undefined) { - this.__values__ = toArray(this.value()); + var sortBy = baseRest(function(collection, iteratees) { + if (collection == null) { + return []; } - var done = this.__index__ >= this.__values__.length, - value = done ? undefined : this.__values__[this.__index__++]; + var length = iteratees.length; + if (length > 1 && isIterateeCall(collection, iteratees[0], iteratees[1])) { + iteratees = []; + } else if (length > 2 && isIterateeCall(iteratees[0], iteratees[1], iteratees[2])) { + iteratees = [iteratees[0]]; + } + return baseOrderBy(collection, baseFlatten(iteratees, 1), []); + }); - return { 'done': done, 'value': value }; - } + /*------------------------------------------------------------------------*/ /** - * Enables the wrapper to be iterable. + * Gets the timestamp of the number of milliseconds that have elapsed since + * the Unix epoch (1 January 1970 00:00:00 UTC). * - * @name Symbol.iterator + * @static * @memberOf _ - * @since 4.0.0 - * @category Seq - * @returns {Object} Returns the wrapper object. + * @since 2.4.0 + * @category Date + * @returns {number} Returns the timestamp. * @example * - * var wrapped = _([1, 2]); - * - * wrapped[Symbol.iterator]() === wrapped; - * // => true - * - * Array.from(wrapped); - * // => [1, 2] + * _.defer(function(stamp) { + * console.log(_.now() - stamp); + * }, _.now()); + * // => Logs the number of milliseconds it took for the deferred invocation. */ - function wrapperToIterator() { - return this; - } + var now = ctxNow || function() { + return root.Date.now(); + }; + + /*------------------------------------------------------------------------*/ /** - * Creates a clone of the chain sequence planting `value` as the wrapped value. + * The opposite of `_.before`; this method creates a function that invokes + * `func` once it's called `n` or more times. * - * @name plant + * @static * @memberOf _ - * @since 3.2.0 - * @category Seq - * @param {*} value The value to plant. - * @returns {Object} Returns the new `lodash` wrapper instance. + * @since 0.1.0 + * @category Function + * @param {number} n The number of calls before `func` is invoked. + * @param {Function} func The function to restrict. + * @returns {Function} Returns the new restricted function. * @example * - * function square(n) { - * return n * n; - * } - * - * var wrapped = _([1, 2]).map(square); - * var other = wrapped.plant([3, 4]); + * var saves = ['profile', 'settings']; * - * other.value(); - * // => [9, 16] + * var done = _.after(saves.length, function() { + * console.log('done saving!'); + * }); * - * wrapped.value(); - * // => [1, 4] + * _.forEach(saves, function(type) { + * asyncSave({ 'type': type, 'complete': done }); + * }); + * // => Logs 'done saving!' after the two async saves have completed. */ - function wrapperPlant(value) { - var result, - parent = this; - - while (parent instanceof baseLodash) { - var clone = wrapperClone(parent); - clone.__index__ = 0; - clone.__values__ = undefined; - if (result) { - previous.__wrapped__ = clone; - } else { - result = clone; - } - var previous = clone; - parent = parent.__wrapped__; + function after(n, func) { + if (typeof func != 'function') { + throw new TypeError(FUNC_ERROR_TEXT); } - previous.__wrapped__ = value; - return result; + n = toInteger(n); + return function() { + if (--n < 1) { + return func.apply(this, arguments); + } + }; + } + + /** + * Creates a function that invokes `func`, with up to `n` arguments, + * ignoring any additional arguments. + * + * @static + * @memberOf _ + * @since 3.0.0 + * @category Function + * @param {Function} func The function to cap arguments for. + * @param {number} [n=func.length] The arity cap. + * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. + * @returns {Function} Returns the new capped function. + * @example + * + * _.map(['6', '8', '10'], _.ary(parseInt, 1)); + * // => [6, 8, 10] + */ + function ary(func, n, guard) { + n = guard ? undefined : n; + n = (func && n == null) ? func.length : n; + return createWrap(func, WRAP_ARY_FLAG, undefined, undefined, undefined, undefined, n); } /** - * This method is the wrapper version of `_.reverse`. - * - * **Note:** This method mutates the wrapped array. + * Creates a function that invokes `func`, with the `this` binding and arguments + * of the created function, while it's called less than `n` times. Subsequent + * calls to the created function return the result of the last `func` invocation. * - * @name reverse + * @static * @memberOf _ - * @since 0.1.0 - * @category Seq - * @returns {Object} Returns the new `lodash` wrapper instance. + * @since 3.0.0 + * @category Function + * @param {number} n The number of calls at which `func` is no longer invoked. + * @param {Function} func The function to restrict. + * @returns {Function} Returns the new restricted function. * @example * - * var array = [1, 2, 3]; - * - * _(array).reverse().value() - * // => [3, 2, 1] - * - * console.log(array); - * // => [3, 2, 1] + * jQuery(element).on('click', _.before(5, addContactToList)); + * // => Allows adding up to 4 contacts to the list. */ - function wrapperReverse() { - var value = this.__wrapped__; - if (value instanceof LazyWrapper) { - var wrapped = value; - if (this.__actions__.length) { - wrapped = new LazyWrapper(this); - } - wrapped = wrapped.reverse(); - wrapped.__actions__.push({ - 'func': thru, - 'args': [reverse], - 'thisArg': undefined - }); - return new LodashWrapper(wrapped, this.__chain__); + function before(n, func) { + var result; + if (typeof func != 'function') { + throw new TypeError(FUNC_ERROR_TEXT); } - return this.thru(reverse); + n = toInteger(n); + return function() { + if (--n > 0) { + result = func.apply(this, arguments); + } + if (n <= 1) { + func = undefined; + } + return result; + }; } /** - * Executes the chain sequence to resolve the unwrapped value. + * Creates a function that invokes `func` with the `this` binding of `thisArg` + * and `partials` prepended to the arguments it receives. * - * @name value + * The `_.bind.placeholder` value, which defaults to `_` in monolithic builds, + * may be used as a placeholder for partially applied arguments. + * + * **Note:** Unlike native `Function#bind`, this method doesn't set the "length" + * property of bound functions. + * + * @static * @memberOf _ * @since 0.1.0 - * @alias toJSON, valueOf - * @category Seq - * @returns {*} Returns the resolved unwrapped value. + * @category Function + * @param {Function} func The function to bind. + * @param {*} thisArg The `this` binding of `func`. + * @param {...*} [partials] The arguments to be partially applied. + * @returns {Function} Returns the new bound function. * @example * - * _([1, 2, 3]).value(); - * // => [1, 2, 3] + * function greet(greeting, punctuation) { + * return greeting + ' ' + this.user + punctuation; + * } + * + * var object = { 'user': 'fred' }; + * + * var bound = _.bind(greet, object, 'hi'); + * bound('!'); + * // => 'hi fred!' + * + * // Bound with placeholders. + * var bound = _.bind(greet, object, _, '!'); + * bound('hi'); + * // => 'hi fred!' */ - function wrapperValue() { - return baseWrapperValue(this.__wrapped__, this.__actions__); - } - - /*------------------------------------------------------------------------*/ + var bind = baseRest(function(func, thisArg, partials) { + var bitmask = WRAP_BIND_FLAG; + if (partials.length) { + var holders = replaceHolders(partials, getHolder(bind)); + bitmask |= WRAP_PARTIAL_FLAG; + } + return createWrap(func, bitmask, thisArg, partials, holders); + }); /** - * Creates an object composed of keys generated from the results of running - * each element of `collection` thru `iteratee`. The corresponding value of - * each key is the number of times the key was returned by `iteratee`. The - * iteratee is invoked with one argument: (value). + * Creates a function that invokes the method at `object[key]` with `partials` + * prepended to the arguments it receives. + * + * This method differs from `_.bind` by allowing bound functions to reference + * methods that may be redefined or don't yet exist. See + * [Peter Michaux's article](http://peter.michaux.ca/articles/lazy-function-definition-pattern) + * for more details. + * + * The `_.bindKey.placeholder` value, which defaults to `_` in monolithic + * builds, may be used as a placeholder for partially applied arguments. * * @static * @memberOf _ - * @since 0.5.0 - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} [iteratee=_.identity] The iteratee to transform keys. - * @returns {Object} Returns the composed aggregate object. + * @since 0.10.0 + * @category Function + * @param {Object} object The object to invoke the method on. + * @param {string} key The key of the method. + * @param {...*} [partials] The arguments to be partially applied. + * @returns {Function} Returns the new bound function. * @example * - * _.countBy([6.1, 4.2, 6.3], Math.floor); - * // => { '4': 1, '6': 2 } + * var object = { + * 'user': 'fred', + * 'greet': function(greeting, punctuation) { + * return greeting + ' ' + this.user + punctuation; + * } + * }; * - * // The `_.property` iteratee shorthand. - * _.countBy(['one', 'two', 'three'], 'length'); - * // => { '3': 2, '5': 1 } + * var bound = _.bindKey(object, 'greet', 'hi'); + * bound('!'); + * // => 'hi fred!' + * + * object.greet = function(greeting, punctuation) { + * return greeting + 'ya ' + this.user + punctuation; + * }; + * + * bound('!'); + * // => 'hiya fred!' + * + * // Bound with placeholders. + * var bound = _.bindKey(object, 'greet', _, '!'); + * bound('hi'); + * // => 'hiya fred!' */ - var countBy = createAggregator(function(result, value, key) { - if (hasOwnProperty.call(result, key)) { - ++result[key]; - } else { - baseAssignValue(result, key, 1); + var bindKey = baseRest(function(object, key, partials) { + var bitmask = WRAP_BIND_FLAG | WRAP_BIND_KEY_FLAG; + if (partials.length) { + var holders = replaceHolders(partials, getHolder(bindKey)); + bitmask |= WRAP_PARTIAL_FLAG; } + return createWrap(key, bitmask, object, partials, holders); }); /** - * Checks if `predicate` returns truthy for **all** elements of `collection`. - * Iteration is stopped once `predicate` returns falsey. The predicate is - * invoked with three arguments: (value, index|key, collection). + * Creates a function that accepts arguments of `func` and either invokes + * `func` returning its result, if at least `arity` number of arguments have + * been provided, or returns a function that accepts the remaining `func` + * arguments, and so on. The arity of `func` may be specified if `func.length` + * is not sufficient. * - * **Note:** This method returns `true` for - * [empty collections](https://en.wikipedia.org/wiki/Empty_set) because - * [everything is true](https://en.wikipedia.org/wiki/Vacuous_truth) of - * elements of empty collections. + * The `_.curry.placeholder` value, which defaults to `_` in monolithic builds, + * may be used as a placeholder for provided arguments. + * + * **Note:** This method doesn't set the "length" property of curried functions. * * @static * @memberOf _ - * @since 0.1.0 - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} [predicate=_.identity] The function invoked per iteration. + * @since 2.0.0 + * @category Function + * @param {Function} func The function to curry. + * @param {number} [arity=func.length] The arity of `func`. * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. - * @returns {boolean} Returns `true` if all elements pass the predicate check, - * else `false`. + * @returns {Function} Returns the new curried function. * @example * - * _.every([true, 1, null, 'yes'], Boolean); - * // => false + * var abc = function(a, b, c) { + * return [a, b, c]; + * }; * - * var users = [ - * { 'user': 'barney', 'age': 36, 'active': false }, - * { 'user': 'fred', 'age': 40, 'active': false } - * ]; + * var curried = _.curry(abc); * - * // The `_.matches` iteratee shorthand. - * _.every(users, { 'user': 'barney', 'active': false }); - * // => false + * curried(1)(2)(3); + * // => [1, 2, 3] * - * // The `_.matchesProperty` iteratee shorthand. - * _.every(users, ['active', false]); - * // => true + * curried(1, 2)(3); + * // => [1, 2, 3] * - * // The `_.property` iteratee shorthand. - * _.every(users, 'active'); - * // => false + * curried(1, 2, 3); + * // => [1, 2, 3] + * + * // Curried with placeholders. + * curried(1)(_, 3)(2); + * // => [1, 2, 3] */ - function every(collection, predicate, guard) { - var func = isArray(collection) ? arrayEvery : baseEvery; - if (guard && isIterateeCall(collection, predicate, guard)) { - predicate = undefined; - } - return func(collection, getIteratee(predicate, 3)); + function curry(func, arity, guard) { + arity = guard ? undefined : arity; + var result = createWrap(func, WRAP_CURRY_FLAG, undefined, undefined, undefined, undefined, undefined, arity); + result.placeholder = curry.placeholder; + return result; } /** - * Iterates over elements of `collection`, returning an array of all elements - * `predicate` returns truthy for. The predicate is invoked with three - * arguments: (value, index|key, collection). + * This method is like `_.curry` except that arguments are applied to `func` + * in the manner of `_.partialRight` instead of `_.partial`. * - * **Note:** Unlike `_.remove`, this method returns a new array. + * The `_.curryRight.placeholder` value, which defaults to `_` in monolithic + * builds, may be used as a placeholder for provided arguments. + * + * **Note:** This method doesn't set the "length" property of curried functions. * * @static * @memberOf _ - * @since 0.1.0 - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} [predicate=_.identity] The function invoked per iteration. - * @returns {Array} Returns the new filtered array. - * @see _.reject + * @since 3.0.0 + * @category Function + * @param {Function} func The function to curry. + * @param {number} [arity=func.length] The arity of `func`. + * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. + * @returns {Function} Returns the new curried function. * @example * - * var users = [ - * { 'user': 'barney', 'age': 36, 'active': true }, - * { 'user': 'fred', 'age': 40, 'active': false } - * ]; + * var abc = function(a, b, c) { + * return [a, b, c]; + * }; * - * _.filter(users, function(o) { return !o.active; }); - * // => objects for ['fred'] + * var curried = _.curryRight(abc); * - * // The `_.matches` iteratee shorthand. - * _.filter(users, { 'age': 36, 'active': true }); - * // => objects for ['barney'] + * curried(3)(2)(1); + * // => [1, 2, 3] * - * // The `_.matchesProperty` iteratee shorthand. - * _.filter(users, ['active', false]); - * // => objects for ['fred'] + * curried(2, 3)(1); + * // => [1, 2, 3] * - * // The `_.property` iteratee shorthand. - * _.filter(users, 'active'); - * // => objects for ['barney'] + * curried(1, 2, 3); + * // => [1, 2, 3] * - * // Combining several predicates using `_.overEvery` or `_.overSome`. - * _.filter(users, _.overSome([{ 'age': 36 }, ['age', 40]])); - * // => objects for ['fred', 'barney'] + * // Curried with placeholders. + * curried(3)(1, _)(2); + * // => [1, 2, 3] */ - function filter(collection, predicate) { - var func = isArray(collection) ? arrayFilter : baseFilter; - return func(collection, getIteratee(predicate, 3)); + function curryRight(func, arity, guard) { + arity = guard ? undefined : arity; + var result = createWrap(func, WRAP_CURRY_RIGHT_FLAG, undefined, undefined, undefined, undefined, undefined, arity); + result.placeholder = curryRight.placeholder; + return result; } /** - * Iterates over elements of `collection`, returning the first element - * `predicate` returns truthy for. The predicate is invoked with three - * arguments: (value, index|key, collection). + * Creates a debounced function that delays invoking `func` until after `wait` + * milliseconds have elapsed since the last time the debounced function was + * invoked. The debounced function comes with a `cancel` method to cancel + * delayed `func` invocations and a `flush` method to immediately invoke them. + * Provide `options` to indicate whether `func` should be invoked on the + * leading and/or trailing edge of the `wait` timeout. The `func` is invoked + * with the last arguments provided to the debounced function. Subsequent + * calls to the debounced function return the result of the last `func` + * invocation. + * + * **Note:** If `leading` and `trailing` options are `true`, `func` is + * invoked on the trailing edge of the timeout only if the debounced function + * is invoked more than once during the `wait` timeout. + * + * If `wait` is `0` and `leading` is `false`, `func` invocation is deferred + * until to the next tick, similar to `setTimeout` with a timeout of `0`. + * + * See [David Corbacho's article](https://css-tricks.com/debouncing-throttling-explained-examples/) + * for details over the differences between `_.debounce` and `_.throttle`. * * @static * @memberOf _ * @since 0.1.0 - * @category Collection - * @param {Array|Object} collection The collection to inspect. - * @param {Function} [predicate=_.identity] The function invoked per iteration. - * @param {number} [fromIndex=0] The index to search from. - * @returns {*} Returns the matched element, else `undefined`. + * @category Function + * @param {Function} func The function to debounce. + * @param {number} [wait=0] The number of milliseconds to delay. + * @param {Object} [options={}] The options object. + * @param {boolean} [options.leading=false] + * Specify invoking on the leading edge of the timeout. + * @param {number} [options.maxWait] + * The maximum time `func` is allowed to be delayed before it's invoked. + * @param {boolean} [options.trailing=true] + * Specify invoking on the trailing edge of the timeout. + * @returns {Function} Returns the new debounced function. * @example * - * var users = [ - * { 'user': 'barney', 'age': 36, 'active': true }, - * { 'user': 'fred', 'age': 40, 'active': false }, - * { 'user': 'pebbles', 'age': 1, 'active': true } - * ]; + * // Avoid costly calculations while the window size is in flux. + * jQuery(window).on('resize', _.debounce(calculateLayout, 150)); * - * _.find(users, function(o) { return o.age < 40; }); - * // => object for 'barney' + * // Invoke `sendMail` when clicked, debouncing subsequent calls. + * jQuery(element).on('click', _.debounce(sendMail, 300, { + * 'leading': true, + * 'trailing': false + * })); * - * // The `_.matches` iteratee shorthand. - * _.find(users, { 'age': 1, 'active': true }); - * // => object for 'pebbles' + * // Ensure `batchLog` is invoked once after 1 second of debounced calls. + * var debounced = _.debounce(batchLog, 250, { 'maxWait': 1000 }); + * var source = new EventSource('/stream'); + * jQuery(source).on('message', debounced); * - * // The `_.matchesProperty` iteratee shorthand. - * _.find(users, ['active', false]); - * // => object for 'fred' + * // Cancel the trailing debounced invocation. + * jQuery(window).on('popstate', debounced.cancel); + */ + function debounce(func, wait, options) { + var lastArgs, + lastThis, + maxWait, + result, + timerId, + lastCallTime, + lastInvokeTime = 0, + leading = false, + maxing = false, + trailing = true; + + if (typeof func != 'function') { + throw new TypeError(FUNC_ERROR_TEXT); + } + wait = toNumber(wait) || 0; + if (isObject(options)) { + leading = !!options.leading; + maxing = 'maxWait' in options; + maxWait = maxing ? nativeMax(toNumber(options.maxWait) || 0, wait) : maxWait; + trailing = 'trailing' in options ? !!options.trailing : trailing; + } + + function invokeFunc(time) { + var args = lastArgs, + thisArg = lastThis; + + lastArgs = lastThis = undefined; + lastInvokeTime = time; + result = func.apply(thisArg, args); + return result; + } + + function leadingEdge(time) { + // Reset any `maxWait` timer. + lastInvokeTime = time; + // Start the timer for the trailing edge. + timerId = setTimeout(timerExpired, wait); + // Invoke the leading edge. + return leading ? invokeFunc(time) : result; + } + + function remainingWait(time) { + var timeSinceLastCall = time - lastCallTime, + timeSinceLastInvoke = time - lastInvokeTime, + timeWaiting = wait - timeSinceLastCall; + + return maxing + ? nativeMin(timeWaiting, maxWait - timeSinceLastInvoke) + : timeWaiting; + } + + function shouldInvoke(time) { + var timeSinceLastCall = time - lastCallTime, + timeSinceLastInvoke = time - lastInvokeTime; + + // Either this is the first call, activity has stopped and we're at the + // trailing edge, the system time has gone backwards and we're treating + // it as the trailing edge, or we've hit the `maxWait` limit. + return (lastCallTime === undefined || (timeSinceLastCall >= wait) || + (timeSinceLastCall < 0) || (maxing && timeSinceLastInvoke >= maxWait)); + } + + function timerExpired() { + var time = now(); + if (shouldInvoke(time)) { + return trailingEdge(time); + } + // Restart the timer. + timerId = setTimeout(timerExpired, remainingWait(time)); + } + + function trailingEdge(time) { + timerId = undefined; + + // Only invoke if we have `lastArgs` which means `func` has been + // debounced at least once. + if (trailing && lastArgs) { + return invokeFunc(time); + } + lastArgs = lastThis = undefined; + return result; + } + + function cancel() { + if (timerId !== undefined) { + clearTimeout(timerId); + } + lastInvokeTime = 0; + lastArgs = lastCallTime = lastThis = timerId = undefined; + } + + function flush() { + return timerId === undefined ? result : trailingEdge(now()); + } + + function debounced() { + var time = now(), + isInvoking = shouldInvoke(time); + + lastArgs = arguments; + lastThis = this; + lastCallTime = time; + + if (isInvoking) { + if (timerId === undefined) { + return leadingEdge(lastCallTime); + } + if (maxing) { + // Handle invocations in a tight loop. + clearTimeout(timerId); + timerId = setTimeout(timerExpired, wait); + return invokeFunc(lastCallTime); + } + } + if (timerId === undefined) { + timerId = setTimeout(timerExpired, wait); + } + return result; + } + debounced.cancel = cancel; + debounced.flush = flush; + return debounced; + } + + /** + * Defers invoking the `func` until the current call stack has cleared. Any + * additional arguments are provided to `func` when it's invoked. * - * // The `_.property` iteratee shorthand. - * _.find(users, 'active'); - * // => object for 'barney' + * @static + * @memberOf _ + * @since 0.1.0 + * @category Function + * @param {Function} func The function to defer. + * @param {...*} [args] The arguments to invoke `func` with. + * @returns {number} Returns the timer id. + * @example + * + * _.defer(function(text) { + * console.log(text); + * }, 'deferred'); + * // => Logs 'deferred' after one millisecond. */ - var find = createFind(findIndex); + var defer = baseRest(function(func, args) { + return baseDelay(func, 1, args); + }); /** - * This method is like `_.find` except that it iterates over elements of - * `collection` from right to left. + * Invokes `func` after `wait` milliseconds. Any additional arguments are + * provided to `func` when it's invoked. * * @static * @memberOf _ - * @since 2.0.0 - * @category Collection - * @param {Array|Object} collection The collection to inspect. - * @param {Function} [predicate=_.identity] The function invoked per iteration. - * @param {number} [fromIndex=collection.length-1] The index to search from. - * @returns {*} Returns the matched element, else `undefined`. + * @since 0.1.0 + * @category Function + * @param {Function} func The function to delay. + * @param {number} wait The number of milliseconds to delay invocation. + * @param {...*} [args] The arguments to invoke `func` with. + * @returns {number} Returns the timer id. * @example * - * _.findLast([1, 2, 3, 4], function(n) { - * return n % 2 == 1; - * }); - * // => 3 + * _.delay(function(text) { + * console.log(text); + * }, 1000, 'later'); + * // => Logs 'later' after one second. */ - var findLast = createFind(findLastIndex); + var delay = baseRest(function(func, wait, args) { + return baseDelay(func, toNumber(wait) || 0, args); + }); /** - * Creates a flattened array of values by running each element in `collection` - * thru `iteratee` and flattening the mapped results. The iteratee is invoked - * with three arguments: (value, index|key, collection). + * Creates a function that invokes `func` with arguments reversed. * * @static * @memberOf _ * @since 4.0.0 - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} [iteratee=_.identity] The function invoked per iteration. - * @returns {Array} Returns the new flattened array. + * @category Function + * @param {Function} func The function to flip arguments for. + * @returns {Function} Returns the new flipped function. * @example * - * function duplicate(n) { - * return [n, n]; - * } + * var flipped = _.flip(function() { + * return _.toArray(arguments); + * }); * - * _.flatMap([1, 2], duplicate); - * // => [1, 1, 2, 2] + * flipped('a', 'b', 'c', 'd'); + * // => ['d', 'c', 'b', 'a'] */ - function flatMap(collection, iteratee) { - return baseFlatten(map(collection, iteratee), 1); + function flip(func) { + return createWrap(func, WRAP_FLIP_FLAG); } /** - * This method is like `_.flatMap` except that it recursively flattens the - * mapped results. + * Creates a function that memoizes the result of `func`. If `resolver` is + * provided, it determines the cache key for storing the result based on the + * arguments provided to the memoized function. By default, the first argument + * provided to the memoized function is used as the map cache key. The `func` + * is invoked with the `this` binding of the memoized function. + * + * **Note:** The cache is exposed as the `cache` property on the memoized + * function. Its creation may be customized by replacing the `_.memoize.Cache` + * constructor with one whose instances implement the + * [`Map`](http://ecma-international.org/ecma-262/7.0/#sec-properties-of-the-map-prototype-object) + * method interface of `clear`, `delete`, `get`, `has`, and `set`. * * @static * @memberOf _ - * @since 4.7.0 - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} [iteratee=_.identity] The function invoked per iteration. - * @returns {Array} Returns the new flattened array. + * @since 0.1.0 + * @category Function + * @param {Function} func The function to have its output memoized. + * @param {Function} [resolver] The function to resolve the cache key. + * @returns {Function} Returns the new memoized function. * @example * - * function duplicate(n) { - * return [[[n, n]]]; - * } + * var object = { 'a': 1, 'b': 2 }; + * var other = { 'c': 3, 'd': 4 }; * - * _.flatMapDeep([1, 2], duplicate); - * // => [1, 1, 2, 2] + * var values = _.memoize(_.values); + * values(object); + * // => [1, 2] + * + * values(other); + * // => [3, 4] + * + * object.a = 2; + * values(object); + * // => [1, 2] + * + * // Modify the result cache. + * values.cache.set(object, ['a', 'b']); + * values(object); + * // => ['a', 'b'] + * + * // Replace `_.memoize.Cache`. + * _.memoize.Cache = WeakMap; */ - function flatMapDeep(collection, iteratee) { - return baseFlatten(map(collection, iteratee), INFINITY); + function memoize(func, resolver) { + if (typeof func != 'function' || (resolver != null && typeof resolver != 'function')) { + throw new TypeError(FUNC_ERROR_TEXT); + } + var memoized = function() { + var args = arguments, + key = resolver ? resolver.apply(this, args) : args[0], + cache = memoized.cache; + + if (cache.has(key)) { + return cache.get(key); + } + var result = func.apply(this, args); + memoized.cache = cache.set(key, result) || cache; + return result; + }; + memoized.cache = new (memoize.Cache || MapCache); + return memoized; } + // Expose `MapCache`. + memoize.Cache = MapCache; + /** - * This method is like `_.flatMap` except that it recursively flattens the - * mapped results up to `depth` times. + * Creates a function that negates the result of the predicate `func`. The + * `func` predicate is invoked with the `this` binding and arguments of the + * created function. * * @static * @memberOf _ - * @since 4.7.0 - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} [iteratee=_.identity] The function invoked per iteration. - * @param {number} [depth=1] The maximum recursion depth. - * @returns {Array} Returns the new flattened array. + * @since 3.0.0 + * @category Function + * @param {Function} predicate The predicate to negate. + * @returns {Function} Returns the new negated function. * @example * - * function duplicate(n) { - * return [[[n, n]]]; + * function isEven(n) { + * return n % 2 == 0; * } * - * _.flatMapDepth([1, 2], duplicate, 2); - * // => [[1, 1], [2, 2]] + * _.filter([1, 2, 3, 4, 5, 6], _.negate(isEven)); + * // => [1, 3, 5] */ - function flatMapDepth(collection, iteratee, depth) { - depth = depth === undefined ? 1 : toInteger(depth); - return baseFlatten(map(collection, iteratee), depth); + function negate(predicate) { + if (typeof predicate != 'function') { + throw new TypeError(FUNC_ERROR_TEXT); + } + return function() { + var args = arguments; + switch (args.length) { + case 0: return !predicate.call(this); + case 1: return !predicate.call(this, args[0]); + case 2: return !predicate.call(this, args[0], args[1]); + case 3: return !predicate.call(this, args[0], args[1], args[2]); + } + return !predicate.apply(this, args); + }; } /** - * Iterates over elements of `collection` and invokes `iteratee` for each element. - * The iteratee is invoked with three arguments: (value, index|key, collection). - * Iteratee functions may exit iteration early by explicitly returning `false`. - * - * **Note:** As with other "Collections" methods, objects with a "length" - * property are iterated like arrays. To avoid this behavior use `_.forIn` - * or `_.forOwn` for object iteration. + * Creates a function that is restricted to invoking `func` once. Repeat calls + * to the function return the value of the first invocation. The `func` is + * invoked with the `this` binding and arguments of the created function. * * @static * @memberOf _ * @since 0.1.0 - * @alias each - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} [iteratee=_.identity] The function invoked per iteration. - * @returns {Array|Object} Returns `collection`. - * @see _.forEachRight + * @category Function + * @param {Function} func The function to restrict. + * @returns {Function} Returns the new restricted function. * @example * - * _.forEach([1, 2], function(value) { - * console.log(value); - * }); - * // => Logs `1` then `2`. - * - * _.forEach({ 'a': 1, 'b': 2 }, function(value, key) { - * console.log(key); - * }); - * // => Logs 'a' then 'b' (iteration order is not guaranteed). + * var initialize = _.once(createApplication); + * initialize(); + * initialize(); + * // => `createApplication` is invoked once */ - function forEach(collection, iteratee) { - var func = isArray(collection) ? arrayEach : baseEach; - return func(collection, getIteratee(iteratee, 3)); + function once(func) { + return before(2, func); } /** - * This method is like `_.forEach` except that it iterates over elements of - * `collection` from right to left. + * Creates a function that invokes `func` with its arguments transformed. * * @static + * @since 4.0.0 * @memberOf _ - * @since 2.0.0 - * @alias eachRight - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} [iteratee=_.identity] The function invoked per iteration. - * @returns {Array|Object} Returns `collection`. - * @see _.forEach + * @category Function + * @param {Function} func The function to wrap. + * @param {...(Function|Function[])} [transforms=[_.identity]] + * The argument transforms. + * @returns {Function} Returns the new function. * @example * - * _.forEachRight([1, 2], function(value) { - * console.log(value); - * }); - * // => Logs `2` then `1`. + * function doubled(n) { + * return n * 2; + * } + * + * function square(n) { + * return n * n; + * } + * + * var func = _.overArgs(function(x, y) { + * return [x, y]; + * }, [square, doubled]); + * + * func(9, 3); + * // => [81, 6] + * + * func(10, 5); + * // => [100, 10] */ - function forEachRight(collection, iteratee) { - var func = isArray(collection) ? arrayEachRight : baseEachRight; - return func(collection, getIteratee(iteratee, 3)); - } + var overArgs = castRest(function(func, transforms) { + transforms = (transforms.length == 1 && isArray(transforms[0])) + ? arrayMap(transforms[0], baseUnary(getIteratee())) + : arrayMap(baseFlatten(transforms, 1), baseUnary(getIteratee())); + + var funcsLength = transforms.length; + return baseRest(function(args) { + var index = -1, + length = nativeMin(args.length, funcsLength); + + while (++index < length) { + args[index] = transforms[index].call(this, args[index]); + } + return apply(func, this, args); + }); + }); /** - * Creates an object composed of keys generated from the results of running - * each element of `collection` thru `iteratee`. The order of grouped values - * is determined by the order they occur in `collection`. The corresponding - * value of each key is an array of elements responsible for generating the - * key. The iteratee is invoked with one argument: (value). + * Creates a function that invokes `func` with `partials` prepended to the + * arguments it receives. This method is like `_.bind` except it does **not** + * alter the `this` binding. + * + * The `_.partial.placeholder` value, which defaults to `_` in monolithic + * builds, may be used as a placeholder for partially applied arguments. + * + * **Note:** This method doesn't set the "length" property of partially + * applied functions. * * @static * @memberOf _ - * @since 0.1.0 - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} [iteratee=_.identity] The iteratee to transform keys. - * @returns {Object} Returns the composed aggregate object. + * @since 0.2.0 + * @category Function + * @param {Function} func The function to partially apply arguments to. + * @param {...*} [partials] The arguments to be partially applied. + * @returns {Function} Returns the new partially applied function. * @example * - * _.groupBy([6.1, 4.2, 6.3], Math.floor); - * // => { '4': [4.2], '6': [6.1, 6.3] } + * function greet(greeting, name) { + * return greeting + ' ' + name; + * } * - * // The `_.property` iteratee shorthand. - * _.groupBy(['one', 'two', 'three'], 'length'); - * // => { '3': ['one', 'two'], '5': ['three'] } + * var sayHelloTo = _.partial(greet, 'hello'); + * sayHelloTo('fred'); + * // => 'hello fred' + * + * // Partially applied with placeholders. + * var greetFred = _.partial(greet, _, 'fred'); + * greetFred('hi'); + * // => 'hi fred' */ - var groupBy = createAggregator(function(result, value, key) { - if (hasOwnProperty.call(result, key)) { - result[key].push(value); - } else { - baseAssignValue(result, key, [value]); - } + var partial = baseRest(function(func, partials) { + var holders = replaceHolders(partials, getHolder(partial)); + return createWrap(func, WRAP_PARTIAL_FLAG, undefined, partials, holders); }); /** - * Checks if `value` is in `collection`. If `collection` is a string, it's - * checked for a substring of `value`, otherwise - * [`SameValueZero`](http://ecma-international.org/ecma-262/7.0/#sec-samevaluezero) - * is used for equality comparisons. If `fromIndex` is negative, it's used as - * the offset from the end of `collection`. + * This method is like `_.partial` except that partially applied arguments + * are appended to the arguments it receives. + * + * The `_.partialRight.placeholder` value, which defaults to `_` in monolithic + * builds, may be used as a placeholder for partially applied arguments. + * + * **Note:** This method doesn't set the "length" property of partially + * applied functions. * * @static * @memberOf _ - * @since 0.1.0 - * @category Collection - * @param {Array|Object|string} collection The collection to inspect. - * @param {*} value The value to search for. - * @param {number} [fromIndex=0] The index to search from. - * @param- {Object} [guard] Enables use as an iteratee for methods like `_.reduce`. - * @returns {boolean} Returns `true` if `value` is found, else `false`. + * @since 1.0.0 + * @category Function + * @param {Function} func The function to partially apply arguments to. + * @param {...*} [partials] The arguments to be partially applied. + * @returns {Function} Returns the new partially applied function. * @example * - * _.includes([1, 2, 3], 1); - * // => true - * - * _.includes([1, 2, 3], 1, 2); - * // => false + * function greet(greeting, name) { + * return greeting + ' ' + name; + * } * - * _.includes({ 'a': 1, 'b': 2 }, 1); - * // => true + * var greetFred = _.partialRight(greet, 'fred'); + * greetFred('hi'); + * // => 'hi fred' * - * _.includes('abcd', 'bc'); - * // => true + * // Partially applied with placeholders. + * var sayHelloTo = _.partialRight(greet, 'hello', _); + * sayHelloTo('fred'); + * // => 'hello fred' */ - function includes(collection, value, fromIndex, guard) { - collection = isArrayLike(collection) ? collection : values(collection); - fromIndex = (fromIndex && !guard) ? toInteger(fromIndex) : 0; - - var length = collection.length; - if (fromIndex < 0) { - fromIndex = nativeMax(length + fromIndex, 0); - } - return isString(collection) - ? (fromIndex <= length && collection.indexOf(value, fromIndex) > -1) - : (!!length && baseIndexOf(collection, value, fromIndex) > -1); - } + var partialRight = baseRest(function(func, partials) { + var holders = replaceHolders(partials, getHolder(partialRight)); + return createWrap(func, WRAP_PARTIAL_RIGHT_FLAG, undefined, partials, holders); + }); /** - * Invokes the method at `path` of each element in `collection`, returning - * an array of the results of each invoked method. Any additional arguments - * are provided to each invoked method. If `path` is a function, it's invoked - * for, and `this` bound to, each element in `collection`. + * Creates a function that invokes `func` with arguments arranged according + * to the specified `indexes` where the argument value at the first index is + * provided as the first argument, the argument value at the second index is + * provided as the second argument, and so on. * * @static * @memberOf _ - * @since 4.0.0 - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {Array|Function|string} path The path of the method to invoke or - * the function invoked per iteration. - * @param {...*} [args] The arguments to invoke each method with. - * @returns {Array} Returns the array of results. + * @since 3.0.0 + * @category Function + * @param {Function} func The function to rearrange arguments for. + * @param {...(number|number[])} indexes The arranged argument indexes. + * @returns {Function} Returns the new function. * @example * - * _.invokeMap([[5, 1, 7], [3, 2, 1]], 'sort'); - * // => [[1, 5, 7], [1, 2, 3]] + * var rearged = _.rearg(function(a, b, c) { + * return [a, b, c]; + * }, [2, 0, 1]); * - * _.invokeMap([123, 456], String.prototype.split, ''); - * // => [['1', '2', '3'], ['4', '5', '6']] + * rearged('b', 'c', 'a') + * // => ['a', 'b', 'c'] */ - var invokeMap = baseRest(function(collection, path, args) { - var index = -1, - isFunc = typeof path == 'function', - result = isArrayLike(collection) ? Array(collection.length) : []; - - baseEach(collection, function(value) { - result[++index] = isFunc ? apply(path, value, args) : baseInvoke(value, path, args); - }); - return result; + var rearg = flatRest(function(func, indexes) { + return createWrap(func, WRAP_REARG_FLAG, undefined, undefined, undefined, indexes); }); /** - * Creates an object composed of keys generated from the results of running - * each element of `collection` thru `iteratee`. The corresponding value of - * each key is the last element responsible for generating the key. The - * iteratee is invoked with one argument: (value). + * Creates a function that invokes `func` with the `this` binding of the + * created function and arguments from `start` and beyond provided as + * an array. + * + * **Note:** This method is based on the + * [rest parameter](https://mdn.io/rest_parameters). * * @static * @memberOf _ * @since 4.0.0 - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} [iteratee=_.identity] The iteratee to transform keys. - * @returns {Object} Returns the composed aggregate object. + * @category Function + * @param {Function} func The function to apply a rest parameter to. + * @param {number} [start=func.length-1] The start position of the rest parameter. + * @returns {Function} Returns the new function. * @example * - * var array = [ - * { 'dir': 'left', 'code': 97 }, - * { 'dir': 'right', 'code': 100 } - * ]; + * var say = _.rest(function(what, names) { + * return what + ' ' + _.initial(names).join(', ') + + * (_.size(names) > 1 ? ', & ' : '') + _.last(names); + * }); * - * _.keyBy(array, function(o) { - * return String.fromCharCode(o.code); + * say('hello', 'fred', 'barney', 'pebbles'); + * // => 'hello fred, barney, & pebbles' + */ + function rest(func, start) { + if (typeof func != 'function') { + throw new TypeError(FUNC_ERROR_TEXT); + } + start = start === undefined ? start : toInteger(start); + return baseRest(func, start); + } + + /** + * Creates a function that invokes `func` with the `this` binding of the + * create function and an array of arguments much like + * [`Function#apply`](http://www.ecma-international.org/ecma-262/7.0/#sec-function.prototype.apply). + * + * **Note:** This method is based on the + * [spread operator](https://mdn.io/spread_operator). + * + * @static + * @memberOf _ + * @since 3.2.0 + * @category Function + * @param {Function} func The function to spread arguments over. + * @param {number} [start=0] The start position of the spread. + * @returns {Function} Returns the new function. + * @example + * + * var say = _.spread(function(who, what) { + * return who + ' says ' + what; * }); - * // => { 'a': { 'dir': 'left', 'code': 97 }, 'd': { 'dir': 'right', 'code': 100 } } * - * _.keyBy(array, 'dir'); - * // => { 'left': { 'dir': 'left', 'code': 97 }, 'right': { 'dir': 'right', 'code': 100 } } + * say(['fred', 'hello']); + * // => 'fred says hello' + * + * var numbers = Promise.all([ + * Promise.resolve(40), + * Promise.resolve(36) + * ]); + * + * numbers.then(_.spread(function(x, y) { + * return x + y; + * })); + * // => a Promise of 76 */ - var keyBy = createAggregator(function(result, value, key) { - baseAssignValue(result, key, value); - }); + function spread(func, start) { + if (typeof func != 'function') { + throw new TypeError(FUNC_ERROR_TEXT); + } + start = start == null ? 0 : nativeMax(toInteger(start), 0); + return baseRest(function(args) { + var array = args[start], + otherArgs = castSlice(args, 0, start); + + if (array) { + arrayPush(otherArgs, array); + } + return apply(func, this, otherArgs); + }); + } /** - * Creates an array of values by running each element in `collection` thru - * `iteratee`. The iteratee is invoked with three arguments: - * (value, index|key, collection). + * Creates a throttled function that only invokes `func` at most once per + * every `wait` milliseconds. The throttled function comes with a `cancel` + * method to cancel delayed `func` invocations and a `flush` method to + * immediately invoke them. Provide `options` to indicate whether `func` + * should be invoked on the leading and/or trailing edge of the `wait` + * timeout. The `func` is invoked with the last arguments provided to the + * throttled function. Subsequent calls to the throttled function return the + * result of the last `func` invocation. * - * Many lodash methods are guarded to work as iteratees for methods like - * `_.every`, `_.filter`, `_.map`, `_.mapValues`, `_.reject`, and `_.some`. + * **Note:** If `leading` and `trailing` options are `true`, `func` is + * invoked on the trailing edge of the timeout only if the throttled function + * is invoked more than once during the `wait` timeout. + * + * If `wait` is `0` and `leading` is `false`, `func` invocation is deferred + * until to the next tick, similar to `setTimeout` with a timeout of `0`. * - * The guarded methods are: - * `ary`, `chunk`, `curry`, `curryRight`, `drop`, `dropRight`, `every`, - * `fill`, `invert`, `parseInt`, `random`, `range`, `rangeRight`, `repeat`, - * `sampleSize`, `slice`, `some`, `sortBy`, `split`, `take`, `takeRight`, - * `template`, `trim`, `trimEnd`, `trimStart`, and `words` + * See [David Corbacho's article](https://css-tricks.com/debouncing-throttling-explained-examples/) + * for details over the differences between `_.throttle` and `_.debounce`. * * @static * @memberOf _ * @since 0.1.0 - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} [iteratee=_.identity] The function invoked per iteration. - * @returns {Array} Returns the new mapped array. + * @category Function + * @param {Function} func The function to throttle. + * @param {number} [wait=0] The number of milliseconds to throttle invocations to. + * @param {Object} [options={}] The options object. + * @param {boolean} [options.leading=true] + * Specify invoking on the leading edge of the timeout. + * @param {boolean} [options.trailing=true] + * Specify invoking on the trailing edge of the timeout. + * @returns {Function} Returns the new throttled function. * @example * - * function square(n) { - * return n * n; - * } - * - * _.map([4, 8], square); - * // => [16, 64] - * - * _.map({ 'a': 4, 'b': 8 }, square); - * // => [16, 64] (iteration order is not guaranteed) + * // Avoid excessively updating the position while scrolling. + * jQuery(window).on('scroll', _.throttle(updatePosition, 100)); * - * var users = [ - * { 'user': 'barney' }, - * { 'user': 'fred' } - * ]; + * // Invoke `renewToken` when the click event is fired, but not more than once every 5 minutes. + * var throttled = _.throttle(renewToken, 300000, { 'trailing': false }); + * jQuery(element).on('click', throttled); * - * // The `_.property` iteratee shorthand. - * _.map(users, 'user'); - * // => ['barney', 'fred'] + * // Cancel the trailing throttled invocation. + * jQuery(window).on('popstate', throttled.cancel); */ - function map(collection, iteratee) { - var func = isArray(collection) ? arrayMap : baseMap; - return func(collection, getIteratee(iteratee, 3)); + function throttle(func, wait, options) { + var leading = true, + trailing = true; + + if (typeof func != 'function') { + throw new TypeError(FUNC_ERROR_TEXT); + } + if (isObject(options)) { + leading = 'leading' in options ? !!options.leading : leading; + trailing = 'trailing' in options ? !!options.trailing : trailing; + } + return debounce(func, wait, { + 'leading': leading, + 'maxWait': wait, + 'trailing': trailing + }); } /** - * This method is like `_.sortBy` except that it allows specifying the sort - * orders of the iteratees to sort by. If `orders` is unspecified, all values - * are sorted in ascending order. Otherwise, specify an order of "desc" for - * descending or "asc" for ascending sort order of corresponding values. + * Creates a function that accepts up to one argument, ignoring any + * additional arguments. * * @static * @memberOf _ * @since 4.0.0 - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {Array[]|Function[]|Object[]|string[]} [iteratees=[_.identity]] - * The iteratees to sort by. - * @param {string[]} [orders] The sort orders of `iteratees`. - * @param- {Object} [guard] Enables use as an iteratee for methods like `_.reduce`. - * @returns {Array} Returns the new sorted array. + * @category Function + * @param {Function} func The function to cap arguments for. + * @returns {Function} Returns the new capped function. * @example * - * var users = [ - * { 'user': 'fred', 'age': 48 }, - * { 'user': 'barney', 'age': 34 }, - * { 'user': 'fred', 'age': 40 }, - * { 'user': 'barney', 'age': 36 } - * ]; - * - * // Sort by `user` in ascending order and by `age` in descending order. - * _.orderBy(users, ['user', 'age'], ['asc', 'desc']); - * // => objects for [['barney', 36], ['barney', 34], ['fred', 48], ['fred', 40]] + * _.map(['6', '8', '10'], _.unary(parseInt)); + * // => [6, 8, 10] */ - function orderBy(collection, iteratees, orders, guard) { - if (collection == null) { - return []; - } - if (!isArray(iteratees)) { - iteratees = iteratees == null ? [] : [iteratees]; - } - orders = guard ? undefined : orders; - if (!isArray(orders)) { - orders = orders == null ? [] : [orders]; - } - return baseOrderBy(collection, iteratees, orders); + function unary(func) { + return ary(func, 1); } /** - * Creates an array of elements split into two groups, the first of which - * contains elements `predicate` returns truthy for, the second of which - * contains elements `predicate` returns falsey for. The predicate is - * invoked with one argument: (value). + * Creates a function that provides `value` to `wrapper` as its first + * argument. Any additional arguments provided to the function are appended + * to those provided to the `wrapper`. The wrapper is invoked with the `this` + * binding of the created function. * * @static * @memberOf _ - * @since 3.0.0 - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} [predicate=_.identity] The function invoked per iteration. - * @returns {Array} Returns the array of grouped elements. + * @since 0.1.0 + * @category Function + * @param {*} value The value to wrap. + * @param {Function} [wrapper=identity] The wrapper function. + * @returns {Function} Returns the new function. * @example * - * var users = [ - * { 'user': 'barney', 'age': 36, 'active': false }, - * { 'user': 'fred', 'age': 40, 'active': true }, - * { 'user': 'pebbles', 'age': 1, 'active': false } - * ]; - * - * _.partition(users, function(o) { return o.active; }); - * // => objects for [['fred'], ['barney', 'pebbles']] - * - * // The `_.matches` iteratee shorthand. - * _.partition(users, { 'age': 1, 'active': false }); - * // => objects for [['pebbles'], ['barney', 'fred']] - * - * // The `_.matchesProperty` iteratee shorthand. - * _.partition(users, ['active', false]); - * // => objects for [['barney', 'pebbles'], ['fred']] + * var p = _.wrap(_.escape, function(func, text) { + * return '

' + func(text) + '

'; + * }); * - * // The `_.property` iteratee shorthand. - * _.partition(users, 'active'); - * // => objects for [['fred'], ['barney', 'pebbles']] + * p('fred, barney, & pebbles'); + * // => '

fred, barney, & pebbles

' */ - var partition = createAggregator(function(result, value, key) { - result[key ? 0 : 1].push(value); - }, function() { return [[], []]; }); + function wrap(value, wrapper) { + return partial(castFunction(wrapper), value); + } + + /*------------------------------------------------------------------------*/ /** - * Reduces `collection` to a value which is the accumulated result of running - * each element in `collection` thru `iteratee`, where each successive - * invocation is supplied the return value of the previous. If `accumulator` - * is not given, the first element of `collection` is used as the initial - * value. The iteratee is invoked with four arguments: - * (accumulator, value, index|key, collection). - * - * Many lodash methods are guarded to work as iteratees for methods like - * `_.reduce`, `_.reduceRight`, and `_.transform`. - * - * The guarded methods are: - * `assign`, `defaults`, `defaultsDeep`, `includes`, `merge`, `orderBy`, - * and `sortBy` + * Casts `value` as an array if it's not one. * * @static * @memberOf _ - * @since 0.1.0 - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} [iteratee=_.identity] The function invoked per iteration. - * @param {*} [accumulator] The initial value. - * @returns {*} Returns the accumulated value. - * @see _.reduceRight + * @since 4.4.0 + * @category Lang + * @param {*} value The value to inspect. + * @returns {Array} Returns the cast array. * @example * - * _.reduce([1, 2], function(sum, n) { - * return sum + n; - * }, 0); - * // => 3 + * _.castArray(1); + * // => [1] * - * _.reduce({ 'a': 1, 'b': 2, 'c': 1 }, function(result, value, key) { - * (result[value] || (result[value] = [])).push(key); - * return result; - * }, {}); - * // => { '1': ['a', 'c'], '2': ['b'] } (iteration order is not guaranteed) + * _.castArray({ 'a': 1 }); + * // => [{ 'a': 1 }] + * + * _.castArray('abc'); + * // => ['abc'] + * + * _.castArray(null); + * // => [null] + * + * _.castArray(undefined); + * // => [undefined] + * + * _.castArray(); + * // => [] + * + * var array = [1, 2, 3]; + * console.log(_.castArray(array) === array); + * // => true */ - function reduce(collection, iteratee, accumulator) { - var func = isArray(collection) ? arrayReduce : baseReduce, - initAccum = arguments.length < 3; - - return func(collection, getIteratee(iteratee, 4), accumulator, initAccum, baseEach); + function castArray() { + if (!arguments.length) { + return []; + } + var value = arguments[0]; + return isArray(value) ? value : [value]; } /** - * This method is like `_.reduce` except that it iterates over elements of - * `collection` from right to left. + * Creates a shallow clone of `value`. + * + * **Note:** This method is loosely based on the + * [structured clone algorithm](https://mdn.io/Structured_clone_algorithm) + * and supports cloning arrays, array buffers, booleans, date objects, maps, + * numbers, `Object` objects, regexes, sets, strings, symbols, and typed + * arrays. The own enumerable properties of `arguments` objects are cloned + * as plain objects. An empty object is returned for uncloneable values such + * as error objects, functions, DOM nodes, and WeakMaps. * * @static * @memberOf _ * @since 0.1.0 - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} [iteratee=_.identity] The function invoked per iteration. - * @param {*} [accumulator] The initial value. - * @returns {*} Returns the accumulated value. - * @see _.reduce + * @category Lang + * @param {*} value The value to clone. + * @returns {*} Returns the cloned value. + * @see _.cloneDeep * @example * - * var array = [[0, 1], [2, 3], [4, 5]]; + * var objects = [{ 'a': 1 }, { 'b': 2 }]; * - * _.reduceRight(array, function(flattened, other) { - * return flattened.concat(other); - * }, []); - * // => [4, 5, 2, 3, 0, 1] + * var shallow = _.clone(objects); + * console.log(shallow[0] === objects[0]); + * // => true */ - function reduceRight(collection, iteratee, accumulator) { - var func = isArray(collection) ? arrayReduceRight : baseReduce, - initAccum = arguments.length < 3; - - return func(collection, getIteratee(iteratee, 4), accumulator, initAccum, baseEachRight); + function clone(value) { + return baseClone(value, CLONE_SYMBOLS_FLAG); } /** - * The opposite of `_.filter`; this method returns the elements of `collection` - * that `predicate` does **not** return truthy for. + * This method is like `_.clone` except that it accepts `customizer` which + * is invoked to produce the cloned value. If `customizer` returns `undefined`, + * cloning is handled by the method instead. The `customizer` is invoked with + * up to four arguments; (value [, index|key, object, stack]). * * @static * @memberOf _ - * @since 0.1.0 - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} [predicate=_.identity] The function invoked per iteration. - * @returns {Array} Returns the new filtered array. - * @see _.filter + * @since 4.0.0 + * @category Lang + * @param {*} value The value to clone. + * @param {Function} [customizer] The function to customize cloning. + * @returns {*} Returns the cloned value. + * @see _.cloneDeepWith * @example * - * var users = [ - * { 'user': 'barney', 'age': 36, 'active': false }, - * { 'user': 'fred', 'age': 40, 'active': true } - * ]; - * - * _.reject(users, function(o) { return !o.active; }); - * // => objects for ['fred'] - * - * // The `_.matches` iteratee shorthand. - * _.reject(users, { 'age': 40, 'active': true }); - * // => objects for ['barney'] + * function customizer(value) { + * if (_.isElement(value)) { + * return value.cloneNode(false); + * } + * } * - * // The `_.matchesProperty` iteratee shorthand. - * _.reject(users, ['active', false]); - * // => objects for ['fred'] + * var el = _.cloneWith(document.body, customizer); * - * // The `_.property` iteratee shorthand. - * _.reject(users, 'active'); - * // => objects for ['barney'] + * console.log(el === document.body); + * // => false + * console.log(el.nodeName); + * // => 'BODY' + * console.log(el.childNodes.length); + * // => 0 */ - function reject(collection, predicate) { - var func = isArray(collection) ? arrayFilter : baseFilter; - return func(collection, negate(getIteratee(predicate, 3))); + function cloneWith(value, customizer) { + customizer = typeof customizer == 'function' ? customizer : undefined; + return baseClone(value, CLONE_SYMBOLS_FLAG, customizer); } /** - * Gets a random element from `collection`. + * This method is like `_.clone` except that it recursively clones `value`. * * @static * @memberOf _ - * @since 2.0.0 - * @category Collection - * @param {Array|Object} collection The collection to sample. - * @returns {*} Returns the random element. + * @since 1.0.0 + * @category Lang + * @param {*} value The value to recursively clone. + * @returns {*} Returns the deep cloned value. + * @see _.clone * @example * - * _.sample([1, 2, 3, 4]); - * // => 2 + * var objects = [{ 'a': 1 }, { 'b': 2 }]; + * + * var deep = _.cloneDeep(objects); + * console.log(deep[0] === objects[0]); + * // => false */ - function sample(collection) { - var func = isArray(collection) ? arraySample : baseSample; - return func(collection); + function cloneDeep(value) { + return baseClone(value, CLONE_DEEP_FLAG | CLONE_SYMBOLS_FLAG); } /** - * Gets `n` random elements at unique keys from `collection` up to the - * size of `collection`. + * This method is like `_.cloneWith` except that it recursively clones `value`. * * @static * @memberOf _ * @since 4.0.0 - * @category Collection - * @param {Array|Object} collection The collection to sample. - * @param {number} [n=1] The number of elements to sample. - * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. - * @returns {Array} Returns the random elements. + * @category Lang + * @param {*} value The value to recursively clone. + * @param {Function} [customizer] The function to customize cloning. + * @returns {*} Returns the deep cloned value. + * @see _.cloneWith * @example * - * _.sampleSize([1, 2, 3], 2); - * // => [3, 1] + * function customizer(value) { + * if (_.isElement(value)) { + * return value.cloneNode(true); + * } + * } * - * _.sampleSize([1, 2, 3], 4); - * // => [2, 3, 1] + * var el = _.cloneDeepWith(document.body, customizer); + * + * console.log(el === document.body); + * // => false + * console.log(el.nodeName); + * // => 'BODY' + * console.log(el.childNodes.length); + * // => 20 */ - function sampleSize(collection, n, guard) { - if ((guard ? isIterateeCall(collection, n, guard) : n === undefined)) { - n = 1; - } else { - n = toInteger(n); - } - var func = isArray(collection) ? arraySampleSize : baseSampleSize; - return func(collection, n); + function cloneDeepWith(value, customizer) { + customizer = typeof customizer == 'function' ? customizer : undefined; + return baseClone(value, CLONE_DEEP_FLAG | CLONE_SYMBOLS_FLAG, customizer); } /** - * Creates an array of shuffled values, using a version of the - * [Fisher-Yates shuffle](https://en.wikipedia.org/wiki/Fisher-Yates_shuffle). + * Checks if `object` conforms to `source` by invoking the predicate + * properties of `source` with the corresponding property values of `object`. + * + * **Note:** This method is equivalent to `_.conforms` when `source` is + * partially applied. * * @static * @memberOf _ - * @since 0.1.0 - * @category Collection - * @param {Array|Object} collection The collection to shuffle. - * @returns {Array} Returns the new shuffled array. + * @since 4.14.0 + * @category Lang + * @param {Object} object The object to inspect. + * @param {Object} source The object of property predicates to conform to. + * @returns {boolean} Returns `true` if `object` conforms, else `false`. * @example * - * _.shuffle([1, 2, 3, 4]); - * // => [4, 1, 3, 2] + * var object = { 'a': 1, 'b': 2 }; + * + * _.conformsTo(object, { 'b': function(n) { return n > 1; } }); + * // => true + * + * _.conformsTo(object, { 'b': function(n) { return n > 2; } }); + * // => false */ - function shuffle(collection) { - var func = isArray(collection) ? arrayShuffle : baseShuffle; - return func(collection); + function conformsTo(object, source) { + return source == null || baseConformsTo(object, source, keys(source)); } /** - * Gets the size of `collection` by returning its length for array-like - * values or the number of own enumerable string keyed properties for objects. + * Performs a + * [`SameValueZero`](http://ecma-international.org/ecma-262/7.0/#sec-samevaluezero) + * comparison between two values to determine if they are equivalent. * * @static * @memberOf _ - * @since 0.1.0 - * @category Collection - * @param {Array|Object|string} collection The collection to inspect. - * @returns {number} Returns the collection size. + * @since 4.0.0 + * @category Lang + * @param {*} value The value to compare. + * @param {*} other The other value to compare. + * @returns {boolean} Returns `true` if the values are equivalent, else `false`. * @example * - * _.size([1, 2, 3]); - * // => 3 + * var object = { 'a': 1 }; + * var other = { 'a': 1 }; * - * _.size({ 'a': 1, 'b': 2 }); - * // => 2 + * _.eq(object, object); + * // => true * - * _.size('pebbles'); - * // => 7 + * _.eq(object, other); + * // => false + * + * _.eq('a', 'a'); + * // => true + * + * _.eq('a', Object('a')); + * // => false + * + * _.eq(NaN, NaN); + * // => true */ - function size(collection) { - if (collection == null) { - return 0; - } - if (isArrayLike(collection)) { - return isString(collection) ? stringSize(collection) : collection.length; - } - var tag = getTag(collection); - if (tag == mapTag || tag == setTag) { - return collection.size; - } - return baseKeys(collection).length; + function eq(value, other) { + return value === other || (value !== value && other !== other); } /** - * Checks if `predicate` returns truthy for **any** element of `collection`. - * Iteration is stopped once `predicate` returns truthy. The predicate is - * invoked with three arguments: (value, index|key, collection). + * Checks if `value` is greater than `other`. * * @static * @memberOf _ - * @since 0.1.0 - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} [predicate=_.identity] The function invoked per iteration. - * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. - * @returns {boolean} Returns `true` if any element passes the predicate check, + * @since 3.9.0 + * @category Lang + * @param {*} value The value to compare. + * @param {*} other The other value to compare. + * @returns {boolean} Returns `true` if `value` is greater than `other`, * else `false`. + * @see _.lt * @example * - * _.some([null, 0, 'yes', false], Boolean); + * _.gt(3, 1); * // => true * - * var users = [ - * { 'user': 'barney', 'active': true }, - * { 'user': 'fred', 'active': false } - * ]; - * - * // The `_.matches` iteratee shorthand. - * _.some(users, { 'user': 'barney', 'active': false }); + * _.gt(3, 3); * // => false * - * // The `_.matchesProperty` iteratee shorthand. - * _.some(users, ['active', false]); - * // => true - * - * // The `_.property` iteratee shorthand. - * _.some(users, 'active'); - * // => true + * _.gt(1, 3); + * // => false */ - function some(collection, predicate, guard) { - var func = isArray(collection) ? arraySome : baseSome; - if (guard && isIterateeCall(collection, predicate, guard)) { - predicate = undefined; - } - return func(collection, getIteratee(predicate, 3)); - } + var gt = createRelationalOperation(baseGt); /** - * Creates an array of elements, sorted in ascending order by the results of - * running each element in a collection thru each iteratee. This method - * performs a stable sort, that is, it preserves the original sort order of - * equal elements. The iteratees are invoked with one argument: (value). + * Checks if `value` is greater than or equal to `other`. * * @static * @memberOf _ - * @since 0.1.0 - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {...(Function|Function[])} [iteratees=[_.identity]] - * The iteratees to sort by. - * @returns {Array} Returns the new sorted array. + * @since 3.9.0 + * @category Lang + * @param {*} value The value to compare. + * @param {*} other The other value to compare. + * @returns {boolean} Returns `true` if `value` is greater than or equal to + * `other`, else `false`. + * @see _.lte * @example * - * var users = [ - * { 'user': 'fred', 'age': 48 }, - * { 'user': 'barney', 'age': 36 }, - * { 'user': 'fred', 'age': 30 }, - * { 'user': 'barney', 'age': 34 } - * ]; + * _.gte(3, 1); + * // => true * - * _.sortBy(users, [function(o) { return o.user; }]); - * // => objects for [['barney', 36], ['barney', 34], ['fred', 48], ['fred', 30]] + * _.gte(3, 3); + * // => true * - * _.sortBy(users, ['user', 'age']); - * // => objects for [['barney', 34], ['barney', 36], ['fred', 30], ['fred', 48]] + * _.gte(1, 3); + * // => false */ - var sortBy = baseRest(function(collection, iteratees) { - if (collection == null) { - return []; - } - var length = iteratees.length; - if (length > 1 && isIterateeCall(collection, iteratees[0], iteratees[1])) { - iteratees = []; - } else if (length > 2 && isIterateeCall(iteratees[0], iteratees[1], iteratees[2])) { - iteratees = [iteratees[0]]; - } - return baseOrderBy(collection, baseFlatten(iteratees, 1), []); + var gte = createRelationalOperation(function(value, other) { + return value >= other; }); - /*------------------------------------------------------------------------*/ - /** - * Gets the timestamp of the number of milliseconds that have elapsed since - * the Unix epoch (1 January 1970 00:00:00 UTC). + * Checks if `value` is likely an `arguments` object. * * @static * @memberOf _ - * @since 2.4.0 - * @category Date - * @returns {number} Returns the timestamp. + * @since 0.1.0 + * @category Lang + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is an `arguments` object, + * else `false`. * @example * - * _.defer(function(stamp) { - * console.log(_.now() - stamp); - * }, _.now()); - * // => Logs the number of milliseconds it took for the deferred invocation. + * _.isArguments(function() { return arguments; }()); + * // => true + * + * _.isArguments([1, 2, 3]); + * // => false */ - var now = ctxNow || function() { - return root.Date.now(); + var isArguments = baseIsArguments(function() { return arguments; }()) ? baseIsArguments : function(value) { + return isObjectLike(value) && hasOwnProperty.call(value, 'callee') && + !propertyIsEnumerable.call(value, 'callee'); }; - /*------------------------------------------------------------------------*/ - /** - * The opposite of `_.before`; this method creates a function that invokes - * `func` once it's called `n` or more times. + * Checks if `value` is classified as an `Array` object. * * @static * @memberOf _ * @since 0.1.0 - * @category Function - * @param {number} n The number of calls before `func` is invoked. - * @param {Function} func The function to restrict. - * @returns {Function} Returns the new restricted function. + * @category Lang + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is an array, else `false`. * @example * - * var saves = ['profile', 'settings']; + * _.isArray([1, 2, 3]); + * // => true * - * var done = _.after(saves.length, function() { - * console.log('done saving!'); - * }); + * _.isArray(document.body.children); + * // => false * - * _.forEach(saves, function(type) { - * asyncSave({ 'type': type, 'complete': done }); - * }); - * // => Logs 'done saving!' after the two async saves have completed. + * _.isArray('abc'); + * // => false + * + * _.isArray(_.noop); + * // => false */ - function after(n, func) { - if (typeof func != 'function') { - throw new TypeError(FUNC_ERROR_TEXT); - } - n = toInteger(n); - return function() { - if (--n < 1) { - return func.apply(this, arguments); - } - }; - } + var isArray = Array.isArray; /** - * Creates a function that invokes `func`, with up to `n` arguments, - * ignoring any additional arguments. + * Checks if `value` is classified as an `ArrayBuffer` object. * * @static * @memberOf _ - * @since 3.0.0 - * @category Function - * @param {Function} func The function to cap arguments for. - * @param {number} [n=func.length] The arity cap. - * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. - * @returns {Function} Returns the new capped function. + * @since 4.3.0 + * @category Lang + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is an array buffer, else `false`. * @example * - * _.map(['6', '8', '10'], _.ary(parseInt, 1)); - * // => [6, 8, 10] + * _.isArrayBuffer(new ArrayBuffer(2)); + * // => true + * + * _.isArrayBuffer(new Array(2)); + * // => false */ - function ary(func, n, guard) { - n = guard ? undefined : n; - n = (func && n == null) ? func.length : n; - return createWrap(func, WRAP_ARY_FLAG, undefined, undefined, undefined, undefined, n); - } + var isArrayBuffer = nodeIsArrayBuffer ? baseUnary(nodeIsArrayBuffer) : baseIsArrayBuffer; /** - * Creates a function that invokes `func`, with the `this` binding and arguments - * of the created function, while it's called less than `n` times. Subsequent - * calls to the created function return the result of the last `func` invocation. + * Checks if `value` is array-like. A value is considered array-like if it's + * not a function and has a `value.length` that's an integer greater than or + * equal to `0` and less than or equal to `Number.MAX_SAFE_INTEGER`. * * @static * @memberOf _ - * @since 3.0.0 - * @category Function - * @param {number} n The number of calls at which `func` is no longer invoked. - * @param {Function} func The function to restrict. - * @returns {Function} Returns the new restricted function. + * @since 4.0.0 + * @category Lang + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is array-like, else `false`. * @example * - * jQuery(element).on('click', _.before(5, addContactToList)); - * // => Allows adding up to 4 contacts to the list. + * _.isArrayLike([1, 2, 3]); + * // => true + * + * _.isArrayLike(document.body.children); + * // => true + * + * _.isArrayLike('abc'); + * // => true + * + * _.isArrayLike(_.noop); + * // => false */ - function before(n, func) { - var result; - if (typeof func != 'function') { - throw new TypeError(FUNC_ERROR_TEXT); - } - n = toInteger(n); - return function() { - if (--n > 0) { - result = func.apply(this, arguments); - } - if (n <= 1) { - func = undefined; - } - return result; - }; + function isArrayLike(value) { + return value != null && isLength(value.length) && !isFunction(value); } /** - * Creates a function that invokes `func` with the `this` binding of `thisArg` - * and `partials` prepended to the arguments it receives. - * - * The `_.bind.placeholder` value, which defaults to `_` in monolithic builds, - * may be used as a placeholder for partially applied arguments. - * - * **Note:** Unlike native `Function#bind`, this method doesn't set the "length" - * property of bound functions. + * This method is like `_.isArrayLike` except that it also checks if `value` + * is an object. * * @static * @memberOf _ - * @since 0.1.0 - * @category Function - * @param {Function} func The function to bind. - * @param {*} thisArg The `this` binding of `func`. - * @param {...*} [partials] The arguments to be partially applied. - * @returns {Function} Returns the new bound function. + * @since 4.0.0 + * @category Lang + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is an array-like object, + * else `false`. * @example * - * function greet(greeting, punctuation) { - * return greeting + ' ' + this.user + punctuation; - * } + * _.isArrayLikeObject([1, 2, 3]); + * // => true * - * var object = { 'user': 'fred' }; + * _.isArrayLikeObject(document.body.children); + * // => true * - * var bound = _.bind(greet, object, 'hi'); - * bound('!'); - * // => 'hi fred!' + * _.isArrayLikeObject('abc'); + * // => false * - * // Bound with placeholders. - * var bound = _.bind(greet, object, _, '!'); - * bound('hi'); - * // => 'hi fred!' + * _.isArrayLikeObject(_.noop); + * // => false */ - var bind = baseRest(function(func, thisArg, partials) { - var bitmask = WRAP_BIND_FLAG; - if (partials.length) { - var holders = replaceHolders(partials, getHolder(bind)); - bitmask |= WRAP_PARTIAL_FLAG; - } - return createWrap(func, bitmask, thisArg, partials, holders); - }); + function isArrayLikeObject(value) { + return isObjectLike(value) && isArrayLike(value); + } /** - * Creates a function that invokes the method at `object[key]` with `partials` - * prepended to the arguments it receives. - * - * This method differs from `_.bind` by allowing bound functions to reference - * methods that may be redefined or don't yet exist. See - * [Peter Michaux's article](http://peter.michaux.ca/articles/lazy-function-definition-pattern) - * for more details. - * - * The `_.bindKey.placeholder` value, which defaults to `_` in monolithic - * builds, may be used as a placeholder for partially applied arguments. + * Checks if `value` is classified as a boolean primitive or object. * * @static * @memberOf _ - * @since 0.10.0 - * @category Function - * @param {Object} object The object to invoke the method on. - * @param {string} key The key of the method. - * @param {...*} [partials] The arguments to be partially applied. - * @returns {Function} Returns the new bound function. + * @since 0.1.0 + * @category Lang + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is a boolean, else `false`. * @example * - * var object = { - * 'user': 'fred', - * 'greet': function(greeting, punctuation) { - * return greeting + ' ' + this.user + punctuation; - * } - * }; - * - * var bound = _.bindKey(object, 'greet', 'hi'); - * bound('!'); - * // => 'hi fred!' - * - * object.greet = function(greeting, punctuation) { - * return greeting + 'ya ' + this.user + punctuation; - * }; - * - * bound('!'); - * // => 'hiya fred!' + * _.isBoolean(false); + * // => true * - * // Bound with placeholders. - * var bound = _.bindKey(object, 'greet', _, '!'); - * bound('hi'); - * // => 'hiya fred!' + * _.isBoolean(null); + * // => false */ - var bindKey = baseRest(function(object, key, partials) { - var bitmask = WRAP_BIND_FLAG | WRAP_BIND_KEY_FLAG; - if (partials.length) { - var holders = replaceHolders(partials, getHolder(bindKey)); - bitmask |= WRAP_PARTIAL_FLAG; - } - return createWrap(key, bitmask, object, partials, holders); - }); + function isBoolean(value) { + return value === true || value === false || + (isObjectLike(value) && baseGetTag(value) == boolTag); + } /** - * Creates a function that accepts arguments of `func` and either invokes - * `func` returning its result, if at least `arity` number of arguments have - * been provided, or returns a function that accepts the remaining `func` - * arguments, and so on. The arity of `func` may be specified if `func.length` - * is not sufficient. - * - * The `_.curry.placeholder` value, which defaults to `_` in monolithic builds, - * may be used as a placeholder for provided arguments. - * - * **Note:** This method doesn't set the "length" property of curried functions. + * Checks if `value` is a buffer. * * @static * @memberOf _ - * @since 2.0.0 - * @category Function - * @param {Function} func The function to curry. - * @param {number} [arity=func.length] The arity of `func`. - * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. - * @returns {Function} Returns the new curried function. + * @since 4.3.0 + * @category Lang + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is a buffer, else `false`. * @example * - * var abc = function(a, b, c) { - * return [a, b, c]; - * }; - * - * var curried = _.curry(abc); - * - * curried(1)(2)(3); - * // => [1, 2, 3] - * - * curried(1, 2)(3); - * // => [1, 2, 3] - * - * curried(1, 2, 3); - * // => [1, 2, 3] + * _.isBuffer(new Buffer(2)); + * // => true * - * // Curried with placeholders. - * curried(1)(_, 3)(2); - * // => [1, 2, 3] + * _.isBuffer(new Uint8Array(2)); + * // => false */ - function curry(func, arity, guard) { - arity = guard ? undefined : arity; - var result = createWrap(func, WRAP_CURRY_FLAG, undefined, undefined, undefined, undefined, undefined, arity); - result.placeholder = curry.placeholder; - return result; - } + var isBuffer = nativeIsBuffer || stubFalse; /** - * This method is like `_.curry` except that arguments are applied to `func` - * in the manner of `_.partialRight` instead of `_.partial`. - * - * The `_.curryRight.placeholder` value, which defaults to `_` in monolithic - * builds, may be used as a placeholder for provided arguments. - * - * **Note:** This method doesn't set the "length" property of curried functions. + * Checks if `value` is classified as a `Date` object. * * @static * @memberOf _ - * @since 3.0.0 - * @category Function - * @param {Function} func The function to curry. - * @param {number} [arity=func.length] The arity of `func`. - * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. - * @returns {Function} Returns the new curried function. + * @since 0.1.0 + * @category Lang + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is a date object, else `false`. * @example * - * var abc = function(a, b, c) { - * return [a, b, c]; - * }; - * - * var curried = _.curryRight(abc); + * _.isDate(new Date); + * // => true * - * curried(3)(2)(1); - * // => [1, 2, 3] + * _.isDate('Mon April 23 2012'); + * // => false + */ + var isDate = nodeIsDate ? baseUnary(nodeIsDate) : baseIsDate; + + /** + * Checks if `value` is likely a DOM element. * - * curried(2, 3)(1); - * // => [1, 2, 3] + * @static + * @memberOf _ + * @since 0.1.0 + * @category Lang + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is a DOM element, else `false`. + * @example * - * curried(1, 2, 3); - * // => [1, 2, 3] + * _.isElement(document.body); + * // => true * - * // Curried with placeholders. - * curried(3)(1, _)(2); - * // => [1, 2, 3] + * _.isElement(''); + * // => false */ - function curryRight(func, arity, guard) { - arity = guard ? undefined : arity; - var result = createWrap(func, WRAP_CURRY_RIGHT_FLAG, undefined, undefined, undefined, undefined, undefined, arity); - result.placeholder = curryRight.placeholder; - return result; + function isElement(value) { + return isObjectLike(value) && value.nodeType === 1 && !isPlainObject(value); } /** - * Creates a debounced function that delays invoking `func` until after `wait` - * milliseconds have elapsed since the last time the debounced function was - * invoked. The debounced function comes with a `cancel` method to cancel - * delayed `func` invocations and a `flush` method to immediately invoke them. - * Provide `options` to indicate whether `func` should be invoked on the - * leading and/or trailing edge of the `wait` timeout. The `func` is invoked - * with the last arguments provided to the debounced function. Subsequent - * calls to the debounced function return the result of the last `func` - * invocation. - * - * **Note:** If `leading` and `trailing` options are `true`, `func` is - * invoked on the trailing edge of the timeout only if the debounced function - * is invoked more than once during the `wait` timeout. + * Checks if `value` is an empty object, collection, map, or set. * - * If `wait` is `0` and `leading` is `false`, `func` invocation is deferred - * until to the next tick, similar to `setTimeout` with a timeout of `0`. + * Objects are considered empty if they have no own enumerable string keyed + * properties. * - * See [David Corbacho's article](https://css-tricks.com/debouncing-throttling-explained-examples/) - * for details over the differences between `_.debounce` and `_.throttle`. + * Array-like values such as `arguments` objects, arrays, buffers, strings, or + * jQuery-like collections are considered empty if they have a `length` of `0`. + * Similarly, maps and sets are considered empty if they have a `size` of `0`. * * @static * @memberOf _ * @since 0.1.0 - * @category Function - * @param {Function} func The function to debounce. - * @param {number} [wait=0] The number of milliseconds to delay. - * @param {Object} [options={}] The options object. - * @param {boolean} [options.leading=false] - * Specify invoking on the leading edge of the timeout. - * @param {number} [options.maxWait] - * The maximum time `func` is allowed to be delayed before it's invoked. - * @param {boolean} [options.trailing=true] - * Specify invoking on the trailing edge of the timeout. - * @returns {Function} Returns the new debounced function. + * @category Lang + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is empty, else `false`. * @example * - * // Avoid costly calculations while the window size is in flux. - * jQuery(window).on('resize', _.debounce(calculateLayout, 150)); - * - * // Invoke `sendMail` when clicked, debouncing subsequent calls. - * jQuery(element).on('click', _.debounce(sendMail, 300, { - * 'leading': true, - * 'trailing': false - * })); - * - * // Ensure `batchLog` is invoked once after 1 second of debounced calls. - * var debounced = _.debounce(batchLog, 250, { 'maxWait': 1000 }); - * var source = new EventSource('/stream'); - * jQuery(source).on('message', debounced); + * _.isEmpty(null); + * // => true * - * // Cancel the trailing debounced invocation. - * jQuery(window).on('popstate', debounced.cancel); - */ - function debounce(func, wait, options) { - var lastArgs, - lastThis, - maxWait, - result, - timerId, - lastCallTime, - lastInvokeTime = 0, - leading = false, - maxing = false, - trailing = true; - - if (typeof func != 'function') { - throw new TypeError(FUNC_ERROR_TEXT); - } - wait = toNumber(wait) || 0; - if (isObject(options)) { - leading = !!options.leading; - maxing = 'maxWait' in options; - maxWait = maxing ? nativeMax(toNumber(options.maxWait) || 0, wait) : maxWait; - trailing = 'trailing' in options ? !!options.trailing : trailing; - } - - function invokeFunc(time) { - var args = lastArgs, - thisArg = lastThis; - - lastArgs = lastThis = undefined; - lastInvokeTime = time; - result = func.apply(thisArg, args); - return result; - } - - function leadingEdge(time) { - // Reset any `maxWait` timer. - lastInvokeTime = time; - // Start the timer for the trailing edge. - timerId = setTimeout(timerExpired, wait); - // Invoke the leading edge. - return leading ? invokeFunc(time) : result; - } - - function remainingWait(time) { - var timeSinceLastCall = time - lastCallTime, - timeSinceLastInvoke = time - lastInvokeTime, - timeWaiting = wait - timeSinceLastCall; - - return maxing - ? nativeMin(timeWaiting, maxWait - timeSinceLastInvoke) - : timeWaiting; - } - - function shouldInvoke(time) { - var timeSinceLastCall = time - lastCallTime, - timeSinceLastInvoke = time - lastInvokeTime; - - // Either this is the first call, activity has stopped and we're at the - // trailing edge, the system time has gone backwards and we're treating - // it as the trailing edge, or we've hit the `maxWait` limit. - return (lastCallTime === undefined || (timeSinceLastCall >= wait) || - (timeSinceLastCall < 0) || (maxing && timeSinceLastInvoke >= maxWait)); - } - - function timerExpired() { - var time = now(); - if (shouldInvoke(time)) { - return trailingEdge(time); - } - // Restart the timer. - timerId = setTimeout(timerExpired, remainingWait(time)); + * _.isEmpty(true); + * // => true + * + * _.isEmpty(1); + * // => true + * + * _.isEmpty([1, 2, 3]); + * // => false + * + * _.isEmpty({ 'a': 1 }); + * // => false + */ + function isEmpty(value) { + if (value == null) { + return true; } - - function trailingEdge(time) { - timerId = undefined; - - // Only invoke if we have `lastArgs` which means `func` has been - // debounced at least once. - if (trailing && lastArgs) { - return invokeFunc(time); - } - lastArgs = lastThis = undefined; - return result; + if (isArrayLike(value) && + (isArray(value) || typeof value == 'string' || typeof value.splice == 'function' || + isBuffer(value) || isTypedArray(value) || isArguments(value))) { + return !value.length; } - - function cancel() { - if (timerId !== undefined) { - clearTimeout(timerId); - } - lastInvokeTime = 0; - lastArgs = lastCallTime = lastThis = timerId = undefined; + var tag = getTag(value); + if (tag == mapTag || tag == setTag) { + return !value.size; } - - function flush() { - return timerId === undefined ? result : trailingEdge(now()); + if (isPrototype(value)) { + return !baseKeys(value).length; } - - function debounced() { - var time = now(), - isInvoking = shouldInvoke(time); - - lastArgs = arguments; - lastThis = this; - lastCallTime = time; - - if (isInvoking) { - if (timerId === undefined) { - return leadingEdge(lastCallTime); - } - if (maxing) { - // Handle invocations in a tight loop. - clearTimeout(timerId); - timerId = setTimeout(timerExpired, wait); - return invokeFunc(lastCallTime); - } - } - if (timerId === undefined) { - timerId = setTimeout(timerExpired, wait); + for (var key in value) { + if (hasOwnProperty.call(value, key)) { + return false; } - return result; } - debounced.cancel = cancel; - debounced.flush = flush; - return debounced; + return true; } /** - * Defers invoking the `func` until the current call stack has cleared. Any - * additional arguments are provided to `func` when it's invoked. + * Performs a deep comparison between two values to determine if they are + * equivalent. + * + * **Note:** This method supports comparing arrays, array buffers, booleans, + * date objects, error objects, maps, numbers, `Object` objects, regexes, + * sets, strings, symbols, and typed arrays. `Object` objects are compared + * by their own, not inherited, enumerable properties. Functions and DOM + * nodes are compared by strict equality, i.e. `===`. * * @static * @memberOf _ * @since 0.1.0 - * @category Function - * @param {Function} func The function to defer. - * @param {...*} [args] The arguments to invoke `func` with. - * @returns {number} Returns the timer id. + * @category Lang + * @param {*} value The value to compare. + * @param {*} other The other value to compare. + * @returns {boolean} Returns `true` if the values are equivalent, else `false`. * @example * - * _.defer(function(text) { - * console.log(text); - * }, 'deferred'); - * // => Logs 'deferred' after one millisecond. + * var object = { 'a': 1 }; + * var other = { 'a': 1 }; + * + * _.isEqual(object, other); + * // => true + * + * object === other; + * // => false */ - var defer = baseRest(function(func, args) { - return baseDelay(func, 1, args); - }); + function isEqual(value, other) { + return baseIsEqual(value, other); + } /** - * Invokes `func` after `wait` milliseconds. Any additional arguments are - * provided to `func` when it's invoked. + * This method is like `_.isEqual` except that it accepts `customizer` which + * is invoked to compare values. If `customizer` returns `undefined`, comparisons + * are handled by the method instead. The `customizer` is invoked with up to + * six arguments: (objValue, othValue [, index|key, object, other, stack]). * * @static * @memberOf _ - * @since 0.1.0 - * @category Function - * @param {Function} func The function to delay. - * @param {number} wait The number of milliseconds to delay invocation. - * @param {...*} [args] The arguments to invoke `func` with. - * @returns {number} Returns the timer id. + * @since 4.0.0 + * @category Lang + * @param {*} value The value to compare. + * @param {*} other The other value to compare. + * @param {Function} [customizer] The function to customize comparisons. + * @returns {boolean} Returns `true` if the values are equivalent, else `false`. * @example * - * _.delay(function(text) { - * console.log(text); - * }, 1000, 'later'); - * // => Logs 'later' after one second. + * function isGreeting(value) { + * return /^h(?:i|ello)$/.test(value); + * } + * + * function customizer(objValue, othValue) { + * if (isGreeting(objValue) && isGreeting(othValue)) { + * return true; + * } + * } + * + * var array = ['hello', 'goodbye']; + * var other = ['hi', 'goodbye']; + * + * _.isEqualWith(array, other, customizer); + * // => true */ - var delay = baseRest(function(func, wait, args) { - return baseDelay(func, toNumber(wait) || 0, args); - }); + function isEqualWith(value, other, customizer) { + customizer = typeof customizer == 'function' ? customizer : undefined; + var result = customizer ? customizer(value, other) : undefined; + return result === undefined ? baseIsEqual(value, other, undefined, customizer) : !!result; + } /** - * Creates a function that invokes `func` with arguments reversed. + * Checks if `value` is an `Error`, `EvalError`, `RangeError`, `ReferenceError`, + * `SyntaxError`, `TypeError`, or `URIError` object. * * @static * @memberOf _ - * @since 4.0.0 - * @category Function - * @param {Function} func The function to flip arguments for. - * @returns {Function} Returns the new flipped function. + * @since 3.0.0 + * @category Lang + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is an error object, else `false`. * @example * - * var flipped = _.flip(function() { - * return _.toArray(arguments); - * }); + * _.isError(new Error); + * // => true * - * flipped('a', 'b', 'c', 'd'); - * // => ['d', 'c', 'b', 'a'] + * _.isError(Error); + * // => false */ - function flip(func) { - return createWrap(func, WRAP_FLIP_FLAG); + function isError(value) { + if (!isObjectLike(value)) { + return false; + } + var tag = baseGetTag(value); + return tag == errorTag || tag == domExcTag || + (typeof value.message == 'string' && typeof value.name == 'string' && !isPlainObject(value)); } /** - * Creates a function that memoizes the result of `func`. If `resolver` is - * provided, it determines the cache key for storing the result based on the - * arguments provided to the memoized function. By default, the first argument - * provided to the memoized function is used as the map cache key. The `func` - * is invoked with the `this` binding of the memoized function. + * Checks if `value` is a finite primitive number. * - * **Note:** The cache is exposed as the `cache` property on the memoized - * function. Its creation may be customized by replacing the `_.memoize.Cache` - * constructor with one whose instances implement the - * [`Map`](http://ecma-international.org/ecma-262/7.0/#sec-properties-of-the-map-prototype-object) - * method interface of `clear`, `delete`, `get`, `has`, and `set`. + * **Note:** This method is based on + * [`Number.isFinite`](https://mdn.io/Number/isFinite). * * @static * @memberOf _ * @since 0.1.0 - * @category Function - * @param {Function} func The function to have its output memoized. - * @param {Function} [resolver] The function to resolve the cache key. - * @returns {Function} Returns the new memoized function. + * @category Lang + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is a finite number, else `false`. * @example * - * var object = { 'a': 1, 'b': 2 }; - * var other = { 'c': 3, 'd': 4 }; + * _.isFinite(3); + * // => true * - * var values = _.memoize(_.values); - * values(object); - * // => [1, 2] + * _.isFinite(Number.MIN_VALUE); + * // => true * - * values(other); - * // => [3, 4] + * _.isFinite(Infinity); + * // => false * - * object.a = 2; - * values(object); - * // => [1, 2] + * _.isFinite('3'); + * // => false + */ + function isFinite(value) { + return typeof value == 'number' && nativeIsFinite(value); + } + + /** + * Checks if `value` is classified as a `Function` object. * - * // Modify the result cache. - * values.cache.set(object, ['a', 'b']); - * values(object); - * // => ['a', 'b'] + * @static + * @memberOf _ + * @since 0.1.0 + * @category Lang + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is a function, else `false`. + * @example * - * // Replace `_.memoize.Cache`. - * _.memoize.Cache = WeakMap; + * _.isFunction(_); + * // => true + * + * _.isFunction(/abc/); + * // => false */ - function memoize(func, resolver) { - if (typeof func != 'function' || (resolver != null && typeof resolver != 'function')) { - throw new TypeError(FUNC_ERROR_TEXT); + function isFunction(value) { + if (!isObject(value)) { + return false; } - var memoized = function() { - var args = arguments, - key = resolver ? resolver.apply(this, args) : args[0], - cache = memoized.cache; - - if (cache.has(key)) { - return cache.get(key); - } - var result = func.apply(this, args); - memoized.cache = cache.set(key, result) || cache; - return result; - }; - memoized.cache = new (memoize.Cache || MapCache); - return memoized; + // The use of `Object#toString` avoids issues with the `typeof` operator + // in Safari 9 which returns 'object' for typed arrays and other constructors. + var tag = baseGetTag(value); + return tag == funcTag || tag == genTag || tag == asyncTag || tag == proxyTag; } - // Expose `MapCache`. - memoize.Cache = MapCache; + /** + * Checks if `value` is an integer. + * + * **Note:** This method is based on + * [`Number.isInteger`](https://mdn.io/Number/isInteger). + * + * @static + * @memberOf _ + * @since 4.0.0 + * @category Lang + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is an integer, else `false`. + * @example + * + * _.isInteger(3); + * // => true + * + * _.isInteger(Number.MIN_VALUE); + * // => false + * + * _.isInteger(Infinity); + * // => false + * + * _.isInteger('3'); + * // => false + */ + function isInteger(value) { + return typeof value == 'number' && value == toInteger(value); + } /** - * Creates a function that negates the result of the predicate `func`. The - * `func` predicate is invoked with the `this` binding and arguments of the - * created function. + * Checks if `value` is a valid array-like length. + * + * **Note:** This method is loosely based on + * [`ToLength`](http://ecma-international.org/ecma-262/7.0/#sec-tolength). * * @static * @memberOf _ - * @since 3.0.0 - * @category Function - * @param {Function} predicate The predicate to negate. - * @returns {Function} Returns the new negated function. + * @since 4.0.0 + * @category Lang + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is a valid length, else `false`. * @example * - * function isEven(n) { - * return n % 2 == 0; - * } + * _.isLength(3); + * // => true * - * _.filter([1, 2, 3, 4, 5, 6], _.negate(isEven)); - * // => [1, 3, 5] + * _.isLength(Number.MIN_VALUE); + * // => false + * + * _.isLength(Infinity); + * // => false + * + * _.isLength('3'); + * // => false */ - function negate(predicate) { - if (typeof predicate != 'function') { - throw new TypeError(FUNC_ERROR_TEXT); - } - return function() { - var args = arguments; - switch (args.length) { - case 0: return !predicate.call(this); - case 1: return !predicate.call(this, args[0]); - case 2: return !predicate.call(this, args[0], args[1]); - case 3: return !predicate.call(this, args[0], args[1], args[2]); - } - return !predicate.apply(this, args); - }; + function isLength(value) { + return typeof value == 'number' && + value > -1 && value % 1 == 0 && value <= MAX_SAFE_INTEGER; } /** - * Creates a function that is restricted to invoking `func` once. Repeat calls - * to the function return the value of the first invocation. The `func` is - * invoked with the `this` binding and arguments of the created function. + * Checks if `value` is the + * [language type](http://www.ecma-international.org/ecma-262/7.0/#sec-ecmascript-language-types) + * of `Object`. (e.g. arrays, functions, objects, regexes, `new Number(0)`, and `new String('')`) * * @static * @memberOf _ * @since 0.1.0 - * @category Function - * @param {Function} func The function to restrict. - * @returns {Function} Returns the new restricted function. + * @category Lang + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is an object, else `false`. * @example * - * var initialize = _.once(createApplication); - * initialize(); - * initialize(); - * // => `createApplication` is invoked once + * _.isObject({}); + * // => true + * + * _.isObject([1, 2, 3]); + * // => true + * + * _.isObject(_.noop); + * // => true + * + * _.isObject(null); + * // => false */ - function once(func) { - return before(2, func); + function isObject(value) { + var type = typeof value; + return value != null && (type == 'object' || type == 'function'); } /** - * Creates a function that invokes `func` with its arguments transformed. + * Checks if `value` is object-like. A value is object-like if it's not `null` + * and has a `typeof` result of "object". * * @static - * @since 4.0.0 * @memberOf _ - * @category Function - * @param {Function} func The function to wrap. - * @param {...(Function|Function[])} [transforms=[_.identity]] - * The argument transforms. - * @returns {Function} Returns the new function. + * @since 4.0.0 + * @category Lang + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is object-like, else `false`. * @example * - * function doubled(n) { - * return n * 2; - * } - * - * function square(n) { - * return n * n; - * } + * _.isObjectLike({}); + * // => true * - * var func = _.overArgs(function(x, y) { - * return [x, y]; - * }, [square, doubled]); + * _.isObjectLike([1, 2, 3]); + * // => true * - * func(9, 3); - * // => [81, 6] + * _.isObjectLike(_.noop); + * // => false * - * func(10, 5); - * // => [100, 10] + * _.isObjectLike(null); + * // => false */ - var overArgs = castRest(function(func, transforms) { - transforms = (transforms.length == 1 && isArray(transforms[0])) - ? arrayMap(transforms[0], baseUnary(getIteratee())) - : arrayMap(baseFlatten(transforms, 1), baseUnary(getIteratee())); - - var funcsLength = transforms.length; - return baseRest(function(args) { - var index = -1, - length = nativeMin(args.length, funcsLength); + function isObjectLike(value) { + return value != null && typeof value == 'object'; + } - while (++index < length) { - args[index] = transforms[index].call(this, args[index]); - } - return apply(func, this, args); - }); - }); + /** + * Checks if `value` is classified as a `Map` object. + * + * @static + * @memberOf _ + * @since 4.3.0 + * @category Lang + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is a map, else `false`. + * @example + * + * _.isMap(new Map); + * // => true + * + * _.isMap(new WeakMap); + * // => false + */ + var isMap = nodeIsMap ? baseUnary(nodeIsMap) : baseIsMap; /** - * Creates a function that invokes `func` with `partials` prepended to the - * arguments it receives. This method is like `_.bind` except it does **not** - * alter the `this` binding. + * Performs a partial deep comparison between `object` and `source` to + * determine if `object` contains equivalent property values. * - * The `_.partial.placeholder` value, which defaults to `_` in monolithic - * builds, may be used as a placeholder for partially applied arguments. + * **Note:** This method is equivalent to `_.matches` when `source` is + * partially applied. * - * **Note:** This method doesn't set the "length" property of partially - * applied functions. + * Partial comparisons will match empty array and empty object `source` + * values against any array or object value, respectively. See `_.isEqual` + * for a list of supported value comparisons. * * @static * @memberOf _ - * @since 0.2.0 - * @category Function - * @param {Function} func The function to partially apply arguments to. - * @param {...*} [partials] The arguments to be partially applied. - * @returns {Function} Returns the new partially applied function. + * @since 3.0.0 + * @category Lang + * @param {Object} object The object to inspect. + * @param {Object} source The object of property values to match. + * @returns {boolean} Returns `true` if `object` is a match, else `false`. * @example * - * function greet(greeting, name) { - * return greeting + ' ' + name; + * var object = { 'a': 1, 'b': 2 }; + * + * _.isMatch(object, { 'b': 2 }); + * // => true + * + * _.isMatch(object, { 'b': 1 }); + * // => false + */ + function isMatch(object, source) { + return object === source || baseIsMatch(object, source, getMatchData(source)); + } + + /** + * This method is like `_.isMatch` except that it accepts `customizer` which + * is invoked to compare values. If `customizer` returns `undefined`, comparisons + * are handled by the method instead. The `customizer` is invoked with five + * arguments: (objValue, srcValue, index|key, object, source). + * + * @static + * @memberOf _ + * @since 4.0.0 + * @category Lang + * @param {Object} object The object to inspect. + * @param {Object} source The object of property values to match. + * @param {Function} [customizer] The function to customize comparisons. + * @returns {boolean} Returns `true` if `object` is a match, else `false`. + * @example + * + * function isGreeting(value) { + * return /^h(?:i|ello)$/.test(value); + * } + * + * function customizer(objValue, srcValue) { + * if (isGreeting(objValue) && isGreeting(srcValue)) { + * return true; + * } * } * - * var sayHelloTo = _.partial(greet, 'hello'); - * sayHelloTo('fred'); - * // => 'hello fred' + * var object = { 'greeting': 'hello' }; + * var source = { 'greeting': 'hi' }; * - * // Partially applied with placeholders. - * var greetFred = _.partial(greet, _, 'fred'); - * greetFred('hi'); - * // => 'hi fred' + * _.isMatchWith(object, source, customizer); + * // => true */ - var partial = baseRest(function(func, partials) { - var holders = replaceHolders(partials, getHolder(partial)); - return createWrap(func, WRAP_PARTIAL_FLAG, undefined, partials, holders); - }); + function isMatchWith(object, source, customizer) { + customizer = typeof customizer == 'function' ? customizer : undefined; + return baseIsMatch(object, source, getMatchData(source), customizer); + } /** - * This method is like `_.partial` except that partially applied arguments - * are appended to the arguments it receives. - * - * The `_.partialRight.placeholder` value, which defaults to `_` in monolithic - * builds, may be used as a placeholder for partially applied arguments. + * Checks if `value` is `NaN`. * - * **Note:** This method doesn't set the "length" property of partially - * applied functions. + * **Note:** This method is based on + * [`Number.isNaN`](https://mdn.io/Number/isNaN) and is not the same as + * global [`isNaN`](https://mdn.io/isNaN) which returns `true` for + * `undefined` and other non-number values. * * @static * @memberOf _ - * @since 1.0.0 - * @category Function - * @param {Function} func The function to partially apply arguments to. - * @param {...*} [partials] The arguments to be partially applied. - * @returns {Function} Returns the new partially applied function. + * @since 0.1.0 + * @category Lang + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is `NaN`, else `false`. * @example * - * function greet(greeting, name) { - * return greeting + ' ' + name; - * } + * _.isNaN(NaN); + * // => true * - * var greetFred = _.partialRight(greet, 'fred'); - * greetFred('hi'); - * // => 'hi fred' + * _.isNaN(new Number(NaN)); + * // => true * - * // Partially applied with placeholders. - * var sayHelloTo = _.partialRight(greet, 'hello', _); - * sayHelloTo('fred'); - * // => 'hello fred' + * isNaN(undefined); + * // => true + * + * _.isNaN(undefined); + * // => false */ - var partialRight = baseRest(function(func, partials) { - var holders = replaceHolders(partials, getHolder(partialRight)); - return createWrap(func, WRAP_PARTIAL_RIGHT_FLAG, undefined, partials, holders); - }); + function isNaN(value) { + // An `NaN` primitive is the only value that is not equal to itself. + // Perform the `toStringTag` check first to avoid errors with some + // ActiveX objects in IE. + return isNumber(value) && value != +value; + } /** - * Creates a function that invokes `func` with arguments arranged according - * to the specified `indexes` where the argument value at the first index is - * provided as the first argument, the argument value at the second index is - * provided as the second argument, and so on. + * Checks if `value` is a pristine native function. + * + * **Note:** This method can't reliably detect native functions in the presence + * of the core-js package because core-js circumvents this kind of detection. + * Despite multiple requests, the core-js maintainer has made it clear: any + * attempt to fix the detection will be obstructed. As a result, we're left + * with little choice but to throw an error. Unfortunately, this also affects + * packages, like [babel-polyfill](https://www.npmjs.com/package/babel-polyfill), + * which rely on core-js. * * @static * @memberOf _ * @since 3.0.0 - * @category Function - * @param {Function} func The function to rearrange arguments for. - * @param {...(number|number[])} indexes The arranged argument indexes. - * @returns {Function} Returns the new function. + * @category Lang + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is a native function, + * else `false`. * @example * - * var rearged = _.rearg(function(a, b, c) { - * return [a, b, c]; - * }, [2, 0, 1]); + * _.isNative(Array.prototype.push); + * // => true * - * rearged('b', 'c', 'a') - * // => ['a', 'b', 'c'] + * _.isNative(_); + * // => false */ - var rearg = flatRest(function(func, indexes) { - return createWrap(func, WRAP_REARG_FLAG, undefined, undefined, undefined, indexes); - }); + function isNative(value) { + if (isMaskable(value)) { + throw new Error(CORE_ERROR_TEXT); + } + return baseIsNative(value); + } /** - * Creates a function that invokes `func` with the `this` binding of the - * created function and arguments from `start` and beyond provided as - * an array. - * - * **Note:** This method is based on the - * [rest parameter](https://mdn.io/rest_parameters). + * Checks if `value` is `null`. * * @static * @memberOf _ - * @since 4.0.0 - * @category Function - * @param {Function} func The function to apply a rest parameter to. - * @param {number} [start=func.length-1] The start position of the rest parameter. - * @returns {Function} Returns the new function. + * @since 0.1.0 + * @category Lang + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is `null`, else `false`. * @example * - * var say = _.rest(function(what, names) { - * return what + ' ' + _.initial(names).join(', ') + - * (_.size(names) > 1 ? ', & ' : '') + _.last(names); - * }); + * _.isNull(null); + * // => true * - * say('hello', 'fred', 'barney', 'pebbles'); - * // => 'hello fred, barney, & pebbles' + * _.isNull(void 0); + * // => false */ - function rest(func, start) { - if (typeof func != 'function') { - throw new TypeError(FUNC_ERROR_TEXT); - } - start = start === undefined ? start : toInteger(start); - return baseRest(func, start); + function isNull(value) { + return value === null; } /** - * Creates a function that invokes `func` with the `this` binding of the - * create function and an array of arguments much like - * [`Function#apply`](http://www.ecma-international.org/ecma-262/7.0/#sec-function.prototype.apply). - * - * **Note:** This method is based on the - * [spread operator](https://mdn.io/spread_operator). + * Checks if `value` is `null` or `undefined`. * * @static * @memberOf _ - * @since 3.2.0 - * @category Function - * @param {Function} func The function to spread arguments over. - * @param {number} [start=0] The start position of the spread. - * @returns {Function} Returns the new function. + * @since 4.0.0 + * @category Lang + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is nullish, else `false`. * @example * - * var say = _.spread(function(who, what) { - * return who + ' says ' + what; - * }); - * - * say(['fred', 'hello']); - * // => 'fred says hello' + * _.isNil(null); + * // => true * - * var numbers = Promise.all([ - * Promise.resolve(40), - * Promise.resolve(36) - * ]); + * _.isNil(void 0); + * // => true * - * numbers.then(_.spread(function(x, y) { - * return x + y; - * })); - * // => a Promise of 76 + * _.isNil(NaN); + * // => false */ - function spread(func, start) { - if (typeof func != 'function') { - throw new TypeError(FUNC_ERROR_TEXT); - } - start = start == null ? 0 : nativeMax(toInteger(start), 0); - return baseRest(function(args) { - var array = args[start], - otherArgs = castSlice(args, 0, start); - - if (array) { - arrayPush(otherArgs, array); - } - return apply(func, this, otherArgs); - }); + function isNil(value) { + return value == null; } /** - * Creates a throttled function that only invokes `func` at most once per - * every `wait` milliseconds. The throttled function comes with a `cancel` - * method to cancel delayed `func` invocations and a `flush` method to - * immediately invoke them. Provide `options` to indicate whether `func` - * should be invoked on the leading and/or trailing edge of the `wait` - * timeout. The `func` is invoked with the last arguments provided to the - * throttled function. Subsequent calls to the throttled function return the - * result of the last `func` invocation. - * - * **Note:** If `leading` and `trailing` options are `true`, `func` is - * invoked on the trailing edge of the timeout only if the throttled function - * is invoked more than once during the `wait` timeout. - * - * If `wait` is `0` and `leading` is `false`, `func` invocation is deferred - * until to the next tick, similar to `setTimeout` with a timeout of `0`. + * Checks if `value` is classified as a `Number` primitive or object. * - * See [David Corbacho's article](https://css-tricks.com/debouncing-throttling-explained-examples/) - * for details over the differences between `_.throttle` and `_.debounce`. + * **Note:** To exclude `Infinity`, `-Infinity`, and `NaN`, which are + * classified as numbers, use the `_.isFinite` method. * * @static * @memberOf _ * @since 0.1.0 - * @category Function - * @param {Function} func The function to throttle. - * @param {number} [wait=0] The number of milliseconds to throttle invocations to. - * @param {Object} [options={}] The options object. - * @param {boolean} [options.leading=true] - * Specify invoking on the leading edge of the timeout. - * @param {boolean} [options.trailing=true] - * Specify invoking on the trailing edge of the timeout. - * @returns {Function} Returns the new throttled function. + * @category Lang + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is a number, else `false`. * @example * - * // Avoid excessively updating the position while scrolling. - * jQuery(window).on('scroll', _.throttle(updatePosition, 100)); + * _.isNumber(3); + * // => true * - * // Invoke `renewToken` when the click event is fired, but not more than once every 5 minutes. - * var throttled = _.throttle(renewToken, 300000, { 'trailing': false }); - * jQuery(element).on('click', throttled); + * _.isNumber(Number.MIN_VALUE); + * // => true * - * // Cancel the trailing throttled invocation. - * jQuery(window).on('popstate', throttled.cancel); + * _.isNumber(Infinity); + * // => true + * + * _.isNumber('3'); + * // => false */ - function throttle(func, wait, options) { - var leading = true, - trailing = true; - - if (typeof func != 'function') { - throw new TypeError(FUNC_ERROR_TEXT); - } - if (isObject(options)) { - leading = 'leading' in options ? !!options.leading : leading; - trailing = 'trailing' in options ? !!options.trailing : trailing; - } - return debounce(func, wait, { - 'leading': leading, - 'maxWait': wait, - 'trailing': trailing - }); + function isNumber(value) { + return typeof value == 'number' || + (isObjectLike(value) && baseGetTag(value) == numberTag); } /** - * Creates a function that accepts up to one argument, ignoring any - * additional arguments. + * Checks if `value` is a plain object, that is, an object created by the + * `Object` constructor or one with a `[[Prototype]]` of `null`. * * @static * @memberOf _ - * @since 4.0.0 - * @category Function - * @param {Function} func The function to cap arguments for. - * @returns {Function} Returns the new capped function. + * @since 0.8.0 + * @category Lang + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is a plain object, else `false`. * @example * - * _.map(['6', '8', '10'], _.unary(parseInt)); - * // => [6, 8, 10] + * function Foo() { + * this.a = 1; + * } + * + * _.isPlainObject(new Foo); + * // => false + * + * _.isPlainObject([1, 2, 3]); + * // => false + * + * _.isPlainObject({ 'x': 0, 'y': 0 }); + * // => true + * + * _.isPlainObject(Object.create(null)); + * // => true */ - function unary(func) { - return ary(func, 1); + function isPlainObject(value) { + if (!isObjectLike(value) || baseGetTag(value) != objectTag) { + return false; + } + var proto = getPrototype(value); + if (proto === null) { + return true; + } + var Ctor = hasOwnProperty.call(proto, 'constructor') && proto.constructor; + return typeof Ctor == 'function' && Ctor instanceof Ctor && + funcToString.call(Ctor) == objectCtorString; } /** - * Creates a function that provides `value` to `wrapper` as its first - * argument. Any additional arguments provided to the function are appended - * to those provided to the `wrapper`. The wrapper is invoked with the `this` - * binding of the created function. + * Checks if `value` is classified as a `RegExp` object. * * @static * @memberOf _ * @since 0.1.0 - * @category Function - * @param {*} value The value to wrap. - * @param {Function} [wrapper=identity] The wrapper function. - * @returns {Function} Returns the new function. + * @category Lang + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is a regexp, else `false`. * @example * - * var p = _.wrap(_.escape, function(func, text) { - * return '

' + func(text) + '

'; - * }); + * _.isRegExp(/abc/); + * // => true * - * p('fred, barney, & pebbles'); - * // => '

fred, barney, & pebbles

' + * _.isRegExp('/abc/'); + * // => false */ - function wrap(value, wrapper) { - return partial(castFunction(wrapper), value); - } - - /*------------------------------------------------------------------------*/ + var isRegExp = nodeIsRegExp ? baseUnary(nodeIsRegExp) : baseIsRegExp; /** - * Casts `value` as an array if it's not one. + * Checks if `value` is a safe integer. An integer is safe if it's an IEEE-754 + * double precision number which isn't the result of a rounded unsafe integer. + * + * **Note:** This method is based on + * [`Number.isSafeInteger`](https://mdn.io/Number/isSafeInteger). * * @static * @memberOf _ - * @since 4.4.0 + * @since 4.0.0 * @category Lang - * @param {*} value The value to inspect. - * @returns {Array} Returns the cast array. + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is a safe integer, else `false`. * @example * - * _.castArray(1); - * // => [1] - * - * _.castArray({ 'a': 1 }); - * // => [{ 'a': 1 }] + * _.isSafeInteger(3); + * // => true * - * _.castArray('abc'); - * // => ['abc'] + * _.isSafeInteger(Number.MIN_VALUE); + * // => false * - * _.castArray(null); - * // => [null] + * _.isSafeInteger(Infinity); + * // => false * - * _.castArray(undefined); - * // => [undefined] + * _.isSafeInteger('3'); + * // => false + */ + function isSafeInteger(value) { + return isInteger(value) && value >= -MAX_SAFE_INTEGER && value <= MAX_SAFE_INTEGER; + } + + /** + * Checks if `value` is classified as a `Set` object. * - * _.castArray(); - * // => [] + * @static + * @memberOf _ + * @since 4.3.0 + * @category Lang + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is a set, else `false`. + * @example * - * var array = [1, 2, 3]; - * console.log(_.castArray(array) === array); + * _.isSet(new Set); * // => true + * + * _.isSet(new WeakSet); + * // => false */ - function castArray() { - if (!arguments.length) { - return []; - } - var value = arguments[0]; - return isArray(value) ? value : [value]; - } + var isSet = nodeIsSet ? baseUnary(nodeIsSet) : baseIsSet; /** - * Creates a shallow clone of `value`. - * - * **Note:** This method is loosely based on the - * [structured clone algorithm](https://mdn.io/Structured_clone_algorithm) - * and supports cloning arrays, array buffers, booleans, date objects, maps, - * numbers, `Object` objects, regexes, sets, strings, symbols, and typed - * arrays. The own enumerable properties of `arguments` objects are cloned - * as plain objects. An empty object is returned for uncloneable values such - * as error objects, functions, DOM nodes, and WeakMaps. + * Checks if `value` is classified as a `String` primitive or object. * * @static - * @memberOf _ * @since 0.1.0 + * @memberOf _ * @category Lang - * @param {*} value The value to clone. - * @returns {*} Returns the cloned value. - * @see _.cloneDeep + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is a string, else `false`. * @example * - * var objects = [{ 'a': 1 }, { 'b': 2 }]; - * - * var shallow = _.clone(objects); - * console.log(shallow[0] === objects[0]); + * _.isString('abc'); * // => true + * + * _.isString(1); + * // => false */ - function clone(value) { - return baseClone(value, CLONE_SYMBOLS_FLAG); + function isString(value) { + return typeof value == 'string' || + (!isArray(value) && isObjectLike(value) && baseGetTag(value) == stringTag); } /** - * This method is like `_.clone` except that it accepts `customizer` which - * is invoked to produce the cloned value. If `customizer` returns `undefined`, - * cloning is handled by the method instead. The `customizer` is invoked with - * up to four arguments; (value [, index|key, object, stack]). + * Checks if `value` is classified as a `Symbol` primitive or object. * * @static * @memberOf _ * @since 4.0.0 * @category Lang - * @param {*} value The value to clone. - * @param {Function} [customizer] The function to customize cloning. - * @returns {*} Returns the cloned value. - * @see _.cloneDeepWith + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is a symbol, else `false`. * @example * - * function customizer(value) { - * if (_.isElement(value)) { - * return value.cloneNode(false); - * } - * } - * - * var el = _.cloneWith(document.body, customizer); + * _.isSymbol(Symbol.iterator); + * // => true * - * console.log(el === document.body); + * _.isSymbol('abc'); * // => false - * console.log(el.nodeName); - * // => 'BODY' - * console.log(el.childNodes.length); - * // => 0 */ - function cloneWith(value, customizer) { - customizer = typeof customizer == 'function' ? customizer : undefined; - return baseClone(value, CLONE_SYMBOLS_FLAG, customizer); + function isSymbol(value) { + return typeof value == 'symbol' || + (isObjectLike(value) && baseGetTag(value) == symbolTag); } /** - * This method is like `_.clone` except that it recursively clones `value`. + * Checks if `value` is classified as a typed array. * * @static * @memberOf _ - * @since 1.0.0 + * @since 3.0.0 * @category Lang - * @param {*} value The value to recursively clone. - * @returns {*} Returns the deep cloned value. - * @see _.clone + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is a typed array, else `false`. * @example * - * var objects = [{ 'a': 1 }, { 'b': 2 }]; + * _.isTypedArray(new Uint8Array); + * // => true * - * var deep = _.cloneDeep(objects); - * console.log(deep[0] === objects[0]); + * _.isTypedArray([]); * // => false */ - function cloneDeep(value) { - return baseClone(value, CLONE_DEEP_FLAG | CLONE_SYMBOLS_FLAG); - } + var isTypedArray = nodeIsTypedArray ? baseUnary(nodeIsTypedArray) : baseIsTypedArray; /** - * This method is like `_.cloneWith` except that it recursively clones `value`. + * Checks if `value` is `undefined`. * * @static + * @since 0.1.0 * @memberOf _ - * @since 4.0.0 * @category Lang - * @param {*} value The value to recursively clone. - * @param {Function} [customizer] The function to customize cloning. - * @returns {*} Returns the deep cloned value. - * @see _.cloneWith + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is `undefined`, else `false`. * @example * - * function customizer(value) { - * if (_.isElement(value)) { - * return value.cloneNode(true); - * } - * } - * - * var el = _.cloneDeepWith(document.body, customizer); + * _.isUndefined(void 0); + * // => true * - * console.log(el === document.body); + * _.isUndefined(null); * // => false - * console.log(el.nodeName); - * // => 'BODY' - * console.log(el.childNodes.length); - * // => 20 */ - function cloneDeepWith(value, customizer) { - customizer = typeof customizer == 'function' ? customizer : undefined; - return baseClone(value, CLONE_DEEP_FLAG | CLONE_SYMBOLS_FLAG, customizer); + function isUndefined(value) { + return value === undefined; } /** - * Checks if `object` conforms to `source` by invoking the predicate - * properties of `source` with the corresponding property values of `object`. - * - * **Note:** This method is equivalent to `_.conforms` when `source` is - * partially applied. + * Checks if `value` is classified as a `WeakMap` object. * * @static * @memberOf _ - * @since 4.14.0 + * @since 4.3.0 * @category Lang - * @param {Object} object The object to inspect. - * @param {Object} source The object of property predicates to conform to. - * @returns {boolean} Returns `true` if `object` conforms, else `false`. + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is a weak map, else `false`. * @example * - * var object = { 'a': 1, 'b': 2 }; - * - * _.conformsTo(object, { 'b': function(n) { return n > 1; } }); + * _.isWeakMap(new WeakMap); * // => true * - * _.conformsTo(object, { 'b': function(n) { return n > 2; } }); + * _.isWeakMap(new Map); * // => false */ - function conformsTo(object, source) { - return source == null || baseConformsTo(object, source, keys(source)); + function isWeakMap(value) { + return isObjectLike(value) && getTag(value) == weakMapTag; } /** - * Performs a - * [`SameValueZero`](http://ecma-international.org/ecma-262/7.0/#sec-samevaluezero) - * comparison between two values to determine if they are equivalent. + * Checks if `value` is classified as a `WeakSet` object. * * @static * @memberOf _ - * @since 4.0.0 + * @since 4.3.0 * @category Lang - * @param {*} value The value to compare. - * @param {*} other The other value to compare. - * @returns {boolean} Returns `true` if the values are equivalent, else `false`. + * @param {*} value The value to check. + * @returns {boolean} Returns `true` if `value` is a weak set, else `false`. * @example * - * var object = { 'a': 1 }; - * var other = { 'a': 1 }; - * - * _.eq(object, object); - * // => true - * - * _.eq(object, other); - * // => false - * - * _.eq('a', 'a'); + * _.isWeakSet(new WeakSet); * // => true * - * _.eq('a', Object('a')); + * _.isWeakSet(new Set); * // => false - * - * _.eq(NaN, NaN); - * // => true */ - function eq(value, other) { - return value === other || (value !== value && other !== other); + function isWeakSet(value) { + return isObjectLike(value) && baseGetTag(value) == weakSetTag; } /** - * Checks if `value` is greater than `other`. + * Checks if `value` is less than `other`. * * @static * @memberOf _ @@ -51353,24 +51858,24 @@ exports.Deprecation = Deprecation; * @category Lang * @param {*} value The value to compare. * @param {*} other The other value to compare. - * @returns {boolean} Returns `true` if `value` is greater than `other`, + * @returns {boolean} Returns `true` if `value` is less than `other`, * else `false`. - * @see _.lt + * @see _.gt * @example * - * _.gt(3, 1); + * _.lt(1, 3); * // => true * - * _.gt(3, 3); + * _.lt(3, 3); * // => false * - * _.gt(1, 3); + * _.lt(3, 1); * // => false */ - var gt = createRelationalOperation(baseGt); + var lt = createRelationalOperation(baseLt); /** - * Checks if `value` is greater than or equal to `other`. + * Checks if `value` is less than or equal to `other`. * * @static * @memberOf _ @@ -51378,5975 +51883,6502 @@ exports.Deprecation = Deprecation; * @category Lang * @param {*} value The value to compare. * @param {*} other The other value to compare. - * @returns {boolean} Returns `true` if `value` is greater than or equal to + * @returns {boolean} Returns `true` if `value` is less than or equal to * `other`, else `false`. - * @see _.lte + * @see _.gte * @example * - * _.gte(3, 1); + * _.lte(1, 3); * // => true * - * _.gte(3, 3); + * _.lte(3, 3); * // => true * - * _.gte(1, 3); + * _.lte(3, 1); * // => false */ - var gte = createRelationalOperation(function(value, other) { - return value >= other; + var lte = createRelationalOperation(function(value, other) { + return value <= other; }); /** - * Checks if `value` is likely an `arguments` object. + * Converts `value` to an array. * * @static - * @memberOf _ * @since 0.1.0 + * @memberOf _ * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is an `arguments` object, - * else `false`. + * @param {*} value The value to convert. + * @returns {Array} Returns the converted array. * @example * - * _.isArguments(function() { return arguments; }()); - * // => true + * _.toArray({ 'a': 1, 'b': 2 }); + * // => [1, 2] * - * _.isArguments([1, 2, 3]); - * // => false + * _.toArray('abc'); + * // => ['a', 'b', 'c'] + * + * _.toArray(1); + * // => [] + * + * _.toArray(null); + * // => [] */ - var isArguments = baseIsArguments(function() { return arguments; }()) ? baseIsArguments : function(value) { - return isObjectLike(value) && hasOwnProperty.call(value, 'callee') && - !propertyIsEnumerable.call(value, 'callee'); - }; + function toArray(value) { + if (!value) { + return []; + } + if (isArrayLike(value)) { + return isString(value) ? stringToArray(value) : copyArray(value); + } + if (symIterator && value[symIterator]) { + return iteratorToArray(value[symIterator]()); + } + var tag = getTag(value), + func = tag == mapTag ? mapToArray : (tag == setTag ? setToArray : values); + + return func(value); + } /** - * Checks if `value` is classified as an `Array` object. + * Converts `value` to a finite number. * * @static * @memberOf _ - * @since 0.1.0 + * @since 4.12.0 * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is an array, else `false`. + * @param {*} value The value to convert. + * @returns {number} Returns the converted number. * @example * - * _.isArray([1, 2, 3]); - * // => true + * _.toFinite(3.2); + * // => 3.2 * - * _.isArray(document.body.children); - * // => false + * _.toFinite(Number.MIN_VALUE); + * // => 5e-324 * - * _.isArray('abc'); - * // => false + * _.toFinite(Infinity); + * // => 1.7976931348623157e+308 * - * _.isArray(_.noop); - * // => false + * _.toFinite('3.2'); + * // => 3.2 */ - var isArray = Array.isArray; + function toFinite(value) { + if (!value) { + return value === 0 ? value : 0; + } + value = toNumber(value); + if (value === INFINITY || value === -INFINITY) { + var sign = (value < 0 ? -1 : 1); + return sign * MAX_INTEGER; + } + return value === value ? value : 0; + } /** - * Checks if `value` is classified as an `ArrayBuffer` object. + * Converts `value` to an integer. + * + * **Note:** This method is loosely based on + * [`ToInteger`](http://www.ecma-international.org/ecma-262/7.0/#sec-tointeger). * * @static * @memberOf _ - * @since 4.3.0 + * @since 4.0.0 * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is an array buffer, else `false`. + * @param {*} value The value to convert. + * @returns {number} Returns the converted integer. * @example * - * _.isArrayBuffer(new ArrayBuffer(2)); - * // => true + * _.toInteger(3.2); + * // => 3 * - * _.isArrayBuffer(new Array(2)); - * // => false + * _.toInteger(Number.MIN_VALUE); + * // => 0 + * + * _.toInteger(Infinity); + * // => 1.7976931348623157e+308 + * + * _.toInteger('3.2'); + * // => 3 */ - var isArrayBuffer = nodeIsArrayBuffer ? baseUnary(nodeIsArrayBuffer) : baseIsArrayBuffer; + function toInteger(value) { + var result = toFinite(value), + remainder = result % 1; + + return result === result ? (remainder ? result - remainder : result) : 0; + } /** - * Checks if `value` is array-like. A value is considered array-like if it's - * not a function and has a `value.length` that's an integer greater than or - * equal to `0` and less than or equal to `Number.MAX_SAFE_INTEGER`. + * Converts `value` to an integer suitable for use as the length of an + * array-like object. + * + * **Note:** This method is based on + * [`ToLength`](http://ecma-international.org/ecma-262/7.0/#sec-tolength). * * @static * @memberOf _ * @since 4.0.0 * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is array-like, else `false`. + * @param {*} value The value to convert. + * @returns {number} Returns the converted integer. * @example * - * _.isArrayLike([1, 2, 3]); - * // => true + * _.toLength(3.2); + * // => 3 * - * _.isArrayLike(document.body.children); - * // => true + * _.toLength(Number.MIN_VALUE); + * // => 0 * - * _.isArrayLike('abc'); - * // => true + * _.toLength(Infinity); + * // => 4294967295 * - * _.isArrayLike(_.noop); - * // => false + * _.toLength('3.2'); + * // => 3 */ - function isArrayLike(value) { - return value != null && isLength(value.length) && !isFunction(value); + function toLength(value) { + return value ? baseClamp(toInteger(value), 0, MAX_ARRAY_LENGTH) : 0; } /** - * This method is like `_.isArrayLike` except that it also checks if `value` - * is an object. + * Converts `value` to a number. * * @static * @memberOf _ * @since 4.0.0 * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is an array-like object, - * else `false`. + * @param {*} value The value to process. + * @returns {number} Returns the number. * @example * - * _.isArrayLikeObject([1, 2, 3]); - * // => true + * _.toNumber(3.2); + * // => 3.2 * - * _.isArrayLikeObject(document.body.children); - * // => true + * _.toNumber(Number.MIN_VALUE); + * // => 5e-324 * - * _.isArrayLikeObject('abc'); - * // => false + * _.toNumber(Infinity); + * // => Infinity * - * _.isArrayLikeObject(_.noop); - * // => false + * _.toNumber('3.2'); + * // => 3.2 */ - function isArrayLikeObject(value) { - return isObjectLike(value) && isArrayLike(value); + function toNumber(value) { + if (typeof value == 'number') { + return value; + } + if (isSymbol(value)) { + return NAN; + } + if (isObject(value)) { + var other = typeof value.valueOf == 'function' ? value.valueOf() : value; + value = isObject(other) ? (other + '') : other; + } + if (typeof value != 'string') { + return value === 0 ? value : +value; + } + value = baseTrim(value); + var isBinary = reIsBinary.test(value); + return (isBinary || reIsOctal.test(value)) + ? freeParseInt(value.slice(2), isBinary ? 2 : 8) + : (reIsBadHex.test(value) ? NAN : +value); } /** - * Checks if `value` is classified as a boolean primitive or object. + * Converts `value` to a plain object flattening inherited enumerable string + * keyed properties of `value` to own properties of the plain object. * * @static * @memberOf _ - * @since 0.1.0 + * @since 3.0.0 * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a boolean, else `false`. + * @param {*} value The value to convert. + * @returns {Object} Returns the converted plain object. * @example * - * _.isBoolean(false); - * // => true + * function Foo() { + * this.b = 2; + * } * - * _.isBoolean(null); - * // => false + * Foo.prototype.c = 3; + * + * _.assign({ 'a': 1 }, new Foo); + * // => { 'a': 1, 'b': 2 } + * + * _.assign({ 'a': 1 }, _.toPlainObject(new Foo)); + * // => { 'a': 1, 'b': 2, 'c': 3 } */ - function isBoolean(value) { - return value === true || value === false || - (isObjectLike(value) && baseGetTag(value) == boolTag); + function toPlainObject(value) { + return copyObject(value, keysIn(value)); } /** - * Checks if `value` is a buffer. + * Converts `value` to a safe integer. A safe integer can be compared and + * represented correctly. * * @static * @memberOf _ - * @since 4.3.0 + * @since 4.0.0 * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a buffer, else `false`. + * @param {*} value The value to convert. + * @returns {number} Returns the converted integer. * @example * - * _.isBuffer(new Buffer(2)); - * // => true + * _.toSafeInteger(3.2); + * // => 3 * - * _.isBuffer(new Uint8Array(2)); - * // => false + * _.toSafeInteger(Number.MIN_VALUE); + * // => 0 + * + * _.toSafeInteger(Infinity); + * // => 9007199254740991 + * + * _.toSafeInteger('3.2'); + * // => 3 */ - var isBuffer = nativeIsBuffer || stubFalse; + function toSafeInteger(value) { + return value + ? baseClamp(toInteger(value), -MAX_SAFE_INTEGER, MAX_SAFE_INTEGER) + : (value === 0 ? value : 0); + } /** - * Checks if `value` is classified as a `Date` object. + * Converts `value` to a string. An empty string is returned for `null` + * and `undefined` values. The sign of `-0` is preserved. * * @static * @memberOf _ - * @since 0.1.0 + * @since 4.0.0 * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a date object, else `false`. + * @param {*} value The value to convert. + * @returns {string} Returns the converted string. * @example * - * _.isDate(new Date); - * // => true + * _.toString(null); + * // => '' * - * _.isDate('Mon April 23 2012'); - * // => false + * _.toString(-0); + * // => '-0' + * + * _.toString([1, 2, 3]); + * // => '1,2,3' */ - var isDate = nodeIsDate ? baseUnary(nodeIsDate) : baseIsDate; + function toString(value) { + return value == null ? '' : baseToString(value); + } + + /*------------------------------------------------------------------------*/ /** - * Checks if `value` is likely a DOM element. + * Assigns own enumerable string keyed properties of source objects to the + * destination object. Source objects are applied from left to right. + * Subsequent sources overwrite property assignments of previous sources. + * + * **Note:** This method mutates `object` and is loosely based on + * [`Object.assign`](https://mdn.io/Object/assign). * * @static * @memberOf _ - * @since 0.1.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a DOM element, else `false`. + * @since 0.10.0 + * @category Object + * @param {Object} object The destination object. + * @param {...Object} [sources] The source objects. + * @returns {Object} Returns `object`. + * @see _.assignIn * @example * - * _.isElement(document.body); - * // => true + * function Foo() { + * this.a = 1; + * } * - * _.isElement(''); - * // => false + * function Bar() { + * this.c = 3; + * } + * + * Foo.prototype.b = 2; + * Bar.prototype.d = 4; + * + * _.assign({ 'a': 0 }, new Foo, new Bar); + * // => { 'a': 1, 'c': 3 } */ - function isElement(value) { - return isObjectLike(value) && value.nodeType === 1 && !isPlainObject(value); - } + var assign = createAssigner(function(object, source) { + if (isPrototype(source) || isArrayLike(source)) { + copyObject(source, keys(source), object); + return; + } + for (var key in source) { + if (hasOwnProperty.call(source, key)) { + assignValue(object, key, source[key]); + } + } + }); + + /** + * This method is like `_.assign` except that it iterates over own and + * inherited source properties. + * + * **Note:** This method mutates `object`. + * + * @static + * @memberOf _ + * @since 4.0.0 + * @alias extend + * @category Object + * @param {Object} object The destination object. + * @param {...Object} [sources] The source objects. + * @returns {Object} Returns `object`. + * @see _.assign + * @example + * + * function Foo() { + * this.a = 1; + * } + * + * function Bar() { + * this.c = 3; + * } + * + * Foo.prototype.b = 2; + * Bar.prototype.d = 4; + * + * _.assignIn({ 'a': 0 }, new Foo, new Bar); + * // => { 'a': 1, 'b': 2, 'c': 3, 'd': 4 } + */ + var assignIn = createAssigner(function(object, source) { + copyObject(source, keysIn(source), object); + }); + + /** + * This method is like `_.assignIn` except that it accepts `customizer` + * which is invoked to produce the assigned values. If `customizer` returns + * `undefined`, assignment is handled by the method instead. The `customizer` + * is invoked with five arguments: (objValue, srcValue, key, object, source). + * + * **Note:** This method mutates `object`. + * + * @static + * @memberOf _ + * @since 4.0.0 + * @alias extendWith + * @category Object + * @param {Object} object The destination object. + * @param {...Object} sources The source objects. + * @param {Function} [customizer] The function to customize assigned values. + * @returns {Object} Returns `object`. + * @see _.assignWith + * @example + * + * function customizer(objValue, srcValue) { + * return _.isUndefined(objValue) ? srcValue : objValue; + * } + * + * var defaults = _.partialRight(_.assignInWith, customizer); + * + * defaults({ 'a': 1 }, { 'b': 2 }, { 'a': 3 }); + * // => { 'a': 1, 'b': 2 } + */ + var assignInWith = createAssigner(function(object, source, srcIndex, customizer) { + copyObject(source, keysIn(source), object, customizer); + }); + + /** + * This method is like `_.assign` except that it accepts `customizer` + * which is invoked to produce the assigned values. If `customizer` returns + * `undefined`, assignment is handled by the method instead. The `customizer` + * is invoked with five arguments: (objValue, srcValue, key, object, source). + * + * **Note:** This method mutates `object`. + * + * @static + * @memberOf _ + * @since 4.0.0 + * @category Object + * @param {Object} object The destination object. + * @param {...Object} sources The source objects. + * @param {Function} [customizer] The function to customize assigned values. + * @returns {Object} Returns `object`. + * @see _.assignInWith + * @example + * + * function customizer(objValue, srcValue) { + * return _.isUndefined(objValue) ? srcValue : objValue; + * } + * + * var defaults = _.partialRight(_.assignWith, customizer); + * + * defaults({ 'a': 1 }, { 'b': 2 }, { 'a': 3 }); + * // => { 'a': 1, 'b': 2 } + */ + var assignWith = createAssigner(function(object, source, srcIndex, customizer) { + copyObject(source, keys(source), object, customizer); + }); + + /** + * Creates an array of values corresponding to `paths` of `object`. + * + * @static + * @memberOf _ + * @since 1.0.0 + * @category Object + * @param {Object} object The object to iterate over. + * @param {...(string|string[])} [paths] The property paths to pick. + * @returns {Array} Returns the picked values. + * @example + * + * var object = { 'a': [{ 'b': { 'c': 3 } }, 4] }; + * + * _.at(object, ['a[0].b.c', 'a[1]']); + * // => [3, 4] + */ + var at = flatRest(baseAt); /** - * Checks if `value` is an empty object, collection, map, or set. - * - * Objects are considered empty if they have no own enumerable string keyed - * properties. - * - * Array-like values such as `arguments` objects, arrays, buffers, strings, or - * jQuery-like collections are considered empty if they have a `length` of `0`. - * Similarly, maps and sets are considered empty if they have a `size` of `0`. + * Creates an object that inherits from the `prototype` object. If a + * `properties` object is given, its own enumerable string keyed properties + * are assigned to the created object. * * @static * @memberOf _ - * @since 0.1.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is empty, else `false`. + * @since 2.3.0 + * @category Object + * @param {Object} prototype The object to inherit from. + * @param {Object} [properties] The properties to assign to the object. + * @returns {Object} Returns the new object. * @example * - * _.isEmpty(null); - * // => true + * function Shape() { + * this.x = 0; + * this.y = 0; + * } * - * _.isEmpty(true); - * // => true + * function Circle() { + * Shape.call(this); + * } * - * _.isEmpty(1); - * // => true + * Circle.prototype = _.create(Shape.prototype, { + * 'constructor': Circle + * }); * - * _.isEmpty([1, 2, 3]); - * // => false + * var circle = new Circle; + * circle instanceof Circle; + * // => true * - * _.isEmpty({ 'a': 1 }); - * // => false + * circle instanceof Shape; + * // => true */ - function isEmpty(value) { - if (value == null) { - return true; - } - if (isArrayLike(value) && - (isArray(value) || typeof value == 'string' || typeof value.splice == 'function' || - isBuffer(value) || isTypedArray(value) || isArguments(value))) { - return !value.length; - } - var tag = getTag(value); - if (tag == mapTag || tag == setTag) { - return !value.size; - } - if (isPrototype(value)) { - return !baseKeys(value).length; - } - for (var key in value) { - if (hasOwnProperty.call(value, key)) { - return false; - } - } - return true; + function create(prototype, properties) { + var result = baseCreate(prototype); + return properties == null ? result : baseAssign(result, properties); } /** - * Performs a deep comparison between two values to determine if they are - * equivalent. + * Assigns own and inherited enumerable string keyed properties of source + * objects to the destination object for all destination properties that + * resolve to `undefined`. Source objects are applied from left to right. + * Once a property is set, additional values of the same property are ignored. * - * **Note:** This method supports comparing arrays, array buffers, booleans, - * date objects, error objects, maps, numbers, `Object` objects, regexes, - * sets, strings, symbols, and typed arrays. `Object` objects are compared - * by their own, not inherited, enumerable properties. Functions and DOM - * nodes are compared by strict equality, i.e. `===`. + * **Note:** This method mutates `object`. * * @static - * @memberOf _ * @since 0.1.0 - * @category Lang - * @param {*} value The value to compare. - * @param {*} other The other value to compare. - * @returns {boolean} Returns `true` if the values are equivalent, else `false`. + * @memberOf _ + * @category Object + * @param {Object} object The destination object. + * @param {...Object} [sources] The source objects. + * @returns {Object} Returns `object`. + * @see _.defaultsDeep * @example * - * var object = { 'a': 1 }; - * var other = { 'a': 1 }; - * - * _.isEqual(object, other); - * // => true - * - * object === other; - * // => false + * _.defaults({ 'a': 1 }, { 'b': 2 }, { 'a': 3 }); + * // => { 'a': 1, 'b': 2 } */ - function isEqual(value, other) { - return baseIsEqual(value, other); - } + var defaults = baseRest(function(object, sources) { + object = Object(object); + + var index = -1; + var length = sources.length; + var guard = length > 2 ? sources[2] : undefined; + + if (guard && isIterateeCall(sources[0], sources[1], guard)) { + length = 1; + } + + while (++index < length) { + var source = sources[index]; + var props = keysIn(source); + var propsIndex = -1; + var propsLength = props.length; + + while (++propsIndex < propsLength) { + var key = props[propsIndex]; + var value = object[key]; + + if (value === undefined || + (eq(value, objectProto[key]) && !hasOwnProperty.call(object, key))) { + object[key] = source[key]; + } + } + } + + return object; + }); /** - * This method is like `_.isEqual` except that it accepts `customizer` which - * is invoked to compare values. If `customizer` returns `undefined`, comparisons - * are handled by the method instead. The `customizer` is invoked with up to - * six arguments: (objValue, othValue [, index|key, object, other, stack]). + * This method is like `_.defaults` except that it recursively assigns + * default properties. + * + * **Note:** This method mutates `object`. * * @static * @memberOf _ - * @since 4.0.0 - * @category Lang - * @param {*} value The value to compare. - * @param {*} other The other value to compare. - * @param {Function} [customizer] The function to customize comparisons. - * @returns {boolean} Returns `true` if the values are equivalent, else `false`. + * @since 3.10.0 + * @category Object + * @param {Object} object The destination object. + * @param {...Object} [sources] The source objects. + * @returns {Object} Returns `object`. + * @see _.defaults * @example * - * function isGreeting(value) { - * return /^h(?:i|ello)$/.test(value); - * } - * - * function customizer(objValue, othValue) { - * if (isGreeting(objValue) && isGreeting(othValue)) { - * return true; - * } - * } - * - * var array = ['hello', 'goodbye']; - * var other = ['hi', 'goodbye']; - * - * _.isEqualWith(array, other, customizer); - * // => true + * _.defaultsDeep({ 'a': { 'b': 2 } }, { 'a': { 'b': 1, 'c': 3 } }); + * // => { 'a': { 'b': 2, 'c': 3 } } */ - function isEqualWith(value, other, customizer) { - customizer = typeof customizer == 'function' ? customizer : undefined; - var result = customizer ? customizer(value, other) : undefined; - return result === undefined ? baseIsEqual(value, other, undefined, customizer) : !!result; - } + var defaultsDeep = baseRest(function(args) { + args.push(undefined, customDefaultsMerge); + return apply(mergeWith, undefined, args); + }); /** - * Checks if `value` is an `Error`, `EvalError`, `RangeError`, `ReferenceError`, - * `SyntaxError`, `TypeError`, or `URIError` object. + * This method is like `_.find` except that it returns the key of the first + * element `predicate` returns truthy for instead of the element itself. * * @static * @memberOf _ - * @since 3.0.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is an error object, else `false`. + * @since 1.1.0 + * @category Object + * @param {Object} object The object to inspect. + * @param {Function} [predicate=_.identity] The function invoked per iteration. + * @returns {string|undefined} Returns the key of the matched element, + * else `undefined`. * @example * - * _.isError(new Error); - * // => true + * var users = { + * 'barney': { 'age': 36, 'active': true }, + * 'fred': { 'age': 40, 'active': false }, + * 'pebbles': { 'age': 1, 'active': true } + * }; * - * _.isError(Error); - * // => false + * _.findKey(users, function(o) { return o.age < 40; }); + * // => 'barney' (iteration order is not guaranteed) + * + * // The `_.matches` iteratee shorthand. + * _.findKey(users, { 'age': 1, 'active': true }); + * // => 'pebbles' + * + * // The `_.matchesProperty` iteratee shorthand. + * _.findKey(users, ['active', false]); + * // => 'fred' + * + * // The `_.property` iteratee shorthand. + * _.findKey(users, 'active'); + * // => 'barney' */ - function isError(value) { - if (!isObjectLike(value)) { - return false; - } - var tag = baseGetTag(value); - return tag == errorTag || tag == domExcTag || - (typeof value.message == 'string' && typeof value.name == 'string' && !isPlainObject(value)); + function findKey(object, predicate) { + return baseFindKey(object, getIteratee(predicate, 3), baseForOwn); } /** - * Checks if `value` is a finite primitive number. - * - * **Note:** This method is based on - * [`Number.isFinite`](https://mdn.io/Number/isFinite). + * This method is like `_.findKey` except that it iterates over elements of + * a collection in the opposite order. * * @static * @memberOf _ - * @since 0.1.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a finite number, else `false`. + * @since 2.0.0 + * @category Object + * @param {Object} object The object to inspect. + * @param {Function} [predicate=_.identity] The function invoked per iteration. + * @returns {string|undefined} Returns the key of the matched element, + * else `undefined`. * @example * - * _.isFinite(3); - * // => true + * var users = { + * 'barney': { 'age': 36, 'active': true }, + * 'fred': { 'age': 40, 'active': false }, + * 'pebbles': { 'age': 1, 'active': true } + * }; * - * _.isFinite(Number.MIN_VALUE); - * // => true + * _.findLastKey(users, function(o) { return o.age < 40; }); + * // => returns 'pebbles' assuming `_.findKey` returns 'barney' * - * _.isFinite(Infinity); - * // => false + * // The `_.matches` iteratee shorthand. + * _.findLastKey(users, { 'age': 36, 'active': true }); + * // => 'barney' * - * _.isFinite('3'); - * // => false + * // The `_.matchesProperty` iteratee shorthand. + * _.findLastKey(users, ['active', false]); + * // => 'fred' + * + * // The `_.property` iteratee shorthand. + * _.findLastKey(users, 'active'); + * // => 'pebbles' */ - function isFinite(value) { - return typeof value == 'number' && nativeIsFinite(value); + function findLastKey(object, predicate) { + return baseFindKey(object, getIteratee(predicate, 3), baseForOwnRight); } /** - * Checks if `value` is classified as a `Function` object. + * Iterates over own and inherited enumerable string keyed properties of an + * object and invokes `iteratee` for each property. The iteratee is invoked + * with three arguments: (value, key, object). Iteratee functions may exit + * iteration early by explicitly returning `false`. * * @static * @memberOf _ - * @since 0.1.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a function, else `false`. + * @since 0.3.0 + * @category Object + * @param {Object} object The object to iterate over. + * @param {Function} [iteratee=_.identity] The function invoked per iteration. + * @returns {Object} Returns `object`. + * @see _.forInRight * @example * - * _.isFunction(_); - * // => true + * function Foo() { + * this.a = 1; + * this.b = 2; + * } * - * _.isFunction(/abc/); - * // => false + * Foo.prototype.c = 3; + * + * _.forIn(new Foo, function(value, key) { + * console.log(key); + * }); + * // => Logs 'a', 'b', then 'c' (iteration order is not guaranteed). */ - function isFunction(value) { - if (!isObject(value)) { - return false; - } - // The use of `Object#toString` avoids issues with the `typeof` operator - // in Safari 9 which returns 'object' for typed arrays and other constructors. - var tag = baseGetTag(value); - return tag == funcTag || tag == genTag || tag == asyncTag || tag == proxyTag; + function forIn(object, iteratee) { + return object == null + ? object + : baseFor(object, getIteratee(iteratee, 3), keysIn); } /** - * Checks if `value` is an integer. - * - * **Note:** This method is based on - * [`Number.isInteger`](https://mdn.io/Number/isInteger). + * This method is like `_.forIn` except that it iterates over properties of + * `object` in the opposite order. * * @static * @memberOf _ - * @since 4.0.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is an integer, else `false`. + * @since 2.0.0 + * @category Object + * @param {Object} object The object to iterate over. + * @param {Function} [iteratee=_.identity] The function invoked per iteration. + * @returns {Object} Returns `object`. + * @see _.forIn * @example * - * _.isInteger(3); - * // => true - * - * _.isInteger(Number.MIN_VALUE); - * // => false + * function Foo() { + * this.a = 1; + * this.b = 2; + * } * - * _.isInteger(Infinity); - * // => false + * Foo.prototype.c = 3; * - * _.isInteger('3'); - * // => false + * _.forInRight(new Foo, function(value, key) { + * console.log(key); + * }); + * // => Logs 'c', 'b', then 'a' assuming `_.forIn` logs 'a', 'b', then 'c'. */ - function isInteger(value) { - return typeof value == 'number' && value == toInteger(value); + function forInRight(object, iteratee) { + return object == null + ? object + : baseForRight(object, getIteratee(iteratee, 3), keysIn); } /** - * Checks if `value` is a valid array-like length. - * - * **Note:** This method is loosely based on - * [`ToLength`](http://ecma-international.org/ecma-262/7.0/#sec-tolength). + * Iterates over own enumerable string keyed properties of an object and + * invokes `iteratee` for each property. The iteratee is invoked with three + * arguments: (value, key, object). Iteratee functions may exit iteration + * early by explicitly returning `false`. * * @static * @memberOf _ - * @since 4.0.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a valid length, else `false`. + * @since 0.3.0 + * @category Object + * @param {Object} object The object to iterate over. + * @param {Function} [iteratee=_.identity] The function invoked per iteration. + * @returns {Object} Returns `object`. + * @see _.forOwnRight * @example * - * _.isLength(3); - * // => true - * - * _.isLength(Number.MIN_VALUE); - * // => false + * function Foo() { + * this.a = 1; + * this.b = 2; + * } * - * _.isLength(Infinity); - * // => false + * Foo.prototype.c = 3; * - * _.isLength('3'); - * // => false + * _.forOwn(new Foo, function(value, key) { + * console.log(key); + * }); + * // => Logs 'a' then 'b' (iteration order is not guaranteed). */ - function isLength(value) { - return typeof value == 'number' && - value > -1 && value % 1 == 0 && value <= MAX_SAFE_INTEGER; + function forOwn(object, iteratee) { + return object && baseForOwn(object, getIteratee(iteratee, 3)); } /** - * Checks if `value` is the - * [language type](http://www.ecma-international.org/ecma-262/7.0/#sec-ecmascript-language-types) - * of `Object`. (e.g. arrays, functions, objects, regexes, `new Number(0)`, and `new String('')`) + * This method is like `_.forOwn` except that it iterates over properties of + * `object` in the opposite order. * * @static * @memberOf _ - * @since 0.1.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is an object, else `false`. + * @since 2.0.0 + * @category Object + * @param {Object} object The object to iterate over. + * @param {Function} [iteratee=_.identity] The function invoked per iteration. + * @returns {Object} Returns `object`. + * @see _.forOwn * @example * - * _.isObject({}); - * // => true - * - * _.isObject([1, 2, 3]); - * // => true + * function Foo() { + * this.a = 1; + * this.b = 2; + * } * - * _.isObject(_.noop); - * // => true + * Foo.prototype.c = 3; * - * _.isObject(null); - * // => false + * _.forOwnRight(new Foo, function(value, key) { + * console.log(key); + * }); + * // => Logs 'b' then 'a' assuming `_.forOwn` logs 'a' then 'b'. */ - function isObject(value) { - var type = typeof value; - return value != null && (type == 'object' || type == 'function'); + function forOwnRight(object, iteratee) { + return object && baseForOwnRight(object, getIteratee(iteratee, 3)); } /** - * Checks if `value` is object-like. A value is object-like if it's not `null` - * and has a `typeof` result of "object". + * Creates an array of function property names from own enumerable properties + * of `object`. * * @static + * @since 0.1.0 * @memberOf _ - * @since 4.0.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is object-like, else `false`. + * @category Object + * @param {Object} object The object to inspect. + * @returns {Array} Returns the function names. + * @see _.functionsIn * @example * - * _.isObjectLike({}); - * // => true - * - * _.isObjectLike([1, 2, 3]); - * // => true + * function Foo() { + * this.a = _.constant('a'); + * this.b = _.constant('b'); + * } * - * _.isObjectLike(_.noop); - * // => false + * Foo.prototype.c = _.constant('c'); * - * _.isObjectLike(null); - * // => false + * _.functions(new Foo); + * // => ['a', 'b'] */ - function isObjectLike(value) { - return value != null && typeof value == 'object'; + function functions(object) { + return object == null ? [] : baseFunctions(object, keys(object)); } /** - * Checks if `value` is classified as a `Map` object. + * Creates an array of function property names from own and inherited + * enumerable properties of `object`. * * @static * @memberOf _ - * @since 4.3.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a map, else `false`. + * @since 4.0.0 + * @category Object + * @param {Object} object The object to inspect. + * @returns {Array} Returns the function names. + * @see _.functions * @example * - * _.isMap(new Map); - * // => true + * function Foo() { + * this.a = _.constant('a'); + * this.b = _.constant('b'); + * } * - * _.isMap(new WeakMap); - * // => false + * Foo.prototype.c = _.constant('c'); + * + * _.functionsIn(new Foo); + * // => ['a', 'b', 'c'] */ - var isMap = nodeIsMap ? baseUnary(nodeIsMap) : baseIsMap; + function functionsIn(object) { + return object == null ? [] : baseFunctions(object, keysIn(object)); + } /** - * Performs a partial deep comparison between `object` and `source` to - * determine if `object` contains equivalent property values. - * - * **Note:** This method is equivalent to `_.matches` when `source` is - * partially applied. - * - * Partial comparisons will match empty array and empty object `source` - * values against any array or object value, respectively. See `_.isEqual` - * for a list of supported value comparisons. + * Gets the value at `path` of `object`. If the resolved value is + * `undefined`, the `defaultValue` is returned in its place. * * @static - * @memberOf _ - * @since 3.0.0 - * @category Lang - * @param {Object} object The object to inspect. - * @param {Object} source The object of property values to match. - * @returns {boolean} Returns `true` if `object` is a match, else `false`. + * @memberOf _ + * @since 3.7.0 + * @category Object + * @param {Object} object The object to query. + * @param {Array|string} path The path of the property to get. + * @param {*} [defaultValue] The value returned for `undefined` resolved values. + * @returns {*} Returns the resolved value. * @example * - * var object = { 'a': 1, 'b': 2 }; + * var object = { 'a': [{ 'b': { 'c': 3 } }] }; * - * _.isMatch(object, { 'b': 2 }); - * // => true + * _.get(object, 'a[0].b.c'); + * // => 3 * - * _.isMatch(object, { 'b': 1 }); - * // => false + * _.get(object, ['a', '0', 'b', 'c']); + * // => 3 + * + * _.get(object, 'a.b.c', 'default'); + * // => 'default' */ - function isMatch(object, source) { - return object === source || baseIsMatch(object, source, getMatchData(source)); + function get(object, path, defaultValue) { + var result = object == null ? undefined : baseGet(object, path); + return result === undefined ? defaultValue : result; } /** - * This method is like `_.isMatch` except that it accepts `customizer` which - * is invoked to compare values. If `customizer` returns `undefined`, comparisons - * are handled by the method instead. The `customizer` is invoked with five - * arguments: (objValue, srcValue, index|key, object, source). + * Checks if `path` is a direct property of `object`. * * @static + * @since 0.1.0 * @memberOf _ - * @since 4.0.0 - * @category Lang - * @param {Object} object The object to inspect. - * @param {Object} source The object of property values to match. - * @param {Function} [customizer] The function to customize comparisons. - * @returns {boolean} Returns `true` if `object` is a match, else `false`. + * @category Object + * @param {Object} object The object to query. + * @param {Array|string} path The path to check. + * @returns {boolean} Returns `true` if `path` exists, else `false`. * @example * - * function isGreeting(value) { - * return /^h(?:i|ello)$/.test(value); - * } + * var object = { 'a': { 'b': 2 } }; + * var other = _.create({ 'a': _.create({ 'b': 2 }) }); * - * function customizer(objValue, srcValue) { - * if (isGreeting(objValue) && isGreeting(srcValue)) { - * return true; - * } - * } + * _.has(object, 'a'); + * // => true * - * var object = { 'greeting': 'hello' }; - * var source = { 'greeting': 'hi' }; + * _.has(object, 'a.b'); + * // => true * - * _.isMatchWith(object, source, customizer); + * _.has(object, ['a', 'b']); * // => true + * + * _.has(other, 'a'); + * // => false */ - function isMatchWith(object, source, customizer) { - customizer = typeof customizer == 'function' ? customizer : undefined; - return baseIsMatch(object, source, getMatchData(source), customizer); + function has(object, path) { + return object != null && hasPath(object, path, baseHas); } /** - * Checks if `value` is `NaN`. - * - * **Note:** This method is based on - * [`Number.isNaN`](https://mdn.io/Number/isNaN) and is not the same as - * global [`isNaN`](https://mdn.io/isNaN) which returns `true` for - * `undefined` and other non-number values. + * Checks if `path` is a direct or inherited property of `object`. * * @static * @memberOf _ - * @since 0.1.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is `NaN`, else `false`. + * @since 4.0.0 + * @category Object + * @param {Object} object The object to query. + * @param {Array|string} path The path to check. + * @returns {boolean} Returns `true` if `path` exists, else `false`. * @example * - * _.isNaN(NaN); + * var object = _.create({ 'a': _.create({ 'b': 2 }) }); + * + * _.hasIn(object, 'a'); * // => true * - * _.isNaN(new Number(NaN)); + * _.hasIn(object, 'a.b'); * // => true * - * isNaN(undefined); + * _.hasIn(object, ['a', 'b']); * // => true * - * _.isNaN(undefined); + * _.hasIn(object, 'b'); * // => false */ - function isNaN(value) { - // An `NaN` primitive is the only value that is not equal to itself. - // Perform the `toStringTag` check first to avoid errors with some - // ActiveX objects in IE. - return isNumber(value) && value != +value; + function hasIn(object, path) { + return object != null && hasPath(object, path, baseHasIn); } /** - * Checks if `value` is a pristine native function. - * - * **Note:** This method can't reliably detect native functions in the presence - * of the core-js package because core-js circumvents this kind of detection. - * Despite multiple requests, the core-js maintainer has made it clear: any - * attempt to fix the detection will be obstructed. As a result, we're left - * with little choice but to throw an error. Unfortunately, this also affects - * packages, like [babel-polyfill](https://www.npmjs.com/package/babel-polyfill), - * which rely on core-js. + * Creates an object composed of the inverted keys and values of `object`. + * If `object` contains duplicate values, subsequent values overwrite + * property assignments of previous values. * * @static * @memberOf _ - * @since 3.0.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a native function, - * else `false`. + * @since 0.7.0 + * @category Object + * @param {Object} object The object to invert. + * @returns {Object} Returns the new inverted object. * @example * - * _.isNative(Array.prototype.push); - * // => true + * var object = { 'a': 1, 'b': 2, 'c': 1 }; * - * _.isNative(_); - * // => false + * _.invert(object); + * // => { '1': 'c', '2': 'b' } */ - function isNative(value) { - if (isMaskable(value)) { - throw new Error(CORE_ERROR_TEXT); + var invert = createInverter(function(result, value, key) { + if (value != null && + typeof value.toString != 'function') { + value = nativeObjectToString.call(value); } - return baseIsNative(value); - } + + result[value] = key; + }, constant(identity)); /** - * Checks if `value` is `null`. + * This method is like `_.invert` except that the inverted object is generated + * from the results of running each element of `object` thru `iteratee`. The + * corresponding inverted value of each inverted key is an array of keys + * responsible for generating the inverted value. The iteratee is invoked + * with one argument: (value). * * @static * @memberOf _ - * @since 0.1.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is `null`, else `false`. + * @since 4.1.0 + * @category Object + * @param {Object} object The object to invert. + * @param {Function} [iteratee=_.identity] The iteratee invoked per element. + * @returns {Object} Returns the new inverted object. * @example * - * _.isNull(null); - * // => true + * var object = { 'a': 1, 'b': 2, 'c': 1 }; * - * _.isNull(void 0); - * // => false + * _.invertBy(object); + * // => { '1': ['a', 'c'], '2': ['b'] } + * + * _.invertBy(object, function(value) { + * return 'group' + value; + * }); + * // => { 'group1': ['a', 'c'], 'group2': ['b'] } */ - function isNull(value) { - return value === null; - } + var invertBy = createInverter(function(result, value, key) { + if (value != null && + typeof value.toString != 'function') { + value = nativeObjectToString.call(value); + } + + if (hasOwnProperty.call(result, value)) { + result[value].push(key); + } else { + result[value] = [key]; + } + }, getIteratee); /** - * Checks if `value` is `null` or `undefined`. + * Invokes the method at `path` of `object`. * * @static * @memberOf _ * @since 4.0.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is nullish, else `false`. + * @category Object + * @param {Object} object The object to query. + * @param {Array|string} path The path of the method to invoke. + * @param {...*} [args] The arguments to invoke the method with. + * @returns {*} Returns the result of the invoked method. * @example * - * _.isNil(null); - * // => true - * - * _.isNil(void 0); - * // => true + * var object = { 'a': [{ 'b': { 'c': [1, 2, 3, 4] } }] }; * - * _.isNil(NaN); - * // => false + * _.invoke(object, 'a[0].b.c.slice', 1, 3); + * // => [2, 3] */ - function isNil(value) { - return value == null; - } + var invoke = baseRest(baseInvoke); /** - * Checks if `value` is classified as a `Number` primitive or object. + * Creates an array of the own enumerable property names of `object`. * - * **Note:** To exclude `Infinity`, `-Infinity`, and `NaN`, which are - * classified as numbers, use the `_.isFinite` method. + * **Note:** Non-object values are coerced to objects. See the + * [ES spec](http://ecma-international.org/ecma-262/7.0/#sec-object.keys) + * for more details. * * @static - * @memberOf _ * @since 0.1.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a number, else `false`. + * @memberOf _ + * @category Object + * @param {Object} object The object to query. + * @returns {Array} Returns the array of property names. * @example * - * _.isNumber(3); - * // => true + * function Foo() { + * this.a = 1; + * this.b = 2; + * } * - * _.isNumber(Number.MIN_VALUE); - * // => true + * Foo.prototype.c = 3; * - * _.isNumber(Infinity); - * // => true + * _.keys(new Foo); + * // => ['a', 'b'] (iteration order is not guaranteed) * - * _.isNumber('3'); - * // => false + * _.keys('hi'); + * // => ['0', '1'] */ - function isNumber(value) { - return typeof value == 'number' || - (isObjectLike(value) && baseGetTag(value) == numberTag); + function keys(object) { + return isArrayLike(object) ? arrayLikeKeys(object) : baseKeys(object); } /** - * Checks if `value` is a plain object, that is, an object created by the - * `Object` constructor or one with a `[[Prototype]]` of `null`. + * Creates an array of the own and inherited enumerable property names of `object`. + * + * **Note:** Non-object values are coerced to objects. * * @static * @memberOf _ - * @since 0.8.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a plain object, else `false`. + * @since 3.0.0 + * @category Object + * @param {Object} object The object to query. + * @returns {Array} Returns the array of property names. * @example * * function Foo() { * this.a = 1; + * this.b = 2; * } * - * _.isPlainObject(new Foo); - * // => false + * Foo.prototype.c = 3; * - * _.isPlainObject([1, 2, 3]); - * // => false + * _.keysIn(new Foo); + * // => ['a', 'b', 'c'] (iteration order is not guaranteed) + */ + function keysIn(object) { + return isArrayLike(object) ? arrayLikeKeys(object, true) : baseKeysIn(object); + } + + /** + * The opposite of `_.mapValues`; this method creates an object with the + * same values as `object` and keys generated by running each own enumerable + * string keyed property of `object` thru `iteratee`. The iteratee is invoked + * with three arguments: (value, key, object). * - * _.isPlainObject({ 'x': 0, 'y': 0 }); - * // => true + * @static + * @memberOf _ + * @since 3.8.0 + * @category Object + * @param {Object} object The object to iterate over. + * @param {Function} [iteratee=_.identity] The function invoked per iteration. + * @returns {Object} Returns the new mapped object. + * @see _.mapValues + * @example * - * _.isPlainObject(Object.create(null)); - * // => true + * _.mapKeys({ 'a': 1, 'b': 2 }, function(value, key) { + * return key + value; + * }); + * // => { 'a1': 1, 'b2': 2 } */ - function isPlainObject(value) { - if (!isObjectLike(value) || baseGetTag(value) != objectTag) { - return false; - } - var proto = getPrototype(value); - if (proto === null) { - return true; - } - var Ctor = hasOwnProperty.call(proto, 'constructor') && proto.constructor; - return typeof Ctor == 'function' && Ctor instanceof Ctor && - funcToString.call(Ctor) == objectCtorString; + function mapKeys(object, iteratee) { + var result = {}; + iteratee = getIteratee(iteratee, 3); + + baseForOwn(object, function(value, key, object) { + baseAssignValue(result, iteratee(value, key, object), value); + }); + return result; } /** - * Checks if `value` is classified as a `RegExp` object. + * Creates an object with the same keys as `object` and values generated + * by running each own enumerable string keyed property of `object` thru + * `iteratee`. The iteratee is invoked with three arguments: + * (value, key, object). * * @static * @memberOf _ - * @since 0.1.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a regexp, else `false`. + * @since 2.4.0 + * @category Object + * @param {Object} object The object to iterate over. + * @param {Function} [iteratee=_.identity] The function invoked per iteration. + * @returns {Object} Returns the new mapped object. + * @see _.mapKeys * @example * - * _.isRegExp(/abc/); - * // => true + * var users = { + * 'fred': { 'user': 'fred', 'age': 40 }, + * 'pebbles': { 'user': 'pebbles', 'age': 1 } + * }; * - * _.isRegExp('/abc/'); - * // => false + * _.mapValues(users, function(o) { return o.age; }); + * // => { 'fred': 40, 'pebbles': 1 } (iteration order is not guaranteed) + * + * // The `_.property` iteratee shorthand. + * _.mapValues(users, 'age'); + * // => { 'fred': 40, 'pebbles': 1 } (iteration order is not guaranteed) */ - var isRegExp = nodeIsRegExp ? baseUnary(nodeIsRegExp) : baseIsRegExp; + function mapValues(object, iteratee) { + var result = {}; + iteratee = getIteratee(iteratee, 3); + + baseForOwn(object, function(value, key, object) { + baseAssignValue(result, key, iteratee(value, key, object)); + }); + return result; + } /** - * Checks if `value` is a safe integer. An integer is safe if it's an IEEE-754 - * double precision number which isn't the result of a rounded unsafe integer. + * This method is like `_.assign` except that it recursively merges own and + * inherited enumerable string keyed properties of source objects into the + * destination object. Source properties that resolve to `undefined` are + * skipped if a destination value exists. Array and plain object properties + * are merged recursively. Other objects and value types are overridden by + * assignment. Source objects are applied from left to right. Subsequent + * sources overwrite property assignments of previous sources. * - * **Note:** This method is based on - * [`Number.isSafeInteger`](https://mdn.io/Number/isSafeInteger). + * **Note:** This method mutates `object`. * * @static * @memberOf _ - * @since 4.0.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a safe integer, else `false`. + * @since 0.5.0 + * @category Object + * @param {Object} object The destination object. + * @param {...Object} [sources] The source objects. + * @returns {Object} Returns `object`. * @example * - * _.isSafeInteger(3); - * // => true - * - * _.isSafeInteger(Number.MIN_VALUE); - * // => false + * var object = { + * 'a': [{ 'b': 2 }, { 'd': 4 }] + * }; * - * _.isSafeInteger(Infinity); - * // => false + * var other = { + * 'a': [{ 'c': 3 }, { 'e': 5 }] + * }; * - * _.isSafeInteger('3'); - * // => false + * _.merge(object, other); + * // => { 'a': [{ 'b': 2, 'c': 3 }, { 'd': 4, 'e': 5 }] } */ - function isSafeInteger(value) { - return isInteger(value) && value >= -MAX_SAFE_INTEGER && value <= MAX_SAFE_INTEGER; - } + var merge = createAssigner(function(object, source, srcIndex) { + baseMerge(object, source, srcIndex); + }); /** - * Checks if `value` is classified as a `Set` object. + * This method is like `_.merge` except that it accepts `customizer` which + * is invoked to produce the merged values of the destination and source + * properties. If `customizer` returns `undefined`, merging is handled by the + * method instead. The `customizer` is invoked with six arguments: + * (objValue, srcValue, key, object, source, stack). + * + * **Note:** This method mutates `object`. * * @static * @memberOf _ - * @since 4.3.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a set, else `false`. + * @since 4.0.0 + * @category Object + * @param {Object} object The destination object. + * @param {...Object} sources The source objects. + * @param {Function} customizer The function to customize assigned values. + * @returns {Object} Returns `object`. * @example * - * _.isSet(new Set); - * // => true + * function customizer(objValue, srcValue) { + * if (_.isArray(objValue)) { + * return objValue.concat(srcValue); + * } + * } * - * _.isSet(new WeakSet); - * // => false + * var object = { 'a': [1], 'b': [2] }; + * var other = { 'a': [3], 'b': [4] }; + * + * _.mergeWith(object, other, customizer); + * // => { 'a': [1, 3], 'b': [2, 4] } */ - var isSet = nodeIsSet ? baseUnary(nodeIsSet) : baseIsSet; + var mergeWith = createAssigner(function(object, source, srcIndex, customizer) { + baseMerge(object, source, srcIndex, customizer); + }); /** - * Checks if `value` is classified as a `String` primitive or object. + * The opposite of `_.pick`; this method creates an object composed of the + * own and inherited enumerable property paths of `object` that are not omitted. + * + * **Note:** This method is considerably slower than `_.pick`. * * @static * @since 0.1.0 * @memberOf _ - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a string, else `false`. + * @category Object + * @param {Object} object The source object. + * @param {...(string|string[])} [paths] The property paths to omit. + * @returns {Object} Returns the new object. * @example * - * _.isString('abc'); - * // => true + * var object = { 'a': 1, 'b': '2', 'c': 3 }; * - * _.isString(1); - * // => false + * _.omit(object, ['a', 'c']); + * // => { 'b': '2' } */ - function isString(value) { - return typeof value == 'string' || - (!isArray(value) && isObjectLike(value) && baseGetTag(value) == stringTag); - } + var omit = flatRest(function(object, paths) { + var result = {}; + if (object == null) { + return result; + } + var isDeep = false; + paths = arrayMap(paths, function(path) { + path = castPath(path, object); + isDeep || (isDeep = path.length > 1); + return path; + }); + copyObject(object, getAllKeysIn(object), result); + if (isDeep) { + result = baseClone(result, CLONE_DEEP_FLAG | CLONE_FLAT_FLAG | CLONE_SYMBOLS_FLAG, customOmitClone); + } + var length = paths.length; + while (length--) { + baseUnset(result, paths[length]); + } + return result; + }); /** - * Checks if `value` is classified as a `Symbol` primitive or object. + * The opposite of `_.pickBy`; this method creates an object composed of + * the own and inherited enumerable string keyed properties of `object` that + * `predicate` doesn't return truthy for. The predicate is invoked with two + * arguments: (value, key). * * @static * @memberOf _ * @since 4.0.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a symbol, else `false`. + * @category Object + * @param {Object} object The source object. + * @param {Function} [predicate=_.identity] The function invoked per property. + * @returns {Object} Returns the new object. * @example * - * _.isSymbol(Symbol.iterator); - * // => true + * var object = { 'a': 1, 'b': '2', 'c': 3 }; * - * _.isSymbol('abc'); - * // => false + * _.omitBy(object, _.isNumber); + * // => { 'b': '2' } */ - function isSymbol(value) { - return typeof value == 'symbol' || - (isObjectLike(value) && baseGetTag(value) == symbolTag); + function omitBy(object, predicate) { + return pickBy(object, negate(getIteratee(predicate))); } /** - * Checks if `value` is classified as a typed array. + * Creates an object composed of the picked `object` properties. + * + * @static + * @since 0.1.0 + * @memberOf _ + * @category Object + * @param {Object} object The source object. + * @param {...(string|string[])} [paths] The property paths to pick. + * @returns {Object} Returns the new object. + * @example + * + * var object = { 'a': 1, 'b': '2', 'c': 3 }; + * + * _.pick(object, ['a', 'c']); + * // => { 'a': 1, 'c': 3 } + */ + var pick = flatRest(function(object, paths) { + return object == null ? {} : basePick(object, paths); + }); + + /** + * Creates an object composed of the `object` properties `predicate` returns + * truthy for. The predicate is invoked with two arguments: (value, key). * * @static * @memberOf _ - * @since 3.0.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a typed array, else `false`. + * @since 4.0.0 + * @category Object + * @param {Object} object The source object. + * @param {Function} [predicate=_.identity] The function invoked per property. + * @returns {Object} Returns the new object. * @example * - * _.isTypedArray(new Uint8Array); - * // => true + * var object = { 'a': 1, 'b': '2', 'c': 3 }; * - * _.isTypedArray([]); - * // => false + * _.pickBy(object, _.isNumber); + * // => { 'a': 1, 'c': 3 } */ - var isTypedArray = nodeIsTypedArray ? baseUnary(nodeIsTypedArray) : baseIsTypedArray; + function pickBy(object, predicate) { + if (object == null) { + return {}; + } + var props = arrayMap(getAllKeysIn(object), function(prop) { + return [prop]; + }); + predicate = getIteratee(predicate); + return basePickBy(object, props, function(value, path) { + return predicate(value, path[0]); + }); + } /** - * Checks if `value` is `undefined`. + * This method is like `_.get` except that if the resolved value is a + * function it's invoked with the `this` binding of its parent object and + * its result is returned. * * @static * @since 0.1.0 * @memberOf _ - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is `undefined`, else `false`. + * @category Object + * @param {Object} object The object to query. + * @param {Array|string} path The path of the property to resolve. + * @param {*} [defaultValue] The value returned for `undefined` resolved values. + * @returns {*} Returns the resolved value. * @example * - * _.isUndefined(void 0); - * // => true + * var object = { 'a': [{ 'b': { 'c1': 3, 'c2': _.constant(4) } }] }; * - * _.isUndefined(null); - * // => false + * _.result(object, 'a[0].b.c1'); + * // => 3 + * + * _.result(object, 'a[0].b.c2'); + * // => 4 + * + * _.result(object, 'a[0].b.c3', 'default'); + * // => 'default' + * + * _.result(object, 'a[0].b.c3', _.constant('default')); + * // => 'default' */ - function isUndefined(value) { - return value === undefined; + function result(object, path, defaultValue) { + path = castPath(path, object); + + var index = -1, + length = path.length; + + // Ensure the loop is entered when path is empty. + if (!length) { + length = 1; + object = undefined; + } + while (++index < length) { + var value = object == null ? undefined : object[toKey(path[index])]; + if (value === undefined) { + index = length; + value = defaultValue; + } + object = isFunction(value) ? value.call(object) : value; + } + return object; } /** - * Checks if `value` is classified as a `WeakMap` object. + * Sets the value at `path` of `object`. If a portion of `path` doesn't exist, + * it's created. Arrays are created for missing index properties while objects + * are created for all other missing properties. Use `_.setWith` to customize + * `path` creation. + * + * **Note:** This method mutates `object`. * * @static * @memberOf _ - * @since 4.3.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a weak map, else `false`. + * @since 3.7.0 + * @category Object + * @param {Object} object The object to modify. + * @param {Array|string} path The path of the property to set. + * @param {*} value The value to set. + * @returns {Object} Returns `object`. * @example * - * _.isWeakMap(new WeakMap); - * // => true + * var object = { 'a': [{ 'b': { 'c': 3 } }] }; * - * _.isWeakMap(new Map); - * // => false + * _.set(object, 'a[0].b.c', 4); + * console.log(object.a[0].b.c); + * // => 4 + * + * _.set(object, ['x', '0', 'y', 'z'], 5); + * console.log(object.x[0].y.z); + * // => 5 */ - function isWeakMap(value) { - return isObjectLike(value) && getTag(value) == weakMapTag; + function set(object, path, value) { + return object == null ? object : baseSet(object, path, value); } /** - * Checks if `value` is classified as a `WeakSet` object. + * This method is like `_.set` except that it accepts `customizer` which is + * invoked to produce the objects of `path`. If `customizer` returns `undefined` + * path creation is handled by the method instead. The `customizer` is invoked + * with three arguments: (nsValue, key, nsObject). + * + * **Note:** This method mutates `object`. * * @static * @memberOf _ - * @since 4.3.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a weak set, else `false`. + * @since 4.0.0 + * @category Object + * @param {Object} object The object to modify. + * @param {Array|string} path The path of the property to set. + * @param {*} value The value to set. + * @param {Function} [customizer] The function to customize assigned values. + * @returns {Object} Returns `object`. * @example * - * _.isWeakSet(new WeakSet); - * // => true + * var object = {}; * - * _.isWeakSet(new Set); - * // => false + * _.setWith(object, '[0][1]', 'a', Object); + * // => { '0': { '1': 'a' } } */ - function isWeakSet(value) { - return isObjectLike(value) && baseGetTag(value) == weakSetTag; + function setWith(object, path, value, customizer) { + customizer = typeof customizer == 'function' ? customizer : undefined; + return object == null ? object : baseSet(object, path, value, customizer); } /** - * Checks if `value` is less than `other`. + * Creates an array of own enumerable string keyed-value pairs for `object` + * which can be consumed by `_.fromPairs`. If `object` is a map or set, its + * entries are returned. * * @static * @memberOf _ - * @since 3.9.0 - * @category Lang - * @param {*} value The value to compare. - * @param {*} other The other value to compare. - * @returns {boolean} Returns `true` if `value` is less than `other`, - * else `false`. - * @see _.gt + * @since 4.0.0 + * @alias entries + * @category Object + * @param {Object} object The object to query. + * @returns {Array} Returns the key-value pairs. * @example * - * _.lt(1, 3); - * // => true + * function Foo() { + * this.a = 1; + * this.b = 2; + * } * - * _.lt(3, 3); - * // => false + * Foo.prototype.c = 3; * - * _.lt(3, 1); - * // => false + * _.toPairs(new Foo); + * // => [['a', 1], ['b', 2]] (iteration order is not guaranteed) */ - var lt = createRelationalOperation(baseLt); + var toPairs = createToPairs(keys); /** - * Checks if `value` is less than or equal to `other`. + * Creates an array of own and inherited enumerable string keyed-value pairs + * for `object` which can be consumed by `_.fromPairs`. If `object` is a map + * or set, its entries are returned. * * @static * @memberOf _ - * @since 3.9.0 - * @category Lang - * @param {*} value The value to compare. - * @param {*} other The other value to compare. - * @returns {boolean} Returns `true` if `value` is less than or equal to - * `other`, else `false`. - * @see _.gte + * @since 4.0.0 + * @alias entriesIn + * @category Object + * @param {Object} object The object to query. + * @returns {Array} Returns the key-value pairs. * @example * - * _.lte(1, 3); - * // => true + * function Foo() { + * this.a = 1; + * this.b = 2; + * } * - * _.lte(3, 3); - * // => true + * Foo.prototype.c = 3; * - * _.lte(3, 1); - * // => false + * _.toPairsIn(new Foo); + * // => [['a', 1], ['b', 2], ['c', 3]] (iteration order is not guaranteed) */ - var lte = createRelationalOperation(function(value, other) { - return value <= other; - }); + var toPairsIn = createToPairs(keysIn); /** - * Converts `value` to an array. + * An alternative to `_.reduce`; this method transforms `object` to a new + * `accumulator` object which is the result of running each of its own + * enumerable string keyed properties thru `iteratee`, with each invocation + * potentially mutating the `accumulator` object. If `accumulator` is not + * provided, a new object with the same `[[Prototype]]` will be used. The + * iteratee is invoked with four arguments: (accumulator, value, key, object). + * Iteratee functions may exit iteration early by explicitly returning `false`. * * @static - * @since 0.1.0 * @memberOf _ - * @category Lang - * @param {*} value The value to convert. - * @returns {Array} Returns the converted array. + * @since 1.3.0 + * @category Object + * @param {Object} object The object to iterate over. + * @param {Function} [iteratee=_.identity] The function invoked per iteration. + * @param {*} [accumulator] The custom accumulator value. + * @returns {*} Returns the accumulated value. * @example * - * _.toArray({ 'a': 1, 'b': 2 }); - * // => [1, 2] - * - * _.toArray('abc'); - * // => ['a', 'b', 'c'] - * - * _.toArray(1); - * // => [] + * _.transform([2, 3, 4], function(result, n) { + * result.push(n *= n); + * return n % 2 == 0; + * }, []); + * // => [4, 9] * - * _.toArray(null); - * // => [] + * _.transform({ 'a': 1, 'b': 2, 'c': 1 }, function(result, value, key) { + * (result[value] || (result[value] = [])).push(key); + * }, {}); + * // => { '1': ['a', 'c'], '2': ['b'] } */ - function toArray(value) { - if (!value) { - return []; - } - if (isArrayLike(value)) { - return isString(value) ? stringToArray(value) : copyArray(value); - } - if (symIterator && value[symIterator]) { - return iteratorToArray(value[symIterator]()); - } - var tag = getTag(value), - func = tag == mapTag ? mapToArray : (tag == setTag ? setToArray : values); + function transform(object, iteratee, accumulator) { + var isArr = isArray(object), + isArrLike = isArr || isBuffer(object) || isTypedArray(object); - return func(value); + iteratee = getIteratee(iteratee, 4); + if (accumulator == null) { + var Ctor = object && object.constructor; + if (isArrLike) { + accumulator = isArr ? new Ctor : []; + } + else if (isObject(object)) { + accumulator = isFunction(Ctor) ? baseCreate(getPrototype(object)) : {}; + } + else { + accumulator = {}; + } + } + (isArrLike ? arrayEach : baseForOwn)(object, function(value, index, object) { + return iteratee(accumulator, value, index, object); + }); + return accumulator; } /** - * Converts `value` to a finite number. + * Removes the property at `path` of `object`. + * + * **Note:** This method mutates `object`. * * @static * @memberOf _ - * @since 4.12.0 - * @category Lang - * @param {*} value The value to convert. - * @returns {number} Returns the converted number. + * @since 4.0.0 + * @category Object + * @param {Object} object The object to modify. + * @param {Array|string} path The path of the property to unset. + * @returns {boolean} Returns `true` if the property is deleted, else `false`. * @example * - * _.toFinite(3.2); - * // => 3.2 + * var object = { 'a': [{ 'b': { 'c': 7 } }] }; + * _.unset(object, 'a[0].b.c'); + * // => true * - * _.toFinite(Number.MIN_VALUE); - * // => 5e-324 + * console.log(object); + * // => { 'a': [{ 'b': {} }] }; * - * _.toFinite(Infinity); - * // => 1.7976931348623157e+308 + * _.unset(object, ['a', '0', 'b', 'c']); + * // => true * - * _.toFinite('3.2'); - * // => 3.2 + * console.log(object); + * // => { 'a': [{ 'b': {} }] }; */ - function toFinite(value) { - if (!value) { - return value === 0 ? value : 0; - } - value = toNumber(value); - if (value === INFINITY || value === -INFINITY) { - var sign = (value < 0 ? -1 : 1); - return sign * MAX_INTEGER; - } - return value === value ? value : 0; + function unset(object, path) { + return object == null ? true : baseUnset(object, path); } /** - * Converts `value` to an integer. + * This method is like `_.set` except that accepts `updater` to produce the + * value to set. Use `_.updateWith` to customize `path` creation. The `updater` + * is invoked with one argument: (value). * - * **Note:** This method is loosely based on - * [`ToInteger`](http://www.ecma-international.org/ecma-262/7.0/#sec-tointeger). + * **Note:** This method mutates `object`. * * @static * @memberOf _ - * @since 4.0.0 - * @category Lang - * @param {*} value The value to convert. - * @returns {number} Returns the converted integer. + * @since 4.6.0 + * @category Object + * @param {Object} object The object to modify. + * @param {Array|string} path The path of the property to set. + * @param {Function} updater The function to produce the updated value. + * @returns {Object} Returns `object`. * @example * - * _.toInteger(3.2); - * // => 3 - * - * _.toInteger(Number.MIN_VALUE); - * // => 0 + * var object = { 'a': [{ 'b': { 'c': 3 } }] }; * - * _.toInteger(Infinity); - * // => 1.7976931348623157e+308 + * _.update(object, 'a[0].b.c', function(n) { return n * n; }); + * console.log(object.a[0].b.c); + * // => 9 * - * _.toInteger('3.2'); - * // => 3 + * _.update(object, 'x[0].y.z', function(n) { return n ? n + 1 : 0; }); + * console.log(object.x[0].y.z); + * // => 0 */ - function toInteger(value) { - var result = toFinite(value), - remainder = result % 1; - - return result === result ? (remainder ? result - remainder : result) : 0; + function update(object, path, updater) { + return object == null ? object : baseUpdate(object, path, castFunction(updater)); } /** - * Converts `value` to an integer suitable for use as the length of an - * array-like object. + * This method is like `_.update` except that it accepts `customizer` which is + * invoked to produce the objects of `path`. If `customizer` returns `undefined` + * path creation is handled by the method instead. The `customizer` is invoked + * with three arguments: (nsValue, key, nsObject). * - * **Note:** This method is based on - * [`ToLength`](http://ecma-international.org/ecma-262/7.0/#sec-tolength). + * **Note:** This method mutates `object`. * * @static * @memberOf _ - * @since 4.0.0 - * @category Lang - * @param {*} value The value to convert. - * @returns {number} Returns the converted integer. + * @since 4.6.0 + * @category Object + * @param {Object} object The object to modify. + * @param {Array|string} path The path of the property to set. + * @param {Function} updater The function to produce the updated value. + * @param {Function} [customizer] The function to customize assigned values. + * @returns {Object} Returns `object`. * @example * - * _.toLength(3.2); - * // => 3 - * - * _.toLength(Number.MIN_VALUE); - * // => 0 - * - * _.toLength(Infinity); - * // => 4294967295 + * var object = {}; * - * _.toLength('3.2'); - * // => 3 + * _.updateWith(object, '[0][1]', _.constant('a'), Object); + * // => { '0': { '1': 'a' } } */ - function toLength(value) { - return value ? baseClamp(toInteger(value), 0, MAX_ARRAY_LENGTH) : 0; + function updateWith(object, path, updater, customizer) { + customizer = typeof customizer == 'function' ? customizer : undefined; + return object == null ? object : baseUpdate(object, path, castFunction(updater), customizer); } /** - * Converts `value` to a number. + * Creates an array of the own enumerable string keyed property values of `object`. + * + * **Note:** Non-object values are coerced to objects. * * @static + * @since 0.1.0 * @memberOf _ - * @since 4.0.0 - * @category Lang - * @param {*} value The value to process. - * @returns {number} Returns the number. + * @category Object + * @param {Object} object The object to query. + * @returns {Array} Returns the array of property values. * @example * - * _.toNumber(3.2); - * // => 3.2 + * function Foo() { + * this.a = 1; + * this.b = 2; + * } * - * _.toNumber(Number.MIN_VALUE); - * // => 5e-324 + * Foo.prototype.c = 3; * - * _.toNumber(Infinity); - * // => Infinity + * _.values(new Foo); + * // => [1, 2] (iteration order is not guaranteed) * - * _.toNumber('3.2'); - * // => 3.2 + * _.values('hi'); + * // => ['h', 'i'] */ - function toNumber(value) { - if (typeof value == 'number') { - return value; - } - if (isSymbol(value)) { - return NAN; - } - if (isObject(value)) { - var other = typeof value.valueOf == 'function' ? value.valueOf() : value; - value = isObject(other) ? (other + '') : other; - } - if (typeof value != 'string') { - return value === 0 ? value : +value; - } - value = baseTrim(value); - var isBinary = reIsBinary.test(value); - return (isBinary || reIsOctal.test(value)) - ? freeParseInt(value.slice(2), isBinary ? 2 : 8) - : (reIsBadHex.test(value) ? NAN : +value); + function values(object) { + return object == null ? [] : baseValues(object, keys(object)); } /** - * Converts `value` to a plain object flattening inherited enumerable string - * keyed properties of `value` to own properties of the plain object. + * Creates an array of the own and inherited enumerable string keyed property + * values of `object`. + * + * **Note:** Non-object values are coerced to objects. * * @static * @memberOf _ * @since 3.0.0 - * @category Lang - * @param {*} value The value to convert. - * @returns {Object} Returns the converted plain object. + * @category Object + * @param {Object} object The object to query. + * @returns {Array} Returns the array of property values. * @example * * function Foo() { + * this.a = 1; * this.b = 2; * } * * Foo.prototype.c = 3; * - * _.assign({ 'a': 1 }, new Foo); - * // => { 'a': 1, 'b': 2 } + * _.valuesIn(new Foo); + * // => [1, 2, 3] (iteration order is not guaranteed) + */ + function valuesIn(object) { + return object == null ? [] : baseValues(object, keysIn(object)); + } + + /*------------------------------------------------------------------------*/ + + /** + * Clamps `number` within the inclusive `lower` and `upper` bounds. * - * _.assign({ 'a': 1 }, _.toPlainObject(new Foo)); - * // => { 'a': 1, 'b': 2, 'c': 3 } + * @static + * @memberOf _ + * @since 4.0.0 + * @category Number + * @param {number} number The number to clamp. + * @param {number} [lower] The lower bound. + * @param {number} upper The upper bound. + * @returns {number} Returns the clamped number. + * @example + * + * _.clamp(-10, -5, 5); + * // => -5 + * + * _.clamp(10, -5, 5); + * // => 5 */ - function toPlainObject(value) { - return copyObject(value, keysIn(value)); + function clamp(number, lower, upper) { + if (upper === undefined) { + upper = lower; + lower = undefined; + } + if (upper !== undefined) { + upper = toNumber(upper); + upper = upper === upper ? upper : 0; + } + if (lower !== undefined) { + lower = toNumber(lower); + lower = lower === lower ? lower : 0; + } + return baseClamp(toNumber(number), lower, upper); } /** - * Converts `value` to a safe integer. A safe integer can be compared and - * represented correctly. + * Checks if `n` is between `start` and up to, but not including, `end`. If + * `end` is not specified, it's set to `start` with `start` then set to `0`. + * If `start` is greater than `end` the params are swapped to support + * negative ranges. * * @static * @memberOf _ - * @since 4.0.0 - * @category Lang - * @param {*} value The value to convert. - * @returns {number} Returns the converted integer. + * @since 3.3.0 + * @category Number + * @param {number} number The number to check. + * @param {number} [start=0] The start of the range. + * @param {number} end The end of the range. + * @returns {boolean} Returns `true` if `number` is in the range, else `false`. + * @see _.range, _.rangeRight * @example * - * _.toSafeInteger(3.2); - * // => 3 + * _.inRange(3, 2, 4); + * // => true * - * _.toSafeInteger(Number.MIN_VALUE); - * // => 0 + * _.inRange(4, 8); + * // => true * - * _.toSafeInteger(Infinity); - * // => 9007199254740991 + * _.inRange(4, 2); + * // => false * - * _.toSafeInteger('3.2'); - * // => 3 + * _.inRange(2, 2); + * // => false + * + * _.inRange(1.2, 2); + * // => true + * + * _.inRange(5.2, 4); + * // => false + * + * _.inRange(-3, -2, -6); + * // => true */ - function toSafeInteger(value) { - return value - ? baseClamp(toInteger(value), -MAX_SAFE_INTEGER, MAX_SAFE_INTEGER) - : (value === 0 ? value : 0); + function inRange(number, start, end) { + start = toFinite(start); + if (end === undefined) { + end = start; + start = 0; + } else { + end = toFinite(end); + } + number = toNumber(number); + return baseInRange(number, start, end); } /** - * Converts `value` to a string. An empty string is returned for `null` - * and `undefined` values. The sign of `-0` is preserved. + * Produces a random number between the inclusive `lower` and `upper` bounds. + * If only one argument is provided a number between `0` and the given number + * is returned. If `floating` is `true`, or either `lower` or `upper` are + * floats, a floating-point number is returned instead of an integer. + * + * **Note:** JavaScript follows the IEEE-754 standard for resolving + * floating-point values which can produce unexpected results. * * @static * @memberOf _ - * @since 4.0.0 - * @category Lang - * @param {*} value The value to convert. - * @returns {string} Returns the converted string. + * @since 0.7.0 + * @category Number + * @param {number} [lower=0] The lower bound. + * @param {number} [upper=1] The upper bound. + * @param {boolean} [floating] Specify returning a floating-point number. + * @returns {number} Returns the random number. * @example * - * _.toString(null); - * // => '' + * _.random(0, 5); + * // => an integer between 0 and 5 * - * _.toString(-0); - * // => '-0' + * _.random(5); + * // => also an integer between 0 and 5 * - * _.toString([1, 2, 3]); - * // => '1,2,3' + * _.random(5, true); + * // => a floating-point number between 0 and 5 + * + * _.random(1.2, 5.2); + * // => a floating-point number between 1.2 and 5.2 */ - function toString(value) { - return value == null ? '' : baseToString(value); + function random(lower, upper, floating) { + if (floating && typeof floating != 'boolean' && isIterateeCall(lower, upper, floating)) { + upper = floating = undefined; + } + if (floating === undefined) { + if (typeof upper == 'boolean') { + floating = upper; + upper = undefined; + } + else if (typeof lower == 'boolean') { + floating = lower; + lower = undefined; + } + } + if (lower === undefined && upper === undefined) { + lower = 0; + upper = 1; + } + else { + lower = toFinite(lower); + if (upper === undefined) { + upper = lower; + lower = 0; + } else { + upper = toFinite(upper); + } + } + if (lower > upper) { + var temp = lower; + lower = upper; + upper = temp; + } + if (floating || lower % 1 || upper % 1) { + var rand = nativeRandom(); + return nativeMin(lower + (rand * (upper - lower + freeParseFloat('1e-' + ((rand + '').length - 1)))), upper); + } + return baseRandom(lower, upper); } /*------------------------------------------------------------------------*/ /** - * Assigns own enumerable string keyed properties of source objects to the - * destination object. Source objects are applied from left to right. - * Subsequent sources overwrite property assignments of previous sources. - * - * **Note:** This method mutates `object` and is loosely based on - * [`Object.assign`](https://mdn.io/Object/assign). + * Converts `string` to [camel case](https://en.wikipedia.org/wiki/CamelCase). * * @static * @memberOf _ - * @since 0.10.0 - * @category Object - * @param {Object} object The destination object. - * @param {...Object} [sources] The source objects. - * @returns {Object} Returns `object`. - * @see _.assignIn + * @since 3.0.0 + * @category String + * @param {string} [string=''] The string to convert. + * @returns {string} Returns the camel cased string. * @example * - * function Foo() { - * this.a = 1; - * } - * - * function Bar() { - * this.c = 3; - * } + * _.camelCase('Foo Bar'); + * // => 'fooBar' * - * Foo.prototype.b = 2; - * Bar.prototype.d = 4; + * _.camelCase('--foo-bar--'); + * // => 'fooBar' * - * _.assign({ 'a': 0 }, new Foo, new Bar); - * // => { 'a': 1, 'c': 3 } + * _.camelCase('__FOO_BAR__'); + * // => 'fooBar' */ - var assign = createAssigner(function(object, source) { - if (isPrototype(source) || isArrayLike(source)) { - copyObject(source, keys(source), object); - return; - } - for (var key in source) { - if (hasOwnProperty.call(source, key)) { - assignValue(object, key, source[key]); - } - } + var camelCase = createCompounder(function(result, word, index) { + word = word.toLowerCase(); + return result + (index ? capitalize(word) : word); }); /** - * This method is like `_.assign` except that it iterates over own and - * inherited source properties. - * - * **Note:** This method mutates `object`. + * Converts the first character of `string` to upper case and the remaining + * to lower case. * * @static * @memberOf _ - * @since 4.0.0 - * @alias extend - * @category Object - * @param {Object} object The destination object. - * @param {...Object} [sources] The source objects. - * @returns {Object} Returns `object`. - * @see _.assign + * @since 3.0.0 + * @category String + * @param {string} [string=''] The string to capitalize. + * @returns {string} Returns the capitalized string. * @example * - * function Foo() { - * this.a = 1; - * } - * - * function Bar() { - * this.c = 3; - * } - * - * Foo.prototype.b = 2; - * Bar.prototype.d = 4; - * - * _.assignIn({ 'a': 0 }, new Foo, new Bar); - * // => { 'a': 1, 'b': 2, 'c': 3, 'd': 4 } + * _.capitalize('FRED'); + * // => 'Fred' */ - var assignIn = createAssigner(function(object, source) { - copyObject(source, keysIn(source), object); - }); + function capitalize(string) { + return upperFirst(toString(string).toLowerCase()); + } /** - * This method is like `_.assignIn` except that it accepts `customizer` - * which is invoked to produce the assigned values. If `customizer` returns - * `undefined`, assignment is handled by the method instead. The `customizer` - * is invoked with five arguments: (objValue, srcValue, key, object, source). - * - * **Note:** This method mutates `object`. + * Deburrs `string` by converting + * [Latin-1 Supplement](https://en.wikipedia.org/wiki/Latin-1_Supplement_(Unicode_block)#Character_table) + * and [Latin Extended-A](https://en.wikipedia.org/wiki/Latin_Extended-A) + * letters to basic Latin letters and removing + * [combining diacritical marks](https://en.wikipedia.org/wiki/Combining_Diacritical_Marks). * * @static * @memberOf _ - * @since 4.0.0 - * @alias extendWith - * @category Object - * @param {Object} object The destination object. - * @param {...Object} sources The source objects. - * @param {Function} [customizer] The function to customize assigned values. - * @returns {Object} Returns `object`. - * @see _.assignWith + * @since 3.0.0 + * @category String + * @param {string} [string=''] The string to deburr. + * @returns {string} Returns the deburred string. * @example * - * function customizer(objValue, srcValue) { - * return _.isUndefined(objValue) ? srcValue : objValue; - * } - * - * var defaults = _.partialRight(_.assignInWith, customizer); - * - * defaults({ 'a': 1 }, { 'b': 2 }, { 'a': 3 }); - * // => { 'a': 1, 'b': 2 } + * _.deburr('déjà vu'); + * // => 'deja vu' */ - var assignInWith = createAssigner(function(object, source, srcIndex, customizer) { - copyObject(source, keysIn(source), object, customizer); - }); + function deburr(string) { + string = toString(string); + return string && string.replace(reLatin, deburrLetter).replace(reComboMark, ''); + } /** - * This method is like `_.assign` except that it accepts `customizer` - * which is invoked to produce the assigned values. If `customizer` returns - * `undefined`, assignment is handled by the method instead. The `customizer` - * is invoked with five arguments: (objValue, srcValue, key, object, source). - * - * **Note:** This method mutates `object`. + * Checks if `string` ends with the given target string. * * @static * @memberOf _ - * @since 4.0.0 - * @category Object - * @param {Object} object The destination object. - * @param {...Object} sources The source objects. - * @param {Function} [customizer] The function to customize assigned values. - * @returns {Object} Returns `object`. - * @see _.assignInWith + * @since 3.0.0 + * @category String + * @param {string} [string=''] The string to inspect. + * @param {string} [target] The string to search for. + * @param {number} [position=string.length] The position to search up to. + * @returns {boolean} Returns `true` if `string` ends with `target`, + * else `false`. * @example * - * function customizer(objValue, srcValue) { - * return _.isUndefined(objValue) ? srcValue : objValue; - * } + * _.endsWith('abc', 'c'); + * // => true * - * var defaults = _.partialRight(_.assignWith, customizer); + * _.endsWith('abc', 'b'); + * // => false * - * defaults({ 'a': 1 }, { 'b': 2 }, { 'a': 3 }); - * // => { 'a': 1, 'b': 2 } + * _.endsWith('abc', 'b', 2); + * // => true */ - var assignWith = createAssigner(function(object, source, srcIndex, customizer) { - copyObject(source, keys(source), object, customizer); - }); + function endsWith(string, target, position) { + string = toString(string); + target = baseToString(target); + + var length = string.length; + position = position === undefined + ? length + : baseClamp(toInteger(position), 0, length); + + var end = position; + position -= target.length; + return position >= 0 && string.slice(position, end) == target; + } /** - * Creates an array of values corresponding to `paths` of `object`. + * Converts the characters "&", "<", ">", '"', and "'" in `string` to their + * corresponding HTML entities. + * + * **Note:** No other characters are escaped. To escape additional + * characters use a third-party library like [_he_](https://mths.be/he). + * + * Though the ">" character is escaped for symmetry, characters like + * ">" and "/" don't need escaping in HTML and have no special meaning + * unless they're part of a tag or unquoted attribute value. See + * [Mathias Bynens's article](https://mathiasbynens.be/notes/ambiguous-ampersands) + * (under "semi-related fun fact") for more details. + * + * When working with HTML you should always + * [quote attribute values](http://wonko.com/post/html-escaping) to reduce + * XSS vectors. * * @static + * @since 0.1.0 * @memberOf _ - * @since 1.0.0 - * @category Object - * @param {Object} object The object to iterate over. - * @param {...(string|string[])} [paths] The property paths to pick. - * @returns {Array} Returns the picked values. + * @category String + * @param {string} [string=''] The string to escape. + * @returns {string} Returns the escaped string. * @example * - * var object = { 'a': [{ 'b': { 'c': 3 } }, 4] }; - * - * _.at(object, ['a[0].b.c', 'a[1]']); - * // => [3, 4] + * _.escape('fred, barney, & pebbles'); + * // => 'fred, barney, & pebbles' */ - var at = flatRest(baseAt); + function escape(string) { + string = toString(string); + return (string && reHasUnescapedHtml.test(string)) + ? string.replace(reUnescapedHtml, escapeHtmlChar) + : string; + } /** - * Creates an object that inherits from the `prototype` object. If a - * `properties` object is given, its own enumerable string keyed properties - * are assigned to the created object. + * Escapes the `RegExp` special characters "^", "$", "\", ".", "*", "+", + * "?", "(", ")", "[", "]", "{", "}", and "|" in `string`. * * @static * @memberOf _ - * @since 2.3.0 - * @category Object - * @param {Object} prototype The object to inherit from. - * @param {Object} [properties] The properties to assign to the object. - * @returns {Object} Returns the new object. + * @since 3.0.0 + * @category String + * @param {string} [string=''] The string to escape. + * @returns {string} Returns the escaped string. * @example * - * function Shape() { - * this.x = 0; - * this.y = 0; - * } - * - * function Circle() { - * Shape.call(this); - * } - * - * Circle.prototype = _.create(Shape.prototype, { - * 'constructor': Circle - * }); - * - * var circle = new Circle; - * circle instanceof Circle; - * // => true - * - * circle instanceof Shape; - * // => true + * _.escapeRegExp('[lodash](https://lodash.com/)'); + * // => '\[lodash\]\(https://lodash\.com/\)' */ - function create(prototype, properties) { - var result = baseCreate(prototype); - return properties == null ? result : baseAssign(result, properties); + function escapeRegExp(string) { + string = toString(string); + return (string && reHasRegExpChar.test(string)) + ? string.replace(reRegExpChar, '\\$&') + : string; } /** - * Assigns own and inherited enumerable string keyed properties of source - * objects to the destination object for all destination properties that - * resolve to `undefined`. Source objects are applied from left to right. - * Once a property is set, additional values of the same property are ignored. - * - * **Note:** This method mutates `object`. + * Converts `string` to + * [kebab case](https://en.wikipedia.org/wiki/Letter_case#Special_case_styles). * * @static - * @since 0.1.0 * @memberOf _ - * @category Object - * @param {Object} object The destination object. - * @param {...Object} [sources] The source objects. - * @returns {Object} Returns `object`. - * @see _.defaultsDeep + * @since 3.0.0 + * @category String + * @param {string} [string=''] The string to convert. + * @returns {string} Returns the kebab cased string. * @example * - * _.defaults({ 'a': 1 }, { 'b': 2 }, { 'a': 3 }); - * // => { 'a': 1, 'b': 2 } + * _.kebabCase('Foo Bar'); + * // => 'foo-bar' + * + * _.kebabCase('fooBar'); + * // => 'foo-bar' + * + * _.kebabCase('__FOO_BAR__'); + * // => 'foo-bar' */ - var defaults = baseRest(function(object, sources) { - object = Object(object); - - var index = -1; - var length = sources.length; - var guard = length > 2 ? sources[2] : undefined; - - if (guard && isIterateeCall(sources[0], sources[1], guard)) { - length = 1; - } - - while (++index < length) { - var source = sources[index]; - var props = keysIn(source); - var propsIndex = -1; - var propsLength = props.length; - - while (++propsIndex < propsLength) { - var key = props[propsIndex]; - var value = object[key]; - - if (value === undefined || - (eq(value, objectProto[key]) && !hasOwnProperty.call(object, key))) { - object[key] = source[key]; - } - } - } - - return object; + var kebabCase = createCompounder(function(result, word, index) { + return result + (index ? '-' : '') + word.toLowerCase(); }); /** - * This method is like `_.defaults` except that it recursively assigns - * default properties. - * - * **Note:** This method mutates `object`. + * Converts `string`, as space separated words, to lower case. * * @static * @memberOf _ - * @since 3.10.0 - * @category Object - * @param {Object} object The destination object. - * @param {...Object} [sources] The source objects. - * @returns {Object} Returns `object`. - * @see _.defaults + * @since 4.0.0 + * @category String + * @param {string} [string=''] The string to convert. + * @returns {string} Returns the lower cased string. * @example * - * _.defaultsDeep({ 'a': { 'b': 2 } }, { 'a': { 'b': 1, 'c': 3 } }); - * // => { 'a': { 'b': 2, 'c': 3 } } + * _.lowerCase('--Foo-Bar--'); + * // => 'foo bar' + * + * _.lowerCase('fooBar'); + * // => 'foo bar' + * + * _.lowerCase('__FOO_BAR__'); + * // => 'foo bar' */ - var defaultsDeep = baseRest(function(args) { - args.push(undefined, customDefaultsMerge); - return apply(mergeWith, undefined, args); + var lowerCase = createCompounder(function(result, word, index) { + return result + (index ? ' ' : '') + word.toLowerCase(); }); /** - * This method is like `_.find` except that it returns the key of the first - * element `predicate` returns truthy for instead of the element itself. + * Converts the first character of `string` to lower case. * * @static * @memberOf _ - * @since 1.1.0 - * @category Object - * @param {Object} object The object to inspect. - * @param {Function} [predicate=_.identity] The function invoked per iteration. - * @returns {string|undefined} Returns the key of the matched element, - * else `undefined`. + * @since 4.0.0 + * @category String + * @param {string} [string=''] The string to convert. + * @returns {string} Returns the converted string. * @example * - * var users = { - * 'barney': { 'age': 36, 'active': true }, - * 'fred': { 'age': 40, 'active': false }, - * 'pebbles': { 'age': 1, 'active': true } - * }; + * _.lowerFirst('Fred'); + * // => 'fred' * - * _.findKey(users, function(o) { return o.age < 40; }); - * // => 'barney' (iteration order is not guaranteed) + * _.lowerFirst('FRED'); + * // => 'fRED' + */ + var lowerFirst = createCaseFirst('toLowerCase'); + + /** + * Pads `string` on the left and right sides if it's shorter than `length`. + * Padding characters are truncated if they can't be evenly divided by `length`. + * + * @static + * @memberOf _ + * @since 3.0.0 + * @category String + * @param {string} [string=''] The string to pad. + * @param {number} [length=0] The padding length. + * @param {string} [chars=' '] The string used as padding. + * @returns {string} Returns the padded string. + * @example * - * // The `_.matches` iteratee shorthand. - * _.findKey(users, { 'age': 1, 'active': true }); - * // => 'pebbles' + * _.pad('abc', 8); + * // => ' abc ' * - * // The `_.matchesProperty` iteratee shorthand. - * _.findKey(users, ['active', false]); - * // => 'fred' + * _.pad('abc', 8, '_-'); + * // => '_-abc_-_' * - * // The `_.property` iteratee shorthand. - * _.findKey(users, 'active'); - * // => 'barney' + * _.pad('abc', 3); + * // => 'abc' */ - function findKey(object, predicate) { - return baseFindKey(object, getIteratee(predicate, 3), baseForOwn); + function pad(string, length, chars) { + string = toString(string); + length = toInteger(length); + + var strLength = length ? stringSize(string) : 0; + if (!length || strLength >= length) { + return string; + } + var mid = (length - strLength) / 2; + return ( + createPadding(nativeFloor(mid), chars) + + string + + createPadding(nativeCeil(mid), chars) + ); } /** - * This method is like `_.findKey` except that it iterates over elements of - * a collection in the opposite order. + * Pads `string` on the right side if it's shorter than `length`. Padding + * characters are truncated if they exceed `length`. * * @static * @memberOf _ - * @since 2.0.0 - * @category Object - * @param {Object} object The object to inspect. - * @param {Function} [predicate=_.identity] The function invoked per iteration. - * @returns {string|undefined} Returns the key of the matched element, - * else `undefined`. + * @since 4.0.0 + * @category String + * @param {string} [string=''] The string to pad. + * @param {number} [length=0] The padding length. + * @param {string} [chars=' '] The string used as padding. + * @returns {string} Returns the padded string. * @example * - * var users = { - * 'barney': { 'age': 36, 'active': true }, - * 'fred': { 'age': 40, 'active': false }, - * 'pebbles': { 'age': 1, 'active': true } - * }; - * - * _.findLastKey(users, function(o) { return o.age < 40; }); - * // => returns 'pebbles' assuming `_.findKey` returns 'barney' - * - * // The `_.matches` iteratee shorthand. - * _.findLastKey(users, { 'age': 36, 'active': true }); - * // => 'barney' + * _.padEnd('abc', 6); + * // => 'abc ' * - * // The `_.matchesProperty` iteratee shorthand. - * _.findLastKey(users, ['active', false]); - * // => 'fred' + * _.padEnd('abc', 6, '_-'); + * // => 'abc_-_' * - * // The `_.property` iteratee shorthand. - * _.findLastKey(users, 'active'); - * // => 'pebbles' + * _.padEnd('abc', 3); + * // => 'abc' */ - function findLastKey(object, predicate) { - return baseFindKey(object, getIteratee(predicate, 3), baseForOwnRight); + function padEnd(string, length, chars) { + string = toString(string); + length = toInteger(length); + + var strLength = length ? stringSize(string) : 0; + return (length && strLength < length) + ? (string + createPadding(length - strLength, chars)) + : string; } /** - * Iterates over own and inherited enumerable string keyed properties of an - * object and invokes `iteratee` for each property. The iteratee is invoked - * with three arguments: (value, key, object). Iteratee functions may exit - * iteration early by explicitly returning `false`. + * Pads `string` on the left side if it's shorter than `length`. Padding + * characters are truncated if they exceed `length`. * * @static * @memberOf _ - * @since 0.3.0 - * @category Object - * @param {Object} object The object to iterate over. - * @param {Function} [iteratee=_.identity] The function invoked per iteration. - * @returns {Object} Returns `object`. - * @see _.forInRight + * @since 4.0.0 + * @category String + * @param {string} [string=''] The string to pad. + * @param {number} [length=0] The padding length. + * @param {string} [chars=' '] The string used as padding. + * @returns {string} Returns the padded string. * @example * - * function Foo() { - * this.a = 1; - * this.b = 2; - * } + * _.padStart('abc', 6); + * // => ' abc' * - * Foo.prototype.c = 3; + * _.padStart('abc', 6, '_-'); + * // => '_-_abc' * - * _.forIn(new Foo, function(value, key) { - * console.log(key); - * }); - * // => Logs 'a', 'b', then 'c' (iteration order is not guaranteed). + * _.padStart('abc', 3); + * // => 'abc' */ - function forIn(object, iteratee) { - return object == null - ? object - : baseFor(object, getIteratee(iteratee, 3), keysIn); + function padStart(string, length, chars) { + string = toString(string); + length = toInteger(length); + + var strLength = length ? stringSize(string) : 0; + return (length && strLength < length) + ? (createPadding(length - strLength, chars) + string) + : string; } /** - * This method is like `_.forIn` except that it iterates over properties of - * `object` in the opposite order. + * Converts `string` to an integer of the specified radix. If `radix` is + * `undefined` or `0`, a `radix` of `10` is used unless `value` is a + * hexadecimal, in which case a `radix` of `16` is used. + * + * **Note:** This method aligns with the + * [ES5 implementation](https://es5.github.io/#x15.1.2.2) of `parseInt`. * * @static * @memberOf _ - * @since 2.0.0 - * @category Object - * @param {Object} object The object to iterate over. - * @param {Function} [iteratee=_.identity] The function invoked per iteration. - * @returns {Object} Returns `object`. - * @see _.forIn + * @since 1.1.0 + * @category String + * @param {string} string The string to convert. + * @param {number} [radix=10] The radix to interpret `value` by. + * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. + * @returns {number} Returns the converted integer. * @example * - * function Foo() { - * this.a = 1; - * this.b = 2; - * } - * - * Foo.prototype.c = 3; + * _.parseInt('08'); + * // => 8 * - * _.forInRight(new Foo, function(value, key) { - * console.log(key); - * }); - * // => Logs 'c', 'b', then 'a' assuming `_.forIn` logs 'a', 'b', then 'c'. + * _.map(['6', '08', '10'], _.parseInt); + * // => [6, 8, 10] */ - function forInRight(object, iteratee) { - return object == null - ? object - : baseForRight(object, getIteratee(iteratee, 3), keysIn); + function parseInt(string, radix, guard) { + if (guard || radix == null) { + radix = 0; + } else if (radix) { + radix = +radix; + } + return nativeParseInt(toString(string).replace(reTrimStart, ''), radix || 0); } /** - * Iterates over own enumerable string keyed properties of an object and - * invokes `iteratee` for each property. The iteratee is invoked with three - * arguments: (value, key, object). Iteratee functions may exit iteration - * early by explicitly returning `false`. + * Repeats the given string `n` times. * * @static * @memberOf _ - * @since 0.3.0 - * @category Object - * @param {Object} object The object to iterate over. - * @param {Function} [iteratee=_.identity] The function invoked per iteration. - * @returns {Object} Returns `object`. - * @see _.forOwnRight + * @since 3.0.0 + * @category String + * @param {string} [string=''] The string to repeat. + * @param {number} [n=1] The number of times to repeat the string. + * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. + * @returns {string} Returns the repeated string. * @example * - * function Foo() { - * this.a = 1; - * this.b = 2; - * } + * _.repeat('*', 3); + * // => '***' * - * Foo.prototype.c = 3; + * _.repeat('abc', 2); + * // => 'abcabc' * - * _.forOwn(new Foo, function(value, key) { - * console.log(key); - * }); - * // => Logs 'a' then 'b' (iteration order is not guaranteed). + * _.repeat('abc', 0); + * // => '' */ - function forOwn(object, iteratee) { - return object && baseForOwn(object, getIteratee(iteratee, 3)); + function repeat(string, n, guard) { + if ((guard ? isIterateeCall(string, n, guard) : n === undefined)) { + n = 1; + } else { + n = toInteger(n); + } + return baseRepeat(toString(string), n); } /** - * This method is like `_.forOwn` except that it iterates over properties of - * `object` in the opposite order. + * Replaces matches for `pattern` in `string` with `replacement`. + * + * **Note:** This method is based on + * [`String#replace`](https://mdn.io/String/replace). * * @static * @memberOf _ - * @since 2.0.0 - * @category Object - * @param {Object} object The object to iterate over. - * @param {Function} [iteratee=_.identity] The function invoked per iteration. - * @returns {Object} Returns `object`. - * @see _.forOwn + * @since 4.0.0 + * @category String + * @param {string} [string=''] The string to modify. + * @param {RegExp|string} pattern The pattern to replace. + * @param {Function|string} replacement The match replacement. + * @returns {string} Returns the modified string. * @example * - * function Foo() { - * this.a = 1; - * this.b = 2; - * } - * - * Foo.prototype.c = 3; - * - * _.forOwnRight(new Foo, function(value, key) { - * console.log(key); - * }); - * // => Logs 'b' then 'a' assuming `_.forOwn` logs 'a' then 'b'. + * _.replace('Hi Fred', 'Fred', 'Barney'); + * // => 'Hi Barney' */ - function forOwnRight(object, iteratee) { - return object && baseForOwnRight(object, getIteratee(iteratee, 3)); + function replace() { + var args = arguments, + string = toString(args[0]); + + return args.length < 3 ? string : string.replace(args[1], args[2]); } /** - * Creates an array of function property names from own enumerable properties - * of `object`. + * Converts `string` to + * [snake case](https://en.wikipedia.org/wiki/Snake_case). * * @static - * @since 0.1.0 * @memberOf _ - * @category Object - * @param {Object} object The object to inspect. - * @returns {Array} Returns the function names. - * @see _.functionsIn + * @since 3.0.0 + * @category String + * @param {string} [string=''] The string to convert. + * @returns {string} Returns the snake cased string. * @example * - * function Foo() { - * this.a = _.constant('a'); - * this.b = _.constant('b'); - * } + * _.snakeCase('Foo Bar'); + * // => 'foo_bar' * - * Foo.prototype.c = _.constant('c'); + * _.snakeCase('fooBar'); + * // => 'foo_bar' * - * _.functions(new Foo); - * // => ['a', 'b'] + * _.snakeCase('--FOO-BAR--'); + * // => 'foo_bar' */ - function functions(object) { - return object == null ? [] : baseFunctions(object, keys(object)); - } + var snakeCase = createCompounder(function(result, word, index) { + return result + (index ? '_' : '') + word.toLowerCase(); + }); /** - * Creates an array of function property names from own and inherited - * enumerable properties of `object`. + * Splits `string` by `separator`. + * + * **Note:** This method is based on + * [`String#split`](https://mdn.io/String/split). * * @static * @memberOf _ * @since 4.0.0 - * @category Object - * @param {Object} object The object to inspect. - * @returns {Array} Returns the function names. - * @see _.functions + * @category String + * @param {string} [string=''] The string to split. + * @param {RegExp|string} separator The separator pattern to split by. + * @param {number} [limit] The length to truncate results to. + * @returns {Array} Returns the string segments. * @example * - * function Foo() { - * this.a = _.constant('a'); - * this.b = _.constant('b'); - * } - * - * Foo.prototype.c = _.constant('c'); - * - * _.functionsIn(new Foo); - * // => ['a', 'b', 'c'] + * _.split('a-b-c', '-', 2); + * // => ['a', 'b'] */ - function functionsIn(object) { - return object == null ? [] : baseFunctions(object, keysIn(object)); + function split(string, separator, limit) { + if (limit && typeof limit != 'number' && isIterateeCall(string, separator, limit)) { + separator = limit = undefined; + } + limit = limit === undefined ? MAX_ARRAY_LENGTH : limit >>> 0; + if (!limit) { + return []; + } + string = toString(string); + if (string && ( + typeof separator == 'string' || + (separator != null && !isRegExp(separator)) + )) { + separator = baseToString(separator); + if (!separator && hasUnicode(string)) { + return castSlice(stringToArray(string), 0, limit); + } + } + return string.split(separator, limit); } /** - * Gets the value at `path` of `object`. If the resolved value is - * `undefined`, the `defaultValue` is returned in its place. + * Converts `string` to + * [start case](https://en.wikipedia.org/wiki/Letter_case#Stylistic_or_specialised_usage). * * @static * @memberOf _ - * @since 3.7.0 - * @category Object - * @param {Object} object The object to query. - * @param {Array|string} path The path of the property to get. - * @param {*} [defaultValue] The value returned for `undefined` resolved values. - * @returns {*} Returns the resolved value. + * @since 3.1.0 + * @category String + * @param {string} [string=''] The string to convert. + * @returns {string} Returns the start cased string. * @example * - * var object = { 'a': [{ 'b': { 'c': 3 } }] }; - * - * _.get(object, 'a[0].b.c'); - * // => 3 + * _.startCase('--foo-bar--'); + * // => 'Foo Bar' * - * _.get(object, ['a', '0', 'b', 'c']); - * // => 3 + * _.startCase('fooBar'); + * // => 'Foo Bar' * - * _.get(object, 'a.b.c', 'default'); - * // => 'default' + * _.startCase('__FOO_BAR__'); + * // => 'FOO BAR' */ - function get(object, path, defaultValue) { - var result = object == null ? undefined : baseGet(object, path); - return result === undefined ? defaultValue : result; - } + var startCase = createCompounder(function(result, word, index) { + return result + (index ? ' ' : '') + upperFirst(word); + }); /** - * Checks if `path` is a direct property of `object`. + * Checks if `string` starts with the given target string. * * @static - * @since 0.1.0 * @memberOf _ - * @category Object - * @param {Object} object The object to query. - * @param {Array|string} path The path to check. - * @returns {boolean} Returns `true` if `path` exists, else `false`. + * @since 3.0.0 + * @category String + * @param {string} [string=''] The string to inspect. + * @param {string} [target] The string to search for. + * @param {number} [position=0] The position to search from. + * @returns {boolean} Returns `true` if `string` starts with `target`, + * else `false`. * @example * - * var object = { 'a': { 'b': 2 } }; - * var other = _.create({ 'a': _.create({ 'b': 2 }) }); - * - * _.has(object, 'a'); + * _.startsWith('abc', 'a'); * // => true * - * _.has(object, 'a.b'); - * // => true + * _.startsWith('abc', 'b'); + * // => false * - * _.has(object, ['a', 'b']); + * _.startsWith('abc', 'b', 1); * // => true - * - * _.has(other, 'a'); - * // => false */ - function has(object, path) { - return object != null && hasPath(object, path, baseHas); + function startsWith(string, target, position) { + string = toString(string); + position = position == null + ? 0 + : baseClamp(toInteger(position), 0, string.length); + + target = baseToString(target); + return string.slice(position, position + target.length) == target; } /** - * Checks if `path` is a direct or inherited property of `object`. + * Creates a compiled template function that can interpolate data properties + * in "interpolate" delimiters, HTML-escape interpolated data properties in + * "escape" delimiters, and execute JavaScript in "evaluate" delimiters. Data + * properties may be accessed as free variables in the template. If a setting + * object is given, it takes precedence over `_.templateSettings` values. + * + * **Note:** In the development build `_.template` utilizes + * [sourceURLs](http://www.html5rocks.com/en/tutorials/developertools/sourcemaps/#toc-sourceurl) + * for easier debugging. + * + * For more information on precompiling templates see + * [lodash's custom builds documentation](https://lodash.com/custom-builds). + * + * For more information on Chrome extension sandboxes see + * [Chrome's extensions documentation](https://developer.chrome.com/extensions/sandboxingEval). * * @static + * @since 0.1.0 * @memberOf _ - * @since 4.0.0 - * @category Object - * @param {Object} object The object to query. - * @param {Array|string} path The path to check. - * @returns {boolean} Returns `true` if `path` exists, else `false`. + * @category String + * @param {string} [string=''] The template string. + * @param {Object} [options={}] The options object. + * @param {RegExp} [options.escape=_.templateSettings.escape] + * The HTML "escape" delimiter. + * @param {RegExp} [options.evaluate=_.templateSettings.evaluate] + * The "evaluate" delimiter. + * @param {Object} [options.imports=_.templateSettings.imports] + * An object to import into the template as free variables. + * @param {RegExp} [options.interpolate=_.templateSettings.interpolate] + * The "interpolate" delimiter. + * @param {string} [options.sourceURL='lodash.templateSources[n]'] + * The sourceURL of the compiled template. + * @param {string} [options.variable='obj'] + * The data object variable name. + * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. + * @returns {Function} Returns the compiled template function. * @example * - * var object = _.create({ 'a': _.create({ 'b': 2 }) }); + * // Use the "interpolate" delimiter to create a compiled template. + * var compiled = _.template('hello <%= user %>!'); + * compiled({ 'user': 'fred' }); + * // => 'hello fred!' * - * _.hasIn(object, 'a'); - * // => true + * // Use the HTML "escape" delimiter to escape data property values. + * var compiled = _.template('<%- value %>'); + * compiled({ 'value': ' +

Noir app

+
+ + +
+
+

Logs

+

Proof

+
+ + +``` + +It _could_ be a beautiful UI... Depending on which universe you live in. + +## Some good old vanilla Javascript + +Our love for Noir needs undivided attention, so let's just open `main.js` and delete everything (this is where the romantic scenery becomes a bit creepy). + +Start by pasting in this boilerplate code: + +```js +function display(container, msg) { + const c = document.getElementById(container); + const p = document.createElement('p'); + p.textContent = msg; + c.appendChild(p); +} + +document.getElementById('submitGuess').addEventListener('click', async () => { + try { + // here's where love happens + } catch (err) { + display('logs', 'Oh 💔 Wrong guess'); + } +}); +``` + +The display function doesn't do much. We're simply manipulating our website to see stuff happening. For example, if the proof fails, it will simply log a broken heart 😢 + +:::info + +At this point in the tutorial, your folder structure should look like this: + +```tree +. +└── circuit + └── ...same as above +└── vite-project + ├── vite.config.js + ├── main.js + ├── package.json + └── index.html +``` + +You'll see other files and folders showing up (like `package-lock.json`, `node_modules`) but you shouldn't have to care about those. + +::: + +## Some NoirJS + +We're starting with the good stuff now. If you've compiled the circuit as described above, you should have a `json` file we want to import at the very top of our `main.js` file: + +```ts +import circuit from '../circuit/target/circuit.json'; +``` + +[Noir is backend-agnostic](../index.mdx#whats-new-about-noir). We write Noir, but we also need a proving backend. That's why we need to import and instantiate the two dependencies we installed above: `BarretenbergBackend` and `Noir`. Let's import them right below: + +```js +import { BarretenbergBackend, BarretenbergVerifier as Verifier } from '@noir-lang/backend_barretenberg'; +import { Noir } from '@noir-lang/noir_js'; +``` + +And instantiate them inside our try-catch block: + +```ts +// try { +const backend = new BarretenbergBackend(circuit); +const noir = new Noir(circuit); +// } +``` + +:::note + +For the remainder of the tutorial, everything will be happening inside the `try` block + +::: + +## Our app + +Now for the app itself. We're capturing whatever is in the input when people press the submit button. Just add this: + +```js +const x = parseInt(document.getElementById('guessInput').value); +const input = { x, y: 2 }; +``` + +Now we're ready to prove stuff! Let's feed some inputs to our circuit and calculate the proof: + +```js +await setup(); // let's squeeze our wasm inits here + +display('logs', 'Generating proof... ⌛'); +const { witness } = await noir.execute(input); +const proof = await backend.generateProof(witness); +display('logs', 'Generating proof... ✅'); +display('results', proof.proof); +``` + +You're probably eager to see stuff happening, so go and run your app now! + +From your terminal, run `npm run dev`. If it doesn't open a browser for you, just visit `localhost:5173`. You should now see the worst UI ever, with an ugly input. + +![Getting Started 0](@site/static/img/noir_getting_started_1.png) + +Now, our circuit says `fn main(x: Field, y: pub Field)`. This means only the `y` value is public, and it's hardcoded above: `input = { x, y: 2 }`. In other words, you won't need to send your secret`x` to the verifier! + +By inputting any number other than 2 in the input box and clicking "submit", you should get a valid proof. Otherwise the proof won't even generate correctly. By the way, if you're human, you shouldn't be able to understand anything on the "proof" box. That's OK. We like you, human ❤️. + +## Verifying + +Time to celebrate, yes! But we shouldn't trust machines so blindly. Let's add these lines to see our proof being verified: + +```js +display('logs', 'Verifying proof... ⌛'); +const isValid = await backend.verifyProof(proof); + +// or to cache and use the verification key: +// const verificationKey = await backend.getVerificationKey(); +// const verifier = new Verifier(); +// const isValid = await verifier.verifyProof(proof, verificationKey); + +if (isValid) display('logs', 'Verifying proof... ✅'); +``` + +You have successfully generated a client-side Noir web app! + +![coded app without math knowledge](@site/static/img/memes/flextape.jpeg) + +## Further Reading + +You can see how noirjs is used in a full stack Next.js hardhat application in the [noir-starter repo here](https://github.com/noir-lang/noir-starter/tree/main/vite-hardhat). The example shows how to calculate a proof in the browser and verify it with a deployed Solidity verifier contract from noirjs. + +You should also check out the more advanced examples in the [noir-examples repo](https://github.com/noir-lang/noir-examples), where you'll find reference usage for some cool apps. + +## UltraHonk Backend + +Barretenberg has recently exposed a new UltraHonk backend. We can use UltraHonk in NoirJS after version 0.33.0. Everything will be the same as the tutorial above, except that the class we need to import will change: + +```js +import { UltraHonkBackend, UltraHonkVerifier as Verifier } from '@noir-lang/backend_barretenberg'; +``` + +The backend will then be instantiated as such: + +```js +const backend = new UltraHonkBackend(circuit); +``` + +Then all the commands to prove and verify your circuit will be same. + +The only feature currently unsupported with UltraHonk are [recursive proofs](../explainers/explainer-recursion.md). diff --git a/noir/noir-repo/docs/versioned_sidebars/version-v0.39.0-sidebars.json b/noir/noir-repo/docs/versioned_sidebars/version-v0.39.0-sidebars.json new file mode 100644 index 00000000000..b9ad026f69f --- /dev/null +++ b/noir/noir-repo/docs/versioned_sidebars/version-v0.39.0-sidebars.json @@ -0,0 +1,93 @@ +{ + "sidebar": [ + { + "type": "doc", + "id": "index" + }, + { + "type": "category", + "label": "Getting Started", + "items": [ + { + "type": "autogenerated", + "dirName": "getting_started" + } + ] + }, + { + "type": "category", + "label": "The Noir Language", + "items": [ + { + "type": "autogenerated", + "dirName": "noir" + } + ] + }, + { + "type": "html", + "value": "
", + "defaultStyle": true + }, + { + "type": "category", + "label": "How To Guides", + "items": [ + { + "type": "autogenerated", + "dirName": "how_to" + } + ] + }, + { + "type": "category", + "label": "Explainers", + "items": [ + { + "type": "autogenerated", + "dirName": "explainers" + } + ] + }, + { + "type": "category", + "label": "Tutorials", + "items": [ + { + "type": "autogenerated", + "dirName": "tutorials" + } + ] + }, + { + "type": "category", + "label": "Reference", + "items": [ + { + "type": "autogenerated", + "dirName": "reference" + } + ] + }, + { + "type": "category", + "label": "Tooling", + "items": [ + { + "type": "autogenerated", + "dirName": "tooling" + } + ] + }, + { + "type": "html", + "value": "
", + "defaultStyle": true + }, + { + "type": "doc", + "id": "migration_notes", + "label": "Migration notes" + } + ] +} diff --git a/noir/noir-repo/noir_stdlib/src/ec/tecurve.nr b/noir/noir-repo/noir_stdlib/src/ec/tecurve.nr index 08f017c4f91..45a6b322ed1 100644 --- a/noir/noir-repo/noir_stdlib/src/ec/tecurve.nr +++ b/noir/noir-repo/noir_stdlib/src/ec/tecurve.nr @@ -27,7 +27,7 @@ pub mod affine { impl Point { // Point constructor - //#[deprecated = "It's recommmended to use the external noir-edwards library (https://github.com/noir-lang/noir-edwards)"] + // #[deprecated("It's recommmended to use the external noir-edwards library (https://github.com/noir-lang/noir-edwards)")] pub fn new(x: Field, y: Field) -> Self { Self { x, y } } diff --git a/noir/noir-repo/noir_stdlib/src/hash/poseidon/mod.nr b/noir/noir-repo/noir_stdlib/src/hash/poseidon/mod.nr index 6f0e461a610..e2e47959024 100644 --- a/noir/noir-repo/noir_stdlib/src/hash/poseidon/mod.nr +++ b/noir/noir-repo/noir_stdlib/src/hash/poseidon/mod.nr @@ -346,6 +346,7 @@ impl Hasher for PoseidonHasher { result } + #[inline_always] fn write(&mut self, input: Field) { self._state = self._state.push_back(input); } diff --git a/noir/noir-repo/release-please-config.json b/noir/noir-repo/release-please-config.json index 0ba192754a0..963f1af26d9 100644 --- a/noir/noir-repo/release-please-config.json +++ b/noir/noir-repo/release-please-config.json @@ -3,75 +3,69 @@ "bump-minor-pre-major": true, "bump-patch-for-minor-pre-major": true, "prerelease": true, + "prerelease-type": "beta", "pull-request-title-pattern": "chore: Release Noir(${version})", "group-pull-request-title-pattern": "chore: Release Noir(${version})", "packages": { ".": { "release-type": "simple", + "versioning": "prerelease", "component": "noir", "package-name": "noir", "include-component-in-tag": false, "extra-files": [ "Cargo.toml", + "acvm-repo/acir/Cargo.toml", + "acvm-repo/acir_field/Cargo.toml", + "acvm-repo/acvm/Cargo.toml", + "acvm-repo/acvm_js/Cargo.toml", + "acvm-repo/blackbox_solver/Cargo.toml", + "acvm-repo/bn254_blackbox_solver/Cargo.toml", + "acvm-repo/brillig/Cargo.toml", + "acvm-repo/brillig_vm/Cargo.toml", { "type": "json", - "path": "compiler/wasm/package.json", + "path": "acvm-repo/acvm_js/package.json", "jsonpath": "$.version" }, { "type": "json", - "path": "tooling/noir_codegen/package.json", + "path": "acvm-repo/acvm_js/package.json", "jsonpath": "$.version" }, { "type": "json", - "path": "tooling/noir_js/package.json", + "path": "compiler/wasm/package.json", "jsonpath": "$.version" }, { "type": "json", - "path": "tooling/noir_js_backend_barretenberg/package.json", + "path": "tooling/noir_codegen/package.json", "jsonpath": "$.version" }, { "type": "json", - "path": "tooling/noir_js_types/package.json", + "path": "tooling/noir_js/package.json", "jsonpath": "$.version" }, { "type": "json", - "path": "tooling/noirc_abi_wasm/package.json", + "path": "tooling/noir_js_backend_barretenberg/package.json", "jsonpath": "$.version" }, { "type": "json", - "path": "docs/docs/package.json", + "path": "tooling/noir_js_types/package.json", "jsonpath": "$.version" - } - ] - }, - "acvm-repo": { - "release-type": "simple", - "package-name": "acvm", - "component": "acvm", - "include-component-in-tag": false, - "extra-files": [ - "acir/Cargo.toml", - "acir_field/Cargo.toml", - "acvm/Cargo.toml", - "acvm_js/Cargo.toml", - "blackbox_solver/Cargo.toml", - "bn254_blackbox_solver/Cargo.toml", - "brillig/Cargo.toml", - "brillig_vm/Cargo.toml", + }, { "type": "json", - "path": "acvm_js/package.json", + "path": "tooling/noirc_abi_wasm/package.json", "jsonpath": "$.version" }, { "type": "json", - "path": "acvm_js/package.json", + "path": "docs/docs/package.json", "jsonpath": "$.version" } ] diff --git a/noir/noir-repo/scripts/install_bb.sh b/noir/noir-repo/scripts/install_bb.sh index cff81eedbac..db98f17c503 100755 --- a/noir/noir-repo/scripts/install_bb.sh +++ b/noir/noir-repo/scripts/install_bb.sh @@ -1,6 +1,6 @@ #!/bin/bash -VERSION="0.61.0" +VERSION="0.63.0" BBUP_PATH=~/.bb/bbup diff --git a/noir/noir-repo/test_programs/compile_success_empty/comptime_apply_range_constraint/Nargo.toml b/noir/noir-repo/test_programs/compile_success_empty/comptime_apply_range_constraint/Nargo.toml index bfd6fa75728..84d16060440 100644 --- a/noir/noir-repo/test_programs/compile_success_empty/comptime_apply_range_constraint/Nargo.toml +++ b/noir/noir-repo/test_programs/compile_success_empty/comptime_apply_range_constraint/Nargo.toml @@ -2,4 +2,5 @@ name = "comptime_apply_range_constraint" type = "bin" authors = [""] + [dependencies] diff --git a/noir/noir-repo/test_programs/execution_success/brillig_hash_to_field/Nargo.toml b/noir/noir-repo/test_programs/compile_success_empty/embedded_curve_msm_simplification/Nargo.toml similarity index 55% rename from noir/noir-repo/test_programs/execution_success/brillig_hash_to_field/Nargo.toml rename to noir/noir-repo/test_programs/compile_success_empty/embedded_curve_msm_simplification/Nargo.toml index 7cfcc745f0d..9c9bd8de04a 100644 --- a/noir/noir-repo/test_programs/execution_success/brillig_hash_to_field/Nargo.toml +++ b/noir/noir-repo/test_programs/compile_success_empty/embedded_curve_msm_simplification/Nargo.toml @@ -1,5 +1,5 @@ [package] -name = "brillig_hash_to_field" +name = "embedded_curve_msm_simplification" type = "bin" authors = [""] diff --git a/noir/noir-repo/test_programs/compile_success_empty/embedded_curve_msm_simplification/src/main.nr b/noir/noir-repo/test_programs/compile_success_empty/embedded_curve_msm_simplification/src/main.nr new file mode 100644 index 00000000000..e5aaa0f4d15 --- /dev/null +++ b/noir/noir-repo/test_programs/compile_success_empty/embedded_curve_msm_simplification/src/main.nr @@ -0,0 +1,12 @@ +fn main() { + let pub_x = 0x0000000000000000000000000000000000000000000000000000000000000001; + let pub_y = 0x0000000000000002cf135e7506a45d632d270d45f1181294833fc48d823f272c; + + let g1_y = 17631683881184975370165255887551781615748388533673675138860; + let g1 = std::embedded_curve_ops::EmbeddedCurvePoint { x: 1, y: g1_y, is_infinite: false }; + let scalar = std::embedded_curve_ops::EmbeddedCurveScalar { lo: 1, hi: 0 }; + // Test that multi_scalar_mul correctly derives the public key + let res = std::embedded_curve_ops::multi_scalar_mul([g1], [scalar]); + assert(res.x == pub_x); + assert(res.y == pub_y); +} diff --git a/noir/noir-repo/test_programs/execution_success/assert/src/main.nr b/noir/noir-repo/test_programs/execution_success/assert/src/main.nr index 00e94414c0b..b95665d502b 100644 --- a/noir/noir-repo/test_programs/execution_success/assert/src/main.nr +++ b/noir/noir-repo/test_programs/execution_success/assert/src/main.nr @@ -1,3 +1,10 @@ fn main(x: Field) { assert(x == 1); + assert(1 == conditional(x as bool)); +} + +fn conditional(x: bool) -> Field { + assert(x, f"Expected x to be true but got {x}"); + assert_eq(x, true, f"Expected x to be true but got {x}"); + 1 } diff --git a/noir/noir-repo/test_programs/execution_success/brillig_array_to_slice/Nargo.toml b/noir/noir-repo/test_programs/execution_success/brillig_array_to_slice/Nargo.toml deleted file mode 100644 index 58157c38c26..00000000000 --- a/noir/noir-repo/test_programs/execution_success/brillig_array_to_slice/Nargo.toml +++ /dev/null @@ -1,7 +0,0 @@ -[package] -name = "brillig_array_to_slice" -type = "bin" -authors = [""] -compiler_version = ">=0.25.0" - -[dependencies] \ No newline at end of file diff --git a/noir/noir-repo/test_programs/execution_success/brillig_array_to_slice/Prover.toml b/noir/noir-repo/test_programs/execution_success/brillig_array_to_slice/Prover.toml deleted file mode 100644 index 11497a473bc..00000000000 --- a/noir/noir-repo/test_programs/execution_success/brillig_array_to_slice/Prover.toml +++ /dev/null @@ -1 +0,0 @@ -x = "0" diff --git a/noir/noir-repo/test_programs/execution_success/brillig_array_to_slice/src/main.nr b/noir/noir-repo/test_programs/execution_success/brillig_array_to_slice/src/main.nr deleted file mode 100644 index f54adb39963..00000000000 --- a/noir/noir-repo/test_programs/execution_success/brillig_array_to_slice/src/main.nr +++ /dev/null @@ -1,20 +0,0 @@ -unconstrained fn brillig_as_slice(x: Field) -> (u32, Field, Field) { - let mut dynamic: [Field; 1] = [1]; - dynamic[x] = 2; - assert(dynamic[0] == 2); - - let brillig_slice = dynamic.as_slice(); - assert(brillig_slice.len() == 1); - - (brillig_slice.len(), dynamic[0], brillig_slice[0]) -} - -fn main(x: Field) { - unsafe { - let (slice_len, dynamic_0, slice_0) = brillig_as_slice(x); - assert(slice_len == 1); - assert(dynamic_0 == 2); - assert(slice_0 == 2); - } -} - diff --git a/noir/noir-repo/test_programs/execution_success/brillig_assert/Nargo.toml b/noir/noir-repo/test_programs/execution_success/brillig_assert/Nargo.toml deleted file mode 100644 index b7d9231ab75..00000000000 --- a/noir/noir-repo/test_programs/execution_success/brillig_assert/Nargo.toml +++ /dev/null @@ -1,6 +0,0 @@ -[package] -name = "brillig_assert" -type = "bin" -authors = [""] - -[dependencies] diff --git a/noir/noir-repo/test_programs/execution_success/brillig_assert/Prover.toml b/noir/noir-repo/test_programs/execution_success/brillig_assert/Prover.toml deleted file mode 100644 index 4dd6b405159..00000000000 --- a/noir/noir-repo/test_programs/execution_success/brillig_assert/Prover.toml +++ /dev/null @@ -1 +0,0 @@ -x = "1" diff --git a/noir/noir-repo/test_programs/execution_success/brillig_assert/src/main.nr b/noir/noir-repo/test_programs/execution_success/brillig_assert/src/main.nr deleted file mode 100644 index dc0138d3f05..00000000000 --- a/noir/noir-repo/test_programs/execution_success/brillig_assert/src/main.nr +++ /dev/null @@ -1,14 +0,0 @@ -// Tests a very simple program. -// -// The features being tested is using assert on brillig -fn main(x: Field) { - unsafe { - assert(1 == conditional(x as bool)); - } -} - -unconstrained fn conditional(x: bool) -> Field { - assert(x, f"Expected x to be false but got {x}"); - assert_eq(x, true, f"Expected x to be false but got {x}"); - 1 -} diff --git a/noir/noir-repo/test_programs/execution_success/brillig_blake3/Nargo.toml b/noir/noir-repo/test_programs/execution_success/brillig_blake3/Nargo.toml deleted file mode 100644 index 879476dbdcf..00000000000 --- a/noir/noir-repo/test_programs/execution_success/brillig_blake3/Nargo.toml +++ /dev/null @@ -1,7 +0,0 @@ -[package] -name = "brillig_blake3" -type = "bin" -authors = [""] -compiler_version = ">=0.22.0" - -[dependencies] diff --git a/noir/noir-repo/test_programs/execution_success/brillig_blake3/Prover.toml b/noir/noir-repo/test_programs/execution_success/brillig_blake3/Prover.toml deleted file mode 100644 index c807701479b..00000000000 --- a/noir/noir-repo/test_programs/execution_success/brillig_blake3/Prover.toml +++ /dev/null @@ -1,37 +0,0 @@ -# hello as bytes -# https://connor4312.github.io/blake3/index.html -x = [104, 101, 108, 108, 111] -result = [ - 0xea, - 0x8f, - 0x16, - 0x3d, - 0xb3, - 0x86, - 0x82, - 0x92, - 0x5e, - 0x44, - 0x91, - 0xc5, - 0xe5, - 0x8d, - 0x4b, - 0xb3, - 0x50, - 0x6e, - 0xf8, - 0xc1, - 0x4e, - 0xb7, - 0x8a, - 0x86, - 0xe9, - 0x08, - 0xc5, - 0x62, - 0x4a, - 0x67, - 0x20, - 0x0f, -] diff --git a/noir/noir-repo/test_programs/execution_success/brillig_blake3/src/main.nr b/noir/noir-repo/test_programs/execution_success/brillig_blake3/src/main.nr deleted file mode 100644 index 64852d775f4..00000000000 --- a/noir/noir-repo/test_programs/execution_success/brillig_blake3/src/main.nr +++ /dev/null @@ -1,4 +0,0 @@ -unconstrained fn main(x: [u8; 5], result: [u8; 32]) { - let digest = std::hash::blake3(x); - assert(digest == result); -} diff --git a/noir/noir-repo/test_programs/execution_success/brillig_ecdsa_secp256k1/Nargo.toml b/noir/noir-repo/test_programs/execution_success/brillig_ecdsa_secp256k1/Nargo.toml deleted file mode 100644 index 495a49f2247..00000000000 --- a/noir/noir-repo/test_programs/execution_success/brillig_ecdsa_secp256k1/Nargo.toml +++ /dev/null @@ -1,6 +0,0 @@ -[package] -name = "brillig_ecdsa_secp256k1" -type = "bin" -authors = [""] - -[dependencies] diff --git a/noir/noir-repo/test_programs/execution_success/brillig_ecdsa_secp256k1/Prover.toml b/noir/noir-repo/test_programs/execution_success/brillig_ecdsa_secp256k1/Prover.toml deleted file mode 100644 index e78fc19cb71..00000000000 --- a/noir/noir-repo/test_programs/execution_success/brillig_ecdsa_secp256k1/Prover.toml +++ /dev/null @@ -1,169 +0,0 @@ - -hashed_message = [ - 0x3a, - 0x73, - 0xf4, - 0x12, - 0x3a, - 0x5c, - 0xd2, - 0x12, - 0x1f, - 0x21, - 0xcd, - 0x7e, - 0x8d, - 0x35, - 0x88, - 0x35, - 0x47, - 0x69, - 0x49, - 0xd0, - 0x35, - 0xd9, - 0xc2, - 0xda, - 0x68, - 0x06, - 0xb4, - 0x63, - 0x3a, - 0xc8, - 0xc1, - 0xe2, -] -pub_key_x = [ - 0xa0, - 0x43, - 0x4d, - 0x9e, - 0x47, - 0xf3, - 0xc8, - 0x62, - 0x35, - 0x47, - 0x7c, - 0x7b, - 0x1a, - 0xe6, - 0xae, - 0x5d, - 0x34, - 0x42, - 0xd4, - 0x9b, - 0x19, - 0x43, - 0xc2, - 0xb7, - 0x52, - 0xa6, - 0x8e, - 0x2a, - 0x47, - 0xe2, - 0x47, - 0xc7, -] -pub_key_y = [ - 0x89, - 0x3a, - 0xba, - 0x42, - 0x54, - 0x19, - 0xbc, - 0x27, - 0xa3, - 0xb6, - 0xc7, - 0xe6, - 0x93, - 0xa2, - 0x4c, - 0x69, - 0x6f, - 0x79, - 0x4c, - 0x2e, - 0xd8, - 0x77, - 0xa1, - 0x59, - 0x3c, - 0xbe, - 0xe5, - 0x3b, - 0x03, - 0x73, - 0x68, - 0xd7, -] -signature = [ - 0xe5, - 0x08, - 0x1c, - 0x80, - 0xab, - 0x42, - 0x7d, - 0xc3, - 0x70, - 0x34, - 0x6f, - 0x4a, - 0x0e, - 0x31, - 0xaa, - 0x2b, - 0xad, - 0x8d, - 0x97, - 0x98, - 0xc3, - 0x80, - 0x61, - 0xdb, - 0x9a, - 0xe5, - 0x5a, - 0x4e, - 0x8d, - 0xf4, - 0x54, - 0xfd, - 0x28, - 0x11, - 0x98, - 0x94, - 0x34, - 0x4e, - 0x71, - 0xb7, - 0x87, - 0x70, - 0xcc, - 0x93, - 0x1d, - 0x61, - 0xf4, - 0x80, - 0xec, - 0xbb, - 0x0b, - 0x89, - 0xd6, - 0xeb, - 0x69, - 0x69, - 0x01, - 0x61, - 0xe4, - 0x9a, - 0x71, - 0x5f, - 0xcd, - 0x55, -] diff --git a/noir/noir-repo/test_programs/execution_success/brillig_ecdsa_secp256k1/src/main.nr b/noir/noir-repo/test_programs/execution_success/brillig_ecdsa_secp256k1/src/main.nr deleted file mode 100644 index 6bde8ac4ac7..00000000000 --- a/noir/noir-repo/test_programs/execution_success/brillig_ecdsa_secp256k1/src/main.nr +++ /dev/null @@ -1,17 +0,0 @@ -// Tests a very simple program. -// -// The features being tested is ecdsa in brillig -fn main(hashed_message: [u8; 32], pub_key_x: [u8; 32], pub_key_y: [u8; 32], signature: [u8; 64]) { - unsafe { - assert(ecdsa(hashed_message, pub_key_x, pub_key_y, signature)); - } -} - -unconstrained fn ecdsa( - hashed_message: [u8; 32], - pub_key_x: [u8; 32], - pub_key_y: [u8; 32], - signature: [u8; 64], -) -> bool { - std::ecdsa_secp256k1::verify_signature(pub_key_x, pub_key_y, signature, hashed_message) -} diff --git a/noir/noir-repo/test_programs/execution_success/brillig_ecdsa_secp256r1/Nargo.toml b/noir/noir-repo/test_programs/execution_success/brillig_ecdsa_secp256r1/Nargo.toml deleted file mode 100644 index 0a71e782104..00000000000 --- a/noir/noir-repo/test_programs/execution_success/brillig_ecdsa_secp256r1/Nargo.toml +++ /dev/null @@ -1,6 +0,0 @@ -[package] -name = "brillig_ecdsa_secp256r1" -type = "bin" -authors = [""] - -[dependencies] diff --git a/noir/noir-repo/test_programs/execution_success/brillig_ecdsa_secp256r1/Prover.toml b/noir/noir-repo/test_programs/execution_success/brillig_ecdsa_secp256r1/Prover.toml deleted file mode 100644 index a45f799877b..00000000000 --- a/noir/noir-repo/test_programs/execution_success/brillig_ecdsa_secp256r1/Prover.toml +++ /dev/null @@ -1,20 +0,0 @@ -hashed_message = [ - 84, 112, 91, 163, 186, 175, 219, 223, 186, 140, 95, 154, 112, 247, 168, 155, 238, 152, - 217, 6, 181, 62, 49, 7, 77, 167, 186, 236, 220, 13, 169, 173, -] -pub_key_x = [ - 85, 15, 71, 16, 3, 243, 223, 151, 195, 223, 80, 106, 199, 151, 246, 114, 31, 177, 161, - 251, 123, 143, 111, 131, 210, 36, 73, 138, 101, 200, 142, 36, -] -pub_key_y = [ - 19, 96, 147, 215, 1, 46, 80, 154, 115, 113, 92, 189, 11, 0, 163, 204, 15, 244, 181, - 192, 27, 63, 250, 25, 106, 177, 251, 50, 112, 54, 184, 230, -] -signature = [ - 44, 112, 168, 208, 132, 182, 43, 252, 92, 224, 54, 65, 202, 249, 247, 42, - 212, 218, 140, 129, 191, 230, 236, 148, 135, 187, 94, 27, 239, 98, 161, 50, - 24, 173, 158, 226, 158, 175, 53, 31, 220, 80, 241, 82, 12, 66, 94, 155, - 144, 138, 7, 39, 139, 67, 176, 236, 123, 135, 39, 120, 193, 78, 7, 132 -] - - diff --git a/noir/noir-repo/test_programs/execution_success/brillig_ecdsa_secp256r1/src/main.nr b/noir/noir-repo/test_programs/execution_success/brillig_ecdsa_secp256r1/src/main.nr deleted file mode 100644 index 091905a3d01..00000000000 --- a/noir/noir-repo/test_programs/execution_success/brillig_ecdsa_secp256r1/src/main.nr +++ /dev/null @@ -1,17 +0,0 @@ -// Tests a very simple program. -// -// The features being tested is ecdsa in brillig -fn main(hashed_message: [u8; 32], pub_key_x: [u8; 32], pub_key_y: [u8; 32], signature: [u8; 64]) { - unsafe { - assert(ecdsa(hashed_message, pub_key_x, pub_key_y, signature)); - } -} - -unconstrained fn ecdsa( - hashed_message: [u8; 32], - pub_key_x: [u8; 32], - pub_key_y: [u8; 32], - signature: [u8; 64], -) -> bool { - std::ecdsa_secp256r1::verify_signature(pub_key_x, pub_key_y, signature, hashed_message) -} diff --git a/noir/noir-repo/test_programs/execution_success/brillig_hash_to_field/Prover.toml b/noir/noir-repo/test_programs/execution_success/brillig_hash_to_field/Prover.toml deleted file mode 100644 index ecdcfd1fb00..00000000000 --- a/noir/noir-repo/test_programs/execution_success/brillig_hash_to_field/Prover.toml +++ /dev/null @@ -1 +0,0 @@ -input = "27" \ No newline at end of file diff --git a/noir/noir-repo/test_programs/execution_success/brillig_hash_to_field/src/main.nr b/noir/noir-repo/test_programs/execution_success/brillig_hash_to_field/src/main.nr deleted file mode 100644 index 48c628020b6..00000000000 --- a/noir/noir-repo/test_programs/execution_success/brillig_hash_to_field/src/main.nr +++ /dev/null @@ -1,12 +0,0 @@ -// Tests a very simple program. -// -// The features being tested is hash_to_field in brillig -fn main(input: Field) -> pub Field { - unsafe { - hash_to_field(input) - } -} - -unconstrained fn hash_to_field(input: Field) -> Field { - std::hash::hash_to_field(&[input]) -} diff --git a/noir/noir-repo/test_programs/execution_success/brillig_keccak/Nargo.toml b/noir/noir-repo/test_programs/execution_success/brillig_keccak/Nargo.toml deleted file mode 100644 index 8cacf2186b8..00000000000 --- a/noir/noir-repo/test_programs/execution_success/brillig_keccak/Nargo.toml +++ /dev/null @@ -1,6 +0,0 @@ -[package] -name = "brillig_keccak" -type = "bin" -authors = [""] - -[dependencies] diff --git a/noir/noir-repo/test_programs/execution_success/brillig_keccak/Prover.toml b/noir/noir-repo/test_programs/execution_success/brillig_keccak/Prover.toml deleted file mode 100644 index d65c4011d3f..00000000000 --- a/noir/noir-repo/test_programs/execution_success/brillig_keccak/Prover.toml +++ /dev/null @@ -1,35 +0,0 @@ -x = 0xbd -result = [ - 0x5a, - 0x50, - 0x2f, - 0x9f, - 0xca, - 0x46, - 0x7b, - 0x26, - 0x6d, - 0x5b, - 0x78, - 0x33, - 0x65, - 0x19, - 0x37, - 0xe8, - 0x05, - 0x27, - 0x0c, - 0xa3, - 0xf3, - 0xaf, - 0x1c, - 0x0d, - 0xd2, - 0x46, - 0x2d, - 0xca, - 0x4b, - 0x3b, - 0x1a, - 0xbf, -] diff --git a/noir/noir-repo/test_programs/execution_success/brillig_keccak/src/main.nr b/noir/noir-repo/test_programs/execution_success/brillig_keccak/src/main.nr deleted file mode 100644 index e5c8e5f493e..00000000000 --- a/noir/noir-repo/test_programs/execution_success/brillig_keccak/src/main.nr +++ /dev/null @@ -1,26 +0,0 @@ -// Tests a very simple program. -// -// The features being tested is keccak256 in brillig -fn main(x: Field, result: [u8; 32]) { - unsafe { - // We use the `as` keyword here to denote the fact that we want to take just the first byte from the x Field - // The padding is taken care of by the program - let digest = keccak256([x as u8], 1); - assert(digest == result); - //#1399: variable message size - let message_size = 4; - let hash_a = keccak256([1, 2, 3, 4], message_size); - let hash_b = keccak256([1, 2, 3, 4, 0, 0, 0, 0], message_size); - - assert(hash_a == hash_b); - - let message_size_big = 8; - let hash_c = keccak256([1, 2, 3, 4, 0, 0, 0, 0], message_size_big); - - assert(hash_a != hash_c); - } -} - -unconstrained fn keccak256(data: [u8; N], msg_len: u32) -> [u8; 32] { - std::hash::keccak256(data, msg_len) -} diff --git a/noir/noir-repo/test_programs/execution_success/brillig_loop/Nargo.toml b/noir/noir-repo/test_programs/execution_success/brillig_loop/Nargo.toml deleted file mode 100644 index 1212397c4db..00000000000 --- a/noir/noir-repo/test_programs/execution_success/brillig_loop/Nargo.toml +++ /dev/null @@ -1,6 +0,0 @@ -[package] -name = "brillig_loop" -type = "bin" -authors = [""] - -[dependencies] diff --git a/noir/noir-repo/test_programs/execution_success/brillig_loop/Prover.toml b/noir/noir-repo/test_programs/execution_success/brillig_loop/Prover.toml deleted file mode 100644 index 22cd5b7c12f..00000000000 --- a/noir/noir-repo/test_programs/execution_success/brillig_loop/Prover.toml +++ /dev/null @@ -1 +0,0 @@ -sum = "6" diff --git a/noir/noir-repo/test_programs/execution_success/brillig_loop/src/main.nr b/noir/noir-repo/test_programs/execution_success/brillig_loop/src/main.nr deleted file mode 100644 index 2d073afb482..00000000000 --- a/noir/noir-repo/test_programs/execution_success/brillig_loop/src/main.nr +++ /dev/null @@ -1,34 +0,0 @@ -// Tests a very simple program. -// -// The features being tested is basic looping on brillig -fn main(sum: u32) { - unsafe { - assert(loop(4) == sum); - assert(loop_incl(3) == sum); - assert(plain_loop() == sum); - } -} - -unconstrained fn loop(x: u32) -> u32 { - let mut sum = 0; - for i in 0..x { - sum = sum + i; - } - sum -} - -unconstrained fn loop_incl(x: u32) -> u32 { - let mut sum = 0; - for i in 0..=x { - sum = sum + i; - } - sum -} - -unconstrained fn plain_loop() -> u32 { - let mut sum = 0; - for i in 0..4 { - sum = sum + i; - } - sum -} diff --git a/noir/noir-repo/test_programs/execution_success/brillig_references/Nargo.toml b/noir/noir-repo/test_programs/execution_success/brillig_references/Nargo.toml deleted file mode 100644 index 0f64b862ba0..00000000000 --- a/noir/noir-repo/test_programs/execution_success/brillig_references/Nargo.toml +++ /dev/null @@ -1,6 +0,0 @@ -[package] -name = "brillig_references" -type = "bin" -authors = [""] - -[dependencies] diff --git a/noir/noir-repo/test_programs/execution_success/brillig_references/Prover.toml b/noir/noir-repo/test_programs/execution_success/brillig_references/Prover.toml deleted file mode 100644 index 151faa5a9b1..00000000000 --- a/noir/noir-repo/test_programs/execution_success/brillig_references/Prover.toml +++ /dev/null @@ -1 +0,0 @@ -x = "2" \ No newline at end of file diff --git a/noir/noir-repo/test_programs/execution_success/brillig_references/src/main.nr b/noir/noir-repo/test_programs/execution_success/brillig_references/src/main.nr deleted file mode 100644 index 47f263cf557..00000000000 --- a/noir/noir-repo/test_programs/execution_success/brillig_references/src/main.nr +++ /dev/null @@ -1,53 +0,0 @@ -unconstrained fn main(mut x: Field) { - add1(&mut x); - assert(x == 3); - // https://github.com/noir-lang/noir/issues/1899 - // let mut s = S { y: x }; - // s.add2(); - // assert(s.y == 5); - // Test that normal mutable variables are still copied - let mut a = 0; - mutate_copy(a); - assert(a == 0); - // Test something 3 allocations deep - let mut nested_allocations = Nested { y: &mut &mut 0 }; - add1(*nested_allocations.y); - assert(**nested_allocations.y == 1); - // Test nested struct allocations with a mutable reference to an array. - let mut c = C { foo: 0, bar: &mut C2 { array: &mut [1, 2] } }; - *c.bar.array = [3, 4]; - let arr: [Field; 2] = *c.bar.array; - assert(arr[0] == 3); - assert(arr[1] == 4); -} - -unconstrained fn add1(x: &mut Field) { - *x += 1; -} - -struct S { - y: Field, -} - -struct Nested { - y: &mut &mut Field, -} - -struct C { - foo: Field, - bar: &mut C2, -} - -struct C2 { - array: &mut [Field; 2], -} - -impl S { - unconstrained fn add2(&mut self) { - self.y += 2; - } -} - -unconstrained fn mutate_copy(mut a: Field) { - a = 7; -} diff --git a/noir/noir-repo/test_programs/execution_success/brillig_sha256/Nargo.toml b/noir/noir-repo/test_programs/execution_success/brillig_sha256/Nargo.toml deleted file mode 100644 index 7140fa0fd0b..00000000000 --- a/noir/noir-repo/test_programs/execution_success/brillig_sha256/Nargo.toml +++ /dev/null @@ -1,6 +0,0 @@ -[package] -name = "brillig_sha256" -type = "bin" -authors = [""] - -[dependencies] diff --git a/noir/noir-repo/test_programs/execution_success/brillig_sha256/Prover.toml b/noir/noir-repo/test_programs/execution_success/brillig_sha256/Prover.toml deleted file mode 100644 index 374ae90ad78..00000000000 --- a/noir/noir-repo/test_programs/execution_success/brillig_sha256/Prover.toml +++ /dev/null @@ -1,35 +0,0 @@ -x = 0xbd -result = [ - 0x68, - 0x32, - 0x57, - 0x20, - 0xaa, - 0xbd, - 0x7c, - 0x82, - 0xf3, - 0x0f, - 0x55, - 0x4b, - 0x31, - 0x3d, - 0x05, - 0x70, - 0xc9, - 0x5a, - 0xcc, - 0xbb, - 0x7d, - 0xc4, - 0xb5, - 0xaa, - 0xe1, - 0x12, - 0x04, - 0xc0, - 0x8f, - 0xfe, - 0x73, - 0x2b, -] diff --git a/noir/noir-repo/test_programs/execution_success/brillig_sha256/src/main.nr b/noir/noir-repo/test_programs/execution_success/brillig_sha256/src/main.nr deleted file mode 100644 index e574676965d..00000000000 --- a/noir/noir-repo/test_programs/execution_success/brillig_sha256/src/main.nr +++ /dev/null @@ -1,15 +0,0 @@ -// Tests a very simple program. -// -// The features being tested is sha256 in brillig -fn main(x: Field, result: [u8; 32]) { - unsafe { - assert(result == sha256(x)); - } -} - -unconstrained fn sha256(x: Field) -> [u8; 32] { - // We use the `as` keyword here to denote the fact that we want to take just the first byte from the x Field - // The padding is taken care of by the program - std::hash::sha256([x as u8]) -} - diff --git a/noir/noir-repo/test_programs/execution_success/brillig_slices/Nargo.toml b/noir/noir-repo/test_programs/execution_success/brillig_slices/Nargo.toml deleted file mode 100644 index 5f6caad088a..00000000000 --- a/noir/noir-repo/test_programs/execution_success/brillig_slices/Nargo.toml +++ /dev/null @@ -1,5 +0,0 @@ -[package] -name = "brillig_slices" -type = "bin" -authors = [""] -[dependencies] diff --git a/noir/noir-repo/test_programs/execution_success/brillig_slices/Prover.toml b/noir/noir-repo/test_programs/execution_success/brillig_slices/Prover.toml deleted file mode 100644 index f28f2f8cc48..00000000000 --- a/noir/noir-repo/test_programs/execution_success/brillig_slices/Prover.toml +++ /dev/null @@ -1,2 +0,0 @@ -x = "5" -y = "10" diff --git a/noir/noir-repo/test_programs/execution_success/brillig_slices/src/main.nr b/noir/noir-repo/test_programs/execution_success/brillig_slices/src/main.nr deleted file mode 100644 index f3928cbc026..00000000000 --- a/noir/noir-repo/test_programs/execution_success/brillig_slices/src/main.nr +++ /dev/null @@ -1,144 +0,0 @@ -use std::slice; -unconstrained fn main(x: Field, y: Field) { - let mut slice: [Field] = &[y, x]; - assert(slice.len() == 2); - - slice = slice.push_back(7); - assert(slice.len() == 3); - assert(slice[0] == y); - assert(slice[1] == x); - assert(slice[2] == 7); - // Array set on slice target - slice[0] = x; - slice[1] = y; - slice[2] = 1; - - assert(slice[0] == x); - assert(slice[1] == y); - assert(slice[2] == 1); - - slice = push_front_to_slice(slice, 2); - assert(slice.len() == 4); - assert(slice[0] == 2); - assert(slice[1] == x); - assert(slice[2] == y); - assert(slice[3] == 1); - - let (item, popped_front_slice) = slice.pop_front(); - slice = popped_front_slice; - assert(item == 2); - - assert(slice.len() == 3); - assert(slice[0] == x); - assert(slice[1] == y); - assert(slice[2] == 1); - - let (popped_back_slice, another_item) = slice.pop_back(); - slice = popped_back_slice; - assert(another_item == 1); - - assert(slice.len() == 2); - assert(slice[0] == x); - assert(slice[1] == y); - - slice = slice.insert(1, 2); - assert(slice.len() == 3); - assert(slice[0] == x); - assert(slice[1] == 2); - assert(slice[2] == y); - - let (removed_slice, should_be_2) = slice.remove(1); - slice = removed_slice; - assert(should_be_2 == 2); - - assert(slice.len() == 2); - assert(slice[0] == x); - assert(slice[1] == y); - - let (slice_with_only_x, should_be_y) = slice.remove(1); - slice = slice_with_only_x; - assert(should_be_y == y); - - assert(slice.len() == 1); - assert(slice[0] == x); - - let (empty_slice, should_be_x) = slice.remove(0); - assert(should_be_x == x); - assert(empty_slice.len() == 0); - - regression_merge_slices(x, y); -} -// Tests slice passing to/from functions -unconstrained fn push_front_to_slice(slice: [T], item: T) -> [T] { - slice.push_front(item) -} -// The parameters to this function must come from witness values (inputs to main) -unconstrained fn regression_merge_slices(x: Field, y: Field) { - merge_slices_if(x, y); - merge_slices_else(x); -} - -unconstrained fn merge_slices_if(x: Field, y: Field) { - let slice = merge_slices_return(x, y); - assert(slice[2] == 10); - assert(slice.len() == 3); - - let slice = merge_slices_mutate(x, y); - assert(slice[3] == 5); - assert(slice.len() == 4); - - let slice = merge_slices_mutate_in_loop(x, y); - assert(slice[6] == 4); - assert(slice.len() == 7); -} - -unconstrained fn merge_slices_else(x: Field) { - let slice = merge_slices_return(x, 5); - assert(slice[0] == 0); - assert(slice[1] == 0); - assert(slice.len() == 2); - - let slice = merge_slices_mutate(x, 5); - assert(slice[2] == 5); - assert(slice.len() == 3); - - let slice = merge_slices_mutate_in_loop(x, 5); - assert(slice[2] == 5); - assert(slice.len() == 3); -} -// Test returning a merged slice without a mutation -unconstrained fn merge_slices_return(x: Field, y: Field) -> [Field] { - let slice = &[0; 2]; - if x != y { - if x != 20 { - slice.push_back(y) - } else { - slice - } - } else { - slice - } -} -// Test mutating a slice inside of an if statement -unconstrained fn merge_slices_mutate(x: Field, y: Field) -> [Field] { - let mut slice = &[0; 2]; - if x != y { - slice = slice.push_back(y); - slice = slice.push_back(x); - } else { - slice = slice.push_back(x); - } - slice -} -// Test mutating a slice inside of a loop in an if statement -unconstrained fn merge_slices_mutate_in_loop(x: Field, y: Field) -> [Field] { - let mut slice = &[0; 2]; - if x != y { - for i in 0..5 { - slice = slice.push_back(i as Field); - } - } else { - slice = slice.push_back(x); - } - slice -} diff --git a/noir/noir-repo/test_programs/execution_success/loop/src/main.nr b/noir/noir-repo/test_programs/execution_success/loop/src/main.nr index 4482fdb3443..b3be4c4c3ff 100644 --- a/noir/noir-repo/test_programs/execution_success/loop/src/main.nr +++ b/noir/noir-repo/test_programs/execution_success/loop/src/main.nr @@ -4,6 +4,7 @@ fn main(six_as_u32: u32) { assert_eq(loop(4), six_as_u32); assert_eq(loop_incl(3), six_as_u32); + assert(plain_loop() == six_as_u32); } fn loop(x: u32) -> u32 { @@ -21,3 +22,11 @@ fn loop_incl(x: u32) -> u32 { } sum } + +fn plain_loop() -> u32 { + let mut sum = 0; + for i in 0..4 { + sum = sum + i; + } + sum +} diff --git a/noir/noir-repo/test_programs/execution_success/sha256/src/main.nr b/noir/noir-repo/test_programs/execution_success/sha256/src/main.nr index d26d916ccff..8e5e46b9837 100644 --- a/noir/noir-repo/test_programs/execution_success/sha256/src/main.nr +++ b/noir/noir-repo/test_programs/execution_success/sha256/src/main.nr @@ -17,6 +17,9 @@ fn main(x: Field, result: [u8; 32], input: [u8; 2], toggle: bool) { // docs:end:sha256_var assert(digest == result); + let digest = std::hash::sha256([x as u8]); + assert(digest == result); + // variable size let size: Field = 1 + toggle as Field; let var_sha = std::hash::sha256_var(input, size as u64); diff --git a/noir/noir-repo/test_programs/execution_success/sha256_brillig_performance_regression/Nargo.toml b/noir/noir-repo/test_programs/execution_success/sha256_brillig_performance_regression/Nargo.toml new file mode 100644 index 00000000000..f7076311e1d --- /dev/null +++ b/noir/noir-repo/test_programs/execution_success/sha256_brillig_performance_regression/Nargo.toml @@ -0,0 +1,7 @@ +[package] +name = "sha256_brillig_performance_regression" +type = "bin" +authors = [""] +compiler_version = ">=0.33.0" + +[dependencies] diff --git a/noir/noir-repo/test_programs/execution_success/sha256_brillig_performance_regression/Prover.toml b/noir/noir-repo/test_programs/execution_success/sha256_brillig_performance_regression/Prover.toml new file mode 100644 index 00000000000..5bb7f354257 --- /dev/null +++ b/noir/noir-repo/test_programs/execution_success/sha256_brillig_performance_regression/Prover.toml @@ -0,0 +1,16 @@ +input_amount = "1" +minimum_output_amount = "2" +secret_hash_for_L1_to_l2_message = "3" +uniswap_fee_tier = "4" + +[aztec_recipient] +inner = "5" + +[caller_on_L1] +inner = "6" + +[input_asset_bridge_portal_address] +inner = "7" + +[output_asset_bridge_portal_address] +inner = "8" diff --git a/noir/noir-repo/test_programs/execution_success/sha256_brillig_performance_regression/src/main.nr b/noir/noir-repo/test_programs/execution_success/sha256_brillig_performance_regression/src/main.nr new file mode 100644 index 00000000000..42cc6d4ff3b --- /dev/null +++ b/noir/noir-repo/test_programs/execution_success/sha256_brillig_performance_regression/src/main.nr @@ -0,0 +1,104 @@ +// Performance regression extracted from an aztec protocol contract. + +unconstrained fn main( + input_asset_bridge_portal_address: EthAddress, + input_amount: Field, + uniswap_fee_tier: Field, + output_asset_bridge_portal_address: EthAddress, + minimum_output_amount: Field, + aztec_recipient: AztecAddress, + secret_hash_for_L1_to_l2_message: Field, + caller_on_L1: EthAddress, +) -> pub Field { + let mut hash_bytes = [0; 260]; // 8 fields of 32 bytes each + 4 bytes fn selector + let input_token_portal_bytes: [u8; 32] = + input_asset_bridge_portal_address.to_field().to_be_bytes(); + let in_amount_bytes: [u8; 32] = input_amount.to_be_bytes(); + let uniswap_fee_tier_bytes: [u8; 32] = uniswap_fee_tier.to_be_bytes(); + let output_token_portal_bytes: [u8; 32] = + output_asset_bridge_portal_address.to_field().to_be_bytes(); + let amount_out_min_bytes: [u8; 32] = minimum_output_amount.to_be_bytes(); + let aztec_recipient_bytes: [u8; 32] = aztec_recipient.to_field().to_be_bytes(); + let secret_hash_for_L1_to_l2_message_bytes: [u8; 32] = + secret_hash_for_L1_to_l2_message.to_be_bytes(); + let caller_on_L1_bytes: [u8; 32] = caller_on_L1.to_field().to_be_bytes(); + + // The purpose of including the following selector is to make the message unique to that specific call. Note that + // it has nothing to do with calling the function. + let selector = comptime { + std::hash::keccak256( + "swap_public(address,uint256,uint24,address,uint256,bytes32,bytes32,address)".as_bytes(), + 75, + ) + }; + + hash_bytes[0] = selector[0]; + hash_bytes[1] = selector[1]; + hash_bytes[2] = selector[2]; + hash_bytes[3] = selector[3]; + + for i in 0..32 { + hash_bytes[i + 4] = input_token_portal_bytes[i]; + hash_bytes[i + 36] = in_amount_bytes[i]; + hash_bytes[i + 68] = uniswap_fee_tier_bytes[i]; + hash_bytes[i + 100] = output_token_portal_bytes[i]; + hash_bytes[i + 132] = amount_out_min_bytes[i]; + hash_bytes[i + 164] = aztec_recipient_bytes[i]; + hash_bytes[i + 196] = secret_hash_for_L1_to_l2_message_bytes[i]; + hash_bytes[i + 228] = caller_on_L1_bytes[i]; + } + + let content_hash = sha256_to_field(hash_bytes); + content_hash +} + +// Convert a 32 byte array to a field element by truncating the final byte +pub fn field_from_bytes_32_trunc(bytes32: [u8; 32]) -> Field { + // Convert it to a field element + let mut v = 1; + let mut high = 0 as Field; + let mut low = 0 as Field; + + for i in 0..15 { + // covers bytes 16..30 (31 is truncated and ignored) + low = low + (bytes32[15 + 15 - i] as Field) * v; + v = v * 256; + // covers bytes 0..14 + high = high + (bytes32[14 - i] as Field) * v; + } + // covers byte 15 + low = low + (bytes32[15] as Field) * v; + + low + high * v +} + +pub fn sha256_to_field(bytes_to_hash: [u8; N]) -> Field { + let sha256_hashed = std::hash::sha256(bytes_to_hash); + let hash_in_a_field = field_from_bytes_32_trunc(sha256_hashed); + + hash_in_a_field +} + +pub trait ToField { + fn to_field(self) -> Field; +} + +pub struct EthAddress { + inner: Field, +} + +impl ToField for EthAddress { + fn to_field(self) -> Field { + self.inner + } +} + +pub struct AztecAddress { + pub inner: Field, +} + +impl ToField for AztecAddress { + fn to_field(self) -> Field { + self.inner + } +} diff --git a/noir/noir-repo/test_programs/gates_report.sh b/noir/noir-repo/test_programs/gates_report.sh index 6c901ff24bc..12e9ae0f864 100755 --- a/noir/noir-repo/test_programs/gates_report.sh +++ b/noir/noir-repo/test_programs/gates_report.sh @@ -4,7 +4,7 @@ set -e BACKEND=${BACKEND:-bb} # These tests are incompatible with gas reporting -excluded_dirs=("workspace" "workspace_default_member" "databus") +excluded_dirs=("workspace" "workspace_default_member" "databus" "double_verify_honk_proof" "verify_honk_proof") current_dir=$(pwd) artifacts_path="$current_dir/acir_artifacts" diff --git a/noir/noir-repo/tooling/nargo/src/ops/compile.rs b/noir/noir-repo/tooling/nargo/src/ops/compile.rs index cd9ccf67957..8c44bf35243 100644 --- a/noir/noir-repo/tooling/nargo/src/ops/compile.rs +++ b/noir/noir-repo/tooling/nargo/src/ops/compile.rs @@ -75,6 +75,7 @@ pub fn compile_program( ) } +#[tracing::instrument(level = "trace", name = "compile_program" skip_all, fields(package = package.name.to_string()))] pub fn compile_program_with_debug_instrumenter( file_manager: &FileManager, parsed_files: &ParsedFiles, @@ -92,6 +93,7 @@ pub fn compile_program_with_debug_instrumenter( noirc_driver::compile_main(&mut context, crate_id, compile_options, cached_program) } +#[tracing::instrument(level = "trace", skip_all, fields(package_name = package.name.to_string()))] pub fn compile_contract( file_manager: &FileManager, parsed_files: &ParsedFiles, diff --git a/noir/noir-repo/tooling/nargo_cli/build.rs b/noir/noir-repo/tooling/nargo_cli/build.rs index ce46a717113..ad1f82f4e45 100644 --- a/noir/noir-repo/tooling/nargo_cli/build.rs +++ b/noir/noir-repo/tooling/nargo_cli/build.rs @@ -152,7 +152,7 @@ fn generate_test_cases( } cases } else { - vec![Inliner::Max] + vec![Inliner::Default] }; // We can't use a `#[test_matrix(brillig_cases, inliner_cases)` if we only want to limit the @@ -163,7 +163,10 @@ fn generate_test_cases( if *brillig && inliner.value() < matrix_config.min_inliner { continue; } - test_cases.push(format!("#[test_case::test_case({brillig}, {})]", inliner.label())); + test_cases.push(format!( + "#[test_case::test_case(ForceBrillig({brillig}), Inliner({}))]", + inliner.label() + )); } } let test_cases = test_cases.join("\n"); @@ -183,7 +186,7 @@ lazy_static::lazy_static! {{ }} {test_cases} -fn test_{test_name}(force_brillig: bool, inliner_aggressiveness: i64) {{ +fn test_{test_name}(force_brillig: ForceBrillig, inliner_aggressiveness: Inliner) {{ let test_program_dir = PathBuf::from("{test_dir}"); // Ignore poisoning errors if some of the matrix cases failed. @@ -198,8 +201,8 @@ fn test_{test_name}(force_brillig: bool, inliner_aggressiveness: i64) {{ let mut nargo = Command::cargo_bin("nargo").unwrap(); nargo.arg("--program-dir").arg(test_program_dir); nargo.arg("{test_command}").arg("--force"); - nargo.arg("--inliner-aggressiveness").arg(inliner_aggressiveness.to_string()); - if force_brillig {{ + nargo.arg("--inliner-aggressiveness").arg(inliner_aggressiveness.0.to_string()); + if force_brillig.0 {{ nargo.arg("--force-brillig"); }} @@ -237,7 +240,7 @@ fn generate_execution_success_tests(test_file: &mut File, test_data_dir: &Path) "#, &MatrixConfig { vary_brillig: !IGNORED_BRILLIG_TESTS.contains(&test_name.as_str()), - vary_inliner: false, + vary_inliner: true, min_inliner: INLINER_MIN_OVERRIDES .iter() .find(|(n, _)| *n == test_name.as_str()) diff --git a/noir/noir-repo/tooling/nargo_cli/tests/execute.rs b/noir/noir-repo/tooling/nargo_cli/tests/execute.rs index e2bef43b571..561520c57a9 100644 --- a/noir/noir-repo/tooling/nargo_cli/tests/execute.rs +++ b/noir/noir-repo/tooling/nargo_cli/tests/execute.rs @@ -14,6 +14,12 @@ mod tests { test_binary::build_test_binary_once!(mock_backend, "../backend_interface/test-binaries"); + // Utilities to keep the test matrix labels more intuitive. + #[derive(Debug, Clone, Copy)] + struct ForceBrillig(pub bool); + #[derive(Debug, Clone, Copy)] + struct Inliner(pub i64); + // include tests generated by `build.rs` include!(concat!(env!("OUT_DIR"), "/execute.rs")); } diff --git a/noir/noir-repo/tooling/nargo_fmt/build.rs b/noir/noir-repo/tooling/nargo_fmt/build.rs index 47bb375f7d1..bd2db5f5b18 100644 --- a/noir/noir-repo/tooling/nargo_fmt/build.rs +++ b/noir/noir-repo/tooling/nargo_fmt/build.rs @@ -47,7 +47,10 @@ fn generate_formatter_tests(test_file: &mut File, test_data_dir: &Path) { .join("\n"); let output_source_path = outputs_dir.join(file_name).display().to_string(); - let output_source = std::fs::read_to_string(output_source_path.clone()).unwrap(); + let output_source = + std::fs::read_to_string(output_source_path.clone()).unwrap_or_else(|_| { + panic!("expected output source at {:?} was not found", &output_source_path) + }); write!( test_file, diff --git a/noir/noir-repo/tooling/nargo_fmt/src/formatter/trait_impl.rs b/noir/noir-repo/tooling/nargo_fmt/src/formatter/trait_impl.rs index 73d9a61b3d4..b31da8a4101 100644 --- a/noir/noir-repo/tooling/nargo_fmt/src/formatter/trait_impl.rs +++ b/noir/noir-repo/tooling/nargo_fmt/src/formatter/trait_impl.rs @@ -10,6 +10,11 @@ use super::Formatter; impl<'a> Formatter<'a> { pub(super) fn format_trait_impl(&mut self, trait_impl: NoirTraitImpl) { + // skip synthetic trait impl's, e.g. generated from trait aliases + if trait_impl.is_synthetic { + return; + } + let has_where_clause = !trait_impl.where_clause.is_empty(); self.write_indentation(); diff --git a/noir/noir-repo/tooling/nargo_fmt/src/formatter/traits.rs b/noir/noir-repo/tooling/nargo_fmt/src/formatter/traits.rs index 9a6b84c6537..1f192be471e 100644 --- a/noir/noir-repo/tooling/nargo_fmt/src/formatter/traits.rs +++ b/noir/noir-repo/tooling/nargo_fmt/src/formatter/traits.rs @@ -15,9 +15,18 @@ impl<'a> Formatter<'a> { self.write_identifier(noir_trait.name); self.format_generics(noir_trait.generics); + if noir_trait.is_alias { + self.write_space(); + self.write_token(Token::Assign); + } + if !noir_trait.bounds.is_empty() { self.skip_comments_and_whitespace(); - self.write_token(Token::Colon); + + if !noir_trait.is_alias { + self.write_token(Token::Colon); + } + self.write_space(); for (index, trait_bound) in noir_trait.bounds.into_iter().enumerate() { @@ -34,6 +43,12 @@ impl<'a> Formatter<'a> { self.format_where_clause(noir_trait.where_clause, true); } + // aliases have ';' in lieu of '{ items }' + if noir_trait.is_alias { + self.write_semicolon(); + return; + } + self.write_space(); self.write_left_brace(); if noir_trait.items.is_empty() { diff --git a/noir/noir-repo/tooling/nargo_fmt/tests/expected/trait_alias.nr b/noir/noir-repo/tooling/nargo_fmt/tests/expected/trait_alias.nr new file mode 100644 index 00000000000..926f3160279 --- /dev/null +++ b/noir/noir-repo/tooling/nargo_fmt/tests/expected/trait_alias.nr @@ -0,0 +1,86 @@ +trait Foo { + fn foo(self) -> Self; +} + +trait Baz = Foo; + +impl Foo for Field { + fn foo(self) -> Self { + self + } +} + +fn baz(x: T) -> T +where + T: Baz, +{ + x.foo() +} + +pub trait Foo_2 { + fn foo_2(self) -> Self; +} + +pub trait Bar_2 { + fn bar_2(self) -> Self; +} + +pub trait Baz_2 = Foo_2 + Bar_2; + +fn baz_2(x: T) -> T +where + T: Baz_2, +{ + x.foo_2().bar_2() +} + +impl Foo_2 for Field { + fn foo_2(self) -> Self { + self + 1 + } +} + +impl Bar_2 for Field { + fn bar_2(self) -> Self { + self + 2 + } +} + +trait Foo_3 { + fn foo_3(self) -> Self; +} + +trait Bar_3 { + fn bar_3(self) -> T; +} + +trait Baz_3 = Foo_3 + Bar_3; + +fn baz_3(x: T) -> U +where + T: Baz_3, +{ + x.foo_3().bar_3() +} + +impl Foo_3 for Field { + fn foo_3(self) -> Self { + self + 1 + } +} + +impl Bar_3 for Field { + fn bar_3(self) -> bool { + true + } +} + +fn main() { + let x: Field = 0; + let _ = baz(x); + + assert(0.foo_2().bar_2() == baz_2(0)); + + assert(0.foo_3().bar_3() == baz_3(0)); +} + diff --git a/noir/noir-repo/tooling/nargo_fmt/tests/input/trait_alias.nr b/noir/noir-repo/tooling/nargo_fmt/tests/input/trait_alias.nr new file mode 100644 index 00000000000..53ae756795b --- /dev/null +++ b/noir/noir-repo/tooling/nargo_fmt/tests/input/trait_alias.nr @@ -0,0 +1,78 @@ +trait Foo { + fn foo(self) -> Self; +} + +trait Baz = Foo; + +impl Foo for Field { + fn foo(self) -> Self { self } +} + +fn baz(x: T) -> T where T: Baz { + x.foo() +} + + +pub trait Foo_2 { + fn foo_2(self) -> Self; +} + +pub trait Bar_2 { + fn bar_2(self) -> Self; +} + +pub trait Baz_2 = Foo_2 + Bar_2; + +fn baz_2(x: T) -> T where T: Baz_2 { + x.foo_2().bar_2() +} + +impl Foo_2 for Field { + fn foo_2(self) -> Self { + self + 1 + } +} + +impl Bar_2 for Field { + fn bar_2(self) -> Self { + self + 2 + } +} + + +trait Foo_3 { + fn foo_3(self) -> Self; +} + +trait Bar_3 { + fn bar_3(self) -> T; +} + +trait Baz_3 = Foo_3 + Bar_3; + +fn baz_3(x: T) -> U where T: Baz_3 { + x.foo_3().bar_3() +} + +impl Foo_3 for Field { + fn foo_3(self) -> Self { + self + 1 + } +} + +impl Bar_3 for Field { + fn bar_3(self) -> bool { + true + } +} + + +fn main() { + let x: Field = 0; + let _ = baz(x); + + assert(0.foo_2().bar_2() == baz_2(0)); + + assert(0.foo_3().bar_3() == baz_3(0)); +} + diff --git a/noir/noir-repo/tooling/noir_codegen/package.json b/noir/noir-repo/tooling/noir_codegen/package.json index 7ec8ba3df64..3530a0ed6f4 100644 --- a/noir/noir-repo/tooling/noir_codegen/package.json +++ b/noir/noir-repo/tooling/noir_codegen/package.json @@ -3,7 +3,7 @@ "contributors": [ "The Noir Team " ], - "version": "0.38.0", + "version": "0.39.0", "packageManager": "yarn@3.5.1", "license": "(MIT OR Apache-2.0)", "type": "module", diff --git a/noir/noir-repo/tooling/noir_js/package.json b/noir/noir-repo/tooling/noir_js/package.json index a3e5777c3d3..8c1c52af8f0 100644 --- a/noir/noir-repo/tooling/noir_js/package.json +++ b/noir/noir-repo/tooling/noir_js/package.json @@ -3,7 +3,7 @@ "contributors": [ "The Noir Team " ], - "version": "0.38.0", + "version": "0.39.0", "packageManager": "yarn@3.5.1", "license": "(MIT OR Apache-2.0)", "type": "module", diff --git a/noir/noir-repo/tooling/noir_js_types/package.json b/noir/noir-repo/tooling/noir_js_types/package.json index 0eeb900253a..2196bc08249 100644 --- a/noir/noir-repo/tooling/noir_js_types/package.json +++ b/noir/noir-repo/tooling/noir_js_types/package.json @@ -4,7 +4,7 @@ "The Noir Team " ], "packageManager": "yarn@3.5.1", - "version": "0.38.0", + "version": "0.39.0", "license": "(MIT OR Apache-2.0)", "homepage": "https://noir-lang.org/", "repository": { diff --git a/noir/noir-repo/tooling/noir_js_types/src/types.ts b/noir/noir-repo/tooling/noir_js_types/src/types.ts index 03e2a93f851..0cfe5d05a17 100644 --- a/noir/noir-repo/tooling/noir_js_types/src/types.ts +++ b/noir/noir-repo/tooling/noir_js_types/src/types.ts @@ -1,5 +1,5 @@ export type Field = string | number | boolean; -export type InputValue = Field | InputMap | (Field | InputMap)[]; +export type InputValue = Field | InputMap | InputValue[]; export type InputMap = { [key: string]: InputValue }; export type Visibility = 'public' | 'private' | 'databus'; diff --git a/noir/noir-repo/tooling/noirc_abi_wasm/package.json b/noir/noir-repo/tooling/noirc_abi_wasm/package.json index ff121326d1a..5f92ada116e 100644 --- a/noir/noir-repo/tooling/noirc_abi_wasm/package.json +++ b/noir/noir-repo/tooling/noirc_abi_wasm/package.json @@ -3,7 +3,7 @@ "contributors": [ "The Noir Team " ], - "version": "0.38.0", + "version": "0.39.0", "license": "(MIT OR Apache-2.0)", "homepage": "https://noir-lang.org/", "repository": { diff --git a/noir/noir-repo/tooling/profiler/Cargo.toml b/noir/noir-repo/tooling/profiler/Cargo.toml index 604208b5a54..798a21ea0d6 100644 --- a/noir/noir-repo/tooling/profiler/Cargo.toml +++ b/noir/noir-repo/tooling/profiler/Cargo.toml @@ -44,6 +44,3 @@ noirc_abi.workspace = true noirc_driver.workspace = true tempfile.workspace = true -[features] -default = ["bn254"] -bn254 = ["acir/bn254"] diff --git a/noir/noir-repo/tooling/profiler/src/cli/execution_flamegraph_cmd.rs b/noir/noir-repo/tooling/profiler/src/cli/execution_flamegraph_cmd.rs index a0b3d6a3128..981d08a3eb1 100644 --- a/noir/noir-repo/tooling/profiler/src/cli/execution_flamegraph_cmd.rs +++ b/noir/noir-repo/tooling/profiler/src/cli/execution_flamegraph_cmd.rs @@ -1,13 +1,12 @@ use std::path::{Path, PathBuf}; use acir::circuit::OpcodeLocation; -use acir::FieldElement; use clap::Args; use color_eyre::eyre::{self, Context}; -use crate::flamegraph::{FlamegraphGenerator, InfernoFlamegraphGenerator, Sample}; +use crate::flamegraph::{BrilligExecutionSample, FlamegraphGenerator, InfernoFlamegraphGenerator}; use crate::fs::{read_inputs_from_file, read_program_from_file}; -use crate::opcode_formatter::AcirOrBrilligOpcode; +use crate::opcode_formatter::format_brillig_opcode; use bn254_blackbox_solver::Bn254BlackBoxSolver; use nargo::ops::DefaultForeignCallExecutor; use noirc_abi::input_parser::Format; @@ -51,7 +50,7 @@ fn run_with_generator( let initial_witness = program.abi.encode(&inputs_map, None)?; println!("Executing"); - let (_, profiling_samples) = nargo::ops::execute_program_with_profiling( + let (_, mut profiling_samples) = nargo::ops::execute_program_with_profiling( &program.bytecode, initial_witness, &Bn254BlackBoxSolver, @@ -59,11 +58,13 @@ fn run_with_generator( )?; println!("Executed"); - let profiling_samples: Vec> = profiling_samples - .into_iter() + println!("Collecting {} samples", profiling_samples.len()); + + let profiling_samples: Vec = profiling_samples + .iter_mut() .map(|sample| { - let call_stack = sample.call_stack; - let brillig_function_id = sample.brillig_function_id; + let call_stack = std::mem::take(&mut sample.call_stack); + let brillig_function_id = std::mem::take(&mut sample.brillig_function_id); let last_entry = call_stack.last(); let opcode = brillig_function_id .and_then(|id| program.bytecode.unconstrained_functions.get(id.0 as usize)) @@ -74,8 +75,8 @@ fn run_with_generator( None } }) - .map(|opcode| AcirOrBrilligOpcode::Brillig(opcode.clone())); - Sample { opcode, call_stack, count: 1, brillig_function_id } + .map(format_brillig_opcode); + BrilligExecutionSample { opcode, call_stack, brillig_function_id } }) .collect(); diff --git a/noir/noir-repo/tooling/profiler/src/cli/gates_flamegraph_cmd.rs b/noir/noir-repo/tooling/profiler/src/cli/gates_flamegraph_cmd.rs index 20cc1b747c3..c3ae29de058 100644 --- a/noir/noir-repo/tooling/profiler/src/cli/gates_flamegraph_cmd.rs +++ b/noir/noir-repo/tooling/profiler/src/cli/gates_flamegraph_cmd.rs @@ -6,10 +6,10 @@ use color_eyre::eyre::{self, Context}; use noirc_artifacts::debug::DebugArtifact; -use crate::flamegraph::{FlamegraphGenerator, InfernoFlamegraphGenerator, Sample}; +use crate::flamegraph::{CompilationSample, FlamegraphGenerator, InfernoFlamegraphGenerator}; use crate::fs::read_program_from_file; use crate::gates_provider::{BackendGatesProvider, GatesProvider}; -use crate::opcode_formatter::AcirOrBrilligOpcode; +use crate::opcode_formatter::format_acir_opcode; #[derive(Debug, Clone, Args)] pub(crate) struct GatesFlamegraphCommand { @@ -83,8 +83,8 @@ fn run_with_provider( .into_iter() .zip(bytecode.opcodes) .enumerate() - .map(|(index, (gates, opcode))| Sample { - opcode: Some(AcirOrBrilligOpcode::Acir(opcode)), + .map(|(index, (gates, opcode))| CompilationSample { + opcode: Some(format_acir_opcode(&opcode)), call_stack: vec![OpcodeLocation::Acir(index)], count: gates, brillig_function_id: None, @@ -106,10 +106,7 @@ fn run_with_provider( #[cfg(test)] mod tests { - use acir::{ - circuit::{Circuit, Program}, - AcirField, - }; + use acir::circuit::{Circuit, Program}; use color_eyre::eyre::{self}; use fm::codespan_files::Files; use noirc_artifacts::program::ProgramArtifact; @@ -143,9 +140,9 @@ mod tests { struct TestFlamegraphGenerator {} impl super::FlamegraphGenerator for TestFlamegraphGenerator { - fn generate_flamegraph<'files, F: AcirField>( + fn generate_flamegraph<'files, S: Sample>( &self, - _samples: Vec>, + _samples: Vec, _debug_symbols: &DebugInfo, _files: &'files impl Files<'files, FileId = fm::FileId>, _artifact_name: &str, diff --git a/noir/noir-repo/tooling/profiler/src/cli/opcodes_flamegraph_cmd.rs b/noir/noir-repo/tooling/profiler/src/cli/opcodes_flamegraph_cmd.rs index bb3df86c339..4e271c9f838 100644 --- a/noir/noir-repo/tooling/profiler/src/cli/opcodes_flamegraph_cmd.rs +++ b/noir/noir-repo/tooling/profiler/src/cli/opcodes_flamegraph_cmd.rs @@ -7,9 +7,9 @@ use color_eyre::eyre::{self, Context}; use noirc_artifacts::debug::DebugArtifact; -use crate::flamegraph::{FlamegraphGenerator, InfernoFlamegraphGenerator, Sample}; +use crate::flamegraph::{CompilationSample, FlamegraphGenerator, InfernoFlamegraphGenerator}; use crate::fs::read_program_from_file; -use crate::opcode_formatter::AcirOrBrilligOpcode; +use crate::opcode_formatter::{format_acir_opcode, format_brillig_opcode}; #[derive(Debug, Clone, Args)] pub(crate) struct OpcodesFlamegraphCommand { @@ -59,8 +59,8 @@ fn run_with_generator( .opcodes .iter() .enumerate() - .map(|(index, opcode)| Sample { - opcode: Some(AcirOrBrilligOpcode::Acir(opcode.clone())), + .map(|(index, opcode)| CompilationSample { + opcode: Some(format_acir_opcode(opcode)), call_stack: vec![OpcodeLocation::Acir(index)], count: 1, brillig_function_id: None, @@ -96,8 +96,8 @@ fn run_with_generator( .bytecode .into_iter() .enumerate() - .map(|(brillig_index, opcode)| Sample { - opcode: Some(AcirOrBrilligOpcode::Brillig(opcode)), + .map(|(brillig_index, opcode)| CompilationSample { + opcode: Some(format_brillig_opcode(&opcode)), call_stack: vec![OpcodeLocation::Brillig { acir_index: acir_opcode_index, brillig_index, @@ -146,7 +146,7 @@ mod tests { brillig::{BrilligBytecode, BrilligFunctionId}, Circuit, Opcode, Program, }, - AcirField, FieldElement, + FieldElement, }; use color_eyre::eyre::{self}; use fm::codespan_files::Files; @@ -160,9 +160,9 @@ mod tests { struct TestFlamegraphGenerator {} impl super::FlamegraphGenerator for TestFlamegraphGenerator { - fn generate_flamegraph<'files, F: AcirField>( + fn generate_flamegraph<'files, S: Sample>( &self, - _samples: Vec>, + _samples: Vec, _debug_symbols: &DebugInfo, _files: &'files impl Files<'files, FileId = fm::FileId>, _artifact_name: &str, diff --git a/noir/noir-repo/tooling/profiler/src/flamegraph.rs b/noir/noir-repo/tooling/profiler/src/flamegraph.rs index 7882ac903ef..38966902314 100644 --- a/noir/noir-repo/tooling/profiler/src/flamegraph.rs +++ b/noir/noir-repo/tooling/profiler/src/flamegraph.rs @@ -3,7 +3,6 @@ use std::{collections::BTreeMap, io::BufWriter}; use acir::circuit::brillig::BrilligFunctionId; use acir::circuit::OpcodeLocation; -use acir::AcirField; use color_eyre::eyre::{self}; use fm::codespan_files::Files; use fxhash::FxHashMap as HashMap; @@ -13,18 +12,66 @@ use noirc_errors::reporter::line_and_column_from_span; use noirc_errors::Location; use noirc_evaluator::brillig::ProcedureId; -use crate::opcode_formatter::AcirOrBrilligOpcode; +pub(crate) trait Sample { + fn count(&self) -> usize; -use super::opcode_formatter::format_opcode; + fn brillig_function_id(&self) -> Option; + + fn call_stack(&self) -> &[OpcodeLocation]; + + fn opcode(self) -> Option; +} #[derive(Debug)] -pub(crate) struct Sample { - pub(crate) opcode: Option>, +pub(crate) struct CompilationSample { + pub(crate) opcode: Option, pub(crate) call_stack: Vec, pub(crate) count: usize, pub(crate) brillig_function_id: Option, } +impl Sample for CompilationSample { + fn count(&self) -> usize { + self.count + } + + fn brillig_function_id(&self) -> Option { + self.brillig_function_id + } + + fn call_stack(&self) -> &[OpcodeLocation] { + &self.call_stack + } + + fn opcode(self) -> Option { + self.opcode + } +} + +pub(crate) struct BrilligExecutionSample { + pub(crate) opcode: Option, + pub(crate) call_stack: Vec, + pub(crate) brillig_function_id: Option, +} + +impl Sample for BrilligExecutionSample { + fn count(&self) -> usize { + 1 + } + + fn brillig_function_id(&self) -> Option { + self.brillig_function_id + } + + fn call_stack(&self) -> &[OpcodeLocation] { + &self.call_stack + } + + fn opcode(self) -> Option { + self.opcode + } +} + #[derive(Debug, Default)] pub(crate) struct FoldedStackItem { pub(crate) total_samples: usize, @@ -33,9 +80,9 @@ pub(crate) struct FoldedStackItem { pub(crate) trait FlamegraphGenerator { #[allow(clippy::too_many_arguments)] - fn generate_flamegraph<'files, F: AcirField>( + fn generate_flamegraph<'files, S: Sample>( &self, - samples: Vec>, + samples: Vec, debug_symbols: &DebugInfo, files: &'files impl Files<'files, FileId = fm::FileId>, artifact_name: &str, @@ -49,9 +96,9 @@ pub(crate) struct InfernoFlamegraphGenerator { } impl FlamegraphGenerator for InfernoFlamegraphGenerator { - fn generate_flamegraph<'files, F: AcirField>( + fn generate_flamegraph<'files, S: Sample>( &self, - samples: Vec>, + samples: Vec, debug_symbols: &DebugInfo, files: &'files impl Files<'files, FileId = fm::FileId>, artifact_name: &str, @@ -82,8 +129,8 @@ impl FlamegraphGenerator for InfernoFlamegraphGenerator { } } -fn generate_folded_sorted_lines<'files, F: AcirField>( - samples: Vec>, +fn generate_folded_sorted_lines<'files, S: Sample>( + samples: Vec, debug_symbols: &DebugInfo, files: &'files impl Files<'files, FileId = fm::FileId>, ) -> Vec { @@ -92,15 +139,15 @@ fn generate_folded_sorted_lines<'files, F: AcirField>( let mut resolution_cache: HashMap> = HashMap::default(); for sample in samples { - let mut location_names = Vec::with_capacity(sample.call_stack.len()); - for opcode_location in sample.call_stack { + let mut location_names = Vec::with_capacity(sample.call_stack().len()); + for opcode_location in sample.call_stack() { let callsite_labels = resolution_cache - .entry(opcode_location) + .entry(*opcode_location) .or_insert_with(|| { find_callsite_labels( debug_symbols, - &opcode_location, - sample.brillig_function_id, + opcode_location, + sample.brillig_function_id(), files, ) }) @@ -109,11 +156,14 @@ fn generate_folded_sorted_lines<'files, F: AcirField>( location_names.extend(callsite_labels); } - if let Some(opcode) = &sample.opcode { - location_names.push(format_opcode(opcode)); + // We move `sample` by calling `sample.opcode()` so we want to fetch the sample count here. + let count = sample.count(); + + if let Some(opcode) = sample.opcode() { + location_names.push(opcode); } - add_locations_to_folded_stack_items(&mut folded_stack_items, location_names, sample.count); + add_locations_to_folded_stack_items(&mut folded_stack_items, location_names, count); } to_folded_sorted_lines(&folded_stack_items, Default::default()) @@ -251,7 +301,7 @@ mod tests { use noirc_errors::{debug_info::DebugInfo, Location, Span}; use std::{collections::BTreeMap, path::Path}; - use crate::{flamegraph::Sample, opcode_formatter::AcirOrBrilligOpcode}; + use crate::{flamegraph::CompilationSample, opcode_formatter::format_acir_opcode}; use super::generate_folded_sorted_lines; @@ -338,25 +388,25 @@ mod tests { BTreeMap::default(), ); - let samples: Vec> = vec![ - Sample { - opcode: Some(AcirOrBrilligOpcode::Acir(AcirOpcode::AssertZero( + let samples: Vec = vec![ + CompilationSample { + opcode: Some(format_acir_opcode(&AcirOpcode::AssertZero::( Expression::default(), ))), call_stack: vec![OpcodeLocation::Acir(0)], count: 10, brillig_function_id: None, }, - Sample { - opcode: Some(AcirOrBrilligOpcode::Acir(AcirOpcode::AssertZero( + CompilationSample { + opcode: Some(format_acir_opcode(&AcirOpcode::AssertZero::( Expression::default(), ))), call_stack: vec![OpcodeLocation::Acir(1)], count: 20, brillig_function_id: None, }, - Sample { - opcode: Some(AcirOrBrilligOpcode::Acir(AcirOpcode::MemoryInit { + CompilationSample { + opcode: Some(format_acir_opcode(&AcirOpcode::MemoryInit:: { block_id: BlockId(0), init: vec![], block_type: acir::circuit::opcodes::BlockType::Memory, diff --git a/noir/noir-repo/tooling/profiler/src/opcode_formatter.rs b/noir/noir-repo/tooling/profiler/src/opcode_formatter.rs index 3c910fa5401..b4367de9e7e 100644 --- a/noir/noir-repo/tooling/profiler/src/opcode_formatter.rs +++ b/noir/noir-repo/tooling/profiler/src/opcode_formatter.rs @@ -2,12 +2,6 @@ use acir::brillig::{BinaryFieldOp, BinaryIntOp, BlackBoxOp, Opcode as BrilligOpc use acir::circuit::{opcodes::BlackBoxFuncCall, Opcode as AcirOpcode}; use acir::AcirField; -#[derive(Debug)] -pub(crate) enum AcirOrBrilligOpcode { - Acir(AcirOpcode), - Brillig(BrilligOpcode), -} - fn format_blackbox_function(call: &BlackBoxFuncCall) -> String { match call { BlackBoxFuncCall::AES128Encrypt { .. } => "aes128_encrypt".to_string(), @@ -127,11 +121,10 @@ fn format_brillig_opcode_kind(opcode: &BrilligOpcode) -> String { } } -pub(crate) fn format_opcode(opcode: &AcirOrBrilligOpcode) -> String { - match opcode { - AcirOrBrilligOpcode::Acir(opcode) => format!("acir::{}", format_acir_opcode_kind(opcode)), - AcirOrBrilligOpcode::Brillig(opcode) => { - format!("brillig::{}", format_brillig_opcode_kind(opcode)) - } - } +pub(crate) fn format_acir_opcode(opcode: &AcirOpcode) -> String { + format!("acir::{}", format_acir_opcode_kind(opcode)) +} + +pub(crate) fn format_brillig_opcode(opcode: &BrilligOpcode) -> String { + format!("brillig::{}", format_brillig_opcode_kind(opcode)) } diff --git a/scripts/ci/attach_ebs_cache.sh b/scripts/ci/attach_ebs_cache.sh index f87233ea93f..ab5e614b06b 100755 --- a/scripts/ci/attach_ebs_cache.sh +++ b/scripts/ci/attach_ebs_cache.sh @@ -1,7 +1,7 @@ #!/bin/bash set -eu -EBS_CACHE_TAG=$1 +EBS_CACHE_TAG=$1-v2 SIZE=$2 REGION="us-east-2" AVAILABILITY_ZONE="us-east-2a" @@ -83,8 +83,8 @@ if [ "$EXISTING_VOLUME" == "None" ]; then --availability-zone $AVAILABILITY_ZONE \ --size $SIZE \ --volume-type gp3 \ - --throughput 1000 \ - --iops 5000 \ + --throughput 125 \ + --iops 3000 \ --tag-specifications "ResourceType=volume,Tags=[{Key=username,Value=$EBS_CACHE_TAG-$SIZE-gp3}]" \ --query "VolumeId" \ --output text) diff --git a/scripts/ci/get_e2e_jobs.sh b/scripts/ci/get_e2e_jobs.sh index 37ee194d2e8..2dbdb42ac40 100755 --- a/scripts/ci/get_e2e_jobs.sh +++ b/scripts/ci/get_e2e_jobs.sh @@ -25,6 +25,7 @@ allow_list=( "e2e_cross_chain_messaging" "e2e_crowdfunding_and_claim" "e2e_deploy_contract" + "e2e_epochs" "e2e_fees" "e2e_fees_failures" "e2e_fees_gas_estimation" @@ -68,7 +69,6 @@ done # Add the input labels and expanded matches to allow_list allow_list+=("${input_labels[@]}" "${expanded_allow_list[@]}") - # Generate full list of targets, excluding specific entries, on one line test_list=$(echo "${full_list[@]}" | grep -v 'base' | grep -v 'bench' | grep -v "network" | grep -v 'devnet' | xargs echo) diff --git a/spartan/aztec-chaos-scenarios/Chart.yaml b/spartan/aztec-chaos-scenarios/Chart.yaml new file mode 100644 index 00000000000..7bb3bfd6136 --- /dev/null +++ b/spartan/aztec-chaos-scenarios/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: aztec-chaos-scenarios +description: Chaos scenarios for spartan using chaos-mesh +type: application +version: 0.1.0 +appVersion: "1.0.0" diff --git a/spartan/network-shaping/scripts/stop_experiments.sh b/spartan/aztec-chaos-scenarios/scripts/stop_experiments.sh similarity index 100% rename from spartan/network-shaping/scripts/stop_experiments.sh rename to spartan/aztec-chaos-scenarios/scripts/stop_experiments.sh diff --git a/spartan/network-shaping/templates/_helpers.tpl b/spartan/aztec-chaos-scenarios/templates/_helpers.tpl similarity index 74% rename from spartan/network-shaping/templates/_helpers.tpl rename to spartan/aztec-chaos-scenarios/templates/_helpers.tpl index 2239bfc3a7e..76ae9b4d9b4 100644 --- a/spartan/network-shaping/templates/_helpers.tpl +++ b/spartan/aztec-chaos-scenarios/templates/_helpers.tpl @@ -1,7 +1,7 @@ {{/* Create a default fully qualified app name. */}} -{{- define "network-shaping.fullname" -}} +{{- define "aztec-chaos-scenarios.fullname" -}} {{- if .Values.fullnameOverride }} {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} {{- else }} @@ -14,17 +14,12 @@ Create a default fully qualified app name. {{- end }} {{- end }} -{{/* -Selector labels -*/}} -{{- define "chaos-mesh.selectorLabels" -}} -{{- end }} {{/* Common labels */}} -{{- define "network-shaping.labels" -}} -app.kubernetes.io/name: {{ include "network-shaping.fullname" . }} +{{- define "aztec-chaos-scenarios.labels" -}} +app.kubernetes.io/name: {{ include "aztec-chaos-scenarios.fullname" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} diff --git a/spartan/aztec-chaos-scenarios/templates/boot-node-failure.yaml b/spartan/aztec-chaos-scenarios/templates/boot-node-failure.yaml new file mode 100644 index 00000000000..b31fbf99dbc --- /dev/null +++ b/spartan/aztec-chaos-scenarios/templates/boot-node-failure.yaml @@ -0,0 +1,21 @@ +{{- if .Values.bootNodeFailure.enabled }} +--- +apiVersion: chaos-mesh.org/v1alpha1 +kind: PodChaos +metadata: + name: {{ .Values.global.targetNamespace }}-boot-node-failure + namespace: {{ .Values.global.chaosMeshNamespace }} + labels: + {{- include "aztec-chaos-scenarios.labels" . | nindent 4 }} + annotations: + "helm.sh/resource-policy": keep +spec: + action: pod-failure + mode: all + selector: + namespaces: + - {{ .Values.global.targetNamespace }} + labelSelectors: + app: boot-node + duration: {{ .Values.bootNodeFailure.duration }} +{{- end }} diff --git a/spartan/network-shaping/templates/network-chaos.yaml b/spartan/aztec-chaos-scenarios/templates/network-shaping.yaml similarity index 72% rename from spartan/network-shaping/templates/network-chaos.yaml rename to spartan/aztec-chaos-scenarios/templates/network-shaping.yaml index 21c954767cb..6e2a86de5f0 100644 --- a/spartan/network-shaping/templates/network-chaos.yaml +++ b/spartan/aztec-chaos-scenarios/templates/network-shaping.yaml @@ -7,7 +7,7 @@ metadata: name: {{ .Values.global.targetNamespace }}-latency namespace: {{ .Values.global.chaosMeshNamespace }} labels: - {{- include "network-shaping.labels" . | nindent 4 }} + {{- include "aztec-chaos-scenarios.labels" . | nindent 4 }} annotations: "helm.sh/hook": post-install,post-upgrade "helm.sh/hook-weight": "0" @@ -31,7 +31,7 @@ metadata: name: {{ .Values.global.targetNamespace }}-bandwidth namespace: {{ .Values.global.chaosMeshNamespace }} labels: - {{- include "network-shaping.labels" . | nindent 4 }} + {{- include "aztec-chaos-scenarios.labels" . | nindent 4 }} annotations: "helm.sh/hook": post-install,post-upgrade "helm.sh/hook-weight": "0" @@ -57,7 +57,7 @@ metadata: name: {{ .Values.global.targetNamespace }}-packet-loss namespace: {{ .Values.global.chaosMeshNamespace }} labels: - {{- include "network-shaping.labels" . | nindent 4 }} + {{- include "aztec-chaos-scenarios.labels" . | nindent 4 }} annotations: "helm.sh/resource-policy": keep spec: @@ -72,26 +72,4 @@ spec: duration: 8760h {{- end }} -{{- if .Values.networkShaping.conditions.killProvers.enabled }} ---- -apiVersion: chaos-mesh.org/v1alpha1 -kind: PodChaos -metadata: - name: {{ .Values.global.targetNamespace }}-kill-provers - namespace: {{ .Values.global.chaosMeshNamespace }} - labels: - {{- include "network-shaping.labels" . | nindent 4 }} - annotations: - "helm.sh/resource-policy": keep -spec: - action: pod-failure - mode: all - selector: - namespaces: - - {{ .Values.global.targetNamespace }} - labelSelectors: - app: prover-node - duration: {{ .Values.networkShaping.conditions.killProvers.duration }} {{- end }} - -{{- end }} \ No newline at end of file diff --git a/spartan/aztec-chaos-scenarios/templates/prover-failure.yaml b/spartan/aztec-chaos-scenarios/templates/prover-failure.yaml new file mode 100644 index 00000000000..ad02d238350 --- /dev/null +++ b/spartan/aztec-chaos-scenarios/templates/prover-failure.yaml @@ -0,0 +1,21 @@ +{{- if .Values.proverFailure.enabled }} +--- +apiVersion: chaos-mesh.org/v1alpha1 +kind: PodChaos +metadata: + name: {{ .Values.global.targetNamespace }}-prover-failure + namespace: {{ .Values.global.chaosMeshNamespace }} + labels: + {{- include "aztec-chaos-scenarios.labels" . | nindent 4 }} + annotations: + "helm.sh/resource-policy": keep +spec: + action: pod-failure + mode: all + selector: + namespaces: + - {{ .Values.global.targetNamespace }} + labelSelectors: + app: prover-node + duration: {{ .Values.proverFailure.duration }} +{{- end }} diff --git a/spartan/aztec-chaos-scenarios/templates/validator-kill.yaml b/spartan/aztec-chaos-scenarios/templates/validator-kill.yaml new file mode 100644 index 00000000000..11177c404f7 --- /dev/null +++ b/spartan/aztec-chaos-scenarios/templates/validator-kill.yaml @@ -0,0 +1,21 @@ +{{- if .Values.validatorKill.enabled }} +--- +apiVersion: chaos-mesh.org/v1alpha1 +kind: PodChaos +metadata: + name: {{ .Values.global.targetNamespace }}-validator-kill + namespace: {{ .Values.global.chaosMeshNamespace }} + labels: + {{- include "aztec-chaos-scenarios.labels" . | nindent 4 }} + annotations: + "helm.sh/resource-policy": keep +spec: + action: pod-kill + mode: fixed-percent + value: {{ .Values.validatorKill.percent | quote }} + selector: + namespaces: + - {{ .Values.global.targetNamespace }} + labelSelectors: + app: validator +{{- end }} diff --git a/spartan/network-shaping/values.yaml b/spartan/aztec-chaos-scenarios/values.yaml similarity index 89% rename from spartan/network-shaping/values.yaml rename to spartan/aztec-chaos-scenarios/values.yaml index 0b3ed0d698e..cb85d9e008f 100644 --- a/spartan/network-shaping/values.yaml +++ b/spartan/aztec-chaos-scenarios/values.yaml @@ -1,3 +1,6 @@ +nameOverride: null +fullnameOverride: null + global: # When deploying, override the namespace to where spartan will deploy to, this will apply all chaos experiments to all pods within that namespace # run deployment with --values global.namespace=your-namespace @@ -7,12 +10,12 @@ global: # Network shaping configuration networkShaping: # Master switch to enable network shaping - enabled: true + enabled: false # Default settings defaultSettings: mode: all - # Set duration to 1 year so the the experiment will run indefinitely unless overridden + # Set duration to 1 year so the experiment will run indefinitely unless overridden duration: 8760h # Network conditions to apply @@ -62,9 +65,17 @@ networkShaping: # Buffer = smoother bandwidth restriction but higher memory usage buffer: 1000 - killProvers: - enabled: false - duration: 13m +proverFailure: + enabled: false + duration: 13m + +validatorKill: + enabled: false + percent: 30 + +bootNodeFailure: + enabled: false + duration: 60m ## Here are some exciting example configurations created by claude: # Example use cases for different configurations: diff --git a/spartan/aztec-chaos-scenarios/values/boot-node-failure.yaml b/spartan/aztec-chaos-scenarios/values/boot-node-failure.yaml new file mode 100644 index 00000000000..5a965eb76f6 --- /dev/null +++ b/spartan/aztec-chaos-scenarios/values/boot-node-failure.yaml @@ -0,0 +1,6 @@ +global: + namespace: "smoke" + +bootNodeFailure: + enabled: true + duration: 60m diff --git a/spartan/network-shaping/values/hard.yaml b/spartan/aztec-chaos-scenarios/values/hard.yaml similarity index 100% rename from spartan/network-shaping/values/hard.yaml rename to spartan/aztec-chaos-scenarios/values/hard.yaml diff --git a/spartan/network-shaping/values/mild.yaml b/spartan/aztec-chaos-scenarios/values/mild.yaml similarity index 100% rename from spartan/network-shaping/values/mild.yaml rename to spartan/aztec-chaos-scenarios/values/mild.yaml diff --git a/spartan/network-shaping/values/moderate.yaml b/spartan/aztec-chaos-scenarios/values/moderate.yaml similarity index 95% rename from spartan/network-shaping/values/moderate.yaml rename to spartan/aztec-chaos-scenarios/values/moderate.yaml index a32d4a09ff5..5c91eded622 100644 --- a/spartan/network-shaping/values/moderate.yaml +++ b/spartan/aztec-chaos-scenarios/values/moderate.yaml @@ -22,4 +22,4 @@ networkShaping: packetLoss: enabled: true loss: "0.5" - correlation: "60" \ No newline at end of file + correlation: "60" diff --git a/spartan/aztec-chaos-scenarios/values/network-requirements.yaml b/spartan/aztec-chaos-scenarios/values/network-requirements.yaml new file mode 100644 index 00000000000..0f5e4c4d3c9 --- /dev/null +++ b/spartan/aztec-chaos-scenarios/values/network-requirements.yaml @@ -0,0 +1,23 @@ +# Imposes the network conditions that are stated as requirements for node operators +global: + namespace: "smoke" + +networkShaping: + enabled: true + conditions: + latency: + enabled: true + delay: + # Regional network latency (e.g., cross-country) + latency: 100ms + jitter: 20ms + correlation: "75" + bandwidth: + enabled: true + rate: 250mbps + limit: 125000000 + buffer: 25000 + packetLoss: + enabled: true + loss: "0.5" + correlation: "60" diff --git a/spartan/aztec-chaos-scenarios/values/prover-failure.yaml b/spartan/aztec-chaos-scenarios/values/prover-failure.yaml new file mode 100644 index 00000000000..16a52fb1d60 --- /dev/null +++ b/spartan/aztec-chaos-scenarios/values/prover-failure.yaml @@ -0,0 +1,6 @@ +global: + namespace: "smoke" + +proverFailure: + enabled: true + duration: 13m diff --git a/spartan/network-shaping/values/rough.yaml b/spartan/aztec-chaos-scenarios/values/rough.yaml similarity index 100% rename from spartan/network-shaping/values/rough.yaml rename to spartan/aztec-chaos-scenarios/values/rough.yaml diff --git a/spartan/aztec-chaos-scenarios/values/validator-kill.yaml b/spartan/aztec-chaos-scenarios/values/validator-kill.yaml new file mode 100644 index 00000000000..695e4a30d94 --- /dev/null +++ b/spartan/aztec-chaos-scenarios/values/validator-kill.yaml @@ -0,0 +1,6 @@ +global: + namespace: "smoke" + +validatorKill: + enabled: true + percent: 25 diff --git a/spartan/aztec-network/files/config/config-prover-env.sh b/spartan/aztec-network/files/config/config-prover-env.sh index 4ee7106cb73..11c4ad5aef2 100644 --- a/spartan/aztec-network/files/config/config-prover-env.sh +++ b/spartan/aztec-network/files/config/config-prover-env.sh @@ -1,11 +1,9 @@ -#!/bin/sh +#!/bin/bash set -eu -alias aztec='node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js' - # Pass the bootnode url as an argument # Ask the bootnode for l1 contract addresses -output=$(aztec get-node-info -u $1) +output=$(node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js get-node-info -u $1) echo "$output" @@ -22,7 +20,7 @@ governance_proposer_address=$(echo "$output" | grep -oP 'GovernanceProposer Addr governance_address=$(echo "$output" | grep -oP 'Governance Address: \K0x[a-fA-F0-9]{40}') # Write the addresses to a file in the shared volume -cat < /shared/contracts.env +cat < /shared/contracts/contracts.env export BOOTSTRAP_NODES=$boot_node_enr export ROLLUP_CONTRACT_ADDRESS=$rollup_address export REGISTRY_CONTRACT_ADDRESS=$registry_address @@ -36,4 +34,4 @@ export GOVERNANCE_PROPOSER_CONTRACT_ADDRESS=$governance_proposer_address export GOVERNANCE_CONTRACT_ADDRESS=$governance_address EOF -cat /shared/contracts.env +cat /shared/contracts/contracts.env diff --git a/spartan/aztec-network/files/config/config-validator-env.sh b/spartan/aztec-network/files/config/config-validator-env.sh index 174482492c4..71d03fbbc98 100644 --- a/spartan/aztec-network/files/config/config-validator-env.sh +++ b/spartan/aztec-network/files/config/config-validator-env.sh @@ -1,11 +1,10 @@ -#!/bin/sh +#!/bin/bash set -eu -alias aztec='node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js' # Pass the bootnode url as an argument # Ask the bootnode for l1 contract addresses -output=$(aztec get-node-info -u $1) +output=$(node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js get-node-info -u $1) echo "$output" @@ -28,7 +27,7 @@ private_key=$(jq -r ".[$INDEX]" /app/config/keys.json) # Write the addresses to a file in the shared volume -cat < /shared/contracts.env +cat < /shared/contracts/contracts.env export BOOTSTRAP_NODES=$boot_node_enr export ROLLUP_CONTRACT_ADDRESS=$rollup_address export REGISTRY_CONTRACT_ADDRESS=$registry_address @@ -45,4 +44,4 @@ export L1_PRIVATE_KEY=$private_key export SEQ_PUBLISHER_PRIVATE_KEY=$private_key EOF -cat /shared/contracts.env \ No newline at end of file +cat /shared/contracts/contracts.env diff --git a/spartan/aztec-network/files/config/deploy-l1-contracts.sh b/spartan/aztec-network/files/config/deploy-l1-contracts.sh index 66cc107f251..4d976821f04 100644 --- a/spartan/aztec-network/files/config/deploy-l1-contracts.sh +++ b/spartan/aztec-network/files/config/deploy-l1-contracts.sh @@ -1,9 +1,8 @@ -#!/bin/sh +#!/bin/bash set -exu CHAIN_ID=$1 -alias aztec='node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js' # Use default account, it is funded on our dev machine export PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" @@ -12,9 +11,9 @@ export PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4 output="" # if INIT_VALIDATORS is true, then we need to pass the validators flag to the deploy-l1-contracts command if [ "$INIT_VALIDATORS" = "true" ]; then - output=$(aztec deploy-l1-contracts --validators $2 --l1-chain-id $CHAIN_ID) + output=$(node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js deploy-l1-contracts --validators $2 --l1-chain-id $CHAIN_ID) else - output=$(aztec deploy-l1-contracts --l1-chain-id $CHAIN_ID) + output=$(node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js deploy-l1-contracts --l1-chain-id $CHAIN_ID) fi echo "$output" @@ -32,7 +31,7 @@ governance_proposer_address=$(echo "$output" | grep -oP 'GovernanceProposer Addr governance_address=$(echo "$output" | grep -oP 'Governance Address: \K0x[a-fA-F0-9]{40}') # Write the addresses to a file in the shared volume -cat < /shared/contracts.env +cat < /shared/contracts/contracts.env export ROLLUP_CONTRACT_ADDRESS=$rollup_address export REGISTRY_CONTRACT_ADDRESS=$registry_address export INBOX_CONTRACT_ADDRESS=$inbox_address @@ -45,4 +44,4 @@ export GOVERNANCE_PROPOSER_CONTRACT_ADDRESS=$governance_proposer_address export GOVERNANCE_CONTRACT_ADDRESS=$governance_address EOF -cat /shared/contracts.env +cat /shared/contracts/contracts.env diff --git a/spartan/aztec-network/files/config/setup-p2p-addresses.sh b/spartan/aztec-network/files/config/setup-p2p-addresses.sh new file mode 100644 index 00000000000..f4b2afce6f2 --- /dev/null +++ b/spartan/aztec-network/files/config/setup-p2p-addresses.sh @@ -0,0 +1,39 @@ +#!/bin/sh + +POD_NAME=$(echo $HOSTNAME) + +if [ "${NETWORK_PUBLIC}" = "true" ]; then + # First try treating HOSTNAME as a pod name + NODE_NAME=$(kubectl get pod $POD_NAME -n ${NAMESPACE} -o jsonpath='{.spec.nodeName}' 2>/dev/null) + + # If that fails, HOSTNAME might be the node name itself + if [ $? -ne 0 ]; then + echo "Could not find pod $POD_NAME, assuming $POD_NAME is the node name" + NODE_NAME=$POD_NAME + fi + + EXTERNAL_IP=$(kubectl get node $NODE_NAME -o jsonpath='{.status.addresses[?(@.type=="ExternalIP")].address}') + + if [ -z "$EXTERNAL_IP" ]; then + echo "Warning: Could not find ExternalIP, falling back to InternalIP" + EXTERNAL_IP=$(kubectl get node $NODE_NAME -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}') + fi + + TCP_ADDR="${EXTERNAL_IP}:${P2P_TCP_PORT}" + UDP_ADDR="${EXTERNAL_IP}:${P2P_UDP_PORT}" + +else + # Get pod IP for non-public networks + POD_IP=$(hostname -i) + TCP_ADDR="${POD_IP}:${P2P_TCP_PORT}" + UDP_ADDR="${POD_IP}:${P2P_UDP_PORT}" +fi + +# Write addresses to file for sourcing +echo "export P2P_TCP_ANNOUNCE_ADDR=${TCP_ADDR}" > /shared/p2p/p2p-addresses +echo "export P2P_TCP_LISTEN_ADDR=0.0.0.0:${P2P_TCP_PORT}" >> /shared/p2p/p2p-addresses +echo "export P2P_UDP_ANNOUNCE_ADDR=${UDP_ADDR}" >> /shared/p2p/p2p-addresses +echo "export P2P_UDP_LISTEN_ADDR=0.0.0.0:${P2P_UDP_PORT}" >> /shared/p2p/p2p-addresses + +echo "P2P addresses configured:" +cat /shared/p2p/p2p-addresses \ No newline at end of file diff --git a/spartan/aztec-network/files/config/setup-service-addresses.sh b/spartan/aztec-network/files/config/setup-service-addresses.sh new file mode 100644 index 00000000000..4594b7a7740 --- /dev/null +++ b/spartan/aztec-network/files/config/setup-service-addresses.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +set -ex + +# Function to get pod and node details +get_service_address() { + local SERVICE_LABEL=$1 + local PORT=$2 + local MAX_RETRIES=30 + local RETRY_INTERVAL=2 + local attempt=1 + + # Get pod name + while [ $attempt -le $MAX_RETRIES ]; do + POD_NAME=$(kubectl get pods -n ${NAMESPACE} -l app=${SERVICE_LABEL} -o jsonpath='{.items[0].metadata.name}') + if [ -n "$POD_NAME" ]; then + break + fi + echo "Attempt $attempt: Waiting for ${SERVICE_LABEL} pod to be available..." >&2 + sleep $RETRY_INTERVAL + attempt=$((attempt + 1)) + done + + if [ -z "$POD_NAME" ]; then + echo "Error: Failed to get ${SERVICE_LABEL} pod name after $MAX_RETRIES attempts" >&2 + return 1 + fi + echo "Pod name: [${POD_NAME}]" >&2 + + # Get node name + attempt=1 + NODE_NAME="" + while [ $attempt -le $MAX_RETRIES ]; do + NODE_NAME=$(kubectl get pod ${POD_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.nodeName}') + if [ -n "$NODE_NAME" ]; then + break + fi + echo "Attempt $attempt: Waiting for node name to be available..." >&2 + sleep $RETRY_INTERVAL + attempt=$((attempt + 1)) + done + + if [ -z "$NODE_NAME" ]; then + echo "Error: Failed to get node name after $MAX_RETRIES attempts" >&2 + return 1 + fi + echo "Node name: ${NODE_NAME}" >&2 + + # Get the node's external IP + NODE_IP=$(kubectl get node ${NODE_NAME} -o jsonpath='{.status.addresses[?(@.type=="ExternalIP")].address}') + echo "Node IP: ${NODE_IP}" >&2 + echo "http://${NODE_IP}:${PORT}" +} + +# Configure Ethereum address +if [ "${ETHEREUM_EXTERNAL_HOST}" != "" ]; then + ETHEREUM_ADDR="${ETHEREUM_EXTERNAL_HOST}" +elif [ "${NETWORK_PUBLIC}" = "true" ]; then + ETHEREUM_ADDR=$(get_service_address "ethereum" "${ETHEREUM_PORT}") +else + ETHEREUM_ADDR="http://${SERVICE_NAME}-ethereum.${NAMESPACE}:${ETHEREUM_PORT}" +fi + +# Configure Boot Node address +if [ "${BOOT_NODE_EXTERNAL_HOST}" != "" ]; then + BOOT_NODE_ADDR="${BOOT_NODE_EXTERNAL_HOST}" +elif [ "${NETWORK_PUBLIC}" = "true" ]; then + BOOT_NODE_ADDR=$(get_service_address "boot-node" "${BOOT_NODE_PORT}") +else + BOOT_NODE_ADDR="http://${SERVICE_NAME}-boot-node.${NAMESPACE}:${BOOT_NODE_PORT}" +fi + +# Configure Prover Node address +if [ "${PROVER_NODE_EXTERNAL_HOST}" != "" ]; then + PROVER_NODE_ADDR="${PROVER_NODE_EXTERNAL_HOST}" +elif [ "${NETWORK_PUBLIC}" = "true" ]; then + PROVER_NODE_ADDR=$(get_service_address "prover-node" "${PROVER_NODE_PORT}") +else + PROVER_NODE_ADDR="http://${SERVICE_NAME}-prover-node.${NAMESPACE}:${PROVER_NODE_PORT}" +fi + + +# Write addresses to file for sourcing +echo "export ETHEREUM_HOST=${ETHEREUM_ADDR}" >> /shared/config/service-addresses +echo "export BOOT_NODE_HOST=${BOOT_NODE_ADDR}" >> /shared/config/service-addresses +echo "export PROVER_NODE_HOST=${PROVER_NODE_ADDR}" >> /shared/config/service-addresses +echo "Addresses configured:" +cat /shared/config/service-addresses diff --git a/spartan/aztec-network/templates/_helpers.tpl b/spartan/aztec-network/templates/_helpers.tpl index f9be2d9ecaa..8afb0c4636d 100644 --- a/spartan/aztec-network/templates/_helpers.tpl +++ b/spartan/aztec-network/templates/_helpers.tpl @@ -50,36 +50,18 @@ app.kubernetes.io/name: {{ include "aztec-network.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} -{{- define "aztec-network.ethereumHost" -}} -{{- if .Values.ethereum.externalHost -}} -http://{{ .Values.ethereum.externalHost }}:{{ .Values.ethereum.service.port }} -{{- else -}} -http://{{ include "aztec-network.fullname" . }}-ethereum.{{ .Release.Namespace }}:{{ .Values.ethereum.service.port }} -{{- end -}} -{{- end -}} + {{- define "aztec-network.pxeUrl" -}} -{{- if .Values.pxe.externalHost -}} -http://{{ .Values.pxe.externalHost }}:{{ .Values.pxe.service.port }} -{{- else -}} -http://{{ include "aztec-network.fullname" . }}-pxe.{{ .Release.Namespace }}:{{ .Values.pxe.service.port }} -{{- end -}} +http://{{ include "aztec-network.fullname" . }}-pxe.{{ .Release.Namespace }}:{{ .Values.pxe.service.nodePort }} {{- end -}} {{- define "aztec-network.bootNodeUrl" -}} -{{- if .Values.bootNode.externalTcpHost -}} -http://{{ .Values.bootNode.externalTcpHost }}:{{ .Values.bootNode.service.nodePort }} -{{- else -}} http://{{ include "aztec-network.fullname" . }}-boot-node-0.{{ include "aztec-network.fullname" . }}-boot-node.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.bootNode.service.nodePort }} {{- end -}} -{{- end -}} {{- define "aztec-network.validatorUrl" -}} -{{- if .Values.validator.externalTcpHost -}} -http://{{ .Values.validator.externalTcpHost }}:{{ .Values.validator.service.nodePort }} -{{- else -}} -http://{{ include "aztec-network.fullname" . }}-validator-0.{{ include "aztec-network.fullname" . }}-validator.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.validator.service.nodePort }} -{{- end -}} +http://{{ include "aztec-network.fullname" . }}-validator.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.validator.service.nodePort }} {{- end -}} {{- define "aztec-network.metricsHost" -}} @@ -123,3 +105,89 @@ http://{{ include "aztec-network.fullname" . }}-metrics.{{ .Release.Namespace }} {{- end -}} {{- end -}} {{- end -}} + +{{/* +P2P Setup Container +*/}} +{{- define "aztec-network.p2pSetupContainer" -}} +- name: setup-p2p-addresses + image: bitnami/kubectl + command: + - /bin/sh + - -c + - | + cp /scripts/setup-p2p-addresses.sh /tmp/setup-p2p-addresses.sh && \ + chmod +x /tmp/setup-p2p-addresses.sh && \ + /tmp/setup-p2p-addresses.sh + env: + - name: NETWORK_PUBLIC + value: "{{ .Values.network.public }}" + - name: NAMESPACE + value: {{ .Release.Namespace }} + - name: P2P_TCP_PORT + value: "{{ .Values.validator.service.p2pTcpPort }}" + - name: P2P_UDP_PORT + value: "{{ .Values.validator.service.p2pUdpPort }}" + volumeMounts: + - name: scripts + mountPath: /scripts + - name: p2p-addresses + mountPath: /shared/p2p +{{- end -}} + +{{/* +Service Address Setup Container +*/}} +{{- define "aztec-network.serviceAddressSetupContainer" -}} +- name: setup-service-addresses + image: bitnami/kubectl + command: + - /bin/bash + - -c + - | + cp /scripts/setup-service-addresses.sh /tmp/setup-service-addresses.sh && \ + chmod +x /tmp/setup-service-addresses.sh && \ + /tmp/setup-service-addresses.sh + env: + - name: NETWORK_PUBLIC + value: "{{ .Values.network.public }}" + - name: NAMESPACE + value: {{ .Release.Namespace }} + - name: EXTERNAL_ETHEREUM_HOST + value: "{{ .Values.ethereum.externalHost }}" + - name: ETHEREUM_PORT + value: "{{ .Values.ethereum.service.port }}" + - name: EXTERNAL_BOOT_NODE_HOST + value: "{{ .Values.bootNode.externalHost }}" + - name: BOOT_NODE_PORT + value: "{{ .Values.bootNode.service.nodePort }}" + - name: EXTERNAL_PROVER_NODE_HOST + value: "{{ .Values.proverNode.externalHost }}" + - name: PROVER_NODE_PORT + value: "{{ .Values.proverNode.service.nodePort }}" + - name: SERVICE_NAME + value: {{ include "aztec-network.fullname" . }} + volumeMounts: + - name: scripts + mountPath: /scripts + - name: config + mountPath: /shared/config +{{- end -}} + +{{/** +Anti-affinity when running in public network mode +*/}} +{{- define "aztec-network.publicAntiAffinity" -}} +affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - validator + - boot-node + - prover + topologyKey: "kubernetes.io/hostname" +{{- end -}} diff --git a/spartan/aztec-network/templates/boot-node.yaml b/spartan/aztec-network/templates/boot-node.yaml index 5f29df22010..f0ee82aabe0 100644 --- a/spartan/aztec-network/templates/boot-node.yaml +++ b/spartan/aztec-network/templates/boot-node.yaml @@ -17,16 +17,25 @@ spec: {{- include "aztec-network.selectorLabels" . | nindent 8 }} app: boot-node spec: + {{- if .Values.network.public }} + hostNetwork: true + {{- include "aztec-network.publicAntiAffinity" . | nindent 6 }} + {{- end }} + serviceAccountName: {{ include "aztec-network.fullname" . }}-node initContainers: + {{- include "aztec-network.p2pSetupContainer" . | nindent 8 }} + {{- include "aztec-network.serviceAddressSetupContainer" . | nindent 8 }} - name: wait-for-ethereum - image: {{ .Values.images.curl.image }} + image: {{ .Values.images.aztec.image }} command: - - /bin/sh + - /bin/bash - -c - | + source /shared/config/service-addresses + echo "Awaiting ethereum node at ${ETHEREUM_HOST}" until curl -s -X POST -H 'Content-Type: application/json' \ -d '{"jsonrpc":"2.0","method":"web3_clientVersion","params":[],"id":67}' \ - {{ include "aztec-network.ethereumHost" . }} | grep -q reth; do + ${ETHEREUM_HOST} | grep -q reth; do echo "Waiting for Ethereum node..." sleep 5 done @@ -38,25 +47,31 @@ spec: done echo "OpenTelemetry collector is ready!" {{- end }} + volumeMounts: + - name: config + mountPath: /shared/config {{- if .Values.bootNode.deployContracts }} - - name: deploy-contracts + - name: deploy-l1-contracts image: {{ .Values.images.aztec.image }} command: [ - "/bin/sh", + "/bin/bash", "-c", - "cp /scripts/deploy-contracts.sh /tmp/deploy-contracts.sh && chmod +x /tmp/deploy-contracts.sh && /tmp/deploy-contracts.sh {{ .Values.ethereum.chainId }} \"{{ join "," .Values.validator.validatorAddresses }}\"" + "cp /scripts/deploy-l1-contracts.sh /tmp/deploy-l1-contracts.sh && \ + chmod +x /tmp/deploy-l1-contracts.sh && \ + source /shared/config/service-addresses && \ + /tmp/deploy-l1-contracts.sh {{ .Values.ethereum.chainId }} \"{{ join "," .Values.validator.validatorAddresses }}\"" ] volumeMounts: - name: scripts-output - mountPath: /shared + mountPath: /shared/contracts + - name: config + mountPath: /shared/config - name: scripts mountPath: /scripts env: - - name: ETHEREUM_HOST - value: {{ include "aztec-network.ethereumHost" . | quote }} - name: INIT_VALIDATORS - value: {{ not .Values.validator.external | quote }} + value: "true" - name: ETHEREUM_SLOT_DURATION value: "{{ .Values.ethereum.blockTime }}" - name: AZTEC_SLOT_DURATION @@ -70,12 +85,15 @@ spec: - name: boot-node image: {{ .Values.images.aztec.image }} command: - # sleep to allow dns name to be resolvable - [ - "/bin/bash", - "-c", - "sleep 30 && source /shared/contracts.env && env && node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --node --archiver --sequencer --pxe", - ] + - /bin/bash + - -c + - | + sleep 30 && \ + source /shared/contracts/contracts.env && \ + source /shared/p2p/p2p-addresses && \ + source /shared/config/service-addresses && \ + env && \ + node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --node --archiver --sequencer --pxe startupProbe: httpGet: path: /status @@ -91,20 +109,24 @@ spec: timeoutSeconds: 30 failureThreshold: 3 volumeMounts: - {{- if .Values.bootNode.deployContracts }} + - name: p2p-addresses + mountPath: /shared/p2p + - name: config + mountPath: /shared/config + {{- if .Values.bootNode.deployContracts }} - name: scripts-output - mountPath: /shared - {{- else }} + mountPath: /shared/contracts + {{- else }} - name: contracts-env - mountPath: /shared/contracts.env + mountPath: /shared/contracts/contracts.env subPath: contracts.env - {{- end }} + {{- end }} env: - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - - name: PORT + - name: AZTEC_PORT value: "{{ .Values.bootNode.service.nodePort }}" - name: LOG_LEVEL value: "{{ .Values.bootNode.logLevel }}" @@ -112,8 +134,6 @@ spec: value: "1" - name: DEBUG value: "{{ .Values.bootNode.debug }}" - - name: ETHEREUM_HOST - value: {{ include "aztec-network.ethereumHost" . | quote }} - name: P2P_ENABLED value: "{{ .Values.bootNode.p2p.enabled }}" - name: COINBASE @@ -126,22 +146,6 @@ spec: value: "{{ .Values.bootNode.sequencer.maxSecondsBetweenBlocks }}" - name: SEQ_MIN_TX_PER_BLOCK value: "{{ .Values.bootNode.sequencer.minTxsPerBlock }}" - - name: P2P_TCP_ANNOUNCE_ADDR - {{- if .Values.bootNode.externalTcpHost }} - value: "{{ .Values.bootNode.externalTcpHost }}:{{ .Values.bootNode.service.p2pTcpPort }}" - {{- else }} - value: "$(POD_IP):{{ .Values.bootNode.service.p2pTcpPort }}" - {{- end }} - - name: P2P_UDP_ANNOUNCE_ADDR - {{- if .Values.bootNode.externalUdpHost }} - value: "{{ .Values.bootNode.externalUdpHost }}:{{ .Values.bootNode.service.p2pUdpPort }}" - {{- else }} - value: "$(POD_IP):{{ .Values.bootNode.service.p2pUdpPort }}" - {{- end }} - - name: P2P_TCP_LISTEN_ADDR - value: "0.0.0.0:{{ .Values.bootNode.service.p2pTcpPort }}" - - name: P2P_UDP_LISTEN_ADDR - value: "0.0.0.0:{{ .Values.bootNode.service.p2pUdpPort }}" - name: VALIDATOR_PRIVATE_KEY value: "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" - name: OTEL_RESOURCE_ATTRIBUTES @@ -154,6 +158,8 @@ spec: value: {{ include "aztec-network.otelCollectorLogsEndpoint" . | quote }} - name: PROVER_REAL_PROOFS value: "{{ .Values.bootNode.realProofs }}" + - name: PXE_PROVER_ENABLED + value: "{{ .Values.bootNode.realProofs }}" - name: ETHEREUM_SLOT_DURATION value: "{{ .Values.ethereum.blockTime }}" - name: AZTEC_SLOT_DURATION @@ -170,10 +176,14 @@ spec: resources: {{- toYaml .Values.bootNode.resources | nindent 12 }} volumes: + - name: p2p-addresses + emptyDir: {} + - name: config + emptyDir: {} {{- if .Values.bootNode.deployContracts }} - name: scripts configMap: - name: {{ include "aztec-network.fullname" . }}-deploy-contracts-script + name: {{ include "aztec-network.fullname" . }}-scripts - name: scripts-output emptyDir: {} {{- else }} @@ -181,18 +191,7 @@ spec: configMap: name: {{ include "aztec-network.fullname" . }}-contracts-env {{- end }} -{{- if .Values.bootNode.deployContracts }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "aztec-network.fullname" . }}-deploy-contracts-script - labels: - {{- include "aztec-network.labels" . | nindent 4 }} -data: - deploy-contracts.sh: | - {{ .Files.Get "files/config/deploy-l1-contracts.sh" | nindent 4 }} -{{- else }} +{{- if not .Values.bootNode.deployContracts }} --- apiVersion: v1 kind: ConfigMap @@ -209,6 +208,7 @@ data: export FEE_JUICE_CONTRACT_ADDRESS={{ .Values.bootNode.contracts.feeJuiceAddress }} export FEE_JUICE_PORTAL_CONTRACT_ADDRESS={{ .Values.bootNode.contracts.feeJuicePortalAddress }} {{- end }} +{{if not .Values.network.public }} --- # Headless service for StatefulSet DNS entries apiVersion: v1 @@ -230,43 +230,4 @@ spec: protocol: UDP - port: {{ .Values.bootNode.service.nodePort }} name: node ---- -{{if .Values.network.public }} -apiVersion: v1 -kind: Service -metadata: - name: boot-node-lb-tcp - labels: - {{- include "aztec-network.labels" . | nindent 4 }} -spec: - type: LoadBalancer - selector: - {{- include "aztec-network.selectorLabels" . | nindent 4 }} - app: boot-node - ports: - - port: {{ .Values.bootNode.service.p2pTcpPort }} - name: p2p-tpc - - port: {{ .Values.bootNode.service.nodePort }} - name: node ---- -apiVersion: v1 -kind: Service -metadata: - name: boot-node-lb-udp - annotations: - service.beta.kubernetes.io/aws-load-balancer-type: "nlb" - service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip" - service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing" - labels: - {{- include "aztec-network.labels" . | nindent 4 }} -spec: - type: LoadBalancer - selector: - {{- include "aztec-network.selectorLabels" . | nindent 4 }} - app: boot-node - ports: - - port: {{ .Values.bootNode.service.p2pUdpPort }} - name: p2p-udp - protocol: UDP ---- {{ end }} diff --git a/spartan/aztec-network/templates/deploy-l1-verifier.yaml b/spartan/aztec-network/templates/deploy-l1-verifier.yaml index 486db8d24ca..8866dd1ca09 100644 --- a/spartan/aztec-network/templates/deploy-l1-verifier.yaml +++ b/spartan/aztec-network/templates/deploy-l1-verifier.yaml @@ -1,4 +1,4 @@ -{{- if .Values.network.setupL2Contracts }} +{{- if and .Values.network.setupL2Contracts .Values.jobs.deployL1Verifier.enable }} apiVersion: batch/v1 kind: Job metadata: @@ -13,6 +13,13 @@ spec: app: deploy-l1-verifier spec: restartPolicy: OnFailure + serviceAccountName: {{ include "aztec-network.fullname" . }}-node + volumes: + - name: config + emptyDir: {} + - name: scripts + configMap: + name: {{ include "aztec-network.fullname" . }}-scripts containers: - name: deploy-l1-verifier image: {{ .Values.images.aztec.image }} @@ -21,38 +28,69 @@ spec: - -c - | set -e + # Install kubectl + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + chmod +x kubectl + mv kubectl /usr/local/bin/ - [ $ENABLE = "true" ] || exit 0 + # Set up kubeconfig using service account credentials + export KUBECONFIG=/tmp/kubeconfig + kubectl config set-cluster default --server=https://kubernetes.default.svc --certificate-authority=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt + kubectl config set-credentials default --token=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) + kubectl config set-context default --cluster=default --user=default + kubectl config use-context default - until curl -s -X GET "$AZTEC_NODE_URL/status"; do - echo "Waiting for Aztec node $AZTEC_NODE_URL..." + cp /scripts/setup-service-addresses.sh /tmp/setup-service-addresses.sh + chmod +x /tmp/setup-service-addresses.sh + /tmp/setup-service-addresses.sh + source /shared/config/service-addresses + + until curl -s -X GET "$BOOT_NODE_HOST/status"; do + echo "Waiting for Aztec node $BOOT_NODE_HOST..." sleep 5 done echo "Boot node is ready!" export ROLLUP_CONTRACT_ADDRESS=$(curl -X POST -H 'Content-Type: application/json' \ -d '{"jsonrpc":"2.0","method":"node_getL1ContractAddresses","params":[],"id":1}' \ - "$AZTEC_NODE_URL" \ + "$BOOT_NODE_HOST" \ | jq -r '.result.rollupAddress.value') echo "Rollup contract address: $ROLLUP_CONTRACT_ADDRESS" node /usr/src/yarn-project/aztec/dest/bin/index.js deploy-l1-verifier --verifier real echo "L1 verifier deployed" env: - - name: ENABLE - value: {{ .Values.jobs.deployL1Verifier.enable | quote }} - name: NODE_NO_WARNINGS value: "1" - name: DEBUG value: "aztec:*" - name: LOG_LEVEL value: "debug" - - name: ETHEREUM_HOST - value: {{ include "aztec-network.ethereumHost" . | quote }} - name: L1_CHAIN_ID value: {{ .Values.ethereum.chainId | quote }} - name: PRIVATE_KEY value: "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" - - name: AZTEC_NODE_URL - value: {{ include "aztec-network.bootNodeUrl" . | quote }} + - name: NETWORK_PUBLIC + value: "{{ .Values.network.public }}" + - name: NAMESPACE + value: {{ .Release.Namespace }} + - name: EXTERNAL_ETHEREUM_HOST + value: "{{ .Values.ethereum.externalHost }}" + - name: ETHEREUM_PORT + value: "{{ .Values.ethereum.service.port }}" + - name: EXTERNAL_BOOT_NODE_HOST + value: "{{ .Values.bootNode.externalHost }}" + - name: BOOT_NODE_PORT + value: "{{ .Values.bootNode.service.nodePort }}" + - name: EXTERNAL_PROVER_NODE_HOST + value: "{{ .Values.proverNode.externalHost }}" + - name: PROVER_NODE_PORT + value: "{{ .Values.proverNode.service.nodePort }}" + - name: SERVICE_NAME + value: {{ include "aztec-network.fullname" . }} + volumeMounts: + - name: config + mountPath: /shared/config + - name: scripts + mountPath: /scripts {{ end }} diff --git a/spartan/aztec-network/templates/prover-agent.yaml b/spartan/aztec-network/templates/prover-agent.yaml index f929daa4b79..34f9648f3ba 100644 --- a/spartan/aztec-network/templates/prover-agent.yaml +++ b/spartan/aztec-network/templates/prover-agent.yaml @@ -17,20 +17,41 @@ spec: {{- include "aztec-network.selectorLabels" . | nindent 8 }} app: prover-agent spec: - {{- if .Values.proverAgent.nodeSelector }} - nodeSelector: - {{- toYaml .Values.proverAgent.nodeSelector | nindent 8 }} + {{- if .Values.proverAgent.gke.spotEnabled }} + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: cloud.google.com/gke-spot + operator: Exists + tolerations: + - key: "cloud.google.com/gke-spot" + operator: "Equal" + value: "true" + effect: "NoSchedule" {{- end }} - + serviceAccountName: {{ include "aztec-network.fullname" . }}-node + {{- if .Values.network.public }} + hostNetwork: true + {{- end }} + volumes: + - name: config + emptyDir: {} + - name: scripts + configMap: + name: {{ include "aztec-network.fullname" . }}-scripts initContainers: + {{- include "aztec-network.serviceAddressSetupContainer" . | nindent 8 }} - name: wait-for-prover-node - image: {{ .Values.images.curl.image }} + image: {{ .Values.images.aztec.image }} command: - - /bin/sh + - /bin/bash - -c - | - until curl -s -X POST "$PROVER_JOB_SOURCE_URL/status"; do - echo "Waiting for Prover node $PROVER_JOB_SOURCE_URL ..." + source /shared/config/service-addresses + until curl -s -X POST ${PROVER_NODE_HOST}/status; do + echo "Waiting for Prover node ${PROVER_NODE_HOST} ..." sleep 5 done echo "Prover node is ready!" @@ -41,18 +62,26 @@ spec: done echo "OpenTelemetry collector is ready!" {{- end }} - env: - - name: PROVER_JOB_SOURCE_URL - value: "http://{{ include "aztec-network.fullname" . }}-prover-node.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.proverNode.service.nodePort }}" + volumeMounts: + - name: config + mountPath: /shared/config containers: - name: prover-agent image: "{{ .Values.images.aztec.image }}" imagePullPolicy: {{ .Values.images.aztec.pullPolicy }} + volumeMounts: + - name: config + mountPath: /shared/config command: - "/bin/bash" - "-c" - - "node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --prover" + - | + source /shared/config/service-addresses && \ + PROVER_JOB_SOURCE_URL=${PROVER_NODE_HOST} \ + node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --prover env: + - name: AZTEC_PORT + value: "{{ .Values.proverAgent.service.nodePort }}" - name: LOG_LEVEL value: "{{ .Values.proverAgent.logLevel }}" - name: LOG_JSON @@ -61,8 +90,6 @@ spec: value: "{{ .Values.proverAgent.debug }}" - name: PROVER_REAL_PROOFS value: "{{ .Values.proverAgent.realProofs }}" - - name: PROVER_JOB_SOURCE_URL - value: "http://{{ include "aztec-network.fullname" . }}-prover-node.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.proverNode.service.nodePort }}" - name: PROVER_AGENT_ENABLED value: "true" - name: PROVER_AGENT_CONCURRENCY diff --git a/spartan/aztec-network/templates/prover-node.yaml b/spartan/aztec-network/templates/prover-node.yaml index d06322b4b79..6b7506149a2 100644 --- a/spartan/aztec-network/templates/prover-node.yaml +++ b/spartan/aztec-network/templates/prover-node.yaml @@ -17,16 +17,24 @@ spec: {{- include "aztec-network.selectorLabels" . | nindent 8 }} app: prover-node spec: + {{- if .Values.network.public }} + hostNetwork: true + {{- include "aztec-network.publicAntiAffinity" . | nindent 6 }} + {{- end }} + serviceAccountName: {{ include "aztec-network.fullname" . }}-node initContainers: - - name: wait-for-boot-node - image: {{ .Values.images.curl.image }} + {{- include "aztec-network.serviceAddressSetupContainer" . | nindent 8 }} + {{- include "aztec-network.p2pSetupContainer" . | nindent 8 }} + - name: wait-for-services + image: {{ .Values.images.aztec.image }} command: - - /bin/sh + - /bin/bash - -c - | + source /shared/config/service-addresses until curl -s -X POST -H 'Content-Type: application/json' \ -d '{"jsonrpc":"2.0","method":"web3_clientVersion","params":[],"id":67}' \ - {{ include "aztec-network.ethereumHost" . }} | grep -q reth; do + ${ETHEREUM_HOST} | grep -q reth; do echo "Waiting for Ethereum node..." sleep 5 done @@ -38,26 +46,31 @@ spec: done echo "OpenTelemetry collector is ready!" {{- end }} - until curl --head --silent {{ include "aztec-network.bootNodeUrl" . }}/status; do + until curl --head --silent $BOOT_NODE_HOST/status; do echo "Waiting for boot node..." sleep 5 done echo "Boot node is ready!" + volumeMounts: + - name: config + mountPath: /shared/config - name: configure-prover-env image: "{{ .Values.images.aztec.image }}" imagePullPolicy: {{ .Values.images.aztec.pullPolicy }} command: - - "/bin/sh" + - "/bin/bash" - "-c" - - "cp /scripts/configure-prover-env.sh /tmp/configure-prover-env.sh && chmod +x /tmp/configure-prover-env.sh && /tmp/configure-prover-env.sh {{ include "aztec-network.bootNodeUrl" . }}" + - "cp /scripts/configure-prover-env.sh /tmp/configure-prover-env.sh && \ + chmod +x /tmp/configure-prover-env.sh && \ + source /shared/config/service-addresses && \ + /tmp/configure-prover-env.sh ${BOOT_NODE_HOST}" volumeMounts: - - name: shared-volume - mountPath: /shared + - name: contracts-env + mountPath: /shared/contracts - name: scripts mountPath: /scripts - env: - - name: ETHEREUM_HOST - value: {{ include "aztec-network.ethereumHost" . | quote }} + - name: config + mountPath: /shared/config containers: - name: prover-node @@ -66,16 +79,25 @@ spec: command: - "/bin/bash" - "-c" - - "source /shared/contracts.env && env && node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --prover-node --archiver" + - | + source /shared/contracts/contracts.env && \ + source /shared/p2p/p2p-addresses && \ + source /shared/config/service-addresses && \ + env && \ + node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --prover-node --archiver volumeMounts: - - name: shared-volume - mountPath: /shared + - name: contracts-env + mountPath: /shared/contracts + - name: p2p-addresses + mountPath: /shared/p2p + - name: config + mountPath: /shared/config env: - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - - name: PORT + - name: AZTEC_PORT value: "{{ .Values.proverNode.service.nodePort }}" - name: LOG_LEVEL value: "{{ .Values.proverNode.logLevel }}" @@ -83,8 +105,6 @@ spec: value: "1" - name: DEBUG value: "{{ .Values.proverNode.debug }}" - - name: ETHEREUM_HOST - value: {{ include "aztec-network.ethereumHost" . | quote }} - name: PROVER_REAL_PROOFS value: "{{ .Values.proverNode.realProofs }}" - name: PROVER_AGENT_ENABLED @@ -106,18 +126,6 @@ spec: value: "{{ .Values.ethereum.chainId }}" - name: P2P_ENABLED value: "{{ .Values.proverNode.p2pEnabled }}" - - name: P2P_TCP_ANNOUNCE_ADDR - {{- if .Values.proverNode.externalTcpHost }} - value: "{{ .Values.proverNode.externalTcpHost }}:{{ .Values.proverNode.service.p2pTcpPort }}" - {{- else }} - value: "$(POD_IP):{{ .Values.proverNode.service.p2pTcpPort }}" - {{- end }} - - name: P2P_UDP_ANNOUNCE_ADDR - {{- if .Values.proverNode.externalUdpHost }} - value: "{{ .Values.proverNode.externalUdpHost }}:{{ .Values.proverNode.service.p2pUdpPort }}" - {{- else }} - value: "$(POD_IP):{{ .Values.proverNode.service.p2pUdpPort }}" - {{- end }} - name: P2P_TCP_LISTEN_ADDR value: "0.0.0.0:{{ .Values.proverNode.service.p2pTcpPort }}" - name: P2P_UDP_LISTEN_ADDR @@ -140,7 +148,13 @@ spec: volumes: - name: scripts configMap: - name: {{ include "aztec-network.fullname" . }}-configure-prover-env + name: {{ include "aztec-network.fullname" . }}-scripts + - name: contracts-env + emptyDir: {} + - name: p2p-addresses + emptyDir: {} + - name: config + emptyDir: {} volumeClaimTemplates: - metadata: name: shared-volume @@ -151,16 +165,7 @@ spec: resources: requests: storage: {{ .Values.proverNode.storage }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "aztec-network.fullname" . }}-configure-prover-env - labels: - {{- include "aztec-network.labels" . | nindent 4 }} -data: - configure-prover-env.sh: | - {{ .Files.Get "files/config/config-prover-env.sh" | nindent 4 }} +{{if not .Values.network.public }} --- apiVersion: v1 kind: Service @@ -181,43 +186,4 @@ spec: - port: {{ .Values.proverNode.service.p2pUdpPort }} name: p2p-udp protocol: UDP ---- -{{if .Values.proverNode.public }} -apiVersion: v1 -kind: Service -metadata: - name: prover-node-lb-tcp - labels: - {{- include "aztec-network.labels" . | nindent 4 }} -spec: - type: LoadBalancer - selector: - {{- include "aztec-network.selectorLabels" . | nindent 4 }} - app: prover-node - ports: - - port: {{ .Values.proverNode.service.nodePort }} - name: node - - port: {{ .Values.proverNode.service.p2pTcpPort }} - name: p2p-tcp ---- -apiVersion: v1 -kind: Service -metadata: - name: prover-node-lb-udp - annotations: - service.beta.kubernetes.io/aws-load-balancer-type: "nlb" - service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip" - service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing" - labels: - {{- include "aztec-network.labels" . | nindent 4 }} -spec: - type: LoadBalancer - selector: - {{- include "aztec-network.selectorLabels" . | nindent 4 }} - app: prover-node - ports: - - port: {{ .Values.proverNode.service.p2pUdpPort }} - name: p2p-udp - protocol: UDP ---- {{ end }} diff --git a/spartan/aztec-network/templates/pxe.yaml b/spartan/aztec-network/templates/pxe.yaml index dbfa87e8f3f..94a8a87886c 100644 --- a/spartan/aztec-network/templates/pxe.yaml +++ b/spartan/aztec-network/templates/pxe.yaml @@ -16,17 +16,36 @@ spec: {{- include "aztec-network.selectorLabels" . | nindent 8 }} app: pxe spec: + serviceAccountName: {{ include "aztec-network.fullname" . }}-node + {{- if .Values.network.public }} + hostNetwork: true + {{- end }} + volumes: + - name: config + emptyDir: {} + - name: scripts + configMap: + name: {{ include "aztec-network.fullname" . }}-scripts + - name: scripts-output + emptyDir: {} initContainers: + {{- include "aztec-network.serviceAddressSetupContainer" . | nindent 8 }} - name: wait-for-boot-node image: {{ .Values.images.curl.image }} command: - /bin/sh - -c - | - until curl --head --silent {{ include "aztec-network.bootNodeUrl" . }}/status; do + source /shared/config/service-addresses + until curl --head --silent ${BOOT_NODE_HOST}/status; do echo "Waiting for boot node..." sleep 5 done + volumeMounts: + - name: config + mountPath: /shared/config + {{- if not .Values.network.public }} + # We only need to wait for the validator service if the network is not public - name: wait-for-validator-service image: {{ .Values.images.curl.image }} command: @@ -37,19 +56,30 @@ spec: echo "Waiting for validator service..." sleep 5 done + {{- end }} containers: - name: pxe image: "{{ .Values.images.aztec.image }}" + volumeMounts: + - name: config + mountPath: /shared/config command: - "/bin/bash" - "-c" - - > + - | + source /shared/config/service-addresses + {{- if .Values.network.public }} + # If the network is public, we need to use the boot node URL + export AZTEC_NODE_URL=${BOOT_NODE_HOST} + {{- else }} + # If the network is not public, we can use the validator URL + export AZTEC_NODE_URL={{ include "aztec-network.validatorUrl" . }} + {{- end }} + echo "AZTEC_NODE_URL=${AZTEC_NODE_URL}" node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --pxe env: - - name: ETHEREUM_HOST - value: {{ include "aztec-network.ethereumHost" . | quote }} - - name: AZTEC_NODE_URL - value: {{ include "aztec-network.bootNodeUrl" . | quote }} + - name: AZTEC_PORT + value: "{{ .Values.pxe.service.nodePort }}" - name: LOG_JSON value: "1" - name: LOG_LEVEL @@ -60,7 +90,7 @@ spec: value: "{{ .Values.pxe.proverEnabled }}" ports: - name: http - containerPort: {{ .Values.pxe.service.port }} + containerPort: {{ .Values.pxe.service.nodePort }} protocol: TCP readinessProbe: exec: @@ -70,7 +100,7 @@ spec: - | curl -s -X POST -H 'content-type: application/json' \ -d '{"jsonrpc":"2.0","method":"pxe_isGlobalStateSynchronized","params":[],"id":67}' \ - 127.0.0.1:{{ .Values.pxe.service.port }} | grep -q '"result":true' + 127.0.0.1:{{ .Values.pxe.service.nodePort }} | grep -q '"result":true' initialDelaySeconds: {{ .Values.pxe.readinessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.pxe.readinessProbe.periodSeconds }} timeoutSeconds: {{ .Values.pxe.readinessProbe.timeoutSeconds }} @@ -92,8 +122,8 @@ spec: app: pxe ports: - protocol: TCP - port: {{ .Values.pxe.service.port }} - targetPort: {{ .Values.pxe.service.targetPort }} + port: {{ .Values.pxe.service.nodePort }} + targetPort: {{ .Values.pxe.service.nodePort }} {{- if and (eq .Values.pxe.service.type "NodePort") .Values.pxe.service.nodePort }} nodePort: {{ .Values.pxe.service.nodePort }} {{- end }} @@ -103,6 +133,10 @@ apiVersion: v1 kind: Service metadata: name: pxe-lb + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: "nlb" + service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip" + service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing" labels: {{- include "aztec-network.labels" . | nindent 4 }} spec: @@ -112,8 +146,8 @@ spec: app: pxe ports: - protocol: TCP - port: {{ .Values.pxe.service.port }} - targetPort: {{ .Values.pxe.service.targetPort }} + port: {{ .Values.pxe.service.nodePort }} + targetPort: {{ .Values.pxe.service.nodePort }} {{- if and (eq .Values.pxe.service.type "NodePort") .Values.pxe.service.nodePort }} nodePort: {{ .Values.pxe.service.nodePort }} {{- end }} diff --git a/spartan/aztec-network/templates/rbac.yaml b/spartan/aztec-network/templates/rbac.yaml new file mode 100644 index 00000000000..a0e8e68cd11 --- /dev/null +++ b/spartan/aztec-network/templates/rbac.yaml @@ -0,0 +1,58 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "aztec-network.fullname" . }}-node + labels: + {{- include "aztec-network.labels" . | nindent 4 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "aztec-network.fullname" . }}-node + labels: + {{- include "aztec-network.labels" . | nindent 4 }} +rules: +- apiGroups: [""] + resources: ["services", "pods"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "aztec-network.fullname" . }}-node + labels: + {{- include "aztec-network.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "aztec-network.fullname" . }}-node +subjects: +- kind: ServiceAccount + name: {{ include "aztec-network.fullname" . }}-node +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "aztec-network.fullname" . }}-node + labels: + {{- include "aztec-network.labels" . | nindent 4 }} +rules: +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "aztec-network.fullname" . }}-node + labels: + {{- include "aztec-network.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "aztec-network.fullname" . }}-node +subjects: +- kind: ServiceAccount + name: {{ include "aztec-network.fullname" . }}-node + namespace: {{ .Release.Namespace }} diff --git a/spartan/aztec-network/templates/reth.yaml b/spartan/aztec-network/templates/reth.yaml index 7312bab7ad5..d6230ecf0ad 100644 --- a/spartan/aztec-network/templates/reth.yaml +++ b/spartan/aztec-network/templates/reth.yaml @@ -16,6 +16,9 @@ spec: {{- include "aztec-network.selectorLabels" . | nindent 8 }} app: ethereum spec: + {{- if .Values.network.public }} + hostNetwork: true + {{- end }} containers: - name: ethereum image: "{{ .Values.images.reth.image }}" @@ -39,21 +42,6 @@ spec: mountPath: /data - name: genesis mountPath: /genesis - # readinessProbe: - # exec: - # command: - # - sh - # - -c - # - | - # wget -qO- --post-data='{"jsonrpc":"2.0","method":"web3_clientVersion","params":[],"id":67}' \ - # --header='Content-Type: application/json' \ - # 127.0.0.1:{{ .Values.ethereum.service.port }} \ - # | grep -q 'reth' - # initialDelaySeconds: {{ .Values.ethereum.readinessProbe.initialDelaySeconds }} - # periodSeconds: {{ .Values.ethereum.readinessProbe.periodSeconds }} - # timeoutSeconds: {{ .Values.ethereum.readinessProbe.timeoutSeconds }} - # successThreshold: {{ .Values.ethereum.readinessProbe.successThreshold }} - # failureThreshold: {{ .Values.ethereum.readinessProbe.failureThreshold }} resources: {{- toYaml .Values.ethereum.resources | nindent 12 }} volumes: @@ -63,6 +51,7 @@ spec: - name: genesis configMap: name: {{ include "aztec-network.fullname" . }}-reth-genesis +{{if not .Values.network.public }} --- apiVersion: v1 kind: Service @@ -82,26 +71,6 @@ spec: {{- if and (eq .Values.ethereum.service.type "NodePort") .Values.ethereum.service.nodePort }} nodePort: {{ .Values.ethereum.service.nodePort }} {{- end }} ---- -{{if .Values.network.public }} -apiVersion: v1 -kind: Service -metadata: - name: ethereum-lb - labels: - {{- include "aztec-network.labels" . | nindent 4 }} -spec: - type: LoadBalancer - selector: - {{- include "aztec-network.selectorLabels" . | nindent 4 }} - app: ethereum - ports: - - protocol: TCP - port: {{ .Values.ethereum.service.port }} - targetPort: {{ .Values.ethereum.service.targetPort }} - {{- if and (eq .Values.ethereum.service.type "NodePort") .Values.ethereum.service.nodePort }} - nodePort: {{ .Values.ethereum.service.nodePort }} - {{- end }} {{ end }} --- apiVersion: v1 diff --git a/spartan/aztec-network/templates/scripts-configmap.yaml b/spartan/aztec-network/templates/scripts-configmap.yaml new file mode 100644 index 00000000000..bc86aabbd36 --- /dev/null +++ b/spartan/aztec-network/templates/scripts-configmap.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "aztec-network.fullname" . }}-scripts + labels: + {{- include "aztec-network.labels" . | nindent 4 }} +data: + setup-service-addresses.sh: | + {{ .Files.Get "files/config/setup-service-addresses.sh" | nindent 4 }} + setup-p2p-addresses.sh: | + {{ .Files.Get "files/config/setup-p2p-addresses.sh" | nindent 4 }} + configure-validator-env.sh: | + {{ .Files.Get "files/config/config-validator-env.sh" | nindent 4 }} + configure-prover-env.sh: | + {{ .Files.Get "files/config/config-prover-env.sh" | nindent 4 }} + deploy-l1-contracts.sh: | + {{ .Files.Get "files/config/deploy-l1-contracts.sh" | nindent 4 }} diff --git a/spartan/aztec-network/templates/setup-l2-contracts.yaml b/spartan/aztec-network/templates/setup-l2-contracts.yaml index 5f6a42d1806..56cf8fc57f2 100644 --- a/spartan/aztec-network/templates/setup-l2-contracts.yaml +++ b/spartan/aztec-network/templates/setup-l2-contracts.yaml @@ -13,16 +13,46 @@ spec: app: setup-l2-contracts spec: restartPolicy: OnFailure + serviceAccountName: {{ include "aztec-network.fullname" . }}-node + volumes: + - name: scripts + configMap: + name: {{ include "aztec-network.fullname" . }}-scripts + - name: config + emptyDir: {} containers: - name: setup-l2-contracts image: {{ .Values.images.aztec.image }} + volumeMounts: + - name: config + mountPath: /shared/config + - name: scripts + mountPath: /scripts command: - /bin/bash - -c - | + # Install kubectl + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + chmod +x kubectl + mv kubectl /usr/local/bin/ + + # Set up kubeconfig using service account credentials + export KUBECONFIG=/tmp/kubeconfig + kubectl config set-cluster default --server=https://kubernetes.default.svc --certificate-authority=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt + kubectl config set-credentials default --token=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) + kubectl config set-context default --cluster=default --user=default + kubectl config use-context default + + cp /scripts/setup-service-addresses.sh /tmp/setup-service-addresses.sh + chmod +x /tmp/setup-service-addresses.sh + /tmp/setup-service-addresses.sh + source /shared/config/service-addresses + export AZTEC_NODE_URL=$BOOT_NODE_HOST + export PXE_URL=$BOOT_NODE_HOST until curl -s -X POST -H 'content-type: application/json' \ -d '{"jsonrpc":"2.0","method":"pxe_getNodeInfo","params":[],"id":67}' \ - {{ include "aztec-network.pxeUrl" . }} | grep -q '"enr:-'; do + $PXE_URL | grep -q '"enr:-'; do echo "Waiting for PXE service..." sleep 5 done @@ -31,10 +61,26 @@ spec: node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js setup-protocol-contracts --skipProofWait --l1-chain-id {{ .Values.ethereum.chainId }} echo "L2 contracts initialized" env: - - name: PXE_URL - value: {{ include "aztec-network.pxeUrl" . | quote }} - name: DEBUG value: "aztec:*" - name: LOG_LEVEL value: "debug" + - name: NETWORK_PUBLIC + value: "{{ .Values.network.public }}" + - name: NAMESPACE + value: {{ .Release.Namespace }} + - name: EXTERNAL_ETHEREUM_HOST + value: "{{ .Values.ethereum.externalHost }}" + - name: ETHEREUM_PORT + value: "{{ .Values.ethereum.service.port }}" + - name: EXTERNAL_BOOT_NODE_HOST + value: "{{ .Values.bootNode.externalHost }}" + - name: BOOT_NODE_PORT + value: "{{ .Values.bootNode.service.nodePort }}" + - name: EXTERNAL_PROVER_NODE_HOST + value: "{{ .Values.proverNode.externalHost }}" + - name: PROVER_NODE_PORT + value: "{{ .Values.proverNode.service.nodePort }}" + - name: SERVICE_NAME + value: {{ include "aztec-network.fullname" . }} {{ end }} diff --git a/spartan/aztec-network/templates/transaction-bot.yaml b/spartan/aztec-network/templates/transaction-bot.yaml index 598262d3710..cd5b88a13bd 100644 --- a/spartan/aztec-network/templates/transaction-bot.yaml +++ b/spartan/aztec-network/templates/transaction-bot.yaml @@ -17,38 +17,64 @@ spec: {{- include "aztec-network.selectorLabels" . | nindent 8 }} app: bot spec: + {{- if .Values.network.public }} + hostNetwork: true + {{- end }} + serviceAccountName: {{ include "aztec-network.fullname" . }}-node + volumes: + - name: config + emptyDir: {} + - name: scripts + configMap: + name: {{ include "aztec-network.fullname" . }}-scripts + - name: scripts-output + emptyDir: {} initContainers: + {{- include "aztec-network.serviceAddressSetupContainer" . | nindent 8 }} - name: wait-for-aztec-node image: "{{ .Values.images.curl.image }}" - env: - - name: AZTEC_NODE_URL - {{- if .Values.bot.nodeUrl }} - value: "{{ .Values.bot.nodeUrl }}" - {{- else }} - value: {{ include "aztec-network.bootNodeUrl" . | quote }} - {{- end }} command: - /bin/sh - -c - | - until curl -s $(AZTEC_NODE_URL)/status; do echo waiting for aztec-node; sleep 2; done + source /shared/config/service-addresses + {{- if .Values.bot.nodeUrl }} + export AZTEC_NODE_URL={{ .Values.bot.nodeUrl }} + {{- else if .Values.network.public }} + export AZTEC_NODE_URL=${BOOT_NODE_HOST} + {{- else }} + export AZTEC_NODE_URL={{ include "aztec-network.validatorUrl" . }} + {{- end }} + echo "AZTEC_NODE_URL=${AZTEC_NODE_URL}" + until curl -s ${AZTEC_NODE_URL}/status; do echo waiting for aztec-node; sleep 2; done + volumeMounts: + - name: config + mountPath: /shared/config containers: - name: transaction-bot image: "{{ .Values.images.aztec.image }}" + volumeMounts: + - name: config + mountPath: /shared/config + - name: scripts + mountPath: /scripts command: - "/bin/bash" - "-c" - - > - node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --pxe --bot - env: - - name: ETHEREUM_HOST - value: {{ include "aztec-network.ethereumHost" . | quote }} - - name: AZTEC_NODE_URL + - | + source /shared/config/service-addresses {{- if .Values.bot.nodeUrl }} - value: "{{ .Values.bot.nodeUrl }}" + export AZTEC_NODE_URL={{ .Values.bot.nodeUrl }} + {{- else if .Values.network.public }} + export AZTEC_NODE_URL=${BOOT_NODE_HOST} {{- else }} - value: {{ include "aztec-network.bootNodeUrl" . | quote }} + export AZTEC_NODE_URL={{ include "aztec-network.validatorUrl" . }} {{- end }} + echo "AZTEC_NODE_URL=${AZTEC_NODE_URL}" + node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --pxe --bot + env: + - name: AZTEC_PORT + value: "{{ .Values.bot.service.nodePort }}" - name: LOG_JSON value: "1" - name: LOG_LEVEL @@ -77,7 +103,7 @@ spec: value: "{{ .Values.bot.stopIfUnhealthy }}" ports: - name: http - containerPort: {{ .Values.bot.service.port }} + containerPort: {{ .Values.bot.service.nodePort }} protocol: TCP readinessProbe: exec: @@ -87,7 +113,7 @@ spec: - | curl -s -X POST -H 'content-type: application/json' \ -d '{"jsonrpc":"2.0","method":"pxe_getNodeInfo","params":[],"id":67}' \ - 127.0.0.1:{{ .Values.bot.service.port }} > /tmp/probe_output.txt && \ + 127.0.0.1:{{ .Values.bot.service.nodePort }} > /tmp/probe_output.txt && \ cat /tmp/probe_output.txt && \ grep -q '"enr:-' /tmp/probe_output.txt initialDelaySeconds: {{ .Values.bot.readinessProbe.initialDelaySeconds }} @@ -111,8 +137,8 @@ spec: app: bot ports: - protocol: TCP - port: {{ .Values.bot.service.port }} - targetPort: {{ .Values.bot.service.targetPort }} + port: {{ .Values.bot.service.nodePort }} + targetPort: {{ .Values.bot.service.nodePort }} {{- if and (eq .Values.bot.service.type "NodePort") .Values.bot.service.nodePort }} nodePort: {{ .Values.bot.service.nodePort }} {{- end }} diff --git a/spartan/aztec-network/templates/validator.yaml b/spartan/aztec-network/templates/validator.yaml index 4ec630257c3..6f8aba191b2 100644 --- a/spartan/aztec-network/templates/validator.yaml +++ b/spartan/aztec-network/templates/validator.yaml @@ -18,20 +18,30 @@ spec: {{- include "aztec-network.selectorLabels" . | nindent 8 }} app: validator spec: + {{- if .Values.network.public }} + hostNetwork: true + {{- include "aztec-network.publicAntiAffinity" . | nindent 6 }} + {{- end }} + serviceAccountName: {{ include "aztec-network.fullname" . }}-node initContainers: - - name: wait-for-boot-node - image: {{ .Values.images.curl.image }} + {{- include "aztec-network.p2pSetupContainer" . | nindent 8 }} + {{- include "aztec-network.serviceAddressSetupContainer" . | nindent 8 }} + - name: wait-for-services + image: {{ .Values.images.aztec.image }} command: - - /bin/sh + - /bin/bash - -c - | + source /shared/config/service-addresses + # First check ethereum node until curl -s -X POST -H 'Content-Type: application/json' \ -d '{"jsonrpc":"2.0","method":"web3_clientVersion","params":[],"id":67}' \ - {{ include "aztec-network.ethereumHost" . }} | grep -q reth; do + $ETHEREUM_HOST | grep -q reth; do echo "Waiting for Ethereum node..." sleep 5 done echo "Ethereum node is ready!" + {{- if .Values.telemetry.enabled }} until curl --head --silent {{ include "aztec-network.otelCollectorMetricsEndpoint" . }} > /dev/null; do echo "Waiting for OpenTelemetry collector..." @@ -39,30 +49,50 @@ spec: done echo "OpenTelemetry collector is ready!" {{- end }} - until curl --head --silent {{ include "aztec-network.bootNodeUrl" . }}/status; do - echo "Waiting for boot node..." - sleep 5 - done - echo "Boot node is ready!" + + if [ "{{ .Values.validator.dynamicBootNode }}" = "true" ]; then + # Get the list of pod IPs for the validator service + echo "{{ include "aztec-network.pxeUrl" . }}" > /shared/pxe/pxe_url + else + until curl --silent --head --fail "${BOOT_NODE_HOST}/status" > /dev/null; do + echo "Waiting for boot node..." + sleep 5 + done + echo "Boot node is ready!" + echo "${BOOT_NODE_HOST}" > /shared/pxe/pxe_url + fi + volumeMounts: + - name: pxe-url + mountPath: /shared/pxe + - name: scripts + mountPath: /scripts + - name: config + mountPath: /shared/config - name: configure-validator-env image: "{{ .Values.images.aztec.image }}" imagePullPolicy: {{ .Values.images.aztec.pullPolicy }} command: - - "/bin/sh" + - "/bin/bash" - "-c" - - "cp /scripts/configure-validator-env.sh /tmp/configure-validator-env.sh && chmod +x /tmp/configure-validator-env.sh && /tmp/configure-validator-env.sh {{ include "aztec-network.bootNodeUrl" . }}" + - | + source /shared/config/service-addresses && \ + cp /scripts/configure-validator-env.sh /tmp/configure-validator-env.sh && \ + chmod +x /tmp/configure-validator-env.sh && \ + /tmp/configure-validator-env.sh "$(cat /shared/pxe/pxe_url)" volumeMounts: - - name: shared-volume - mountPath: /shared + - name: contracts-env + mountPath: /shared/contracts + - name: pxe-url + mountPath: /shared/pxe - name: scripts mountPath: /scripts - name: validator-keys mountPath: /app/config readOnly: true + - name: config + mountPath: /shared/config env: - - name: ETHEREUM_HOST - value: {{ include "aztec-network.ethereumHost" . | quote }} - name: POD_NAME valueFrom: fieldRef: @@ -74,7 +104,13 @@ spec: command: - "/bin/bash" - "-c" - - "sleep 10 && source /shared/contracts.env && env && node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --node --archiver --sequencer" + - | + sleep 10 && \ + source /shared/contracts/contracts.env && \ + source /shared/p2p/p2p-addresses && \ + source /shared/config/service-addresses && \ + env && \ + node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --node --archiver --sequencer startupProbe: httpGet: path: /status @@ -92,14 +128,18 @@ spec: timeoutSeconds: 30 failureThreshold: 3 volumeMounts: - - name: shared-volume - mountPath: /shared + - name: contracts-env + mountPath: /shared/contracts + - name: p2p-addresses + mountPath: /shared/p2p + - name: config + mountPath: /shared/config env: - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - - name: PORT + - name: AZTEC_PORT value: "{{ .Values.validator.service.nodePort }}" - name: LOG_LEVEL value: "{{ .Values.validator.logLevel }}" @@ -107,12 +147,12 @@ spec: value: "1" - name: DEBUG value: "{{ .Values.validator.debug }}" - - name: ETHEREUM_HOST - value: {{ include "aztec-network.ethereumHost" . | quote }} - name: P2P_ENABLED value: "{{ .Values.validator.p2p.enabled }}" - name: VALIDATOR_DISABLED value: "{{ .Values.validator.validator.disabled }}" + - name: VALIDATOR_REEXECUTE + value: "{{ .Values.validator.validator.reexecute }}" - name: SEQ_MAX_SECONDS_BETWEEN_BLOCKS value: "{{ .Values.validator.sequencer.maxSecondsBetweenBlocks }}" - name: SEQ_MIN_TX_PER_BLOCK @@ -123,22 +163,6 @@ spec: value: "{{ .Values.validator.sequencer.enforceTimeTable }}" - name: L1_CHAIN_ID value: "{{ .Values.ethereum.chainId }}" - - name: P2P_TCP_ANNOUNCE_ADDR - {{- if .Values.validator.externalTcpHost }} - value: "{{ .Values.validator.externalTcpHost }}:{{ .Values.validator.service.p2pTcpPort }}" - {{- else }} - value: "$(POD_IP):{{ .Values.validator.service.p2pTcpPort }}" - {{- end }} - - name: P2P_UDP_ANNOUNCE_ADDR - {{- if .Values.validator.externalUdpHost }} - value: "{{ .Values.validator.externalUdpHost }}:{{ .Values.validator.service.p2pUdpPort }}" - {{- else }} - value: "$(POD_IP):{{ .Values.validator.service.p2pUdpPort }}" - {{- end }} - - name: P2P_TCP_LISTEN_ADDR - value: "0.0.0.0:{{ .Values.validator.service.p2pTcpPort }}" - - name: P2P_UDP_LISTEN_ADDR - value: "0.0.0.0:{{ .Values.validator.service.p2pUdpPort }}" - name: OTEL_RESOURCE_ATTRIBUTES value: service.name={{ .Release.Name }},service.namespace={{ .Release.Namespace }},service.version={{ .Chart.AppVersion }},environment={{ .Values.environment | default "production" }} - name: OTEL_EXPORTER_OTLP_METRICS_ENDPOINT @@ -165,32 +189,21 @@ spec: volumes: - name: scripts configMap: - name: {{ include "aztec-network.fullname" . }}-configure-validator-env + name: {{ include "aztec-network.fullname" . }}-scripts - name: validator-keys configMap: name: {{ include "aztec-network.fullname" . }}-validator-keys - volumeClaimTemplates: - - metadata: - name: shared-volume - labels: - {{- include "aztec-network.labels" . | nindent 8 }} - spec: - accessModes: ["ReadWriteOnce"] - resources: - requests: - storage: {{ .Values.validator.storage }} + - name: contracts-env + emptyDir: {} + - name: p2p-addresses + emptyDir: {} + - name: pxe-url + emptyDir: {} + - name: config + emptyDir: {} --- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "aztec-network.fullname" . }}-configure-validator-env - labels: - {{- include "aztec-network.labels" . | nindent 4 }} -data: - configure-validator-env.sh: | - {{ .Files.Get "files/config/config-validator-env.sh" | nindent 4 }} ---- -# Headless service for StatefulSet DNS entries +# If this is not a public network, create a headless service for StatefulSet DNS entries +{{ if not .Values.network.public }} apiVersion: v1 kind: Service metadata: @@ -212,55 +225,4 @@ spec: - port: {{ .Values.validator.service.nodePort }} name: node protocol: TCP ---- -{{if .Values.network.public }} -{{- range $i, $e := until (int .Values.validator.replicas) }} -# Service template for TCP load balancers -apiVersion: v1 -kind: Service -metadata: - name: validator-{{ $i }}-lb-tcp - annotations: - service.beta.kubernetes.io/aws-load-balancer-type: "nlb" - service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip" - service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing" - labels: - {{- include "aztec-network.labels" $ | nindent 4 }} -spec: - type: LoadBalancer - selector: - statefulset.kubernetes.io/pod-name: {{ include "aztec-network.fullname" $ }}-validator-{{ $i }} - {{- include "aztec-network.selectorLabels" $ | nindent 4 }} - app: validator - ports: - - port: {{ $.Values.validator.service.p2pTcpPort }} - name: p2p-tcp - protocol: TCP - - port: {{ $.Values.validator.service.nodePort }} - name: node - protocol: TCP ---- -# Service template for UDP load balancers -apiVersion: v1 -kind: Service -metadata: - name: validator-{{ $i }}-lb-udp - annotations: - service.beta.kubernetes.io/aws-load-balancer-type: "nlb" - service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip" - service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing" - labels: - {{- include "aztec-network.labels" $ | nindent 4 }} -spec: - type: LoadBalancer - selector: - statefulset.kubernetes.io/pod-name: {{ include "aztec-network.fullname" $ }}-validator-{{ $i }} - {{- include "aztec-network.selectorLabels" $ | nindent 4 }} - app: validator - ports: - - port: {{ $.Values.validator.service.p2pUdpPort }} - name: p2p-udp - protocol: UDP ---- -{{- end }} {{ end }} diff --git a/spartan/aztec-network/values.yaml b/spartan/aztec-network/values.yaml index 3d0172d787c..0be51cd0d26 100644 --- a/spartan/aztec-network/values.yaml +++ b/spartan/aztec-network/values.yaml @@ -2,6 +2,12 @@ nameOverride: "" fullnameOverride: "" network: + # If true, pods will use host networking. + # This is to ensure that nodes are individually addressable from the outside. + # Under the current configuration, this also means that there must be a unique + # physical node in the cluster for each pod that participates in peer-to-peer. + # I.e. the sum of the number of validator, boot node, and prover nodes must be + # less than the number of physical nodes in the cluster. public: false setupL2Contracts: true @@ -29,8 +35,7 @@ aztec: epochProofClaimWindow: 13 # in L2 slots bootNode: - externalTcpHost: "" - externalUdpHost: "" + externalHost: "" replicas: 1 service: p2pTcpPort: 40400 @@ -67,8 +72,10 @@ bootNode: storage: "8Gi" validator: - externalTcpHost: "" - externalUdpHost: "" + # If true, the validator will use its peers to serve as the boot node. + # This cannot be used when the network first starts up. + # But it must be used if the boot node is killed, and the validator is restarted. + dynamicBootNode: false replicas: 1 validatorKeys: - 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 @@ -87,6 +94,7 @@ validator: enforceTimeTable: true validator: disabled: false + reexecute: true p2p: enabled: "true" startupProbe: @@ -98,12 +106,9 @@ validator: requests: memory: "2Gi" cpu: "200m" - storage: "8Gi" proverNode: - public: false - externalTcpHost: "" - externalUdpHost: "" + externalHost: "" replicas: 1 p2pEnabled: true service: @@ -121,15 +126,13 @@ proverNode: storage: "8Gi" pxe: - externalHost: "" + proverEnabled: false logLevel: "debug" proverEnable: false - debug: "aztec:*,-aztec:avm_simulator*,-aztec:libp2p_service*,-aztec:circuits:artifact_hash,-json-rpc*,-aztec:l2_block_stream,-aztec:world-state:database" + debug: "aztec:*,-aztec:avm_simulator*,-aztec:libp2p_service*,-aztec:circuits:artifact_hash,-json-rpc*,-aztec:world-state:database,-aztec:l2_block_stream*" replicas: 1 service: - port: 8080 - targetPort: 8080 - nodePort: "" + nodePort: 8081 readinessProbe: initialDelaySeconds: 5 periodSeconds: 10 @@ -143,6 +146,7 @@ pxe: bot: enabled: true + nodeUrl: "" logLevel: "debug" debug: "aztec:*,-aztec:avm_simulator*,-aztec:libp2p_service*,-aztec:circuits:artifact_hash,-json-rpc*,-aztec:l2_block_stream,-aztec:world-state:database" replicas: 1 @@ -159,9 +163,7 @@ bot: stopIfUnhealthy: true service: type: ClusterIP - port: 8080 - targetPort: 8080 - nodePort: "" + nodePort: 8082 readinessProbe: initialDelaySeconds: 5 periodSeconds: 10 @@ -200,13 +202,20 @@ ethereum: storage: "80Gi" proverAgent: + service: + nodePort: 8083 enabled: true replicas: 1 + gke: + spotEnabled: false + logLevel: "debug" debug: "aztec:*,-aztec:avm_simulator*,-aztec:libp2p_service*,-aztec:circuits:artifact_hash,-json-rpc*,-aztec:world-state:database,-aztec:l2_block_stream*" realProofs: false concurrency: 1 bb: hardwareConcurrency: "" + nodeSelector: {} + resources: {} jobs: deployL1Verifier: diff --git a/spartan/aztec-network/values/4-validators-with-metrics.yaml b/spartan/aztec-network/values/4-validators-with-metrics.yaml new file mode 100644 index 00000000000..47387cd89c1 --- /dev/null +++ b/spartan/aztec-network/values/4-validators-with-metrics.yaml @@ -0,0 +1,28 @@ +########## +# BEWARE # +########## +# You need to deploy the metrics helm chart before using this values file. +# head to spartan/metrics and run `./install.sh` +# (then `./forward.sh` if you want to see it) +telemetry: + enabled: true + otelCollectorEndpoint: http://metrics-opentelemetry-collector.metrics:4318 + +validator: + replicas: 4 + validatorKeys: + - 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 + - 0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d + - 0x5de4111afa1a4b94908f83103eb1f1706367c2e68ca870fc3fb9a804cdab365a + - 0x7c852118294e51e653712a81e05800f419141751be58f605c371e15141b007a6 + validatorAddresses: + - 0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266 + - 0x70997970C51812dc3A010C7d01b50e0d17dc79C8 + - 0x3C44CdDdB6a900fa2b585dd299e03d12FA4293BC + - 0x90F79bf6EB2c4f870365E785982E1f101E93b906 + validator: + disabled: false + +bootNode: + validator: + disabled: true diff --git a/spartan/aztec-network/values/48-validators.yaml b/spartan/aztec-network/values/48-validators.yaml index dd7f399d805..31d48095681 100644 --- a/spartan/aztec-network/values/48-validators.yaml +++ b/spartan/aztec-network/values/48-validators.yaml @@ -9,7 +9,6 @@ telemetry: otelCollectorEndpoint: http://metrics-opentelemetry-collector.metrics:4318 validator: - debug: "aztec:*,-aztec:avm_simulator:*,-aztec:libp2p_service" replicas: 48 validatorKeys: - 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 diff --git a/spartan/aztec-network/values/gcp-proving-test.yaml b/spartan/aztec-network/values/gcp-proving-test.yaml index 70b2612b4d7..6a361ecd025 100644 --- a/spartan/aztec-network/values/gcp-proving-test.yaml +++ b/spartan/aztec-network/values/gcp-proving-test.yaml @@ -45,9 +45,6 @@ proverAgent: realProofs: true bb: hardwareConcurrency: 31 - nodeSelector: - cloud.google.com/compute-class: "Performance" - cloud.google.com/machine-family: "t2d" resources: requests: memory: "116Gi" diff --git a/spartan/aztec-network/values/release.yaml b/spartan/aztec-network/values/release.yaml new file mode 100644 index 00000000000..e3e34dac3b8 --- /dev/null +++ b/spartan/aztec-network/values/release.yaml @@ -0,0 +1,158 @@ +network: + public: true + +images: + aztec: + pullPolicy: Always + +telemetry: + enabled: true + otelCollectorEndpoint: http://34.150.160.154:4318 + +validator: + realProofs: true + replicas: 48 + validatorKeys: + - 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 + - 0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d + - 0x5de4111afa1a4b94908f83103eb1f1706367c2e68ca870fc3fb9a804cdab365a + - 0x7c852118294e51e653712a81e05800f419141751be58f605c371e15141b007a6 + - 0x47e179ec197488593b187f80a00eb0da91f1b9d0b13f8733639f19c30a34926a + - 0x8b3a350cf5c34c9194ca85829a2df0ec3153be0318b5e2d3348e872092edffba + - 0x92db14e403b83dfe3df233f83dfa3a0d7096f21ca9b0d6d6b8d88b2b4ec1564e + - 0x4bbbf85ce3377467afe5d46f804f221813b2bb87f24d81f60f1fcdbf7cbf4356 + - 0xdbda1821b80551c9d65939329250298aa3472ba22feea921c0cf5d620ea67b97 + - 0x2a871d0798f97d79848a013d4936a73bf4cc922c825d33c1cf7073dff6d409c6 + - 0xf214f2b2cd398c806f84e317254e0f0b801d0643303237d97a22a48e01628897 + - 0x701b615bbdfb9de65240bc28bd21bbc0d996645a3dd57e7b12bc2bdf6f192c82 + - 0xa267530f49f8280200edf313ee7af6b827f2a8bce2897751d06a843f644967b1 + - 0x47c99abed3324a2707c28affff1267e45918ec8c3f20b8aa892e8b065d2942dd + - 0xc526ee95bf44d8fc405a158bb884d9d1238d99f0612e9f33d006bb0789009aaa + - 0x8166f546bab6da521a8369cab06c5d2b9e46670292d85c875ee9ec20e84ffb61 + - 0xea6c44ac03bff858b476bba40716402b03e41b8e97e276d1baec7c37d42484a0 + - 0x689af8efa8c651a91ad287602527f3af2fe9f6501a7ac4b061667b5a93e037fd + - 0xde9be858da4a475276426320d5e9262ecfc3ba460bfac56360bfa6c4c28b4ee0 + - 0xdf57089febbacf7ba0bc227dafbffa9fc08a93fdc68e1e42411a14efcf23656e + - 0xeaa861a9a01391ed3d587d8a5a84ca56ee277629a8b02c22093a419bf240e65d + - 0xc511b2aa70776d4ff1d376e8537903dae36896132c90b91d52c1dfbae267cd8b + - 0x224b7eb7449992aac96d631d9677f7bf5888245eef6d6eeda31e62d2f29a83e4 + - 0x4624e0802698b9769f5bdb260a3777fbd4941ad2901f5966b854f953497eec1b + - 0x375ad145df13ed97f8ca8e27bb21ebf2a3819e9e0a06509a812db377e533def7 + - 0x18743e59419b01d1d846d97ea070b5a3368a3e7f6f0242cf497e1baac6972427 + - 0xe383b226df7c8282489889170b0f68f66af6459261f4833a781acd0804fafe7a + - 0xf3a6b71b94f5cd909fb2dbb287da47badaa6d8bcdc45d595e2884835d8749001 + - 0x4e249d317253b9641e477aba8dd5d8f1f7cf5250a5acadd1229693e262720a19 + - 0x233c86e887ac435d7f7dc64979d7758d69320906a0d340d2b6518b0fd20aa998 + - 0x85a74ca11529e215137ccffd9c95b2c72c5fb0295c973eb21032e823329b3d2d + - 0xac8698a440d33b866b6ffe8775621ce1a4e6ebd04ab7980deb97b3d997fc64fb + - 0xf076539fbce50f0513c488f32bf81524d30ca7a29f400d68378cc5b1b17bc8f2 + - 0x5544b8b2010dbdbef382d254802d856629156aba578f453a76af01b81a80104e + - 0x47003709a0a9a4431899d4e014c1fd01c5aad19e873172538a02370a119bae11 + - 0x9644b39377553a920edc79a275f45fa5399cbcf030972f771d0bca8097f9aad3 + - 0xcaa7b4a2d30d1d565716199f068f69ba5df586cf32ce396744858924fdf827f0 + - 0xfc5a028670e1b6381ea876dd444d3faaee96cffae6db8d93ca6141130259247c + - 0x5b92c5fe82d4fabee0bc6d95b4b8a3f9680a0ed7801f631035528f32c9eb2ad5 + - 0xb68ac4aa2137dd31fd0732436d8e59e959bb62b4db2e6107b15f594caf0f405f + - 0xc95eaed402c8bd203ba04d81b35509f17d0719e3f71f40061a2ec2889bc4caa7 + - 0x55afe0ab59c1f7bbd00d5531ddb834c3c0d289a4ff8f318e498cb3f004db0b53 + - 0xc3f9b30f83d660231203f8395762fa4257fa7db32039f739630f87b8836552cc + - 0x3db34a7bcc6424e7eadb8e290ce6b3e1423c6e3ef482dd890a812cd3c12bbede + - 0xae2daaa1ce8a70e510243a77187d2bc8da63f0186074e4a4e3a7bfae7fa0d639 + - 0x5ea5c783b615eb12be1afd2bdd9d96fae56dda0efe894da77286501fd56bac64 + - 0xf702e0ff916a5a76aaf953de7583d128c013e7f13ecee5d701b49917361c5e90 + - 0x7ec49efc632757533404c2139a55b4d60d565105ca930a58709a1c52d86cf5d3 + validatorAddresses: + - 0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266 + - 0x70997970C51812dc3A010C7d01b50e0d17dc79C8 + - 0x3C44CdDdB6a900fa2b585dd299e03d12FA4293BC + - 0x90F79bf6EB2c4f870365E785982E1f101E93b906 + - 0x15d34AAf54267DB7D7c367839AAf71A00a2C6A65 + - 0x9965507D1a55bcC2695C58ba16FB37d819B0A4dc + - 0x976EA74026E726554dB657fA54763abd0C3a0aa9 + - 0x14dC79964da2C08b23698B3D3cc7Ca32193d9955 + - 0x23618e81E3f5cdF7f54C3d65f7FBc0aBf5B21E8f + - 0xa0Ee7A142d267C1f36714E4a8F75612F20a79720 + - 0xBcd4042DE499D14e55001CcbB24a551F3b954096 + - 0x71bE63f3384f5fb98995898A86B02Fb2426c5788 + - 0xFABB0ac9d68B0B445fB7357272Ff202C5651694a + - 0x1CBd3b2770909D4e10f157cABC84C7264073C9Ec + - 0xdF3e18d64BC6A983f673Ab319CCaE4f1a57C7097 + - 0xcd3B766CCDd6AE721141F452C550Ca635964ce71 + - 0x2546BcD3c84621e976D8185a91A922aE77ECEc30 + - 0xbDA5747bFD65F08deb54cb465eB87D40e51B197E + - 0xdD2FD4581271e230360230F9337D5c0430Bf44C0 + - 0x8626f6940E2eb28930eFb4CeF49B2d1F2C9C1199 + - 0x09DB0a93B389bEF724429898f539AEB7ac2Dd55f + - 0x02484cb50AAC86Eae85610D6f4Bf026f30f6627D + - 0x08135Da0A343E492FA2d4282F2AE34c6c5CC1BbE + - 0x5E661B79FE2D3F6cE70F5AAC07d8Cd9abb2743F1 + - 0x61097BA76cD906d2ba4FD106E757f7Eb455fc295 + - 0xDf37F81dAAD2b0327A0A50003740e1C935C70913 + - 0x553BC17A05702530097c3677091C5BB47a3a7931 + - 0x87BdCE72c06C21cd96219BD8521bDF1F42C78b5e + - 0x40Fc963A729c542424cD800349a7E4Ecc4896624 + - 0x9DCCe783B6464611f38631e6C851bf441907c710 + - 0x1BcB8e569EedAb4668e55145Cfeaf190902d3CF2 + - 0x8263Fce86B1b78F95Ab4dae11907d8AF88f841e7 + - 0xcF2d5b3cBb4D7bF04e3F7bFa8e27081B52191f91 + - 0x86c53Eb85D0B7548fea5C4B4F82b4205C8f6Ac18 + - 0x1aac82773CB722166D7dA0d5b0FA35B0307dD99D + - 0x2f4f06d218E426344CFE1A83D53dAd806994D325 + - 0x1003ff39d25F2Ab16dBCc18EcE05a9B6154f65F4 + - 0x9eAF5590f2c84912A08de97FA28d0529361Deb9E + - 0x11e8F3eA3C6FcF12EcfF2722d75CEFC539c51a1C + - 0x7D86687F980A56b832e9378952B738b614A99dc6 + - 0x9eF6c02FB2ECc446146E05F1fF687a788a8BF76d + - 0x08A2DE6F3528319123b25935C92888B16db8913E + - 0xe141C82D99D85098e03E1a1cC1CdE676556fDdE0 + - 0x4b23D303D9e3719D6CDf8d172Ea030F80509ea15 + - 0xC004e69C5C04A223463Ff32042dd36DabF63A25a + - 0x5eb15C0992734B5e77c888D713b4FC67b3D679A2 + - 0x7Ebb637fd68c523613bE51aad27C35C4DB199B9c + - 0x3c3E2E178C69D4baD964568415a0f0c84fd6320A + + resources: + requests: + memory: "512Mi" + validator: + disabled: false + +bootNode: + realProofs: true + validator: + disabled: true + +proverNode: + realProofs: true + +proverAgent: + replicas: 4 + realProofs: true + bb: + hardwareConcurrency: 16 + gke: + spotEnabled: true + resources: + requests: + memory: "64Gi" + cpu: "16" + limits: + memory: "96Gi" + cpu: "16" + +pxe: + proverEnabled: true + +bot: + followChain: "PENDING" + enabled: true + pxeProverEnabled: true + txIntervalSeconds: 200 + +jobs: + deployL1Verifier: + enable: true + +aztec: + slotDuration: 36 + epochDuration: 32 diff --git a/spartan/metrics/install.sh b/spartan/metrics/install-kind.sh similarity index 80% rename from spartan/metrics/install.sh rename to spartan/metrics/install-kind.sh index 46efddd62de..3a9ecfb4ccf 100755 --- a/spartan/metrics/install.sh +++ b/spartan/metrics/install-kind.sh @@ -14,5 +14,4 @@ helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm helm repo add grafana https://grafana.github.io/helm-charts helm repo add prometheus-community https://prometheus-community.github.io/helm-charts helm dependency update -# need to disable node exporter for GKE autopilot to be happy -helm upgrade metrics "$SCRIPT_DIR" -n metrics --install --create-namespace --atomic --set prometheus.prometheus-node-exporter.enabled=false \ No newline at end of file +helm upgrade metrics "$SCRIPT_DIR" -n metrics --install --create-namespace --atomic --timeout 15m --values "$SCRIPT_DIR/values/kind.yaml" diff --git a/spartan/metrics/install-prod.sh b/spartan/metrics/install-prod.sh index 849dbcd1cc9..a61cc2b8ef4 100755 --- a/spartan/metrics/install-prod.sh +++ b/spartan/metrics/install-prod.sh @@ -3,4 +3,4 @@ set -eu cd "$(dirname "${BASH_SOURCE[0]}")" -helm upgrade metrics . -n metrics --values "./values/prod.yaml" --install --create-namespace --atomic $@ +helm upgrade metrics . -n metrics --values "./values/prod.yaml" --install --create-namespace $@ diff --git a/spartan/metrics/values.yaml b/spartan/metrics/values.yaml index f8407b25357..6e75301c12b 100644 --- a/spartan/metrics/values.yaml +++ b/spartan/metrics/values.yaml @@ -51,8 +51,8 @@ opentelemetry-collector: processors: resource: attributes: - - action: preserve - key: k8s.namespace.name + - action: preserve + key: k8s.namespace.name batch: {} receivers: otlp: @@ -92,31 +92,9 @@ opentelemetry-collector: # Enable and configure the Loki subchart # https://artifacthub.io/packages/helm/grafana/loki-simple-scalable -loki: - deploymentMode: SingleBinary - loki: - auth_enabled: false - commonConfig: - replication_factor: 1 - storage: - type: "filesystem" - schemaConfig: - configs: - - from: "2024-01-01" - store: tsdb - index: - prefix: loki_index_ - period: 24h - object_store: filesystem # we're storing on filesystem so there's no real persistence here. - schema: v13 - singleBinary: - replicas: 1 - read: - replicas: 0 - backend: - replicas: 0 - write: - replicas: 0 +# loki: +# Nothing set here, because we need to use values from the values directory; +# otherwise, things don't get overridden correctly. # Enable and configure the Tempo subchart # https://artifacthub.io/packages/helm/grafana/tempo @@ -159,9 +137,10 @@ prometheus: - job_name: aztec static_configs: - targets: ["metrics-opentelemetry-collector.metrics:8889"] - - job_name: 'kube-state-metrics' + - job_name: "kube-state-metrics" static_configs: - - targets: ['metrics-kube-state-metrics.metrics.svc.cluster.local:8080'] + - targets: + ["metrics-kube-state-metrics.metrics.svc.cluster.local:8080"] # Enable and configure Grafana # https://artifacthub.io/packages/helm/grafana/grafana diff --git a/spartan/metrics/values/gke-autopilot.yaml b/spartan/metrics/values/gke-autopilot.yaml new file mode 100644 index 00000000000..bbdc1b095fd --- /dev/null +++ b/spartan/metrics/values/gke-autopilot.yaml @@ -0,0 +1,4 @@ +# This file isn't used by default. It is here if you need to install metrics on GKE autopilot. +prometheus: + prometheus-node-exporter: + enabled: false diff --git a/spartan/metrics/values/kind.yaml b/spartan/metrics/values/kind.yaml new file mode 100644 index 00000000000..c8b8a970b25 --- /dev/null +++ b/spartan/metrics/values/kind.yaml @@ -0,0 +1,25 @@ +loki: + deploymentMode: SingleBinary + loki: + auth_enabled: false + commonConfig: + replication_factor: 1 + storage: + type: "filesystem" + schemaConfig: + configs: + - from: "2024-01-01" + store: tsdb + index: + prefix: loki_index_ + period: 24h + object_store: filesystem # we're storing on filesystem so there's no real persistence here. + schema: v13 + singleBinary: + replicas: 1 + read: + replicas: 0 + backend: + replicas: 0 + write: + replicas: 0 diff --git a/spartan/metrics/values/prod.yaml b/spartan/metrics/values/prod.yaml index 347f87f73b9..2da726d4431 100644 --- a/spartan/metrics/values/prod.yaml +++ b/spartan/metrics/values/prod.yaml @@ -15,3 +15,49 @@ opentelemetry-collector: service: enabled: true type: LoadBalancer + +loki: + loki: + schemaConfig: + configs: + - from: "2024-04-01" + store: tsdb + object_store: s3 + schema: v13 + index: + prefix: loki_index_ + period: 24h + ingester: + chunk_encoding: snappy + querier: + max_concurrent: 4 + pattern_ingester: + enabled: true + limits_config: + allow_structured_metadata: true + volume_enabled: true + retention_period: 336h # 14 days + compactor: + retention_enabled: true + delete_request_store: s3 + auth_enabled: false + + deploymentMode: SimpleScalable + + singleBinary: + replicas: 0 + backend: + replicas: 2 + read: + replicas: 2 + write: + replicas: 3 + + minio: + enabled: true + persistence: + size: 64Gi + + gateway: + service: + type: LoadBalancer diff --git a/spartan/network-shaping/Chart.yaml b/spartan/network-shaping/Chart.yaml deleted file mode 100644 index 854dc4672e8..00000000000 --- a/spartan/network-shaping/Chart.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v2 -name: network-shaping -description: Network shaping for spartan using chaos-mesh -type: application -version: 0.1.0 -appVersion: "1.0.0" \ No newline at end of file diff --git a/spartan/network-shaping/values/kill-provers.yaml b/spartan/network-shaping/values/kill-provers.yaml deleted file mode 100644 index 2128efca00a..00000000000 --- a/spartan/network-shaping/values/kill-provers.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# Simulates congested network conditions -# High latency, limited bandwidth, packet loss -global: - namespace: "smoke" - -networkShaping: - enabled: true - conditions: - latency: - enabled: false - bandwidth: - enabled: false - packetLoss: - enabled: false - killProvers: - enabled: true - duration: 13m diff --git a/spartan/scripts/deploy_spartan.sh b/spartan/scripts/deploy_spartan.sh index ee78d504cf3..96a8ef2c68d 100755 --- a/spartan/scripts/deploy_spartan.sh +++ b/spartan/scripts/deploy_spartan.sh @@ -72,3 +72,11 @@ if ! upgrade | tee "$SCRIPT_DIR/logs/$NAMESPACE-helm.log" ; then upgrade fi fi + +if ! upgrade | tee "$SCRIPT_DIR/logs/$NAMESPACE-helm.log" ; then + if grep 'cannot patch "'$NAMESPACE'-aztec-network-deploy-l1-verifier"' "$SCRIPT_DIR/logs/$NAMESPACE-helm.log" ; then + kubectl delete job $NAMESPACE-aztec-network-deploy-l1-verifier -n $NAMESPACE + upgrade + fi +fi + diff --git a/spartan/scripts/setup_local_k8s.sh b/spartan/scripts/setup_local_k8s.sh index 0571aebfe9e..8068ce867ae 100755 --- a/spartan/scripts/setup_local_k8s.sh +++ b/spartan/scripts/setup_local_k8s.sh @@ -62,4 +62,4 @@ fi kubectl config use-context kind-kind || true "$SCRIPT_DIR"/../chaos-mesh/install.sh -"$SCRIPT_DIR"/../metrics/install.sh +"$SCRIPT_DIR"/../metrics/install-kind.sh diff --git a/spartan/terraform/deploy-release/data.tf b/spartan/terraform/deploy-release/data.tf new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/spartan/terraform/deploy-release/data.tf @@ -0,0 +1 @@ + diff --git a/spartan/terraform/deploy-release/deploy.sh b/spartan/terraform/deploy-release/deploy.sh new file mode 100755 index 00000000000..e9574554524 --- /dev/null +++ b/spartan/terraform/deploy-release/deploy.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +RELEASE_NAME="rough-rhino" +terraform init -backend-config="key=deploy-network/${RELEASE_NAME}/terraform.tfstate" +terraform apply -var-file="release.tfvars" diff --git a/spartan/terraform/deploy-release/main.tf b/spartan/terraform/deploy-release/main.tf new file mode 100644 index 00000000000..3b26f247325 --- /dev/null +++ b/spartan/terraform/deploy-release/main.tf @@ -0,0 +1,54 @@ +terraform { + backend "s3" { + bucket = "aztec-terraform" + region = "eu-west-2" + } + required_providers { + helm = { + source = "hashicorp/helm" + version = "~> 2.12.1" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.24.0" + } + } +} + +provider "kubernetes" { + alias = "gke-cluster" + config_path = "~/.kube/config" + config_context = var.gke_cluster_context +} + +provider "helm" { + alias = "gke-cluster" + kubernetes { + config_path = "~/.kube/config" + config_context = var.gke_cluster_context + } +} + +# Aztec Helm release for gke-cluster +resource "helm_release" "aztec-gke-cluster" { + provider = helm.gke-cluster + name = var.release_name + repository = "../../" + chart = "aztec-network" + namespace = var.release_name + create_namespace = true + + # base values file + values = [file("../../aztec-network/values/${var.values_file}")] + + set { + name = "images.aztec.image" + value = var.aztec_docker_image + } + + # Setting timeout and wait conditions + timeout = 1200 # 20 minutes in seconds + wait = true + wait_for_jobs = true + +} diff --git a/spartan/terraform/deploy-release/outputs.tf b/spartan/terraform/deploy-release/outputs.tf new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/spartan/terraform/deploy-release/outputs.tf @@ -0,0 +1 @@ + diff --git a/spartan/terraform/deploy-release/release.tfvars b/spartan/terraform/deploy-release/release.tfvars new file mode 100644 index 00000000000..f3236423d9f --- /dev/null +++ b/spartan/terraform/deploy-release/release.tfvars @@ -0,0 +1,4 @@ +release_name = "rough-rhino" +values_file = "release.yaml" +aztec_docker_image = "aztecprotocol/aztec:698cd3d62680629a3f1bfc0f82604534cedbccf3-x86_64" + diff --git a/spartan/terraform/deploy-release/variables.tf b/spartan/terraform/deploy-release/variables.tf new file mode 100644 index 00000000000..ebccc9d3f67 --- /dev/null +++ b/spartan/terraform/deploy-release/variables.tf @@ -0,0 +1,20 @@ +variable "gke_cluster_context" { + description = "GKE cluster context" + type = string + default = "gke_testnet-440309_us-east4-a_spartan-gke" +} + +variable "release_name" { + description = "Name of helm deployment and k8s namespace" + type = string +} + +variable "values_file" { + description = "Name of the values file to use for deployment" + type = string +} + +variable "aztec_docker_image" { + description = "Docker image to use for the aztec network" + type = string +} diff --git a/spartan/terraform/gke-cluster/main.tf b/spartan/terraform/gke-cluster/main.tf index fce5b5f02e4..ed5f0ed4baf 100644 --- a/spartan/terraform/gke-cluster/main.tf +++ b/spartan/terraform/gke-cluster/main.tf @@ -96,6 +96,40 @@ resource "google_container_node_pool" "primary_nodes" { } } +# Create node pool for aztec nodes (validators, prover nodes, boot nodes) +resource "google_container_node_pool" "aztec_nodes" { + name = "aztec-node-pool" + location = var.zone + cluster = google_container_cluster.primary.name + + # Enable autoscaling + autoscaling { + min_node_count = 1 + max_node_count = 128 + } + + # Node configuration + node_config { + machine_type = "t2d-standard-8" + + service_account = google_service_account.gke_sa.email + oauth_scopes = [ + "https://www.googleapis.com/auth/cloud-platform" + ] + + labels = { + env = "production" + } + tags = ["gke-node", "aztec"] + } + + # Management configuration + management { + auto_repair = true + auto_upgrade = true + } +} + # Create spot instance node pool with autoscaling resource "google_container_node_pool" "spot_nodes" { name = "spot-node-pool" diff --git a/spartan/terraform/multicloud-deploy/main.tf b/spartan/terraform/multicloud-deploy/main.tf index 4cf66659de0..b946675a9e5 100644 --- a/spartan/terraform/multicloud-deploy/main.tf +++ b/spartan/terraform/multicloud-deploy/main.tf @@ -58,12 +58,28 @@ resource "helm_release" "aztec-eks-cluster" { # base values file values = [file("../../aztec-network/values/${var.values-file}")] - # removing prover nodes + # network customization + set { + name = "images.aztec.image" + value = var.image + } + + set { + name = "telemetry.enabled" + value = var.telemetry + } + set { name = "network.public" value = true } + set { + name = "validator.replicas" + value = 16 + } + + # removing prover nodes set { name = "proverNode.replicas" value = "0" @@ -75,7 +91,7 @@ resource "helm_release" "aztec-eks-cluster" { } # Setting timeout and wait conditions - timeout = 600 # 10 minutes in seconds + timeout = 1200 # 20 minutes in seconds wait = true wait_for_jobs = true } @@ -92,10 +108,15 @@ resource "helm_release" "aztec-gke-cluster" { # base values file values = [file("../../aztec-network/values/${var.values-file}")] - # disabling all nodes except provers + # network customization set { - name = "network.setupL2Contracts" - value = false + name = "images.aztec.image" + value = var.image + } + + set { + name = "telemetry.enabled" + value = var.telemetry } set { @@ -103,6 +124,27 @@ resource "helm_release" "aztec-gke-cluster" { value = true } + set { + name = "proverAgent.replicas" + value = 32 + } + + set { + name = "proverAgent.gke.spotEnabled" + value = true + } + + set { + name = "network.setupL2Contracts" + value = false + } + + set { + name = "proverAgent.bb.hardwareConcurrency" + value = 16 + } + + # disabling all nodes except provers set { name = "bootNode.replicas" value = "0" @@ -160,7 +202,7 @@ resource "helm_release" "aztec-gke-cluster" { } # Setting timeout and wait conditions - timeout = 600 # 10 minutes in seconds + timeout = 1200 # 20 minutes in seconds wait = true wait_for_jobs = true diff --git a/spartan/terraform/multicloud-deploy/variables.tf b/spartan/terraform/multicloud-deploy/variables.tf index 91486bd432a..9a265412a5b 100644 --- a/spartan/terraform/multicloud-deploy/variables.tf +++ b/spartan/terraform/multicloud-deploy/variables.tf @@ -19,5 +19,17 @@ variable "testnet_name" { variable "values-file" { description = "Name of the values file to use for deployment" type = string - default = "multicloud-demo.yaml" + default = "16-validators.yaml" +} + +variable "image" { + description = "Aztec node image" + type = string + default = "aztecprotocol/aztec:master" +} + +variable "telemetry" { + description = "Toogle telemetry on/off" + type = bool + default = false } diff --git a/yarn-project/Earthfile b/yarn-project/Earthfile index 0333fbcec5a..469083c22cf 100644 --- a/yarn-project/Earthfile +++ b/yarn-project/Earthfile @@ -37,7 +37,6 @@ build: BUILD ../noir/+nargo BUILD --pass-args ../noir-projects/+build BUILD ../l1-contracts/+build - BUILD ../barretenberg/ts/+build BUILD ../noir/+packages FROM +deps @@ -53,7 +52,6 @@ build: RUN ./bootstrap.sh full RUN cd ivc-integration && chmod +x run_browser_tests.sh && npx playwright install && npx playwright install-deps - build-dev: FROM +build SAVE ARTIFACT /usr/src /usr/src @@ -243,11 +241,16 @@ end-to-end: WORKDIR /usr/src/yarn-project/end-to-end ENTRYPOINT ["yarn", "test"] -scripts-prod: +scripts-preview: FROM +build RUN yarn workspaces focus @aztec/scripts --production && yarn cache clean SAVE ARTIFACT /usr/src /usr/src +scripts-prod: + FROM +deps + RUN yarn workspaces focus @aztec/scripts --production && yarn cache clean + SAVE ARTIFACT /usr/src /usr/src + all: BUILD +aztec BUILD +end-to-end diff --git a/yarn-project/archiver/package.json b/yarn-project/archiver/package.json index ff0dd170380..3366d4b3f1b 100644 --- a/yarn-project/archiver/package.json +++ b/yarn-project/archiver/package.json @@ -5,6 +5,7 @@ "exports": { ".": "./dest/index.js", "./data-retrieval": "./dest/archiver/data_retrieval.js", + "./epoch": "./dest/archiver/epoch_helpers.js", "./test": "./dest/test/index.js" }, "typedocOptions": { @@ -71,6 +72,7 @@ "@aztec/foundation": "workspace:^", "@aztec/kv-store": "workspace:^", "@aztec/l1-artifacts": "workspace:^", + "@aztec/noir-contracts.js": "workspace:^", "@aztec/protocol-contracts": "workspace:^", "@aztec/telemetry-client": "workspace:^", "@aztec/types": "workspace:^", @@ -83,7 +85,6 @@ "ws": "^8.13.0" }, "devDependencies": { - "@aztec/noir-contracts.js": "workspace:^", "@jest/globals": "^29.5.0", "@types/debug": "^4.1.7", "@types/jest": "^29.5.0", diff --git a/yarn-project/archiver/src/archiver/archiver.test.ts b/yarn-project/archiver/src/archiver/archiver.test.ts index 9d19fae994a..b542ee2475f 100644 --- a/yarn-project/archiver/src/archiver/archiver.test.ts +++ b/yarn-project/archiver/src/archiver/archiver.test.ts @@ -456,7 +456,7 @@ function makeRollupTx(l2Block: L2Block) { const input = encodeFunctionData({ abi: RollupAbi, functionName: 'propose', - args: [header, archive, blockHash, [], [], body], + args: [{ header, archive, blockHash, txHashes: [] }, [], body], }); return { input } as Transaction; } diff --git a/yarn-project/archiver/src/archiver/archiver.ts b/yarn-project/archiver/src/archiver/archiver.ts index 9b59447ddf4..9dc079a4cf0 100644 --- a/yarn-project/archiver/src/archiver/archiver.ts +++ b/yarn-project/archiver/src/archiver/archiver.ts @@ -2,6 +2,7 @@ import { type EncryptedL2Log, type FromLogType, type GetUnencryptedLogsResponse, + type InBlock, type InboxLeaf, type L1ToL2MessageSource, type L2Block, @@ -12,6 +13,7 @@ import { type L2Tips, type LogFilter, type LogType, + type NullifierWithBlockSource, type TxEffect, type TxHash, type TxReceipt, @@ -41,6 +43,7 @@ import { type EthAddress } from '@aztec/foundation/eth-address'; import { Fr } from '@aztec/foundation/fields'; import { type DebugLogger, createDebugLogger } from '@aztec/foundation/log'; import { RunningPromise } from '@aztec/foundation/running-promise'; +import { count } from '@aztec/foundation/string'; import { Timer } from '@aztec/foundation/timer'; import { InboxAbi, RollupAbi } from '@aztec/l1-artifacts'; import { ProtocolContractAddress } from '@aztec/protocol-contracts'; @@ -73,7 +76,11 @@ import { type L1Published } from './structs/published.js'; /** * Helper interface to combine all sources this archiver implementation provides. */ -export type ArchiveSource = L2BlockSource & L2LogsSource & ContractDataSource & L1ToL2MessageSource; +export type ArchiveSource = L2BlockSource & + L2LogsSource & + ContractDataSource & + L1ToL2MessageSource & + NullifierWithBlockSource; /** * Pulls L2 blocks in a non-blocking manner and provides interface for their retrieval. @@ -282,12 +289,14 @@ export class Archiver implements ArchiveSource { (await this.rollup.read.canPruneAtTime([time], { blockNumber: currentL1BlockNumber })); if (canPrune) { - this.log.verbose(`L2 prune will occur on next submission. Rolling back to last proven block.`); const blocksToUnwind = localPendingBlockNumber - provenBlockNumber; this.log.verbose( - `Unwinding ${blocksToUnwind} block${blocksToUnwind > 1n ? 's' : ''} from block ${localPendingBlockNumber}`, + `L2 prune will occur on next submission. ` + + `Unwinding ${count(blocksToUnwind, 'block')} from block ${localPendingBlockNumber} ` + + `to the last proven block ${provenBlockNumber}.`, ); await this.store.unwindBlocks(Number(localPendingBlockNumber), Number(blocksToUnwind)); + this.log.verbose(`Unwound ${count(blocksToUnwind, 'block')}. New L2 block is ${await this.getBlockNumber()}.`); // TODO(palla/reorg): Do we need to set the block synched L1 block number here? // Seems like the next iteration should handle this. // await this.store.setBlockSynchedL1BlockNumber(currentL1BlockNumber); @@ -582,17 +591,14 @@ export class Archiver implements ArchiveSource { if (number === 'latest') { number = await this.store.getSynchedL2BlockNumber(); } - try { - const headers = await this.store.getBlockHeaders(number, 1); - return headers.length === 0 ? undefined : headers[0]; - } catch (e) { - // If the latest is 0, then getBlockHeaders will throw an error - this.log.error(`getBlockHeader: error fetching block number: ${number}`); + if (number === 0) { return undefined; } + const headers = await this.store.getBlockHeaders(number, 1); + return headers.length === 0 ? undefined : headers[0]; } - public getTxEffect(txHash: TxHash): Promise { + public getTxEffect(txHash: TxHash) { return this.store.getTxEffect(txHash); } @@ -646,6 +652,17 @@ export class Archiver implements ArchiveSource { return this.store.getLogsByTags(tags); } + /** + * Returns the provided nullifier indexes scoped to the block + * they were first included in, or undefined if they're not present in the tree + * @param blockNumber Max block number to search for the nullifiers + * @param nullifiers Nullifiers to get + * @returns The block scoped indexes of the provided nullifiers, or undefined if the nullifier doesn't exist in the tree + */ + findNullifiersIndexesWithBlock(blockNumber: number, nullifiers: Fr[]): Promise<(InBlock | undefined)[]> { + return this.store.findNullifiersIndexesWithBlock(blockNumber, nullifiers); + } + /** * Gets unencrypted logs based on the provided filter. * @param filter - The filter to apply to the logs. @@ -715,6 +732,12 @@ export class Archiver implements ArchiveSource { return this.store.getContractClassIds(); } + // TODO(#10007): Remove this method + async addContractClass(contractClass: ContractClassPublic): Promise { + await this.store.addContractClasses([contractClass], 0); + return; + } + addContractArtifact(address: AztecAddress, artifact: ContractArtifact): Promise { return this.store.addContractArtifact(address, artifact); } @@ -767,6 +790,8 @@ class ArchiverStoreHelper ArchiverDataStore, | 'addLogs' | 'deleteLogs' + | 'addNullifiers' + | 'deleteNullifiers' | 'addContractClasses' | 'deleteContractClasses' | 'addContractInstances' @@ -778,6 +803,11 @@ class ArchiverStoreHelper constructor(private readonly store: ArchiverDataStore) {} + // TODO(#10007): Remove this method + addContractClasses(contractClasses: ContractClassPublic[], blockNum: number): Promise { + return this.store.addContractClasses(contractClasses, blockNum); + } + /** * Extracts and stores contract classes out of ContractClassRegistered events emitted by the class registerer contract. * @param allLogs - All logs emitted in a bunch of blocks. @@ -897,6 +927,7 @@ class ArchiverStoreHelper ).every(Boolean); }), )), + this.store.addNullifiers(blocks.map(block => block.data)), this.store.addBlocks(blocks), ].every(Boolean); } @@ -936,7 +967,7 @@ class ArchiverStoreHelper getBlockHeaders(from: number, limit: number): Promise { return this.store.getBlockHeaders(from, limit); } - getTxEffect(txHash: TxHash): Promise { + getTxEffect(txHash: TxHash): Promise | undefined> { return this.store.getTxEffect(txHash); } getSettledTxReceipt(txHash: TxHash): Promise { @@ -961,6 +992,9 @@ class ArchiverStoreHelper getLogsByTags(tags: Fr[]): Promise { return this.store.getLogsByTags(tags); } + findNullifiersIndexesWithBlock(blockNumber: number, nullifiers: Fr[]): Promise<(InBlock | undefined)[]> { + return this.store.findNullifiersIndexesWithBlock(blockNumber, nullifiers); + } getUnencryptedLogs(filter: LogFilter): Promise { return this.store.getUnencryptedLogs(filter); } diff --git a/yarn-project/archiver/src/archiver/archiver_store.ts b/yarn-project/archiver/src/archiver/archiver_store.ts index 110e42f86f3..c7be4ddbd13 100644 --- a/yarn-project/archiver/src/archiver/archiver_store.ts +++ b/yarn-project/archiver/src/archiver/archiver_store.ts @@ -1,6 +1,7 @@ import { type FromLogType, type GetUnencryptedLogsResponse, + type InBlock, type InboxLeaf, type L2Block, type L2BlockL2Logs, @@ -79,7 +80,7 @@ export interface ArchiverDataStore { * @param txHash - The txHash of the tx corresponding to the tx effect. * @returns The requested tx effect (or undefined if not found). */ - getTxEffect(txHash: TxHash): Promise; + getTxEffect(txHash: TxHash): Promise | undefined>; /** * Gets a receipt of a settled tx. @@ -96,6 +97,23 @@ export interface ArchiverDataStore { addLogs(blocks: L2Block[]): Promise; deleteLogs(blocks: L2Block[]): Promise; + /** + * Append new nullifiers to the store's list. + * @param blocks - The blocks for which to add the nullifiers. + * @returns True if the operation is successful. + */ + addNullifiers(blocks: L2Block[]): Promise; + deleteNullifiers(blocks: L2Block[]): Promise; + + /** + * Returns the provided nullifier indexes scoped to the block + * they were first included in, or undefined if they're not present in the tree + * @param blockNumber Max block number to search for the nullifiers + * @param nullifiers Nullifiers to get + * @returns The block scoped indexes of the provided nullifiers, or undefined if the nullifier doesn't exist in the tree + */ + findNullifiersIndexesWithBlock(blockNumber: number, nullifiers: Fr[]): Promise<(InBlock | undefined)[]>; + /** * Append L1 to L2 messages to the store. * @param messages - The L1 to L2 messages to be added to the store and the last processed L1 block. diff --git a/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts b/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts index cd30af56a78..c974942d422 100644 --- a/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts +++ b/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts @@ -1,4 +1,4 @@ -import { InboxLeaf, L2Block, LogId, LogType, TxHash } from '@aztec/circuit-types'; +import { InboxLeaf, L2Block, LogId, LogType, TxHash, wrapInBlock } from '@aztec/circuit-types'; import '@aztec/circuit-types/jest'; import { AztecAddress, @@ -7,6 +7,7 @@ import { Fr, INITIAL_L2_BLOCK_NUM, L1_TO_L2_MSG_SUBTREE_HEIGHT, + MAX_NULLIFIERS_PER_TX, SerializableContractInstance, } from '@aztec/circuits.js'; import { @@ -37,12 +38,18 @@ export function describeArchiverDataStore(testName: string, getStore: () => Arch [5, 2, () => blocks.slice(4, 6)], ]; + const makeL1Published = (block: L2Block, l1BlockNumber: number): L1Published => ({ + data: block, + l1: { + blockNumber: BigInt(l1BlockNumber), + blockHash: `0x${l1BlockNumber}`, + timestamp: BigInt(l1BlockNumber * 1000), + }, + }); + beforeEach(() => { store = getStore(); - blocks = times(10, i => ({ - data: L2Block.random(i + 1), - l1: { blockNumber: BigInt(i + 10), blockHash: `0x${i}`, timestamp: BigInt(i * 1000) }, - })); + blocks = times(10, i => makeL1Published(L2Block.random(i + 1), i + 10)); }); describe('addBlocks', () => { @@ -68,6 +75,21 @@ export function describeArchiverDataStore(testName: string, getStore: () => Arch expect(await store.getSynchedL2BlockNumber()).toBe(blockNumber - 1); expect(await store.getBlocks(blockNumber, 1)).toEqual([]); }); + + it('can unwind multiple empty blocks', async () => { + const emptyBlocks = times(10, i => makeL1Published(L2Block.random(i + 1, 0), i + 10)); + await store.addBlocks(emptyBlocks); + expect(await store.getSynchedL2BlockNumber()).toBe(10); + + await store.unwindBlocks(10, 3); + expect(await store.getSynchedL2BlockNumber()).toBe(7); + expect((await store.getBlocks(1, 10)).map(b => b.data.number)).toEqual([1, 2, 3, 4, 5, 6, 7]); + }); + + it('refuses to unwind blocks if the tip is not the last block', async () => { + await store.addBlocks(blocks); + await expect(store.unwindBlocks(5, 1)).rejects.toThrow(/can only unwind blocks from the tip/i); + }); }); describe('getBlocks', () => { @@ -191,14 +213,14 @@ export function describeArchiverDataStore(testName: string, getStore: () => Arch }); it.each([ - () => blocks[0].data.body.txEffects[0], - () => blocks[9].data.body.txEffects[3], - () => blocks[3].data.body.txEffects[1], - () => blocks[5].data.body.txEffects[2], - () => blocks[1].data.body.txEffects[0], + () => wrapInBlock(blocks[0].data.body.txEffects[0], blocks[0].data), + () => wrapInBlock(blocks[9].data.body.txEffects[3], blocks[9].data), + () => wrapInBlock(blocks[3].data.body.txEffects[1], blocks[3].data), + () => wrapInBlock(blocks[5].data.body.txEffects[2], blocks[5].data), + () => wrapInBlock(blocks[1].data.body.txEffects[0], blocks[1].data), ])('retrieves a previously stored transaction', async getExpectedTx => { const expectedTx = getExpectedTx(); - const actualTx = await store.getTxEffect(expectedTx.txHash); + const actualTx = await store.getTxEffect(expectedTx.data.txHash); expect(actualTx).toEqual(expectedTx); }); @@ -207,16 +229,16 @@ export function describeArchiverDataStore(testName: string, getStore: () => Arch }); it.each([ - () => blocks[0].data.body.txEffects[0], - () => blocks[9].data.body.txEffects[3], - () => blocks[3].data.body.txEffects[1], - () => blocks[5].data.body.txEffects[2], - () => blocks[1].data.body.txEffects[0], + () => wrapInBlock(blocks[0].data.body.txEffects[0], blocks[0].data), + () => wrapInBlock(blocks[9].data.body.txEffects[3], blocks[9].data), + () => wrapInBlock(blocks[3].data.body.txEffects[1], blocks[3].data), + () => wrapInBlock(blocks[5].data.body.txEffects[2], blocks[5].data), + () => wrapInBlock(blocks[1].data.body.txEffects[0], blocks[1].data), ])('tries to retrieves a previously stored transaction after deleted', async getExpectedTx => { await store.unwindBlocks(blocks.length, blocks.length); const expectedTx = getExpectedTx(); - const actualTx = await store.getTxEffect(expectedTx.txHash); + const actualTx = await store.getTxEffect(expectedTx.data.txHash); expect(actualTx).toEqual(undefined); }); @@ -705,5 +727,58 @@ export function describeArchiverDataStore(testName: string, getStore: () => Arch } }); }); + + describe('findNullifiersIndexesWithBlock', () => { + let blocks: L2Block[]; + const numBlocks = 10; + const nullifiersPerBlock = new Map(); + + beforeEach(() => { + blocks = times(numBlocks, (index: number) => L2Block.random(index + 1, 1)); + + blocks.forEach((block, blockIndex) => { + nullifiersPerBlock.set( + blockIndex, + block.body.txEffects.flatMap(txEffect => txEffect.nullifiers), + ); + }); + }); + + it('returns wrapped nullifiers with blocks if they exist', async () => { + await store.addNullifiers(blocks); + const nullifiersToRetrieve = [...nullifiersPerBlock.get(0)!, ...nullifiersPerBlock.get(5)!, Fr.random()]; + const blockScopedNullifiers = await store.findNullifiersIndexesWithBlock(10, nullifiersToRetrieve); + + expect(blockScopedNullifiers).toHaveLength(nullifiersToRetrieve.length); + const [undefinedNullifier] = blockScopedNullifiers.slice(-1); + const realNullifiers = blockScopedNullifiers.slice(0, -1); + realNullifiers.forEach((blockScopedNullifier, index) => { + expect(blockScopedNullifier).not.toBeUndefined(); + const { data, l2BlockNumber } = blockScopedNullifier!; + expect(data).toEqual(expect.any(BigInt)); + expect(l2BlockNumber).toEqual(index < MAX_NULLIFIERS_PER_TX ? 1 : 6); + }); + expect(undefinedNullifier).toBeUndefined(); + }); + + it('returns wrapped nullifiers filtering by blockNumber', async () => { + await store.addNullifiers(blocks); + const nullifiersToRetrieve = [...nullifiersPerBlock.get(0)!, ...nullifiersPerBlock.get(5)!]; + const blockScopedNullifiers = await store.findNullifiersIndexesWithBlock(5, nullifiersToRetrieve); + + expect(blockScopedNullifiers).toHaveLength(nullifiersToRetrieve.length); + const undefinedNullifiers = blockScopedNullifiers.slice(-MAX_NULLIFIERS_PER_TX); + const realNullifiers = blockScopedNullifiers.slice(0, -MAX_NULLIFIERS_PER_TX); + realNullifiers.forEach(blockScopedNullifier => { + expect(blockScopedNullifier).not.toBeUndefined(); + const { data, l2BlockNumber } = blockScopedNullifier!; + expect(data).toEqual(expect.any(BigInt)); + expect(l2BlockNumber).toEqual(1); + }); + undefinedNullifiers.forEach(undefinedNullifier => { + expect(undefinedNullifier).toBeUndefined(); + }); + }); + }); }); } diff --git a/yarn-project/archiver/src/archiver/data_retrieval.ts b/yarn-project/archiver/src/archiver/data_retrieval.ts index 563f0b43c0f..ce3c6cadbd5 100644 --- a/yarn-project/archiver/src/archiver/data_retrieval.ts +++ b/yarn-project/archiver/src/archiver/data_retrieval.ts @@ -139,9 +139,18 @@ async function getBlockFromRollupTx( if (!allowedMethods.includes(functionName)) { throw new Error(`Unexpected method called ${functionName}`); } - const [headerHex, archiveRootHex, , , , bodyHex] = args! as readonly [Hex, Hex, Hex, Hex[], ViemSignature[], Hex]; - - const header = Header.fromBuffer(Buffer.from(hexToBytes(headerHex))); + const [decodedArgs, , bodyHex] = args! as readonly [ + { + header: Hex; + archive: Hex; + blockHash: Hex; + txHashes: Hex[]; + }, + ViemSignature[], + Hex, + ]; + + const header = Header.fromBuffer(Buffer.from(hexToBytes(decodedArgs.header))); const blockBody = Body.fromBuffer(Buffer.from(hexToBytes(bodyHex))); const blockNumberFromHeader = header.globalVariables.blockNumber.toBigInt(); @@ -152,7 +161,7 @@ async function getBlockFromRollupTx( const archive = AppendOnlyTreeSnapshot.fromBuffer( Buffer.concat([ - Buffer.from(hexToBytes(archiveRootHex)), // L2Block.archive.root + Buffer.from(hexToBytes(decodedArgs.archive)), // L2Block.archive.root numToUInt32BE(Number(l2BlockNum + 1n)), // L2Block.archive.nextAvailableLeafIndex ]), ); diff --git a/yarn-project/archiver/src/archiver/epoch_helpers.ts b/yarn-project/archiver/src/archiver/epoch_helpers.ts index f84bafeee38..55fe28e2f0e 100644 --- a/yarn-project/archiver/src/archiver/epoch_helpers.ts +++ b/yarn-project/archiver/src/archiver/epoch_helpers.ts @@ -1,30 +1,54 @@ -type TimeConstants = { +// REFACTOR: This file should go in a package lower in the dependency graph. + +export type EpochConstants = { + l1GenesisBlock: bigint; l1GenesisTime: bigint; epochDuration: number; slotDuration: number; }; /** Returns the slot number for a given timestamp. */ -export function getSlotAtTimestamp(ts: bigint, constants: Pick) { +export function getSlotAtTimestamp(ts: bigint, constants: Pick) { return ts < constants.l1GenesisTime ? 0n : (ts - constants.l1GenesisTime) / BigInt(constants.slotDuration); } /** Returns the epoch number for a given timestamp. */ -export function getEpochNumberAtTimestamp(ts: bigint, constants: TimeConstants) { +export function getEpochNumberAtTimestamp( + ts: bigint, + constants: Pick, +) { return getSlotAtTimestamp(ts, constants) / BigInt(constants.epochDuration); } -/** Returns the range of slots (inclusive) for a given epoch number. */ -export function getSlotRangeForEpoch(epochNumber: bigint, constants: Pick) { +/** Returns the range of L2 slots (inclusive) for a given epoch number. */ +export function getSlotRangeForEpoch(epochNumber: bigint, constants: Pick) { const startSlot = epochNumber * BigInt(constants.epochDuration); return [startSlot, startSlot + BigInt(constants.epochDuration) - 1n]; } /** Returns the range of L1 timestamps (inclusive) for a given epoch number. */ -export function getTimestampRangeForEpoch(epochNumber: bigint, constants: TimeConstants) { +export function getTimestampRangeForEpoch( + epochNumber: bigint, + constants: Pick, +) { const [startSlot, endSlot] = getSlotRangeForEpoch(epochNumber, constants); return [ constants.l1GenesisTime + startSlot * BigInt(constants.slotDuration), constants.l1GenesisTime + endSlot * BigInt(constants.slotDuration), ]; } + +/** + * Returns the range of L1 blocks (inclusive) for a given epoch number. + * @remarks This assumes no time warp has happened. + */ +export function getL1BlockRangeForEpoch( + epochNumber: bigint, + constants: Pick, +) { + const epochDurationInL1Blocks = BigInt(constants.epochDuration) * BigInt(constants.slotDuration); + return [ + epochNumber * epochDurationInL1Blocks + constants.l1GenesisBlock, + (epochNumber + 1n) * epochDurationInL1Blocks + constants.l1GenesisBlock - 1n, + ]; +} diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/block_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/block_store.ts index a4a3ba3281c..91ae9d578c2 100644 --- a/yarn-project/archiver/src/archiver/kv_archiver_store/block_store.ts +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/block_store.ts @@ -1,4 +1,4 @@ -import { Body, L2Block, type TxEffect, type TxHash, TxReceipt } from '@aztec/circuit-types'; +import { Body, type InBlock, L2Block, type TxEffect, type TxHash, TxReceipt } from '@aztec/circuit-types'; import { AppendOnlyTreeSnapshot, type AztecAddress, Header, INITIAL_L2_BLOCK_NUM } from '@aztec/circuits.js'; import { createDebugLogger } from '@aztec/foundation/log'; import { type AztecKVStore, type AztecMap, type AztecSingleton, type Range } from '@aztec/kv-store'; @@ -20,7 +20,7 @@ export class BlockStore { /** Map block number to block data */ #blocks: AztecMap; - /** Map block body hash to block body */ + /** Map block hash to block body */ #blockBodies: AztecMap; /** Stores L1 block number in which the last processed L2 block was included */ @@ -72,7 +72,7 @@ export class BlockStore { void this.#txIndex.set(tx.txHash.toString(), [block.data.number, i]); }); - void this.#blockBodies.set(block.data.body.getTxsEffectsHash().toString('hex'), block.data.body.toBuffer()); + void this.#blockBodies.set(block.data.hash().toString(), block.data.body.toBuffer()); } void this.#lastSynchedL1Block.set(blocks[blocks.length - 1].l1.blockNumber); @@ -92,7 +92,7 @@ export class BlockStore { return this.db.transaction(() => { const last = this.getSynchedL2BlockNumber(); if (from != last) { - throw new Error(`Can only remove from the tip`); + throw new Error(`Can only unwind blocks from the tip (requested ${from} but current tip is ${last})`); } for (let i = 0; i < blocksToUnwind; i++) { @@ -106,7 +106,9 @@ export class BlockStore { block.data.body.txEffects.forEach(tx => { void this.#txIndex.delete(tx.txHash.toString()); }); - void this.#blockBodies.delete(block.data.body.getTxsEffectsHash().toString('hex')); + const blockHash = block.data.hash().toString(); + void this.#blockBodies.delete(blockHash); + this.#log.debug(`Unwound block ${blockNumber} ${blockHash}`); } return true; @@ -154,10 +156,12 @@ export class BlockStore { private getBlockFromBlockStorage(blockStorage: BlockStorage) { const header = Header.fromBuffer(blockStorage.header); const archive = AppendOnlyTreeSnapshot.fromBuffer(blockStorage.archive); - - const blockBodyBuffer = this.#blockBodies.get(header.contentCommitment.txsEffectsHash.toString('hex')); + const blockHash = header.hash().toString(); + const blockBodyBuffer = this.#blockBodies.get(blockHash); if (blockBodyBuffer === undefined) { - throw new Error('Body could not be retrieved'); + throw new Error( + `Could not retrieve body for block ${header.globalVariables.blockNumber.toNumber()} ${blockHash}`, + ); } const body = Body.fromBuffer(blockBodyBuffer); @@ -170,14 +174,22 @@ export class BlockStore { * @param txHash - The txHash of the tx corresponding to the tx effect. * @returns The requested tx effect (or undefined if not found). */ - getTxEffect(txHash: TxHash): TxEffect | undefined { + getTxEffect(txHash: TxHash): InBlock | undefined { const [blockNumber, txIndex] = this.getTxLocation(txHash) ?? []; if (typeof blockNumber !== 'number' || typeof txIndex !== 'number') { return undefined; } const block = this.getBlock(blockNumber); - return block?.data.body.txEffects[txIndex]; + if (!block) { + return undefined; + } + + return { + data: block.data.body.txEffects[txIndex], + l2BlockNumber: block.data.number, + l2BlockHash: block.data.hash().toString(), + }; } /** diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts index 313190e8767..134bbf1c4f7 100644 --- a/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts @@ -1,12 +1,12 @@ import { type FromLogType, type GetUnencryptedLogsResponse, + type InBlock, type InboxLeaf, type L2Block, type L2BlockL2Logs, type LogFilter, type LogType, - type TxEffect, type TxHash, type TxReceipt, type TxScopedL2Log, @@ -33,6 +33,7 @@ import { ContractClassStore } from './contract_class_store.js'; import { ContractInstanceStore } from './contract_instance_store.js'; import { LogStore } from './log_store.js'; import { MessageStore } from './message_store.js'; +import { NullifierStore } from './nullifier_store.js'; /** * LMDB implementation of the ArchiverDataStore interface. @@ -40,6 +41,7 @@ import { MessageStore } from './message_store.js'; export class KVArchiverDataStore implements ArchiverDataStore { #blockStore: BlockStore; #logStore: LogStore; + #nullifierStore: NullifierStore; #messageStore: MessageStore; #contractClassStore: ContractClassStore; #contractInstanceStore: ContractInstanceStore; @@ -54,6 +56,7 @@ export class KVArchiverDataStore implements ArchiverDataStore { this.#contractClassStore = new ContractClassStore(db); this.#contractInstanceStore = new ContractInstanceStore(db); this.#contractArtifactStore = new ContractArtifactsStore(db); + this.#nullifierStore = new NullifierStore(db); } getContractArtifact(address: AztecAddress): Promise { @@ -159,7 +162,7 @@ export class KVArchiverDataStore implements ArchiverDataStore { * @param txHash - The txHash of the tx corresponding to the tx effect. * @returns The requested tx effect (or undefined if not found). */ - getTxEffect(txHash: TxHash): Promise { + getTxEffect(txHash: TxHash) { return Promise.resolve(this.#blockStore.getTxEffect(txHash)); } @@ -185,6 +188,23 @@ export class KVArchiverDataStore implements ArchiverDataStore { return this.#logStore.deleteLogs(blocks); } + /** + * Append new nullifiers to the store's list. + * @param blocks - The blocks for which to add the nullifiers. + * @returns True if the operation is successful. + */ + addNullifiers(blocks: L2Block[]): Promise { + return this.#nullifierStore.addNullifiers(blocks); + } + + deleteNullifiers(blocks: L2Block[]): Promise { + return this.#nullifierStore.deleteNullifiers(blocks); + } + + findNullifiersIndexesWithBlock(blockNumber: number, nullifiers: Fr[]): Promise<(InBlock | undefined)[]> { + return this.#nullifierStore.findNullifiersIndexesWithBlock(blockNumber, nullifiers); + } + getTotalL1ToL2MessageCount(): Promise { return Promise.resolve(this.#messageStore.getTotalL1ToL2MessageCount()); } diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/nullifier_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/nullifier_store.ts new file mode 100644 index 00000000000..716dedc5a0a --- /dev/null +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/nullifier_store.ts @@ -0,0 +1,78 @@ +import { type InBlock, type L2Block } from '@aztec/circuit-types'; +import { type Fr, MAX_NULLIFIERS_PER_TX } from '@aztec/circuits.js'; +import { createDebugLogger } from '@aztec/foundation/log'; +import { type AztecKVStore, type AztecMap } from '@aztec/kv-store'; + +export class NullifierStore { + #nullifiersToBlockNumber: AztecMap; + #nullifiersToBlockHash: AztecMap; + #nullifiersToIndex: AztecMap; + #log = createDebugLogger('aztec:archiver:log_store'); + + constructor(private db: AztecKVStore) { + this.#nullifiersToBlockNumber = db.openMap('archiver_nullifiers_to_block_number'); + this.#nullifiersToBlockHash = db.openMap('archiver_nullifiers_to_block_hash'); + this.#nullifiersToIndex = db.openMap('archiver_nullifiers_to_index'); + } + + async addNullifiers(blocks: L2Block[]): Promise { + await this.db.transaction(() => { + blocks.forEach(block => { + const dataStartIndexForBlock = + block.header.state.partial.nullifierTree.nextAvailableLeafIndex - + block.body.numberOfTxsIncludingPadded * MAX_NULLIFIERS_PER_TX; + block.body.txEffects.forEach((txEffects, txIndex) => { + const dataStartIndexForTx = dataStartIndexForBlock + txIndex * MAX_NULLIFIERS_PER_TX; + txEffects.nullifiers.forEach((nullifier, nullifierIndex) => { + void this.#nullifiersToBlockNumber.set(nullifier.toString(), block.number); + void this.#nullifiersToBlockHash.set(nullifier.toString(), block.hash().toString()); + void this.#nullifiersToIndex.set(nullifier.toString(), dataStartIndexForTx + nullifierIndex); + }); + }); + }); + }); + return true; + } + + async deleteNullifiers(blocks: L2Block[]): Promise { + await this.db.transaction(() => { + for (const block of blocks) { + for (const nullifier of block.body.txEffects.flatMap(tx => tx.nullifiers)) { + void this.#nullifiersToBlockNumber.delete(nullifier.toString()); + void this.#nullifiersToBlockHash.delete(nullifier.toString()); + void this.#nullifiersToIndex.delete(nullifier.toString()); + } + } + }); + return true; + } + + async findNullifiersIndexesWithBlock( + blockNumber: number, + nullifiers: Fr[], + ): Promise<(InBlock | undefined)[]> { + const maybeNullifiers = await this.db.transaction(() => { + return nullifiers.map(nullifier => ({ + data: this.#nullifiersToIndex.get(nullifier.toString()), + l2BlockNumber: this.#nullifiersToBlockNumber.get(nullifier.toString()), + l2BlockHash: this.#nullifiersToBlockHash.get(nullifier.toString()), + })); + }); + return maybeNullifiers.map(({ data, l2BlockNumber, l2BlockHash }) => { + if ( + data === undefined || + l2BlockNumber === undefined || + l2BlockHash === undefined || + l2BlockNumber > blockNumber + ) { + return undefined; + } else { + return { + data: BigInt(data), + l2BlockNumber, + l2BlockHash, + } as InBlock; + } + }); + } +} diff --git a/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts b/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts index 1f20956380b..f103343da35 100644 --- a/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts +++ b/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts @@ -6,6 +6,7 @@ import { ExtendedUnencryptedL2Log, type FromLogType, type GetUnencryptedLogsResponse, + type InBlock, type InboxLeaf, type L2Block, type L2BlockL2Logs, @@ -17,6 +18,7 @@ import { TxReceipt, TxScopedL2Log, type UnencryptedL2BlockL2Logs, + wrapInBlock, } from '@aztec/circuit-types'; import { type ContractClassPublic, @@ -27,6 +29,7 @@ import { type Header, INITIAL_L2_BLOCK_NUM, MAX_NOTE_HASHES_PER_TX, + MAX_NULLIFIERS_PER_TX, type UnconstrainedFunctionWithMembershipProof, } from '@aztec/circuits.js'; import { type ContractArtifact } from '@aztec/foundation/abi'; @@ -50,7 +53,7 @@ export class MemoryArchiverStore implements ArchiverDataStore { /** * An array containing all the tx effects in the L2 blocks that have been fetched so far. */ - private txEffects: TxEffect[] = []; + private txEffects: InBlock[] = []; private noteEncryptedLogsPerBlock: Map = new Map(); @@ -64,6 +67,8 @@ export class MemoryArchiverStore implements ArchiverDataStore { private contractClassLogsPerBlock: Map = new Map(); + private blockScopedNullifiers: Map = new Map(); + /** * Contains all L1 to L2 messages. */ @@ -181,7 +186,7 @@ export class MemoryArchiverStore implements ArchiverDataStore { this.lastL1BlockNewBlocks = blocks[blocks.length - 1].l1.blockNumber; this.l2Blocks.push(...blocks); - this.txEffects.push(...blocks.flatMap(b => b.data.body.txEffects)); + this.txEffects.push(...blocks.flatMap(b => b.data.body.txEffects.map(txEffect => wrapInBlock(txEffect, b.data)))); return Promise.resolve(true); } @@ -196,7 +201,7 @@ export class MemoryArchiverStore implements ArchiverDataStore { public async unwindBlocks(from: number, blocksToUnwind: number): Promise { const last = await this.getSynchedL2BlockNumber(); if (from != last) { - throw new Error(`Can only remove the tip`); + throw new Error(`Can only unwind blocks from the tip (requested ${from} but current tip is ${last})`); } const stopAt = from - blocksToUnwind; @@ -297,6 +302,51 @@ export class MemoryArchiverStore implements ArchiverDataStore { return Promise.resolve(true); } + addNullifiers(blocks: L2Block[]): Promise { + blocks.forEach(block => { + const dataStartIndexForBlock = + block.header.state.partial.nullifierTree.nextAvailableLeafIndex - + block.body.numberOfTxsIncludingPadded * MAX_NULLIFIERS_PER_TX; + block.body.txEffects.forEach((txEffects, txIndex) => { + const dataStartIndexForTx = dataStartIndexForBlock + txIndex * MAX_NULLIFIERS_PER_TX; + txEffects.nullifiers.forEach((nullifier, nullifierIndex) => { + this.blockScopedNullifiers.set(nullifier.toString(), { + index: BigInt(dataStartIndexForTx + nullifierIndex), + blockNumber: block.number, + blockHash: block.hash().toString(), + }); + }); + }); + }); + return Promise.resolve(true); + } + + deleteNullifiers(blocks: L2Block[]): Promise { + blocks.forEach(block => { + block.body.txEffects.forEach(txEffect => { + txEffect.nullifiers.forEach(nullifier => { + this.blockScopedNullifiers.delete(nullifier.toString()); + }); + }); + }); + return Promise.resolve(true); + } + + findNullifiersIndexesWithBlock(blockNumber: number, nullifiers: Fr[]): Promise<(InBlock | undefined)[]> { + const blockScopedNullifiers = nullifiers.map(nullifier => { + const nullifierData = this.blockScopedNullifiers.get(nullifier.toString()); + if (nullifierData !== undefined && nullifierData.blockNumber <= blockNumber) { + return { + data: nullifierData.index, + l2BlockHash: nullifierData.blockHash, + l2BlockNumber: nullifierData.blockNumber, + } as InBlock; + } + return undefined; + }); + return Promise.resolve(blockScopedNullifiers); + } + getTotalL1ToL2MessageCount(): Promise { return Promise.resolve(this.l1ToL2Messages.getTotalL1ToL2MessageCount()); } @@ -365,8 +415,8 @@ export class MemoryArchiverStore implements ArchiverDataStore { * @param txHash - The txHash of the tx effect. * @returns The requested tx effect. */ - public getTxEffect(txHash: TxHash): Promise { - const txEffect = this.txEffects.find(tx => tx.txHash.equals(txHash)); + public getTxEffect(txHash: TxHash): Promise | undefined> { + const txEffect = this.txEffects.find(tx => tx.data.txHash.equals(txHash)); return Promise.resolve(txEffect); } diff --git a/yarn-project/archiver/src/factory.ts b/yarn-project/archiver/src/factory.ts index e439ab370d5..28ed2ec035b 100644 --- a/yarn-project/archiver/src/factory.ts +++ b/yarn-project/archiver/src/factory.ts @@ -1,9 +1,10 @@ import { type ArchiverApi, type Service } from '@aztec/circuit-types'; -import { type ContractClassPublic } from '@aztec/circuits.js'; +import { type ContractClassPublic, getContractClassFromArtifact } from '@aztec/circuits.js'; import { createDebugLogger } from '@aztec/foundation/log'; import { type Maybe } from '@aztec/foundation/types'; import { type DataStoreConfig } from '@aztec/kv-store/config'; import { createStore } from '@aztec/kv-store/utils'; +import { TokenBridgeContractArtifact, TokenContractArtifact } from '@aztec/noir-contracts.js'; import { getCanonicalProtocolContract, protocolContractNames } from '@aztec/protocol-contracts'; import { type TelemetryClient } from '@aztec/telemetry-client'; import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; @@ -21,14 +22,15 @@ export async function createArchiver( if (!config.archiverUrl) { const store = await createStore('archiver', config, createDebugLogger('aztec:archiver:lmdb')); const archiverStore = new KVArchiverDataStore(store, config.maxLogs); - await initWithProtocolContracts(archiverStore); + await registerProtocolContracts(archiverStore); + await registerCommonContracts(archiverStore); return Archiver.createAndSync(config, archiverStore, telemetry, opts.blockUntilSync); } else { return createArchiverClient(config.archiverUrl); } } -async function initWithProtocolContracts(store: KVArchiverDataStore) { +async function registerProtocolContracts(store: KVArchiverDataStore) { const blockNumber = 0; for (const name of protocolContractNames) { const contract = getCanonicalProtocolContract(name); @@ -42,3 +44,19 @@ async function initWithProtocolContracts(store: KVArchiverDataStore) { await store.addContractInstances([contract.instance], blockNumber); } } + +// TODO(#10007): Remove this method. We are explicitly registering these contracts +// here to ensure they are available to all nodes and all prover nodes, since the PXE +// was tweaked to automatically push contract classes to the node it is registered, +// but other nodes in the network may require the contract classes to be registered as well. +// TODO(#10007): Remove the dependency on noir-contracts.js from this package once we remove this. +async function registerCommonContracts(store: KVArchiverDataStore) { + const blockNumber = 0; + const artifacts = [TokenBridgeContractArtifact, TokenContractArtifact]; + const classes = artifacts.map(artifact => ({ + ...getContractClassFromArtifact(artifact), + privateFunctions: [], + unconstrainedFunctions: [], + })); + await store.addContractClasses(classes, blockNumber); +} diff --git a/yarn-project/archiver/src/test/mock_l2_block_source.ts b/yarn-project/archiver/src/test/mock_l2_block_source.ts index 86ffce4ae50..2ab843cb42a 100644 --- a/yarn-project/archiver/src/test/mock_l2_block_source.ts +++ b/yarn-project/archiver/src/test/mock_l2_block_source.ts @@ -119,8 +119,14 @@ export class MockL2BlockSource implements L2BlockSource { * @returns The requested tx effect. */ public getTxEffect(txHash: TxHash) { - const txEffect = this.l2Blocks.flatMap(b => b.body.txEffects).find(tx => tx.txHash.equals(txHash)); - return Promise.resolve(txEffect); + const match = this.l2Blocks + .flatMap(b => b.body.txEffects.map(tx => [tx, b] as const)) + .find(([tx]) => tx.txHash.equals(txHash)); + if (!match) { + return Promise.resolve(undefined); + } + const [txEffect, block] = match; + return Promise.resolve({ data: txEffect, l2BlockNumber: block.number, l2BlockHash: block.hash().toString() }); } /** diff --git a/yarn-project/archiver/tsconfig.json b/yarn-project/archiver/tsconfig.json index dbe9915c010..901d60f657d 100644 --- a/yarn-project/archiver/tsconfig.json +++ b/yarn-project/archiver/tsconfig.json @@ -24,6 +24,9 @@ { "path": "../l1-artifacts" }, + { + "path": "../noir-contracts.js" + }, { "path": "../protocol-contracts" }, @@ -32,9 +35,6 @@ }, { "path": "../types" - }, - { - "path": "../noir-contracts.js" } ], "include": ["src"] diff --git a/yarn-project/aztec-node/src/aztec-node/server.test.ts b/yarn-project/aztec-node/src/aztec-node/server.test.ts index 09b1a2e081a..29bbf20e274 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.test.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.test.ts @@ -6,6 +6,7 @@ import { type L2LogsSource, MerkleTreeId, type MerkleTreeReadOperations, + type NullifierWithBlockSource, type WorldStateSynchronizer, mockTxForRollup, } from '@aztec/circuit-types'; @@ -53,6 +54,8 @@ describe('aztec node', () => { // all txs use the same allowed FPC class const contractSource = mock(); + const nullifierWithBlockSource = mock(); + const aztecNodeConfig: AztecNodeConfig = getConfigEnvVars(); node = new AztecNodeService( @@ -72,6 +75,7 @@ describe('aztec node', () => { l2LogsSource, contractSource, l1ToL2MessageSource, + nullifierWithBlockSource, worldState, undefined, 12345, diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index eec371b96fb..279638eb734 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -6,6 +6,7 @@ import { type EpochProofQuote, type FromLogType, type GetUnencryptedLogsResponse, + type InBlock, type L1ToL2MessageSource, type L2Block, type L2BlockL2Logs, @@ -16,6 +17,7 @@ import { LogType, MerkleTreeId, NullifierMembershipWitness, + type NullifierWithBlockSource, type ProcessedTx, type ProverConfig, PublicDataWitness, @@ -55,7 +57,7 @@ import { type L1ContractAddresses, createEthereumChain } from '@aztec/ethereum'; import { type ContractArtifact } from '@aztec/foundation/abi'; import { AztecAddress } from '@aztec/foundation/aztec-address'; import { padArrayEnd } from '@aztec/foundation/collection'; -import { createDebugLogger } from '@aztec/foundation/log'; +import { type DebugLogger, createDebugLogger } from '@aztec/foundation/log'; import { Timer } from '@aztec/foundation/timer'; import { type AztecKVStore } from '@aztec/kv-store'; import { openTmpStore } from '@aztec/kv-store/utils'; @@ -70,8 +72,8 @@ import { createP2PClient, } from '@aztec/p2p'; import { ProtocolContractAddress } from '@aztec/protocol-contracts'; -import { GlobalVariableBuilder, SequencerClient } from '@aztec/sequencer-client'; -import { PublicProcessorFactory, WASMSimulator, createSimulationProvider } from '@aztec/simulator'; +import { GlobalVariableBuilder, type L1Publisher, SequencerClient } from '@aztec/sequencer-client'; +import { PublicProcessorFactory } from '@aztec/simulator'; import { type TelemetryClient } from '@aztec/telemetry-client'; import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { createValidatorClient } from '@aztec/validator-client'; @@ -96,6 +98,7 @@ export class AztecNodeService implements AztecNode { protected readonly unencryptedLogsSource: L2LogsSource, protected readonly contractDataSource: ContractDataSource, protected readonly l1ToL2MessageSource: L1ToL2MessageSource, + protected readonly nullifierSource: NullifierWithBlockSource, protected readonly worldStateSynchronizer: WorldStateSynchronizer, protected readonly sequencer: SequencerClient | undefined, protected readonly l1ChainId: number, @@ -117,14 +120,18 @@ export class AztecNodeService implements AztecNode { this.log.info(message); } - addEpochProofQuote(quote: EpochProofQuote): Promise { + public addEpochProofQuote(quote: EpochProofQuote): Promise { return Promise.resolve(this.p2pClient.addEpochProofQuote(quote)); } - getEpochProofQuotes(epoch: bigint): Promise { + public getEpochProofQuotes(epoch: bigint): Promise { return this.p2pClient.getEpochProofQuotes(epoch); } + public getL2Tips() { + return this.blockSource.getL2Tips(); + } + /** * initializes the Aztec Node, wait for component to sync. * @param config - The configuration to be used by the aztec node. @@ -132,10 +139,14 @@ export class AztecNodeService implements AztecNode { */ public static async createAndSync( config: AztecNodeConfig, - telemetry?: TelemetryClient, - log = createDebugLogger('aztec:node'), + deps: { + telemetry?: TelemetryClient; + logger?: DebugLogger; + publisher?: L1Publisher; + } = {}, ): Promise { - telemetry ??= new NoopTelemetryClient(); + const telemetry = deps.telemetry ?? new NoopTelemetryClient(); + const log = deps.logger ?? createDebugLogger('aztec:node'); const ethereumChain = createEthereumChain(config.l1RpcUrl, config.l1ChainId); //validate that the actual chain id matches that specified in configuration if (config.l1ChainId !== ethereumChain.chainInfo.id) { @@ -160,24 +171,21 @@ export class AztecNodeService implements AztecNode { // start both and wait for them to sync from the block source await Promise.all([p2pClient.start(), worldStateSynchronizer.start()]); - const simulationProvider = await createSimulationProvider(config, log); - const validatorClient = createValidatorClient(config, p2pClient, telemetry); // now create the sequencer const sequencer = config.disableValidator ? undefined - : await SequencerClient.new( - config, + : await SequencerClient.new(config, { validatorClient, p2pClient, worldStateSynchronizer, - archiver, - archiver, - archiver, - simulationProvider, + contractDataSource: archiver, + l2BlockSource: archiver, + l1ToL2MessageSource: archiver, telemetry, - ); + ...deps, + }); return new AztecNodeService( config, @@ -187,6 +195,7 @@ export class AztecNodeService implements AztecNode { archiver, archiver, archiver, + archiver, worldStateSynchronizer, sequencer, ethereumChain.chainInfo.id, @@ -372,7 +381,7 @@ export class AztecNodeService implements AztecNode { return txReceipt; } - public getTxEffect(txHash: TxHash): Promise { + public getTxEffect(txHash: TxHash): Promise | undefined> { return this.blockSource.getTxEffect(txHash); } @@ -426,6 +435,16 @@ export class AztecNodeService implements AztecNode { return await Promise.all(leafValues.map(leafValue => committedDb.findLeafIndex(treeId, leafValue.toBuffer()))); } + public async findNullifiersIndexesWithBlock( + blockNumber: L2BlockNumber, + nullifiers: Fr[], + ): Promise<(InBlock | undefined)[]> { + if (blockNumber === 'latest') { + blockNumber = await this.getBlockNumber(); + } + return this.nullifierSource.findNullifiersIndexesWithBlock(blockNumber, nullifiers); + } + /** * Returns a sibling path for the given index in the nullifier tree. * @param blockNumber - The block number at which to get the data. @@ -708,7 +727,7 @@ export class AztecNodeService implements AztecNode { * Returns the currently committed block header, or the initial header if no blocks have been produced. * @returns The current committed block header. */ - public async getHeader(blockNumber: L2BlockNumber = 'latest'): Promise
{ + public async getBlockHeader(blockNumber: L2BlockNumber = 'latest'): Promise
{ return ( (await this.getBlock(blockNumber === 'latest' ? -1 : blockNumber))?.header ?? this.worldStateSynchronizer.getCommitted().getInitialHeader() @@ -733,11 +752,7 @@ export class AztecNodeService implements AztecNode { feeRecipient, ); const prevHeader = (await this.blockSource.getBlock(-1))?.header; - const publicProcessorFactory = new PublicProcessorFactory( - this.contractDataSource, - new WASMSimulator(), - this.telemetry, - ); + const publicProcessorFactory = new PublicProcessorFactory(this.contractDataSource, this.telemetry); const fork = await this.worldStateSynchronizer.fork(); @@ -819,7 +834,13 @@ export class AztecNodeService implements AztecNode { }); } + // TODO(#10007): Remove this method + public addContractClass(contractClass: ContractClassPublic): Promise { + return this.contractDataSource.addContractClass(contractClass); + } + public addContractArtifact(address: AztecAddress, artifact: ContractArtifact): Promise { + // TODO: Node should validate the artifact before accepting it return this.contractDataSource.addContractArtifact(address, artifact); } diff --git a/yarn-project/aztec-node/src/bin/index.ts b/yarn-project/aztec-node/src/bin/index.ts index 41aba729aeb..e1688b79198 100644 --- a/yarn-project/aztec-node/src/bin/index.ts +++ b/yarn-project/aztec-node/src/bin/index.ts @@ -1,6 +1,5 @@ #!/usr/bin/env -S node --no-warnings import { createDebugLogger } from '@aztec/foundation/log'; -import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import http from 'http'; @@ -16,7 +15,7 @@ const logger = createDebugLogger('aztec:node'); async function createAndDeployAztecNode() { const aztecNodeConfig: AztecNodeConfig = { ...getConfigEnvVars() }; - return await AztecNodeService.createAndSync(aztecNodeConfig, new NoopTelemetryClient()); + return await AztecNodeService.createAndSync(aztecNodeConfig); } /** diff --git a/yarn-project/aztec.js/src/contract/sent_tx.test.ts b/yarn-project/aztec.js/src/contract/sent_tx.test.ts index 13a2def98cd..c2bf65adb6a 100644 --- a/yarn-project/aztec.js/src/contract/sent_tx.test.ts +++ b/yarn-project/aztec.js/src/contract/sent_tx.test.ts @@ -39,7 +39,7 @@ describe('SentTx', () => { it('throws if tx is dropped', async () => { pxe.getTxReceipt.mockResolvedValue({ ...txReceipt, status: TxStatus.DROPPED } as TxReceipt); pxe.getSyncStatus.mockResolvedValue({ blocks: 19 }); - await expect(sentTx.wait({ timeout: 1, interval: 0.4 })).rejects.toThrow(/dropped/); + await expect(sentTx.wait({ timeout: 1, interval: 0.4, ignoreDroppedReceiptsFor: 0 })).rejects.toThrow(/dropped/); }); it('waits for the tx to be proven', async () => { diff --git a/yarn-project/aztec.js/src/contract/sent_tx.ts b/yarn-project/aztec.js/src/contract/sent_tx.ts index 7b85b3fd853..b7803975b16 100644 --- a/yarn-project/aztec.js/src/contract/sent_tx.ts +++ b/yarn-project/aztec.js/src/contract/sent_tx.ts @@ -4,6 +4,8 @@ import { type FieldsOf } from '@aztec/foundation/types'; /** Options related to waiting for a tx. */ export type WaitOpts = { + /** The amount of time to ignore TxStatus.DROPPED receipts (in seconds) due to the presumption that it is being propagated by the p2p network. Defaults to 5. */ + ignoreDroppedReceiptsFor?: number; /** The maximum time (in seconds) to wait for the transaction to be mined. Defaults to 60. */ timeout?: number; /** The maximum time (in seconds) to wait for the transaction to be proven. Defaults to 600. */ @@ -24,6 +26,7 @@ export type WaitOpts = { }; export const DefaultWaitOpts: WaitOpts = { + ignoreDroppedReceiptsFor: 5, timeout: 60, provenTimeout: 600, interval: 1, @@ -78,7 +81,7 @@ export class SentTx { } if (opts?.debug) { const txHash = await this.getTxHash(); - const tx = (await this.pxe.getTxEffect(txHash))!; + const { data: tx } = (await this.pxe.getTxEffect(txHash))!; receipt.debugInfo = { noteHashes: tx.noteHashes, nullifiers: tx.nullifiers, @@ -101,6 +104,9 @@ export class SentTx { protected async waitForReceipt(opts?: WaitOpts): Promise { const txHash = await this.getTxHash(); + const startTime = Date.now(); + const ignoreDroppedReceiptsFor = opts?.ignoreDroppedReceiptsFor ?? DefaultWaitOpts.ignoreDroppedReceiptsFor; + return await retryUntil( async () => { const txReceipt = await this.pxe.getTxReceipt(txHash); @@ -108,9 +114,15 @@ export class SentTx { if (txReceipt.status === TxStatus.PENDING) { return undefined; } - // If the tx was dropped, return it + // If the tx was "dropped", either return it or ignore based on timing. + // We can ignore it at first because the transaction may have been sent to node 1, and now we're asking node 2 for the receipt. + // If we don't allow a short grace period, we could incorrectly return a TxReceipt with status DROPPED. if (txReceipt.status === TxStatus.DROPPED) { - return txReceipt; + const elapsedSeconds = (Date.now() - startTime) / 1000; + if (!ignoreDroppedReceiptsFor || elapsedSeconds > ignoreDroppedReceiptsFor) { + return txReceipt; + } + return undefined; } // If we don't care about waiting for notes to be synced, return the receipt const waitForNotesAvailable = opts?.waitForNotesAvailable ?? DefaultWaitOpts.waitForNotesAvailable; diff --git a/yarn-project/aztec.js/src/wallet/base_wallet.ts b/yarn-project/aztec.js/src/wallet/base_wallet.ts index 00cc9127a38..94d7e479504 100644 --- a/yarn-project/aztec.js/src/wallet/base_wallet.ts +++ b/yarn-project/aztec.js/src/wallet/base_wallet.ts @@ -13,7 +13,6 @@ import { type SiblingPath, type SyncStatus, type Tx, - type TxEffect, type TxExecutionRequest, type TxHash, type TxProvingResult, @@ -122,7 +121,7 @@ export abstract class BaseWallet implements Wallet { sendTx(tx: Tx): Promise { return this.pxe.sendTx(tx); } - getTxEffect(txHash: TxHash): Promise { + getTxEffect(txHash: TxHash) { return this.pxe.getTxEffect(txHash); } getTxReceipt(txHash: TxHash): Promise { diff --git a/yarn-project/aztec/CHANGELOG.md b/yarn-project/aztec/CHANGELOG.md index fc34213d288..b25ffc2410b 100644 --- a/yarn-project/aztec/CHANGELOG.md +++ b/yarn-project/aztec/CHANGELOG.md @@ -1,5 +1,34 @@ # Changelog +## [0.63.1](https://github.com/AztecProtocol/aztec-packages/compare/aztec-package-v0.63.0...aztec-package-v0.63.1) (2024-11-19) + + +### Miscellaneous + +* **aztec-package:** Synchronize aztec-packages versions + +## [0.63.0](https://github.com/AztecProtocol/aztec-packages/compare/aztec-package-v0.62.0...aztec-package-v0.63.0) (2024-11-19) + + +### Features + +* Extract gossipsub / discv5 dependency gauge prometheus metrics ([#9710](https://github.com/AztecProtocol/aztec-packages/issues/9710)) ([58e75cd](https://github.com/AztecProtocol/aztec-packages/commit/58e75cdcc13de19bf5c1aedf7e66ce16e8cd4aaf)) + + +### Bug Fixes + +* Telemetry stopping on shutdown ([#9740](https://github.com/AztecProtocol/aztec-packages/issues/9740)) ([23b8d8b](https://github.com/AztecProtocol/aztec-packages/commit/23b8d8b15e2a8f5473e6bfedf62e9f1b35c9919c)) + + +### Miscellaneous + +* Clean up data configuration ([#9973](https://github.com/AztecProtocol/aztec-packages/issues/9973)) ([b660739](https://github.com/AztecProtocol/aztec-packages/commit/b66073986827731ce59e6f962ff1fb051677a094)) +* Move epoch and slot durations to config ([#9861](https://github.com/AztecProtocol/aztec-packages/issues/9861)) ([bfd4f2c](https://github.com/AztecProtocol/aztec-packages/commit/bfd4f2ce49393c4629563c07a89f19ebaf9aaab2)) +* Revert "chore: Validate RPC inputs" ([#9875](https://github.com/AztecProtocol/aztec-packages/issues/9875)) ([dd83d52](https://github.com/AztecProtocol/aztec-packages/commit/dd83d520c9925f00de155bddf0cf95852c971995)) +* Token partial notes refactor pt. 2 - bridging ([#9600](https://github.com/AztecProtocol/aztec-packages/issues/9600)) ([d513099](https://github.com/AztecProtocol/aztec-packages/commit/d51309954ab4a5ae1c829c86185b02c156baf3c7)) +* Validate RPC inputs ([#9672](https://github.com/AztecProtocol/aztec-packages/issues/9672)) ([6554122](https://github.com/AztecProtocol/aztec-packages/commit/6554122bdcd6d3840d03fdf1e7896f3961021e1f)), closes [#9455](https://github.com/AztecProtocol/aztec-packages/issues/9455) +* Validate RPC inputs reloaded ([#9878](https://github.com/AztecProtocol/aztec-packages/issues/9878)) ([70ab7c4](https://github.com/AztecProtocol/aztec-packages/commit/70ab7c4a905a33b518c773aa244f4a85064cfde3)) + ## [0.62.0](https://github.com/AztecProtocol/aztec-packages/compare/aztec-package-v0.61.0...aztec-package-v0.62.0) (2024-11-01) diff --git a/yarn-project/aztec/docker-compose.yml b/yarn-project/aztec/docker-compose.yml index 2d554c946c9..c161deb8a2e 100644 --- a/yarn-project/aztec/docker-compose.yml +++ b/yarn-project/aztec/docker-compose.yml @@ -1,18 +1,20 @@ version: '3' services: ethereum: - image: aztecprotocol/foundry-nightly-25f24e677a6a32a62512ad4f561995589ac2c7dc:latest + image: aztecprotocol/foundry:25f24e677a6a32a62512ad4f561995589ac2c7dc-${ARCH_TAG:-amd64} entrypoint: > sh -c ' - if [ -n "$FORK_BLOCK_NUMBER" ] && [ -n "$FORK_URL" ]; then - exec anvil -p 8545 --host 0.0.0.0 --chain-id 31337 --silent --fork-url "$FORK_URL" --fork-block-number "$FORK_BLOCK_NUMBER" - elif [ -n "$FORK_URL" ]; then - exec anvil -p 8545 --host 0.0.0.0 --chain-id 31337 --silent --fork-url "$FORK_URL" + if [ -n "$$FORK_BLOCK_NUMBER" ] && [ -n "$$FORK_URL" ]; then + exec anvil --silent -p "$$ANVIL_PORT" --host 0.0.0.0 --chain-id 31337 --fork-url "$$FORK_URL" --fork-block-number "$$FORK_BLOCK_NUMBER" else - exec anvil -p 8545 --host 0.0.0.0 --chain-id 31337 --silent + exec anvil --silent -p "$$ANVIL_PORT" --host 0.0.0.0 --chain-id 31337 fi' ports: - - '${SANDBOX_ANVIL_PORT:-8545}:8545' + - "${ANVIL_PORT:-8545}:${ANVIL_PORT:-8545}" + environment: + FORK_URL: + FORK_BLOCK_NUMBER: + ANVIL_PORT: ${ANVIL_PORT:-8545} aztec: image: 'aztecprotocol/aztec:${SANDBOX_VERSION:-latest}' diff --git a/yarn-project/aztec/package.json b/yarn-project/aztec/package.json index dae1278e53c..252535c182b 100644 --- a/yarn-project/aztec/package.json +++ b/yarn-project/aztec/package.json @@ -1,6 +1,6 @@ { "name": "@aztec/aztec", - "version": "0.62.0", + "version": "0.63.1", "type": "module", "exports": { ".": "./dest/index.js" diff --git a/yarn-project/aztec/src/sandbox.ts b/yarn-project/aztec/src/sandbox.ts index f1afd126b82..cd2a799df92 100644 --- a/yarn-project/aztec/src/sandbox.ts +++ b/yarn-project/aztec/src/sandbox.ts @@ -171,7 +171,7 @@ export async function createSandbox(config: Partial = {}) { */ export async function createAztecNode(config: Partial = {}, telemetryClient?: TelemetryClient) { const aztecNodeConfig: AztecNodeConfig = { ...getConfigEnvVars(), ...config }; - const node = await AztecNodeService.createAndSync(aztecNodeConfig, telemetryClient); + const node = await AztecNodeService.createAndSync(aztecNodeConfig, { telemetry: telemetryClient }); return node; } diff --git a/yarn-project/bb-prover/src/avm_proving.test.ts b/yarn-project/bb-prover/src/avm_proving.test.ts index e3d7607fc82..b5580a4d97f 100644 --- a/yarn-project/bb-prover/src/avm_proving.test.ts +++ b/yarn-project/bb-prover/src/avm_proving.test.ts @@ -1,41 +1,16 @@ -import { - AvmCircuitInputs, - AvmCircuitPublicInputs, - Gas, - GlobalVariables, - type PublicFunction, - PublicKeys, - SerializableContractInstance, - VerificationKeyData, -} from '@aztec/circuits.js'; -import { makeContractClassPublic, makeContractInstanceFromClassId } from '@aztec/circuits.js/testing'; -import { AztecAddress } from '@aztec/foundation/aztec-address'; -import { Fr, Point } from '@aztec/foundation/fields'; +import { VerificationKeyData } from '@aztec/circuits.js'; +import { Fr } from '@aztec/foundation/fields'; import { createDebugLogger } from '@aztec/foundation/log'; -import { openTmpStore } from '@aztec/kv-store/utils'; -import { AvmSimulator, PublicSideEffectTrace, type WorldStateDB } from '@aztec/simulator'; -import { - getAvmTestContractBytecode, - getAvmTestContractFunctionSelector, - initContext, - initExecutionEnvironment, - initPersistableStateManager, - resolveAvmTestContractAssertionMessage, -} from '@aztec/simulator/avm/fixtures'; -import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; -import { MerkleTrees } from '@aztec/world-state'; +import { simulateAvmTestContractGenerateCircuitInputs } from '@aztec/simulator/public/fixtures'; -import { mock } from 'jest-mock-extended'; import fs from 'node:fs/promises'; import { tmpdir } from 'node:os'; import path from 'path'; import { type BBSuccess, BB_RESULT, generateAvmProof, verifyAvmProof } from './bb/execute.js'; -import { getPublicInputs } from './test/test_avm.js'; import { extractAvmVkData } from './verification_key/verification_key_data.js'; const TIMEOUT = 180_000; -const TIMESTAMP = new Fr(99833); describe('AVM WitGen, proof generation and verification', () => { it( @@ -50,77 +25,8 @@ describe('AVM WitGen, proof generation and verification', () => { ); }); -/************************************************************************ - * Helpers - ************************************************************************/ - -/** - * If assertionErrString is set, we expect a (non exceptional halting) revert due to a failing assertion and - * we check that the revert reason error contains this string. However, the circuit must correctly prove the - * execution. - */ -const proveAndVerifyAvmTestContract = async ( - functionName: string, - calldata: Fr[] = [], - assertionErrString?: string, -) => { - const startSideEffectCounter = 0; - const functionSelector = getAvmTestContractFunctionSelector(functionName); - calldata = [functionSelector.toField(), ...calldata]; - const globals = GlobalVariables.empty(); - globals.timestamp = TIMESTAMP; - - const worldStateDB = mock(); - // - // Top level contract call - const bytecode = getAvmTestContractBytecode('public_dispatch'); - const fnSelector = getAvmTestContractFunctionSelector('public_dispatch'); - const publicFn: PublicFunction = { bytecode, selector: fnSelector }; - const contractClass = makeContractClassPublic(0, publicFn); - const contractInstance = makeContractInstanceFromClassId(contractClass.id); - - // The values here should match those in `avm_simulator.test.ts` - const instanceGet = new SerializableContractInstance({ - version: 1, - salt: new Fr(0x123), - deployer: new AztecAddress(new Fr(0x456)), - contractClassId: new Fr(0x789), - initializationHash: new Fr(0x101112), - publicKeys: new PublicKeys( - new Point(new Fr(0x131415), new Fr(0x161718), false), - new Point(new Fr(0x192021), new Fr(0x222324), false), - new Point(new Fr(0x252627), new Fr(0x282930), false), - new Point(new Fr(0x313233), new Fr(0x343536), false), - ), - }).withAddress(contractInstance.address); - - worldStateDB.getContractInstance - .mockResolvedValueOnce(contractInstance) - .mockResolvedValueOnce(instanceGet) // test gets deployer - .mockResolvedValueOnce(instanceGet) // test gets class id - .mockResolvedValueOnce(instanceGet) // test gets init hash - .mockResolvedValue(contractInstance); - worldStateDB.getContractClass.mockResolvedValue(contractClass); - - const storageValue = new Fr(5); - worldStateDB.storageRead.mockResolvedValue(Promise.resolve(storageValue)); - - const trace = new PublicSideEffectTrace(startSideEffectCounter); - const telemetry = new NoopTelemetryClient(); - const merkleTrees = await (await MerkleTrees.new(openTmpStore(), telemetry)).fork(); - worldStateDB.getMerkleInterface.mockReturnValue(merkleTrees); - const persistableState = initPersistableStateManager({ worldStateDB, trace, merkleTrees, doMerkleOperations: true }); - const environment = initExecutionEnvironment({ - functionSelector, - calldata, - globals, - address: contractInstance.address, - }); - const context = initContext({ env: environment, persistableState }); - - worldStateDB.getBytecode.mockResolvedValue(bytecode); - - const startGas = new Gas(context.machineState.gasLeft.daGas, context.machineState.gasLeft.l2Gas); +async function proveAndVerifyAvmTestContract(functionName: string, calldata: Fr[] = []) { + const avmCircuitInputs = await simulateAvmTestContractGenerateCircuitInputs(functionName, calldata); const internalLogger = createDebugLogger('aztec:avm-proving-test'); const logger = (msg: string, _data?: any) => internalLogger.verbose(msg); @@ -129,40 +35,11 @@ const proveAndVerifyAvmTestContract = async ( const bbPath = path.resolve('../../barretenberg/cpp/build/bin/bb'); const bbWorkingDirectory = await fs.mkdtemp(path.join(tmpdir(), 'bb-')); - // First we simulate (though it's not needed in this simple case). - const simulator = new AvmSimulator(context); - const avmResult = await simulator.execute(); - - if (assertionErrString == undefined) { - expect(avmResult.reverted).toBe(false); - } else { - // Explicit revert when an assertion failed. - expect(avmResult.reverted).toBe(true); - expect(avmResult.revertReason).toBeDefined(); - expect(resolveAvmTestContractAssertionMessage(functionName, avmResult.revertReason!, avmResult.output)).toContain( - assertionErrString, - ); - } - - const pxResult = trace.toPublicFunctionCallResult( - environment, - startGas, - /*endGasLeft=*/ Gas.from(context.machineState.gasLeft), - /*bytecode=*/ simulator.getBytecode()!, - avmResult, - functionName, - ); - - const avmCircuitInputs = new AvmCircuitInputs( - functionName, - /*calldata=*/ context.environment.calldata, - /*publicInputs=*/ getPublicInputs(pxResult), - /*avmHints=*/ pxResult.avmCircuitHints, - /*output*/ AvmCircuitPublicInputs.empty(), - ); - // Then we prove. const proofRes = await generateAvmProof(bbPath, bbWorkingDirectory, avmCircuitInputs, logger); + if (proofRes.status === BB_RESULT.FAILURE) { + internalLogger.error(`Proof generation failed: ${proofRes.reason}`); + } expect(proofRes.status).toEqual(BB_RESULT.SUCCESS); // Then we test VK extraction and serialization. @@ -174,4 +51,4 @@ const proveAndVerifyAvmTestContract = async ( const rawVkPath = path.join(succeededRes.vkPath!, 'vk'); const verificationRes = await verifyAvmProof(bbPath, succeededRes.proofPath!, rawVkPath, logger); expect(verificationRes.status).toBe(BB_RESULT.SUCCESS); -}; +} diff --git a/yarn-project/bb-prover/src/bb/execute.ts b/yarn-project/bb-prover/src/bb/execute.ts index 11ef630a42a..e5159fb55e7 100644 --- a/yarn-project/bb-prover/src/bb/execute.ts +++ b/yarn-project/bb-prover/src/bb/execute.ts @@ -43,6 +43,7 @@ export type BBSuccess = { export type BBFailure = { status: BB_RESULT.FAILURE; reason: string; + retry?: boolean; }; export type BBResult = BBSuccess | BBFailure; @@ -175,6 +176,7 @@ export async function generateKeyForNoirCircuit( return { status: BB_RESULT.FAILURE, reason: `Failed to generate key. Exit code: ${result.exitCode}. Signal ${result.signal}.`, + retry: !!result.signal, }; } catch (error) { return { status: BB_RESULT.FAILURE, reason: `${error}` }; @@ -245,6 +247,7 @@ export async function executeBbClientIvcProof( return { status: BB_RESULT.FAILURE, reason: `Failed to generate proof. Exit code ${result.exitCode}. Signal ${result.signal}.`, + retry: !!result.signal, }; } catch (error) { return { status: BB_RESULT.FAILURE, reason: `${error}` }; @@ -324,6 +327,7 @@ export async function computeVerificationKey( return { status: BB_RESULT.FAILURE, reason: `Failed to write VK. Exit code ${result.exitCode}. Signal ${result.signal}.`, + retry: !!result.signal, }; } catch (error) { return { status: BB_RESULT.FAILURE, reason: `${error}` }; @@ -396,6 +400,7 @@ export async function generateProof( return { status: BB_RESULT.FAILURE, reason: `Failed to generate proof. Exit code ${result.exitCode}. Signal ${result.signal}.`, + retry: !!result.signal, }; } catch (error) { return { status: BB_RESULT.FAILURE, reason: `${error}` }; @@ -470,6 +475,7 @@ export async function generateTubeProof( return { status: BB_RESULT.FAILURE, reason: `Failed to generate proof. Exit code ${result.exitCode}. Signal ${result.signal}.`, + retry: !!result.signal, }; } catch (error) { return { status: BB_RESULT.FAILURE, reason: `${error}` }; @@ -573,6 +579,7 @@ export async function generateAvmProof( return { status: BB_RESULT.FAILURE, reason: `Failed to generate proof. Exit code ${result.exitCode}. Signal ${result.signal}.`, + retry: !!result.signal, }; } catch (error) { return { status: BB_RESULT.FAILURE, reason: `${error}` }; @@ -648,6 +655,7 @@ export async function verifyClientIvcProof( return { status: BB_RESULT.FAILURE, reason: `Failed to verify proof. Exit code ${result.exitCode}. Signal ${result.signal}.`, + retry: !!result.signal, }; } catch (error) { return { status: BB_RESULT.FAILURE, reason: `${error}` }; @@ -690,6 +698,7 @@ async function verifyProofInternal( return { status: BB_RESULT.FAILURE, reason: `Failed to verify proof. Exit code ${result.exitCode}. Signal ${result.signal}.`, + retry: !!result.signal, }; } catch (error) { return { status: BB_RESULT.FAILURE, reason: `${error}` }; @@ -730,6 +739,7 @@ export async function writeVkAsFields( return { status: BB_RESULT.FAILURE, reason: `Failed to create vk as fields. Exit code ${result.exitCode}. Signal ${result.signal}.`, + retry: !!result.signal, }; } catch (error) { return { status: BB_RESULT.FAILURE, reason: `${error}` }; @@ -772,6 +782,7 @@ export async function writeProofAsFields( return { status: BB_RESULT.FAILURE, reason: `Failed to create proof as fields. Exit code ${result.exitCode}. Signal ${result.signal}.`, + retry: !!result.signal, }; } catch (error) { return { status: BB_RESULT.FAILURE, reason: `${error}` }; @@ -813,6 +824,7 @@ export async function generateContractForVerificationKey( return { status: BB_RESULT.FAILURE, reason: `Failed to write verifier contract. Exit code ${result.exitCode}. Signal ${result.signal}.`, + retry: !!result.signal, }; } catch (error) { return { status: BB_RESULT.FAILURE, reason: `${error}` }; diff --git a/yarn-project/bb-prover/src/prover/bb_private_kernel_prover.ts b/yarn-project/bb-prover/src/prover/bb_private_kernel_prover.ts index 4de354a0b56..7f16b0292fd 100644 --- a/yarn-project/bb-prover/src/prover/bb_private_kernel_prover.ts +++ b/yarn-project/bb-prover/src/prover/bb_private_kernel_prover.ts @@ -338,7 +338,7 @@ export class BBNativePrivateKernelProver implements PrivateKernelProver { ); if (vkResult.status === BB_RESULT.FAILURE) { - this.log.error(`Failed to generate proof for ${circuitType}${dbgCircuitName}: ${vkResult.reason}`); + this.log.error(`Failed to generate verification key for ${circuitType}${dbgCircuitName}: ${vkResult.reason}`); throw new Error(vkResult.reason); } diff --git a/yarn-project/bb-prover/src/prover/bb_prover.ts b/yarn-project/bb-prover/src/prover/bb_prover.ts index a8475793cba..f737b093a38 100644 --- a/yarn-project/bb-prover/src/prover/bb_prover.ts +++ b/yarn-project/bb-prover/src/prover/bb_prover.ts @@ -1,6 +1,7 @@ /* eslint-disable require-await */ import { type ProofAndVerificationKey, + ProvingError, type PublicInputsAndRecursiveProof, type ServerCircuitProver, makeProofAndVerificationKey, @@ -477,7 +478,7 @@ export class BBNativeRollupProver implements ServerCircuitProver { if (provingResult.status === BB_RESULT.FAILURE) { logger.error(`Failed to generate proof for ${circuitType}: ${provingResult.reason}`); - throw new Error(provingResult.reason); + throw new ProvingError(provingResult.reason, provingResult, provingResult.retry); } // Ensure our vk cache is up to date @@ -538,7 +539,7 @@ export class BBNativeRollupProver implements ServerCircuitProver { if (provingResult.status === BB_RESULT.FAILURE) { logger.error(`Failed to generate AVM proof for ${input.functionName}: ${provingResult.reason}`); - throw new Error(provingResult.reason); + throw new ProvingError(provingResult.reason, provingResult, provingResult.retry); } return provingResult; @@ -554,8 +555,8 @@ export class BBNativeRollupProver implements ServerCircuitProver { const provingResult = await generateTubeProof(this.config.bbBinaryPath, bbWorkingDirectory, logger.verbose); if (provingResult.status === BB_RESULT.FAILURE) { - logger.error(`Failed to generate proof for tube proof: ${provingResult.reason}`); - throw new Error(provingResult.reason); + logger.error(`Failed to generate proof for tube circuit: ${provingResult.reason}`); + throw new ProvingError(provingResult.reason, provingResult, provingResult.retry); } return provingResult; } @@ -724,7 +725,7 @@ export class BBNativeRollupProver implements ServerCircuitProver { if (result.status === BB_RESULT.FAILURE) { const errorMessage = `Failed to verify proof from key!`; - throw new Error(errorMessage); + throw new ProvingError(errorMessage, result, result.retry); } logger.info(`Successfully verified proof from key in ${result.durationMs} ms`); @@ -785,7 +786,7 @@ export class BBNativeRollupProver implements ServerCircuitProver { if (result.status === BB_RESULT.FAILURE) { const errorMessage = `Failed to convert ${circuit} proof to fields, ${result.reason}`; - throw new Error(errorMessage); + throw new ProvingError(errorMessage, result, result.retry); } const proofString = await fs.readFile(path.join(bbWorkingDirectory, PROOF_FIELDS_FILENAME), { @@ -825,7 +826,11 @@ export class BBNativeRollupProver implements ServerCircuitProver { logger.debug, ).then(result => { if (result.status === BB_RESULT.FAILURE) { - throw new Error(`Failed to generate verification key for ${circuitType}, ${result.reason}`); + throw new ProvingError( + `Failed to generate verification key for ${circuitType}, ${result.reason}`, + result, + result.retry, + ); } return extractVkData(result.vkPath!); }); diff --git a/yarn-project/circuit-types/src/in_block.ts b/yarn-project/circuit-types/src/in_block.ts new file mode 100644 index 00000000000..5e5205b4237 --- /dev/null +++ b/yarn-project/circuit-types/src/in_block.ts @@ -0,0 +1,36 @@ +import { Fr } from '@aztec/circuits.js'; +import { schemas } from '@aztec/foundation/schemas'; + +import { type ZodTypeAny, z } from 'zod'; + +import { type L2Block } from './l2_block.js'; + +export type InBlock = { + l2BlockNumber: number; + l2BlockHash: string; + data: T; +}; + +export function randomInBlock(data: T): InBlock { + return { + data, + l2BlockNumber: Math.floor(Math.random() * 1000), + l2BlockHash: Fr.random().toString(), + }; +} + +export function wrapInBlock(data: T, block: L2Block): InBlock { + return { + data, + l2BlockNumber: block.number, + l2BlockHash: block.hash().toString(), + }; +} + +export function inBlockSchemaFor(schema: T) { + return z.object({ + data: schema, + l2BlockNumber: schemas.Integer, + l2BlockHash: z.string(), + }); +} diff --git a/yarn-project/circuit-types/src/index.ts b/yarn-project/circuit-types/src/index.ts index 86eade75e74..829e43cf8e3 100644 --- a/yarn-project/circuit-types/src/index.ts +++ b/yarn-project/circuit-types/src/index.ts @@ -22,3 +22,6 @@ export * from './simulation_error.js'; export * from './tx/index.js'; export * from './tx_effect.js'; export * from './tx_execution_request.js'; +export * from './in_block.js'; +export * from './nullifier_with_block_source.js'; +export * from './proving_error.js'; diff --git a/yarn-project/circuit-types/src/interfaces/archiver.test.ts b/yarn-project/circuit-types/src/interfaces/archiver.test.ts index b6b411c3177..36947324e24 100644 --- a/yarn-project/circuit-types/src/interfaces/archiver.test.ts +++ b/yarn-project/circuit-types/src/interfaces/archiver.test.ts @@ -20,6 +20,7 @@ import { readFileSync } from 'fs'; import omit from 'lodash.omit'; import { resolve } from 'path'; +import { type InBlock, randomInBlock } from '../in_block.js'; import { L2Block } from '../l2_block.js'; import { type L2Tips } from '../l2_block_source.js'; import { ExtendedUnencryptedL2Log } from '../logs/extended_unencrypted_l2_log.js'; @@ -106,7 +107,7 @@ describe('ArchiverApiSchema', () => { it('getTxEffect', async () => { const result = await context.client.getTxEffect(new TxHash(Buffer.alloc(32, 1))); - expect(result).toBeInstanceOf(TxEffect); + expect(result!.data).toBeInstanceOf(TxEffect); }); it('getSettledTxReceipt', async () => { @@ -143,6 +144,18 @@ describe('ArchiverApiSchema', () => { }); }); + it('findNullifiersIndexesWithBlock', async () => { + const result = await context.client.findNullifiersIndexesWithBlock(1, [Fr.random(), Fr.random()]); + expect(result).toEqual([ + { + data: expect.any(BigInt), + l2BlockNumber: expect.any(Number), + l2BlockHash: expect.any(String), + }, + undefined, + ]); + }); + it('getLogs(Encrypted)', async () => { const result = await context.client.getLogs(1, 1, LogType.ENCRYPTED); expect(result).toEqual([expect.any(EncryptedL2BlockL2Logs)]); @@ -217,7 +230,7 @@ describe('ArchiverApiSchema', () => { it('addContractArtifact', async () => { await context.client.addContractArtifact(AztecAddress.random(), artifact); - }); + }, 20_000); it('getContract', async () => { const address = AztecAddress.random(); @@ -232,6 +245,15 @@ describe('ArchiverApiSchema', () => { version: 1, }); }); + + it('addContractClass', async () => { + const contractClass = getContractClassFromArtifact(artifact); + await context.client.addContractClass({ + ...omit(contractClass, 'publicBytecodeCommitment'), + unconstrainedFunctions: [], + privateFunctions: [], + }); + }); }); class MockArchiver implements ArchiverApi { @@ -261,9 +283,9 @@ class MockArchiver implements ArchiverApi { getBlocks(from: number, _limit: number, _proven?: boolean | undefined): Promise { return Promise.resolve([L2Block.random(from)]); } - getTxEffect(_txHash: TxHash): Promise { + getTxEffect(_txHash: TxHash): Promise | undefined> { expect(_txHash).toBeInstanceOf(TxHash); - return Promise.resolve(TxEffect.empty()); + return Promise.resolve({ l2BlockNumber: 1, l2BlockHash: '0x12', data: TxEffect.random() }); } getSettledTxReceipt(txHash: TxHash): Promise { expect(txHash).toBeInstanceOf(TxHash); @@ -290,6 +312,13 @@ class MockArchiver implements ArchiverApi { finalized: { number: 1, hash: `0x01` }, }); } + findNullifiersIndexesWithBlock(blockNumber: number, nullifiers: Fr[]): Promise<(InBlock | undefined)[]> { + expect(blockNumber).toEqual(1); + expect(nullifiers).toHaveLength(2); + expect(nullifiers[0]).toBeInstanceOf(Fr); + expect(nullifiers[1]).toBeInstanceOf(Fr); + return Promise.resolve([randomInBlock(Fr.random().toBigInt()), undefined]); + } getLogs( _from: number, _limit: number, @@ -362,4 +391,7 @@ class MockArchiver implements ArchiverApi { expect(l1ToL2Message).toBeInstanceOf(Fr); return Promise.resolve(1n); } + addContractClass(_contractClass: ContractClassPublic): Promise { + return Promise.resolve(); + } } diff --git a/yarn-project/circuit-types/src/interfaces/archiver.ts b/yarn-project/circuit-types/src/interfaces/archiver.ts index 0dc118875aa..f5a212f5099 100644 --- a/yarn-project/circuit-types/src/interfaces/archiver.ts +++ b/yarn-project/circuit-types/src/interfaces/archiver.ts @@ -10,6 +10,7 @@ import { type ApiSchemaFor, optional, schemas } from '@aztec/foundation/schemas' import { z } from 'zod'; +import { inBlockSchemaFor } from '../in_block.js'; import { L2Block } from '../l2_block.js'; import { type L2BlockSource, L2TipsSchema } from '../l2_block_source.js'; import { GetUnencryptedLogsResponseSchema, TxScopedL2Log } from '../logs/get_logs_response.js'; @@ -18,12 +19,13 @@ import { type L2LogsSource } from '../logs/l2_logs_source.js'; import { LogFilterSchema } from '../logs/log_filter.js'; import { LogType } from '../logs/log_type.js'; import { type L1ToL2MessageSource } from '../messaging/l1_to_l2_message_source.js'; +import { type NullifierWithBlockSource } from '../nullifier_with_block_source.js'; import { TxHash } from '../tx/tx_hash.js'; import { TxReceipt } from '../tx/tx_receipt.js'; import { TxEffect } from '../tx_effect.js'; export type ArchiverApi = Omit< - L2BlockSource & L2LogsSource & ContractDataSource & L1ToL2MessageSource, + L2BlockSource & L2LogsSource & ContractDataSource & L1ToL2MessageSource & NullifierWithBlockSource, 'start' | 'stop' >; @@ -42,7 +44,7 @@ export const ArchiverApiSchema: ApiSchemaFor = { .function() .args(schemas.Integer, schemas.Integer, optional(z.boolean())) .returns(z.array(L2Block.schema)), - getTxEffect: z.function().args(TxHash.schema).returns(TxEffect.schema.optional()), + getTxEffect: z.function().args(TxHash.schema).returns(inBlockSchemaFor(TxEffect.schema).optional()), getSettledTxReceipt: z.function().args(TxHash.schema).returns(TxReceipt.schema.optional()), getL2SlotNumber: z.function().args().returns(schemas.BigInt), getL2EpochNumber: z.function().args().returns(schemas.BigInt), @@ -57,6 +59,10 @@ export const ArchiverApiSchema: ApiSchemaFor = { .function() .args(z.array(schemas.Fr)) .returns(z.array(z.array(TxScopedL2Log.schema))), + findNullifiersIndexesWithBlock: z + .function() + .args(z.number(), z.array(schemas.Fr)) + .returns(z.array(optional(inBlockSchemaFor(schemas.BigInt)))), getUnencryptedLogs: z.function().args(LogFilterSchema).returns(GetUnencryptedLogsResponseSchema), getContractClassLogs: z.function().args(LogFilterSchema).returns(GetUnencryptedLogsResponseSchema), getPublicFunction: z @@ -70,4 +76,6 @@ export const ArchiverApiSchema: ApiSchemaFor = { addContractArtifact: z.function().args(schemas.AztecAddress, ContractArtifactSchema).returns(z.void()), getL1ToL2Messages: z.function().args(schemas.BigInt).returns(z.array(schemas.Fr)), getL1ToL2MessageIndex: z.function().args(schemas.Fr).returns(schemas.BigInt.optional()), + // TODO(#10007): Remove this method + addContractClass: z.function().args(ContractClassPublicSchema).returns(z.void()), }; diff --git a/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts b/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts index d415249c3f4..5b32cdd1d1e 100644 --- a/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts +++ b/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts @@ -28,7 +28,9 @@ import omit from 'lodash.omit'; import times from 'lodash.times'; import { resolve } from 'path'; +import { type InBlock, randomInBlock } from '../in_block.js'; import { L2Block } from '../l2_block.js'; +import { type L2Tips } from '../l2_block_source.js'; import { ExtendedUnencryptedL2Log } from '../logs/extended_unencrypted_l2_log.js'; import { type GetUnencryptedLogsResponse, TxScopedL2Log } from '../logs/get_logs_response.js'; import { @@ -80,11 +82,28 @@ describe('AztecNodeApiSchema', () => { expect([...tested].sort()).toEqual(all.sort()); }); + it('getL2Tips', async () => { + const result = await context.client.getL2Tips(); + expect(result).toEqual({ + latest: { number: 1, hash: `0x01` }, + proven: { number: 1, hash: `0x01` }, + finalized: { number: 1, hash: `0x01` }, + }); + }); + it('findLeavesIndexes', async () => { const response = await context.client.findLeavesIndexes(1, MerkleTreeId.ARCHIVE, [Fr.random(), Fr.random()]); expect(response).toEqual([1n, undefined]); }); + it('findNullifiersIndexesWithBlock', async () => { + const response = await context.client.findNullifiersIndexesWithBlock(1, [Fr.random(), Fr.random()]); + expect(response).toEqual([ + { data: 1n, l2BlockNumber: expect.any(Number), l2BlockHash: expect.any(String) }, + undefined, + ]); + }); + it('getNullifierSiblingPath', async () => { const response = await context.client.getNullifierSiblingPath(1, 1n); expect(response).toBeInstanceOf(SiblingPath); @@ -188,7 +207,7 @@ describe('AztecNodeApiSchema', () => { it('addContractArtifact', async () => { await context.client.addContractArtifact(AztecAddress.random(), artifact); - }); + }, 20_000); it('getLogs(Encrypted)', async () => { const response = await context.client.getLogs(1, 1, LogType.ENCRYPTED); @@ -231,7 +250,7 @@ describe('AztecNodeApiSchema', () => { it('getTxEffect', async () => { const response = await context.client.getTxEffect(TxHash.random()); - expect(response).toBeInstanceOf(TxEffect); + expect(response!.data).toBeInstanceOf(TxEffect); }); it('getPendingTxs', async () => { @@ -254,8 +273,8 @@ describe('AztecNodeApiSchema', () => { expect(response).toBeInstanceOf(Fr); }); - it('getHeader', async () => { - const response = await context.client.getHeader(); + it('getBlockHeader', async () => { + const response = await context.client.getBlockHeader(); expect(response).toBeInstanceOf(Header); }); @@ -313,11 +332,23 @@ describe('AztecNodeApiSchema', () => { const response = await context.client.getEpochProofQuotes(1n); expect(response).toEqual([expect.any(EpochProofQuote)]); }); + + it('addContractClass', async () => { + const contractClass = getContractClassFromArtifact(artifact); + await context.client.addContractClass({ ...contractClass, unconstrainedFunctions: [], privateFunctions: [] }); + }); }); class MockAztecNode implements AztecNode { constructor(private artifact: ContractArtifact) {} + getL2Tips(): Promise { + return Promise.resolve({ + latest: { number: 1, hash: `0x01` }, + proven: { number: 1, hash: `0x01` }, + finalized: { number: 1, hash: `0x01` }, + }); + } findLeavesIndexes( blockNumber: number | 'latest', treeId: MerkleTreeId, @@ -328,6 +359,15 @@ class MockAztecNode implements AztecNode { expect(leafValues[1]).toBeInstanceOf(Fr); return Promise.resolve([1n, undefined]); } + findNullifiersIndexesWithBlock( + blockNumber: number | 'latest', + nullifiers: Fr[], + ): Promise<(InBlock | undefined)[]> { + expect(nullifiers).toHaveLength(2); + expect(nullifiers[0]).toBeInstanceOf(Fr); + expect(nullifiers[1]).toBeInstanceOf(Fr); + return Promise.resolve([randomInBlock(1n), undefined]); + } getNullifierSiblingPath( blockNumber: number | 'latest', leafIndex: bigint, @@ -472,9 +512,9 @@ class MockAztecNode implements AztecNode { expect(txHash).toBeInstanceOf(TxHash); return Promise.resolve(TxReceipt.empty()); } - getTxEffect(txHash: TxHash): Promise { + getTxEffect(txHash: TxHash): Promise | undefined> { expect(txHash).toBeInstanceOf(TxHash); - return Promise.resolve(TxEffect.random()); + return Promise.resolve({ l2BlockNumber: 1, l2BlockHash: '0x12', data: TxEffect.random() }); } getPendingTxs(): Promise { return Promise.resolve([Tx.random()]); @@ -491,7 +531,7 @@ class MockAztecNode implements AztecNode { expect(slot).toBeInstanceOf(Fr); return Promise.resolve(Fr.random()); } - getHeader(_blockNumber?: number | 'latest' | undefined): Promise
{ + getBlockHeader(_blockNumber?: number | 'latest' | undefined): Promise
{ return Promise.resolve(Header.empty()); } simulatePublicCalls(tx: Tx): Promise { @@ -538,4 +578,7 @@ class MockAztecNode implements AztecNode { expect(epoch).toEqual(1n); return Promise.resolve([EpochProofQuote.random()]); } + addContractClass(_contractClass: ContractClassPublic): Promise { + return Promise.resolve(); + } } diff --git a/yarn-project/circuit-types/src/interfaces/aztec-node.ts b/yarn-project/circuit-types/src/interfaces/aztec-node.ts index 91b7dd8d0ca..deeae772391 100644 --- a/yarn-project/circuit-types/src/interfaces/aztec-node.ts +++ b/yarn-project/circuit-types/src/interfaces/aztec-node.ts @@ -21,7 +21,9 @@ import { type ApiSchemaFor, optional, schemas } from '@aztec/foundation/schemas' import { z } from 'zod'; +import { type InBlock, inBlockSchemaFor } from '../in_block.js'; import { L2Block } from '../l2_block.js'; +import { type L2BlockSource, type L2Tips, L2TipsSchema } from '../l2_block_source.js'; import { type FromLogType, type GetUnencryptedLogsResponse, @@ -48,7 +50,14 @@ import { type ProverCoordination, ProverCoordinationApiSchema } from './prover-c * The aztec node. * We will probably implement the additional interfaces by means other than Aztec Node as it's currently a privacy leak */ -export interface AztecNode extends ProverCoordination { +export interface AztecNode + extends ProverCoordination, + Pick { + /** + * Returns the tips of the L2 chain. + */ + getL2Tips(): Promise; + /** * Find the indexes of the given leaves in the given tree. * @param blockNumber - The block number at which to get the data or 'latest' for latest data @@ -62,6 +71,18 @@ export interface AztecNode extends ProverCoordination { leafValues: Fr[], ): Promise<(bigint | undefined)[]>; + /** + * Returns the indexes of the given nullifiers in the nullifier tree, + * scoped to the block they were included in. + * @param blockNumber - The block number at which to get the data. + * @param nullifiers - The nullifiers to search for. + * @returns The block scoped indexes of the given nullifiers in the nullifier tree, or undefined if not found. + */ + findNullifiersIndexesWithBlock( + blockNumber: L2BlockNumber, + nullifiers: Fr[], + ): Promise<(InBlock | undefined)[]>; + /** * Returns a sibling path for the given index in the nullifier tree. * @param blockNumber - The block number at which to get the data. @@ -298,7 +319,7 @@ export interface AztecNode extends ProverCoordination { * @param txHash - The hash of a transaction which resulted in the returned tx effect. * @returns The requested tx effect. */ - getTxEffect(txHash: TxHash): Promise; + getTxEffect(txHash: TxHash): Promise | undefined>; /** * Method to retrieve pending txs. @@ -336,7 +357,7 @@ export interface AztecNode extends ProverCoordination { * Returns the currently committed block header. * @returns The current committed block header. */ - getHeader(blockNumber?: L2BlockNumber): Promise
; + getBlockHeader(blockNumber?: L2BlockNumber): Promise
; /** * Simulates the public part of a transaction with the current state. @@ -391,16 +412,29 @@ export interface AztecNode extends ProverCoordination { * @param epoch - The epoch for which to get the quotes */ getEpochProofQuotes(epoch: bigint): Promise; + + /** + * Adds a contract class bypassing the registerer. + * TODO(#10007): Remove this method. + * @param contractClass - The class to register. + */ + addContractClass(contractClass: ContractClassPublic): Promise; } export const AztecNodeApiSchema: ApiSchemaFor = { ...ProverCoordinationApiSchema, + getL2Tips: z.function().args().returns(L2TipsSchema), findLeavesIndexes: z .function() .args(L2BlockNumberSchema, z.nativeEnum(MerkleTreeId), z.array(schemas.Fr)) .returns(z.array(optional(schemas.BigInt))), + findNullifiersIndexesWithBlock: z + .function() + .args(L2BlockNumberSchema, z.array(schemas.Fr)) + .returns(z.array(optional(inBlockSchemaFor(schemas.BigInt)))), + getNullifierSiblingPath: z .function() .args(L2BlockNumberSchema, schemas.BigInt) @@ -485,7 +519,7 @@ export const AztecNodeApiSchema: ApiSchemaFor = { getTxReceipt: z.function().args(TxHash.schema).returns(TxReceipt.schema), - getTxEffect: z.function().args(TxHash.schema).returns(TxEffect.schema.optional()), + getTxEffect: z.function().args(TxHash.schema).returns(inBlockSchemaFor(TxEffect.schema).optional()), getPendingTxs: z.function().returns(z.array(Tx.schema)), @@ -495,7 +529,7 @@ export const AztecNodeApiSchema: ApiSchemaFor = { getPublicStorageAt: z.function().args(schemas.AztecAddress, schemas.Fr, L2BlockNumberSchema).returns(schemas.Fr), - getHeader: z.function().args(optional(L2BlockNumberSchema)).returns(Header.schema), + getBlockHeader: z.function().args(optional(L2BlockNumberSchema)).returns(Header.schema), simulatePublicCalls: z.function().args(Tx.schema).returns(PublicSimulationOutput.schema), @@ -514,6 +548,9 @@ export const AztecNodeApiSchema: ApiSchemaFor = { addEpochProofQuote: z.function().args(EpochProofQuote.schema).returns(z.void()), getEpochProofQuotes: z.function().args(schemas.BigInt).returns(z.array(EpochProofQuote.schema)), + + // TODO(#10007): Remove this method + addContractClass: z.function().args(ContractClassPublicSchema).returns(z.void()), }; export function createAztecNodeClient(url: string, fetch = defaultFetch): AztecNode { diff --git a/yarn-project/circuit-types/src/interfaces/pxe.test.ts b/yarn-project/circuit-types/src/interfaces/pxe.test.ts index 8aefa059edb..e2aa6c1cca5 100644 --- a/yarn-project/circuit-types/src/interfaces/pxe.test.ts +++ b/yarn-project/circuit-types/src/interfaces/pxe.test.ts @@ -30,6 +30,7 @@ import times from 'lodash.times'; import { resolve } from 'path'; import { AuthWitness } from '../auth_witness.js'; +import { type InBlock } from '../in_block.js'; import { L2Block } from '../l2_block.js'; import { ExtendedUnencryptedL2Log, type GetUnencryptedLogsResponse, type LogFilter } from '../logs/index.js'; import { type IncomingNotesFilter } from '../notes/incoming_notes_filter.js'; @@ -178,8 +179,10 @@ describe('PXESchema', () => { }); it('getTxEffect', async () => { - const result = await context.client.getTxEffect(TxHash.random()); - expect(result).toBeInstanceOf(TxEffect); + const { l2BlockHash, l2BlockNumber, data } = (await context.client.getTxEffect(TxHash.random()))!; + expect(data).toBeInstanceOf(TxEffect); + expect(l2BlockHash).toMatch(/0x[a-fA-F0-9]{64}/); + expect(l2BlockNumber).toBe(1); }); it('getPublicStorageAt', async () => { @@ -401,9 +404,9 @@ class MockPXE implements PXE { expect(txHash).toBeInstanceOf(TxHash); return Promise.resolve(TxReceipt.empty()); } - getTxEffect(txHash: TxHash): Promise { + getTxEffect(txHash: TxHash): Promise | undefined> { expect(txHash).toBeInstanceOf(TxHash); - return Promise.resolve(TxEffect.random()); + return Promise.resolve({ data: TxEffect.random(), l2BlockHash: Fr.random().toString(), l2BlockNumber: 1 }); } getPublicStorageAt(contract: AztecAddress, slot: Fr): Promise { expect(contract).toBeInstanceOf(AztecAddress); diff --git a/yarn-project/circuit-types/src/interfaces/pxe.ts b/yarn-project/circuit-types/src/interfaces/pxe.ts index 4bfa62f74bb..a2a6f6940fe 100644 --- a/yarn-project/circuit-types/src/interfaces/pxe.ts +++ b/yarn-project/circuit-types/src/interfaces/pxe.ts @@ -27,6 +27,7 @@ import { AbiDecodedSchema, type ApiSchemaFor, type ZodFor, optional, schemas } f import { z } from 'zod'; import { AuthWitness } from '../auth_witness.js'; +import { type InBlock, inBlockSchemaFor } from '../in_block.js'; import { L2Block } from '../l2_block.js'; import { type GetUnencryptedLogsResponse, @@ -216,7 +217,7 @@ export interface PXE { * @param txHash - The hash of a transaction which resulted in the returned tx effect. * @returns The requested tx effect. */ - getTxEffect(txHash: TxHash): Promise; + getTxEffect(txHash: TxHash): Promise | undefined>; /** * Gets the storage value at the given contract storage slot. @@ -500,7 +501,7 @@ export const PXESchema: ApiSchemaFor = { getTxEffect: z .function() .args(TxHash.schema) - .returns(z.union([TxEffect.schema, z.undefined()])), + .returns(z.union([inBlockSchemaFor(TxEffect.schema), z.undefined()])), getPublicStorageAt: z.function().args(schemas.AztecAddress, schemas.Fr).returns(schemas.Fr), getIncomingNotes: z.function().args(IncomingNotesFilterSchema).returns(z.array(UniqueNote.schema)), getL1ToL2MembershipWitness: z diff --git a/yarn-project/circuit-types/src/l2_block_downloader/l2_block_stream.ts b/yarn-project/circuit-types/src/l2_block_downloader/l2_block_stream.ts index 7a1d179dd16..41eb4581346 100644 --- a/yarn-project/circuit-types/src/l2_block_downloader/l2_block_stream.ts +++ b/yarn-project/circuit-types/src/l2_block_downloader/l2_block_stream.ts @@ -5,21 +5,22 @@ import { RunningPromise } from '@aztec/foundation/running-promise'; import { type L2Block } from '../l2_block.js'; import { type L2BlockId, type L2BlockSource, type L2Tips } from '../l2_block_source.js'; -/** Creates a stream of events for new blocks, chain tips updates, and reorgs, out of polling an archiver. */ +/** Creates a stream of events for new blocks, chain tips updates, and reorgs, out of polling an archiver or a node. */ export class L2BlockStream { private readonly runningPromise: RunningPromise; private readonly log = createDebugLogger('aztec:l2_block_stream'); constructor( - private l2BlockSource: L2BlockSource, + private l2BlockSource: Pick, private localData: L2BlockStreamLocalDataProvider, private handler: L2BlockStreamEventHandler, private opts: { proven?: boolean; pollIntervalMS?: number; batchSize?: number; - }, + startingBlock?: number; + } = {}, ) { this.runningPromise = new RunningPromise(() => this.work(), this.opts.pollIntervalMS ?? 1000); } @@ -70,6 +71,11 @@ export class L2BlockStream { await this.emitEvent({ type: 'chain-pruned', blockNumber: latestBlockNumber }); } + // If we are just starting, use the starting block number from the options. + if (latestBlockNumber === 0 && this.opts.startingBlock !== undefined) { + latestBlockNumber = Math.max(this.opts.startingBlock - 1, 0); + } + // Request new blocks from the source. while (latestBlockNumber < sourceTips.latest.number) { const from = latestBlockNumber + 1; @@ -113,7 +119,12 @@ export class L2BlockStream { const sourceBlockHash = args.sourceCache.find(id => id.number === blockNumber && id.hash)?.hash ?? (await this.l2BlockSource.getBlockHeader(blockNumber).then(h => h?.hash().toString())); - this.log.debug(`Comparing block hashes for block ${blockNumber}`, { localBlockHash, sourceBlockHash }); + this.log.debug(`Comparing block hashes for block ${blockNumber}`, { + localBlockHash, + sourceBlockHash, + sourceCacheNumber: args.sourceCache[0]?.number, + sourceCacheHash: args.sourceCache[0]?.hash, + }); return localBlockHash === sourceBlockHash; } diff --git a/yarn-project/circuit-types/src/l2_block_source.ts b/yarn-project/circuit-types/src/l2_block_source.ts index 4dac6da0db9..6f749b28189 100644 --- a/yarn-project/circuit-types/src/l2_block_source.ts +++ b/yarn-project/circuit-types/src/l2_block_source.ts @@ -2,6 +2,7 @@ import { type EthAddress, type Header } from '@aztec/circuits.js'; import { z } from 'zod'; +import { type InBlock } from './in_block.js'; import { type L2Block } from './l2_block.js'; import { type TxHash } from './tx/tx_hash.js'; import { type TxReceipt } from './tx/tx_receipt.js'; @@ -69,7 +70,7 @@ export interface L2BlockSource { * @param txHash - The hash of a transaction which resulted in the returned tx effect. * @returns The requested tx effect. */ - getTxEffect(txHash: TxHash): Promise; + getTxEffect(txHash: TxHash): Promise | undefined>; /** * Gets a receipt of a settled tx. @@ -121,6 +122,7 @@ export type L2Tips = Record; /** Identifies a block by number and hash. */ export type L2BlockId = z.infer; +// TODO(palla/schemas): This package should know what is the block hash of the genesis block 0. const L2BlockIdSchema = z.union([ z.object({ number: z.literal(0), diff --git a/yarn-project/circuit-types/src/logs/get_logs_response.ts b/yarn-project/circuit-types/src/logs/get_logs_response.ts index c62b5af965c..6f1d156be0b 100644 --- a/yarn-project/circuit-types/src/logs/get_logs_response.ts +++ b/yarn-project/circuit-types/src/logs/get_logs_response.ts @@ -84,4 +84,14 @@ export class TxScopedL2Log { static random() { return new TxScopedL2Log(TxHash.random(), 1, 1, false, Fr.random().toBuffer()); } + + equals(other: TxScopedL2Log) { + return ( + this.txHash.equals(other.txHash) && + this.dataStartIndexForTx === other.dataStartIndexForTx && + this.blockNumber === other.blockNumber && + this.isFromPublic === other.isFromPublic && + this.logData.equals(other.logData) + ); + } } diff --git a/yarn-project/circuit-types/src/logs/l1_payload/encrypted_log_payload.test.ts b/yarn-project/circuit-types/src/logs/l1_payload/encrypted_log_payload.test.ts index 9560fb54ddd..a5beb331492 100644 --- a/yarn-project/circuit-types/src/logs/l1_payload/encrypted_log_payload.test.ts +++ b/yarn-project/circuit-types/src/logs/l1_payload/encrypted_log_payload.test.ts @@ -6,7 +6,6 @@ import { PRIVATE_LOG_SIZE_IN_BYTES, computeAddressSecret, computeOvskApp, - computePoint, deriveKeys, derivePublicKeyFromSecretKey, } from '@aztec/circuits.js'; @@ -83,7 +82,7 @@ describe('EncryptedLogPayload', () => { ephSk.hi, ephSk.lo, recipient, - computePoint(recipient).toCompressedBuffer(), + recipient.toAddressPoint().toCompressedBuffer(), ); const outgoingBodyCiphertext = encrypt( outgoingBodyPlaintext, diff --git a/yarn-project/circuit-types/src/logs/l1_payload/encrypted_log_payload.ts b/yarn-project/circuit-types/src/logs/l1_payload/encrypted_log_payload.ts index c585f0a4147..2647121c3be 100644 --- a/yarn-project/circuit-types/src/logs/l1_payload/encrypted_log_payload.ts +++ b/yarn-project/circuit-types/src/logs/l1_payload/encrypted_log_payload.ts @@ -8,7 +8,6 @@ import { Point, type PublicKey, computeOvskApp, - computePoint, derivePublicKeyFromSecretKey, } from '@aztec/circuits.js'; import { randomBytes } from '@aztec/foundation/crypto'; @@ -59,7 +58,7 @@ export class EncryptedLogPayload { ovKeys: KeyValidationRequest, rand: (len: number) => Buffer = randomBytes, ): Buffer { - const addressPoint = computePoint(recipient); + const addressPoint = recipient.toAddressPoint(); const ephPk = derivePublicKeyFromSecretKey(ephSk); const incomingHeaderCiphertext = encrypt(this.contractAddress.toBuffer(), ephSk, addressPoint); diff --git a/yarn-project/circuit-types/src/mocks.ts b/yarn-project/circuit-types/src/mocks.ts index 7b4a3f13ffd..24cc07d0c3a 100644 --- a/yarn-project/circuit-types/src/mocks.ts +++ b/yarn-project/circuit-types/src/mocks.ts @@ -155,11 +155,11 @@ export const mockTx = ( data.forPublic.nonRevertibleAccumulatedData = nonRevertibleBuilder .pushNullifier(firstNullifier.value) - .withPublicCallStack(publicCallRequests.slice(numberOfRevertiblePublicCallRequests)) + .withPublicCallRequests(publicCallRequests.slice(numberOfRevertiblePublicCallRequests)) .build(); data.forPublic.revertibleAccumulatedData = revertibleBuilder - .withPublicCallStack(publicCallRequests.slice(0, numberOfRevertiblePublicCallRequests)) + .withPublicCallRequests(publicCallRequests.slice(0, numberOfRevertiblePublicCallRequests)) .build(); if (hasLogs) { diff --git a/yarn-project/circuit-types/src/nullifier_with_block_source.ts b/yarn-project/circuit-types/src/nullifier_with_block_source.ts new file mode 100644 index 00000000000..d8716c6f3a4 --- /dev/null +++ b/yarn-project/circuit-types/src/nullifier_with_block_source.ts @@ -0,0 +1,7 @@ +import { type Fr } from '@aztec/circuits.js'; + +import { type InBlock } from './index.js'; + +export interface NullifierWithBlockSource { + findNullifiersIndexesWithBlock(blockNumber: number, nullifiers: Fr[]): Promise<(InBlock | undefined)[]>; +} diff --git a/yarn-project/circuit-types/src/p2p/consensus_payload.ts b/yarn-project/circuit-types/src/p2p/consensus_payload.ts index f6b70a5f354..8b020397e4d 100644 --- a/yarn-project/circuit-types/src/p2p/consensus_payload.ts +++ b/yarn-project/circuit-types/src/p2p/consensus_payload.ts @@ -25,9 +25,12 @@ export class ConsensusPayload implements Signable { } getPayloadToSign(domainSeperator: SignatureDomainSeperator): Buffer { - const abi = parseAbiParameters('uint8, bytes32, bytes32[]'); + const abi = parseAbiParameters('uint8, (bytes32, bytes32, bytes, bytes32[])'); const txArray = this.txHashes.map(tx => tx.to0xString()); - const encodedData = encodeAbiParameters(abi, [domainSeperator, this.archive.toString(), txArray] as const); + const encodedData = encodeAbiParameters(abi, [ + domainSeperator, + [this.archive.toString(), this.header.hash().toString(), `0x${this.header.toString()}`, txArray], + ] as const); return Buffer.from(encodedData.slice(2), 'hex'); } diff --git a/yarn-project/circuit-types/src/proving_error.ts b/yarn-project/circuit-types/src/proving_error.ts new file mode 100644 index 00000000000..7207270958d --- /dev/null +++ b/yarn-project/circuit-types/src/proving_error.ts @@ -0,0 +1,18 @@ +/** + * An error thrown when generating a proof fails. + */ +export class ProvingError extends Error { + public static readonly NAME = 'ProvingError'; + + /** + * Creates a new instance + * @param message - The error message. + * @param cause - The cause of the error. + * @param retry - Whether the proof should be retried. + */ + constructor(message: string, cause?: unknown, public readonly retry: boolean = false) { + super(message); + this.name = ProvingError.NAME; + this.cause = cause; + } +} diff --git a/yarn-project/circuit-types/src/public_execution_request.ts b/yarn-project/circuit-types/src/public_execution_request.ts index df60e8e8213..7cf834cb1c8 100644 --- a/yarn-project/circuit-types/src/public_execution_request.ts +++ b/yarn-project/circuit-types/src/public_execution_request.ts @@ -1,4 +1,4 @@ -import { CallContext, type PublicCallRequest, Vector } from '@aztec/circuits.js'; +import { CallContext, PublicCallRequest, Vector } from '@aztec/circuits.js'; import { computeVarArgsHash } from '@aztec/circuits.js/hash'; import { Fr } from '@aztec/foundation/fields'; import { schemas } from '@aztec/foundation/schemas'; @@ -82,9 +82,19 @@ export class PublicExecutionRequest { ); } + toCallRequest(): PublicCallRequest { + return new PublicCallRequest( + this.callContext.msgSender, + this.callContext.contractAddress, + this.callContext.functionSelector, + this.callContext.isStaticCall, + computeVarArgsHash(this.args), + ); + } + [inspect.custom]() { return `PublicExecutionRequest { - callContext: ${this.callContext} + callContext: ${inspect(this.callContext)} args: ${this.args} }`; } diff --git a/yarn-project/circuit-types/src/stats/metrics.ts b/yarn-project/circuit-types/src/stats/metrics.ts index 70d41f4a999..c1af812a2b9 100644 --- a/yarn-project/circuit-types/src/stats/metrics.ts +++ b/yarn-project/circuit-types/src/stats/metrics.ts @@ -37,12 +37,6 @@ export const Metrics = [ description: 'Time to simulate an AVM program.', events: ['avm-simulation'], }, - { - name: 'avm_simulation_bytecode_size_in_bytes', - groupBy: 'app-circuit-name', - description: 'Uncompressed bytecode size for an AVM program.', - events: ['avm-simulation'], - }, { name: 'proof_construction_time_sha256_ms', groupBy: 'threads', diff --git a/yarn-project/circuit-types/src/stats/stats.ts b/yarn-project/circuit-types/src/stats/stats.ts index f9f94709845..a200e9a588c 100644 --- a/yarn-project/circuit-types/src/stats/stats.ts +++ b/yarn-project/circuit-types/src/stats/stats.ts @@ -123,8 +123,6 @@ export type AvmSimulationStats = { appCircuitName: string; /** Duration in ms. */ duration: number; - /** Uncompressed bytecode size. */ - bytecodeSize: number; }; /** Stats for witness generation. */ diff --git a/yarn-project/circuit-types/src/tx/processed_tx.ts b/yarn-project/circuit-types/src/tx/processed_tx.ts index 58c12927287..69da14b2f12 100644 --- a/yarn-project/circuit-types/src/tx/processed_tx.ts +++ b/yarn-project/circuit-types/src/tx/processed_tx.ts @@ -7,7 +7,6 @@ import { type Header, PrivateKernelTailCircuitPublicInputs, type PublicDataWrite, - type PublicKernelCircuitPublicInputs, RevertCode, } from '@aztec/circuits.js'; import { siloL2ToL1Message } from '@aztec/circuits.js/hash'; @@ -68,32 +67,6 @@ export type ProcessedTx = { isEmpty: boolean; }; -export type RevertedTx = ProcessedTx & { - data: PublicKernelCircuitPublicInputs & { - reverted: true; - }; - - revertReason: SimulationError; -}; - -export function isRevertedTx(tx: ProcessedTx): tx is RevertedTx { - return !tx.txEffect.revertCode.isOK(); -} - -export function partitionReverts(txs: ProcessedTx[]): { reverted: RevertedTx[]; nonReverted: ProcessedTx[] } { - return txs.reduce( - ({ reverted, nonReverted }, tx) => { - if (isRevertedTx(tx)) { - reverted.push(tx); - } else { - nonReverted.push(tx); - } - return { reverted, nonReverted }; - }, - { reverted: [], nonReverted: [] } as ReturnType, - ); -} - /** * Represents a tx that failed to be processed by the sequencer public processor. */ diff --git a/yarn-project/circuit-types/src/tx/tx.ts b/yarn-project/circuit-types/src/tx/tx.ts index f53f70d7dcc..e290ad2a46f 100644 --- a/yarn-project/circuit-types/src/tx/tx.ts +++ b/yarn-project/circuit-types/src/tx/tx.ts @@ -2,7 +2,8 @@ import { ClientIvcProof, ContractClassRegisteredEvent, PrivateKernelTailCircuitPublicInputs, - type PublicKernelCircuitPublicInputs, + type PrivateToPublicAccumulatedData, + type ScopedLogHash, } from '@aztec/circuits.js'; import { type Buffer32 } from '@aztec/foundation/buffer'; import { arraySerializedSizeOfNonEmpty } from '@aztec/foundation/collection'; @@ -118,6 +119,22 @@ export class Tx extends Gossipable { ); } + static newWithTxData( + data: PrivateKernelTailCircuitPublicInputs, + publicTeardownExecutionRequest?: PublicExecutionRequest, + ) { + return new Tx( + data, + ClientIvcProof.empty(), + EncryptedNoteTxL2Logs.empty(), + EncryptedTxL2Logs.empty(), + UnencryptedTxL2Logs.empty(), + ContractClassTxL2Logs.empty(), + [], + publicTeardownExecutionRequest ? publicTeardownExecutionRequest : PublicExecutionRequest.empty(), + ); + } + /** * Serializes the Tx object into a Buffer. * @returns Buffer representation of the Tx object. @@ -344,29 +361,26 @@ export class Tx extends Gossipable { * @param logHashes the individual log hashes we want to keep * @param out the output to put passing logs in, to keep this function abstract */ - public filterRevertedLogs(kernelOutput: PublicKernelCircuitPublicInputs) { + public filterRevertedLogs( + privateNonRevertible: PrivateToPublicAccumulatedData, + unencryptedLogsHashes: ScopedLogHash[], + ) { this.encryptedLogs = this.encryptedLogs.filterScoped( - kernelOutput.endNonRevertibleData.encryptedLogsHashes, + privateNonRevertible.encryptedLogsHashes, EncryptedTxL2Logs.empty(), ); - this.unencryptedLogs = this.unencryptedLogs.filterScoped( - kernelOutput.endNonRevertibleData.unencryptedLogsHashes, - UnencryptedTxL2Logs.empty(), - ); - this.noteEncryptedLogs = this.noteEncryptedLogs.filter( - kernelOutput.endNonRevertibleData.noteEncryptedLogsHashes, + privateNonRevertible.noteEncryptedLogsHashes, EncryptedNoteTxL2Logs.empty(), ); - // See comment in enqueued_calls_processor.ts -> tx.filterRevertedLogs() - if (this.data.forPublic) { - this.contractClassLogs = this.contractClassLogs.filterScoped( - this.data.forPublic?.nonRevertibleAccumulatedData.contractClassLogsHashes, - ContractClassTxL2Logs.empty(), - ); - } + this.contractClassLogs = this.contractClassLogs.filterScoped( + privateNonRevertible.contractClassLogsHashes, + ContractClassTxL2Logs.empty(), + ); + + this.unencryptedLogs = this.unencryptedLogs.filterScoped(unencryptedLogsHashes, UnencryptedTxL2Logs.empty()); } } diff --git a/yarn-project/circuits.js/src/constants.gen.ts b/yarn-project/circuits.js/src/constants.gen.ts index c4c26d55c39..ff6911cf4f3 100644 --- a/yarn-project/circuits.js/src/constants.gen.ts +++ b/yarn-project/circuits.js/src/constants.gen.ts @@ -38,7 +38,6 @@ export const PUBLIC_DATA_SUBTREE_HEIGHT = 6; export const L1_TO_L2_MSG_SUBTREE_HEIGHT = 4; export const NOTE_HASH_SUBTREE_SIBLING_PATH_LENGTH = 34; export const NULLIFIER_SUBTREE_SIBLING_PATH_LENGTH = 34; -export const PUBLIC_DATA_SUBTREE_SIBLING_PATH_LENGTH = 34; export const L1_TO_L2_MSG_SUBTREE_SIBLING_PATH_LENGTH = 35; export const MAX_NOTE_HASHES_PER_TX = 64; export const MAX_NULLIFIERS_PER_TX = 64; @@ -58,7 +57,6 @@ export const MAX_NOTE_ENCRYPTED_LOGS_PER_TX = 64; export const MAX_ENCRYPTED_LOGS_PER_TX = 8; export const MAX_UNENCRYPTED_LOGS_PER_TX = 8; export const MAX_CONTRACT_CLASS_LOGS_PER_TX = 1; -export const MAX_PUBLIC_DATA_HINTS = 128; export const NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP = 16; export const EMPTY_NESTED_INDEX = 0; export const PRIVATE_KERNEL_EMPTY_INDEX = 1; @@ -185,29 +183,21 @@ export const HEADER_LENGTH = 24; export const PRIVATE_CIRCUIT_PUBLIC_INPUTS_LENGTH = 490; export const PUBLIC_CIRCUIT_PUBLIC_INPUTS_LENGTH = 866; export const PRIVATE_CONTEXT_INPUTS_LENGTH = 37; -export const PUBLIC_CONTEXT_INPUTS_LENGTH = 41; export const FEE_RECIPIENT_LENGTH = 2; export const AGGREGATION_OBJECT_LENGTH = 16; export const SCOPED_READ_REQUEST_LEN = 3; export const PUBLIC_DATA_READ_LENGTH = 3; export const PRIVATE_VALIDATION_REQUESTS_LENGTH = 772; -export const NUM_PUBLIC_VALIDATION_REQUEST_ARRAYS = 5; -export const PUBLIC_VALIDATION_REQUESTS_LENGTH = 834; -export const PUBLIC_DATA_UPDATE_REQUEST_LENGTH = 3; export const COMBINED_ACCUMULATED_DATA_LENGTH = 550; export const TX_CONSTANT_DATA_LENGTH = 34; export const COMBINED_CONSTANT_DATA_LENGTH = 43; export const PRIVATE_ACCUMULATED_DATA_LENGTH = 1036; export const PRIVATE_KERNEL_CIRCUIT_PUBLIC_INPUTS_LENGTH = 1849; -export const PUBLIC_ACCUMULATED_DATA_LENGTH = 1023; -export const NUM_PUBLIC_ACCUMULATED_DATA_ARRAYS = 8; export const PRIVATE_TO_PUBLIC_ACCUMULATED_DATA_LENGTH = 548; export const PRIVATE_TO_AVM_ACCUMULATED_DATA_LENGTH = 160; export const NUM_PRIVATE_TO_AVM_ACCUMULATED_DATA_ARRAYS = 3; export const AVM_ACCUMULATED_DATA_LENGTH = 318; export const PRIVATE_TO_PUBLIC_KERNEL_CIRCUIT_PUBLIC_INPUTS_LENGTH = 1140; -export const PUBLIC_KERNEL_CIRCUIT_PUBLIC_INPUTS_LENGTH = 2931; -export const VM_CIRCUIT_PUBLIC_INPUTS_LENGTH = 2340; export const KERNEL_CIRCUIT_PUBLIC_INPUTS_LENGTH = 605; export const AVM_CIRCUIT_PUBLIC_INPUTS_LENGTH = 1006; export const CONSTANT_ROLLUP_DATA_LENGTH = 13; diff --git a/yarn-project/circuits.js/src/contract/artifact_hash.test.ts b/yarn-project/circuits.js/src/contract/artifact_hash.test.ts index c6261e83bd1..74b7db7c485 100644 --- a/yarn-project/circuits.js/src/contract/artifact_hash.test.ts +++ b/yarn-project/circuits.js/src/contract/artifact_hash.test.ts @@ -1,5 +1,10 @@ import { type ContractArtifact } from '@aztec/foundation/abi'; +import { loadContractArtifact } from '@aztec/types/abi'; +import type { NoirCompiledContract } from '@aztec/types/noir'; +import { readFileSync } from 'fs'; + +import { getPathToFixture, getTestContractArtifact } from '../tests/fixtures.js'; import { computeArtifactHash } from './artifact_hash.js'; describe('ArtifactHash', () => { @@ -16,7 +21,29 @@ describe('ArtifactHash', () => { notes: {}, }; expect(computeArtifactHash(emptyArtifact).toString()).toMatchInlineSnapshot( - `"0x0c6fd9b48570721c5d36f978d084d77cacbfd2814f1344985f40e62bea6e61be"`, + `"0x0dea64e7fa0688017f77bcb7075485485afb4a5f1f8508483398869439f82fdf"`, + ); + }); + + it('calculates the test contract artifact hash multiple times to ensure deterministic hashing', () => { + const testArtifact = getTestContractArtifact(); + + for (let i = 0; i < 1000; i++) { + expect(computeArtifactHash(testArtifact).toString()).toMatchInlineSnapshot( + `"0x21070d88558fdc3906322f267cf6f0f632caf3949295520fe1f71f156fbb0d0b"`, + ); + } + }); + + it('calculates the test contract artifact hash', () => { + const path = getPathToFixture('Test.test.json'); + const content = JSON.parse(readFileSync(path).toString()) as NoirCompiledContract; + content.outputs.structs.functions.reverse(); + + const testArtifact = loadContractArtifact(content); + + expect(computeArtifactHash(testArtifact).toString()).toMatchInlineSnapshot( + `"0x21070d88558fdc3906322f267cf6f0f632caf3949295520fe1f71f156fbb0d0b"`, ); }); }); diff --git a/yarn-project/circuits.js/src/contract/artifact_hash.ts b/yarn-project/circuits.js/src/contract/artifact_hash.ts index de8ada1782c..a7bc52ae7ad 100644 --- a/yarn-project/circuits.js/src/contract/artifact_hash.ts +++ b/yarn-project/circuits.js/src/contract/artifact_hash.ts @@ -59,11 +59,7 @@ export function computeArtifactHashPreimage(artifact: ContractArtifact) { } export function computeArtifactMetadataHash(artifact: ContractArtifact) { - // TODO: #6021 We need to make sure the artifact is deterministic from any specific compiler run. This relates to selectors not being sorted and being - // apparently random in the order they appear after compiled w/ nargo. We can try to sort this upon loading an artifact. - // TODO: #6021: Should we use the sorted event selectors instead? They'd need to be unique for that. - // Response - The output selectors need to be sorted, because if not noir makes no guarantees on the order of outputs for some reason - return sha256Fr(Buffer.from(JSON.stringify({ name: artifact.name }), 'utf-8')); + return sha256Fr(Buffer.from(JSON.stringify({ name: artifact.name, outputs: artifact.outputs }), 'utf-8')); } export function computeArtifactFunctionTreeRoot(artifact: ContractArtifact, fnType: FunctionType) { @@ -96,11 +92,10 @@ export function computeFunctionArtifactHash( | (Pick & { functionMetadataHash: Fr; selector: FunctionSelector }), ) { const selector = 'selector' in fn ? fn.selector : FunctionSelector.fromNameAndParameters(fn); - // TODO(#5860): make bytecode part of artifact hash preimage again - // const bytecodeHash = sha256Fr(fn.bytecode).toBuffer(); - // const metadataHash = 'functionMetadataHash' in fn ? fn.functionMetadataHash : computeFunctionMetadataHash(fn); - // return sha256Fr(Buffer.concat([numToUInt8(VERSION), selector.toBuffer(), metadataHash.toBuffer(), bytecodeHash])); - return sha256Fr(Buffer.concat([numToUInt8(VERSION), selector.toBuffer()])); + + const bytecodeHash = sha256Fr(fn.bytecode).toBuffer(); + const metadataHash = 'functionMetadataHash' in fn ? fn.functionMetadataHash : computeFunctionMetadataHash(fn); + return sha256Fr(Buffer.concat([numToUInt8(VERSION), selector.toBuffer(), metadataHash.toBuffer(), bytecodeHash])); } export function computeFunctionMetadataHash(fn: FunctionArtifact) { diff --git a/yarn-project/circuits.js/src/contract/interfaces/contract_data_source.ts b/yarn-project/circuits.js/src/contract/interfaces/contract_data_source.ts index 6f545df85db..b67afc8ac50 100644 --- a/yarn-project/circuits.js/src/contract/interfaces/contract_data_source.ts +++ b/yarn-project/circuits.js/src/contract/interfaces/contract_data_source.ts @@ -26,6 +26,12 @@ export interface ContractDataSource { */ getContractClass(id: Fr): Promise; + /** + * Adds a contract class to the database. + * TODO(#10007): Remove this method + */ + addContractClass(contractClass: ContractClassPublic): Promise; + /** * Returns a publicly deployed contract instance given its address. * @param address - Address of the deployed contract. diff --git a/yarn-project/circuits.js/src/contract/private_function_membership_proof.test.ts b/yarn-project/circuits.js/src/contract/private_function_membership_proof.test.ts index 2135967d9d4..3a57e9787b2 100644 --- a/yarn-project/circuits.js/src/contract/private_function_membership_proof.test.ts +++ b/yarn-project/circuits.js/src/contract/private_function_membership_proof.test.ts @@ -31,8 +31,7 @@ describe('private_function_membership_proof', () => { expect(isValidPrivateFunctionMembershipProof(fn, contractClass)).toBeTruthy(); }); - // TODO(#5860): Re-enable this test once noir non-determinism is addressed - test.skip.each([ + test.each([ 'artifactTreeSiblingPath', 'artifactMetadataHash', 'functionMetadataHash', diff --git a/yarn-project/circuits.js/src/contract/unconstrained_function_membership_proof.test.ts b/yarn-project/circuits.js/src/contract/unconstrained_function_membership_proof.test.ts index 57495f78aea..f795c6f29b6 100644 --- a/yarn-project/circuits.js/src/contract/unconstrained_function_membership_proof.test.ts +++ b/yarn-project/circuits.js/src/contract/unconstrained_function_membership_proof.test.ts @@ -50,8 +50,7 @@ describe('unconstrained_function_membership_proof', () => { expect(isValidUnconstrainedFunctionMembershipProof(fn, contractClass)).toBeTruthy(); }); - // TODO(#5860): Re-enable this test once noir non-determinism is addressed - test.skip.each(['artifactTreeSiblingPath', 'artifactMetadataHash', 'functionMetadataHash'] as const)( + test.each(['artifactTreeSiblingPath', 'artifactMetadataHash', 'functionMetadataHash'] as const)( 'fails proof if %s is mangled', field => { const proof = createUnconstrainedFunctionMembershipProof(selector, artifact); diff --git a/yarn-project/circuits.js/src/hints/build_nullifier_non_existent_read_request_hints.test.ts b/yarn-project/circuits.js/src/hints/build_nullifier_non_existent_read_request_hints.test.ts deleted file mode 100644 index beb031fb259..00000000000 --- a/yarn-project/circuits.js/src/hints/build_nullifier_non_existent_read_request_hints.test.ts +++ /dev/null @@ -1,138 +0,0 @@ -import { makeTuple } from '@aztec/foundation/array'; -import { AztecAddress } from '@aztec/foundation/aztec-address'; -import { padArrayEnd } from '@aztec/foundation/collection'; -import { Fr } from '@aztec/foundation/fields'; - -import { MAX_NULLIFIERS_PER_TX, MAX_NULLIFIER_READ_REQUESTS_PER_TX } from '../constants.gen.js'; -import { siloNullifier } from '../hash/index.js'; -import { - Nullifier, - NullifierNonExistentReadRequestHintsBuilder, - ReadRequest, - ScopedReadRequest, -} from '../structs/index.js'; -import { buildNullifierNonExistentReadRequestHints } from './build_nullifier_non_existent_read_request_hints.js'; - -describe('buildNullifierNonExistentReadRequestHints', () => { - const contractAddress = AztecAddress.random(); - const oracle = { - getLowNullifierMembershipWitness: () => ({ membershipWitness: {}, leafPreimage: {} } as any), - }; - const nonExistentReadRequests = makeTuple(MAX_NULLIFIER_READ_REQUESTS_PER_TX, ScopedReadRequest.empty); - let nullifiers = makeTuple(MAX_NULLIFIERS_PER_TX, Nullifier.empty); - - const innerNullifier = (index: number) => index + 1; - - const makeReadRequest = (value: number, counter = 2) => - new ReadRequest(new Fr(value), counter).scope(contractAddress); - - const makeNullifier = (value: number, counter = 1) => { - const siloedValue = siloNullifier(contractAddress, new Fr(value)); - return new Nullifier(siloedValue, 0, new Fr(counter)); - }; - - interface TestNullifier { - value: number; - siloedValue: Fr; - } - - const populateNullifiers = (numNullifiers = MAX_NULLIFIERS_PER_TX) => { - nullifiers = makeTuple(MAX_NULLIFIERS_PER_TX, i => - i < numNullifiers ? makeNullifier(innerNullifier(i)) : Nullifier.empty(), - ); - }; - - const generateSortedNullifiers = (numNullifiers: number) => { - const nullifiers: TestNullifier[] = []; - for (let i = 0; i < numNullifiers; ++i) { - const value = i; - nullifiers.push({ - value, - siloedValue: siloNullifier(contractAddress, new Fr(value)), - }); - } - return nullifiers.sort((a, b) => (b.siloedValue.lt(a.siloedValue) ? 1 : -1)); - }; - - const buildHints = () => buildNullifierNonExistentReadRequestHints(oracle, nonExistentReadRequests, nullifiers); - - it('builds empty hints', async () => { - const hints = await buildHints(); - const emptyHints = NullifierNonExistentReadRequestHintsBuilder.empty(); - expect(hints).toEqual(emptyHints); - }); - - it('builds hints for full sorted nullifiers', async () => { - populateNullifiers(); - - const hints = await buildHints(); - const { sortedPendingValues, sortedPendingValueHints } = hints; - for (let i = 0; i < sortedPendingValues.length - 1; ++i) { - expect(sortedPendingValues[i].value.lt(sortedPendingValues[i + 1].value)).toBe(true); - } - for (let i = 0; i < nullifiers.length; ++i) { - const index = sortedPendingValueHints[i]; - expect(nullifiers[i].value.equals(sortedPendingValues[index].value)).toBe(true); - } - }); - - it('builds hints for half-full sorted nullifiers', async () => { - const numNonEmptyNullifiers = MAX_NULLIFIERS_PER_TX / 2; - populateNullifiers(numNonEmptyNullifiers); - - const hints = await buildHints(); - const { sortedPendingValues, sortedPendingValueHints } = hints; - - // The first half contains sorted values. - for (let i = 0; i < numNonEmptyNullifiers - 1; ++i) { - expect(sortedPendingValues[i]).not.toEqual(Nullifier.empty()); - expect(sortedPendingValues[i].value.lt(sortedPendingValues[i + 1].value)).toBe(true); - } - for (let i = 0; i < numNonEmptyNullifiers; ++i) { - const index = sortedPendingValueHints[i]; - expect(nullifiers[i].value.equals(sortedPendingValues[index].value)).toBe(true); - } - - // The second half is empty. - for (let i = numNonEmptyNullifiers; i < sortedPendingValues.length; ++i) { - expect(sortedPendingValues[i]).toEqual(Nullifier.empty()); - } - for (let i = numNonEmptyNullifiers; i < sortedPendingValueHints.length; ++i) { - expect(sortedPendingValueHints[i]).toBe(0); - } - }); - - it('builds hints for read requests', async () => { - const numNonEmptyNullifiers = MAX_NULLIFIERS_PER_TX / 2; - expect(numNonEmptyNullifiers > 1).toBe(true); // Need at least 2 nullifiers to test a value in the middle. - - const sortedNullifiers = generateSortedNullifiers(numNonEmptyNullifiers + 3); - const minNullifier = sortedNullifiers.splice(0, 1)[0]; - const maxNullifier = sortedNullifiers.pop()!; - const midIndex = Math.floor(numNonEmptyNullifiers / 2); - const midNullifier = sortedNullifiers.splice(midIndex, 1)[0]; - - nonExistentReadRequests[0] = makeReadRequest(midNullifier.value); - nonExistentReadRequests[1] = makeReadRequest(maxNullifier.value); - nonExistentReadRequests[2] = makeReadRequest(minNullifier.value); - nullifiers = padArrayEnd( - sortedNullifiers.map(n => makeNullifier(n.value)), - Nullifier.empty(), - MAX_NULLIFIERS_PER_TX, - ); - - const hints = await buildHints(); - const { nextPendingValueIndices } = hints; - expect(nextPendingValueIndices.slice(0, 3)).toEqual([midIndex, numNonEmptyNullifiers, 0]); - }); - - it('throws if reading existing value', async () => { - populateNullifiers(); - - nonExistentReadRequests[0] = makeReadRequest(innerNullifier(2)); - - await expect(() => buildHints()).rejects.toThrow( - 'Nullifier DOES exists in the pending set at the time of reading, but there is a NonExistentReadRequest for it.', - ); - }); -}); diff --git a/yarn-project/circuits.js/src/hints/build_nullifier_non_existent_read_request_hints.ts b/yarn-project/circuits.js/src/hints/build_nullifier_non_existent_read_request_hints.ts deleted file mode 100644 index ea142b12ccd..00000000000 --- a/yarn-project/circuits.js/src/hints/build_nullifier_non_existent_read_request_hints.ts +++ /dev/null @@ -1,86 +0,0 @@ -import { padArrayEnd } from '@aztec/foundation/collection'; -import { type Fr } from '@aztec/foundation/fields'; -import { type Tuple } from '@aztec/foundation/serialize'; - -import { - MAX_NULLIFIERS_PER_TX, - type MAX_NULLIFIER_NON_EXISTENT_READ_REQUESTS_PER_TX, - type NULLIFIER_TREE_HEIGHT, -} from '../constants.gen.js'; -import { siloNullifier } from '../hash/index.js'; -import { Nullifier, type NullifierLeafPreimage } from '../structs/index.js'; -import { type MembershipWitness } from '../structs/membership_witness.js'; -import { NullifierNonExistentReadRequestHintsBuilder } from '../structs/non_existent_read_request_hints.js'; -import { type ScopedReadRequest } from '../structs/read_request.js'; -import { countAccumulatedItems } from '../utils/index.js'; - -interface NullifierMembershipWitnessWithPreimage { - membershipWitness: MembershipWitness; - leafPreimage: NullifierLeafPreimage; -} - -interface SortedResult { - sortedValues: Tuple; - sortedIndexHints: Tuple; -} - -function sortNullifiersByValues( - nullifiers: Tuple, -): SortedResult { - const numNullifiers = countAccumulatedItems(nullifiers); - const sorted = nullifiers - .slice(0, numNullifiers) - .map((nullifier, originalIndex) => ({ nullifier, originalIndex })) - .sort((a, b) => (b.nullifier.value.lt(a.nullifier.value) ? 1 : -1)); - - const sortedIndexHints: number[] = []; - for (let i = 0; i < numNullifiers; ++i) { - sortedIndexHints[sorted[i].originalIndex] = i; - } - - return { - sortedValues: padArrayEnd( - sorted.map(s => s.nullifier), - Nullifier.empty(), - MAX_NULLIFIERS_PER_TX, - ), - sortedIndexHints: padArrayEnd(sortedIndexHints, 0, MAX_NULLIFIERS_PER_TX), - }; -} - -export async function buildNullifierNonExistentReadRequestHints( - oracle: { - getLowNullifierMembershipWitness(nullifier: Fr): Promise; - }, - nullifierNonExistentReadRequests: Tuple, - pendingNullifiers: Tuple, -) { - const { sortedValues, sortedIndexHints } = sortNullifiersByValues(pendingNullifiers); - - const builder = new NullifierNonExistentReadRequestHintsBuilder(sortedValues, sortedIndexHints); - - const numPendingNullifiers = countAccumulatedItems(pendingNullifiers); - const numReadRequests = countAccumulatedItems(nullifierNonExistentReadRequests); - for (let i = 0; i < numReadRequests; ++i) { - const readRequest = nullifierNonExistentReadRequests[i]; - const siloedValue = siloNullifier(readRequest.contractAddress, readRequest.value); - - const { membershipWitness, leafPreimage } = await oracle.getLowNullifierMembershipWitness(siloedValue); - - let nextPendingValueIndex = sortedValues.findIndex(v => !v.value.lt(siloedValue)); - if (nextPendingValueIndex == -1) { - nextPendingValueIndex = numPendingNullifiers; - } else if ( - sortedValues[nextPendingValueIndex].value.equals(siloedValue) && - sortedValues[nextPendingValueIndex].counter < readRequest.counter - ) { - throw new Error( - 'Nullifier DOES exists in the pending set at the time of reading, but there is a NonExistentReadRequest for it.', - ); - } - - builder.addHint(membershipWitness, leafPreimage, nextPendingValueIndex); - } - - return builder.toHints(); -} diff --git a/yarn-project/circuits.js/src/hints/build_public_data_hints.test.ts b/yarn-project/circuits.js/src/hints/build_public_data_hints.test.ts deleted file mode 100644 index d88703a1d75..00000000000 --- a/yarn-project/circuits.js/src/hints/build_public_data_hints.test.ts +++ /dev/null @@ -1,116 +0,0 @@ -import { makeTuple } from '@aztec/foundation/array'; -import { padArrayEnd } from '@aztec/foundation/collection'; -import { Fr } from '@aztec/foundation/fields'; -import { type Tuple } from '@aztec/foundation/serialize'; - -import { - MAX_PUBLIC_DATA_HINTS, - MAX_PUBLIC_DATA_READS_PER_TX, - MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, -} from '../constants.gen.js'; -import { PublicDataRead, PublicDataTreeLeafPreimage, PublicDataUpdateRequest } from '../structs/index.js'; -import { buildPublicDataHints } from './build_public_data_hints.js'; - -describe('buildPublicDataHints', () => { - let publicDataReads: Tuple; - let publicDataUpdateRequests: Tuple; - - const publicDataLeaves = [ - new PublicDataTreeLeafPreimage(new Fr(0), new Fr(0), new Fr(11), 2n), - new PublicDataTreeLeafPreimage(new Fr(22), new Fr(200), new Fr(0), 0n), - new PublicDataTreeLeafPreimage(new Fr(11), new Fr(100), new Fr(22), 1n), - ]; - - const makePublicDataRead = (leafSlot: number, value: number) => - new PublicDataRead(new Fr(leafSlot), new Fr(value), 0); - const makePublicDataWrite = (leafSlot: number, value: number) => - new PublicDataUpdateRequest(new Fr(leafSlot), new Fr(value), 0); - - const oracle = { - getMatchOrLowPublicDataMembershipWitness: (leafSlot: bigint) => { - const leafPreimage = publicDataLeaves.find( - l => l.slot.toBigInt() <= leafSlot && (l.nextSlot.isZero() || l.nextSlot.toBigInt() > leafSlot), - ); - return { membershipWitness: {}, leafPreimage } as any; - }, - }; - - const buildAndCheckHints = async (expectedSlots: number[]) => { - const hints = await buildPublicDataHints(oracle, publicDataReads, publicDataUpdateRequests); - const partialHints = expectedSlots.map(s => - expect.objectContaining({ - preimage: publicDataLeaves.find(l => l.slot.equals(new Fr(s))), - }), - ); - const emptyPartialHint = expect.objectContaining({ preimage: PublicDataTreeLeafPreimage.empty() }); - expect(hints).toEqual(padArrayEnd(partialHints, emptyPartialHint, MAX_PUBLIC_DATA_HINTS)); - }; - - beforeEach(() => { - publicDataReads = makeTuple(MAX_PUBLIC_DATA_READS_PER_TX, PublicDataRead.empty); - publicDataUpdateRequests = makeTuple(MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, PublicDataUpdateRequest.empty); - }); - - it('returns empty hints', async () => { - await buildAndCheckHints([]); - }); - - it('builds hints for reads for uninitialized slots', async () => { - publicDataReads[0] = makePublicDataRead(12, 0); - publicDataReads[1] = makePublicDataRead(39, 0); - await buildAndCheckHints([11, 22]); - }); - - it('builds hints for reads for initialized slots', async () => { - publicDataReads[0] = makePublicDataRead(22, 200); - publicDataReads[1] = makePublicDataRead(11, 100); - await buildAndCheckHints([22, 11]); - }); - - it('builds hints for writes to uninitialized slots', async () => { - publicDataUpdateRequests[0] = makePublicDataWrite(5, 500); - publicDataUpdateRequests[1] = makePublicDataWrite(17, 700); - await buildAndCheckHints([0, 11]); - }); - - it('builds hints for writes to initialized slots', async () => { - publicDataUpdateRequests[0] = makePublicDataWrite(11, 111); - publicDataUpdateRequests[1] = makePublicDataWrite(22, 222); - await buildAndCheckHints([11, 22]); - }); - - it('skip hints for repeated reads', async () => { - publicDataReads[0] = makePublicDataRead(22, 200); // 22 - publicDataReads[1] = makePublicDataRead(39, 0); // 22 - publicDataReads[2] = makePublicDataRead(22, 200); // No hint needed because slot 22 was read. - publicDataReads[3] = makePublicDataRead(39, 0); // No hint needed because slot 39 was read. - publicDataReads[4] = makePublicDataRead(12, 0); // 11 - publicDataReads[5] = makePublicDataRead(39, 0); // // No hint needed because slot 39 was read. - await buildAndCheckHints([22, 22, 11]); - }); - - it('skip hints for repeated writes', async () => { - publicDataUpdateRequests[0] = makePublicDataWrite(11, 111); // 11 - publicDataUpdateRequests[1] = makePublicDataWrite(5, 500); // 0 - publicDataUpdateRequests[2] = makePublicDataWrite(11, 112); // No hint needed because slot 11 was written. - publicDataUpdateRequests[3] = makePublicDataWrite(17, 700); // 11 - publicDataUpdateRequests[4] = makePublicDataWrite(11, 113); // No hint needed because slot 11 was written. - publicDataUpdateRequests[5] = makePublicDataWrite(5, 222); // No hint needed because slot 5 was written. - publicDataUpdateRequests[6] = makePublicDataWrite(37, 700); // 22 - await buildAndCheckHints([11, 0, 11, 22]); - }); - - it('builds hints for mixed reads and writes', async () => { - publicDataReads[0] = makePublicDataRead(22, 200); // 22 - publicDataReads[1] = makePublicDataRead(7, 0); // 0 - publicDataReads[2] = makePublicDataRead(41, 0); // 22 - publicDataReads[3] = makePublicDataRead(11, 100); // 11 - publicDataReads[4] = makePublicDataRead(39, 0); // 22 - publicDataUpdateRequests[0] = makePublicDataWrite(11, 111); // No hint needed because slot 11 was read. - publicDataUpdateRequests[1] = makePublicDataWrite(5, 500); // 0 - publicDataUpdateRequests[2] = makePublicDataWrite(17, 700); // 11 - publicDataUpdateRequests[3] = makePublicDataWrite(22, 222); // No hint needed because slot 22 was read. - publicDataUpdateRequests[4] = makePublicDataWrite(39, 700); // No hint needed because slot 39 was read. - await buildAndCheckHints([22, 0, 22, 11, 22, 0, 11]); - }); -}); diff --git a/yarn-project/circuits.js/src/hints/build_public_data_hints.ts b/yarn-project/circuits.js/src/hints/build_public_data_hints.ts deleted file mode 100644 index 53200bab364..00000000000 --- a/yarn-project/circuits.js/src/hints/build_public_data_hints.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { padArrayEnd } from '@aztec/foundation/collection'; -import { Fr } from '@aztec/foundation/fields'; -import { type Tuple } from '@aztec/foundation/serialize'; - -import { - MAX_PUBLIC_DATA_HINTS, - type MAX_PUBLIC_DATA_READS_PER_TX, - type MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, - type PUBLIC_DATA_TREE_HEIGHT, -} from '../constants.gen.js'; -import { - PublicDataLeafHint, - type PublicDataRead, - type PublicDataTreeLeafPreimage, - type PublicDataUpdateRequest, -} from '../structs/index.js'; -import { type MembershipWitness } from '../structs/membership_witness.js'; -import { PublicDataHint } from '../structs/public_data_hint.js'; - -interface PublicDataMembershipWitnessWithPreimage { - membershipWitness: MembershipWitness; - leafPreimage: PublicDataTreeLeafPreimage; -} - -type PublicDataMembershipWitnessOracle = { - getMatchOrLowPublicDataMembershipWitness(leafSlot: bigint): Promise; -}; - -async function buildPublicDataLeafHint(oracle: PublicDataMembershipWitnessOracle, leafSlot: bigint) { - const { membershipWitness, leafPreimage } = await oracle.getMatchOrLowPublicDataMembershipWitness(leafSlot); - return new PublicDataLeafHint(leafPreimage, membershipWitness); -} - -export async function buildPublicDataHints( - oracle: PublicDataMembershipWitnessOracle, - publicDataReads: Tuple, - publicDataUpdateRequests: Tuple, -): Promise> { - const leafSlots = [...publicDataReads.map(r => r.leafSlot), ...publicDataUpdateRequests.map(w => w.leafSlot)] - .filter(s => !s.isZero()) - .map(s => s.toBigInt()); - const uniqueLeafSlots = [...new Set(leafSlots)]; - const hints = await Promise.all(uniqueLeafSlots.map(slot => buildPublicDataLeafHint(oracle, slot))); - return padArrayEnd(hints, PublicDataLeafHint.empty(), MAX_PUBLIC_DATA_HINTS); -} - -export async function buildPublicDataHint(oracle: PublicDataMembershipWitnessOracle, leafSlot: bigint) { - const { membershipWitness, leafPreimage } = await oracle.getMatchOrLowPublicDataMembershipWitness(leafSlot); - const exists = leafPreimage.slot.toBigInt() === leafSlot; - const value = exists ? leafPreimage.value : Fr.ZERO; - return new PublicDataHint(new Fr(leafSlot), value, 0, membershipWitness, leafPreimage); -} diff --git a/yarn-project/circuits.js/src/hints/index.ts b/yarn-project/circuits.js/src/hints/index.ts index 2da0098c29b..5d673ff520a 100644 --- a/yarn-project/circuits.js/src/hints/index.ts +++ b/yarn-project/circuits.js/src/hints/index.ts @@ -1,6 +1,4 @@ export * from './build_note_hash_read_request_hints.js'; -export * from './build_nullifier_non_existent_read_request_hints.js'; export * from './build_nullifier_read_request_hints.js'; -export * from './build_public_data_hints.js'; export * from './build_transient_data_hints.js'; export * from './find_private_kernel_reset_dimensions.js'; diff --git a/yarn-project/circuits.js/src/keys/derivation.ts b/yarn-project/circuits.js/src/keys/derivation.ts index 6f5d8b5a59f..495ea964cf8 100644 --- a/yarn-project/circuits.js/src/keys/derivation.ts +++ b/yarn-project/circuits.js/src/keys/derivation.ts @@ -1,6 +1,6 @@ import { AztecAddress } from '@aztec/foundation/aztec-address'; import { poseidon2HashWithSeparator, sha512ToGrumpkinScalar } from '@aztec/foundation/crypto'; -import { Fq, Fr, GrumpkinScalar, Point } from '@aztec/foundation/fields'; +import { Fq, Fr, GrumpkinScalar } from '@aztec/foundation/fields'; import { Grumpkin } from '../barretenberg/crypto/grumpkin/index.js'; import { GeneratorIndex } from '../constants.gen.js'; @@ -82,10 +82,6 @@ export function computeAddressSecret(preaddress: Fr, ivsk: Fq) { return addressSecretCandidate; } -export function computePoint(address: AztecAddress) { - return Point.fromXAndSign(address.toField(), true); -} - export function derivePublicKeyFromSecretKey(secretKey: Fq) { const curve = new Grumpkin(); return curve.mul(curve.generator(), secretKey); @@ -130,7 +126,7 @@ export function deriveKeys(secretKey: Fr) { export function computeTaggingSecret(knownAddress: CompleteAddress, ivsk: Fq, externalAddress: AztecAddress) { const knownPreaddress = computePreaddress(knownAddress.publicKeys.hash(), knownAddress.partialAddress); // TODO: #8970 - Computation of address point from x coordinate might fail - const externalAddressPoint = computePoint(externalAddress); + const externalAddressPoint = externalAddress.toAddressPoint(); const curve = new Grumpkin(); // Given A (known complete address) -> B (external address) and h == preaddress // Compute shared secret as S = (h_A + ivsk_A) * Addr_Point_B diff --git a/yarn-project/circuits.js/src/scripts/constants.in.ts b/yarn-project/circuits.js/src/scripts/constants.in.ts index 369a9c94ae9..0b3aff93742 100644 --- a/yarn-project/circuits.js/src/scripts/constants.in.ts +++ b/yarn-project/circuits.js/src/scripts/constants.in.ts @@ -21,7 +21,6 @@ const CPP_CONSTANTS = [ 'STATE_REFERENCE_LENGTH', 'HEADER_LENGTH', 'CALL_CONTEXT_LENGTH', - 'PUBLIC_CONTEXT_INPUTS_LENGTH', 'PUBLIC_CIRCUIT_PUBLIC_INPUTS_LENGTH', 'READ_REQUEST_LENGTH', 'MAX_NOTE_HASH_READ_REQUESTS_PER_CALL', diff --git a/yarn-project/circuits.js/src/structs/call_context.ts b/yarn-project/circuits.js/src/structs/call_context.ts index f2761bc4d1f..e16c6d1a96c 100644 --- a/yarn-project/circuits.js/src/structs/call_context.ts +++ b/yarn-project/circuits.js/src/structs/call_context.ts @@ -5,6 +5,7 @@ import { schemas } from '@aztec/foundation/schemas'; import { BufferReader, FieldReader, serializeToBuffer, serializeToFields } from '@aztec/foundation/serialize'; import { type FieldsOf } from '@aztec/foundation/types'; +import { inspect } from 'util'; import { z } from 'zod'; import { CALL_CONTEXT_LENGTH } from '../constants.gen.js'; @@ -125,4 +126,13 @@ export class CallContext { callContext.isStaticCall === this.isStaticCall ); } + + [inspect.custom]() { + return `CallContext { + msgSender: ${this.msgSender} + contractAddress: ${this.contractAddress} + functionSelector: ${this.functionSelector} + isStaticCall: ${this.isStaticCall} + }`; + } } diff --git a/yarn-project/circuits.js/src/structs/index.ts b/yarn-project/circuits.js/src/structs/index.ts index 39b77654dce..96b87bbbc6f 100644 --- a/yarn-project/circuits.js/src/structs/index.ts +++ b/yarn-project/circuits.js/src/structs/index.ts @@ -18,7 +18,6 @@ export * from './header.js'; export * from './tagging_secret.js'; export * from './kernel/combined_accumulated_data.js'; export * from './kernel/combined_constant_data.js'; -export * from './kernel/enqueued_call_data.js'; export * from './kernel/private_kernel_empty_inputs.js'; export * from './kernel/kernel_circuit_public_inputs.js'; export * from './kernel/private_accumulated_data.js'; @@ -36,20 +35,13 @@ export * from './kernel/private_to_avm_accumulated_data.js'; export * from './kernel/private_to_public_accumulated_data.js'; export * from './kernel/private_to_public_accumulated_data_builder.js'; export * from './kernel/private_to_public_kernel_circuit_public_inputs.js'; -export * from './kernel/public_accumulated_data.js'; -export * from './kernel/public_kernel_circuit_private_inputs.js'; -export * from './kernel/public_kernel_circuit_public_inputs.js'; -export * from './kernel/public_kernel_data.js'; -export * from './kernel/public_kernel_tail_circuit_private_inputs.js'; export * from './kernel/tx_constant_data.js'; -export * from './kernel/vm_circuit_public_inputs.js'; export * from './key_validation_request.js'; export * from './key_validation_request_and_generator.js'; export * from './l2_to_l1_message.js'; export * from './log_hash.js'; export * from './max_block_number.js'; export * from './membership_witness.js'; -export * from './non_existent_read_request_hints.js'; export * from './note_hash.js'; export * from './nullifier.js'; export * from './optional_number.js'; @@ -66,12 +58,10 @@ export * from './public_call_request.js'; export * from './public_call_stack_item_compressed.js'; export * from './public_circuit_public_inputs.js'; export * from './public_data_hint.js'; -export * from './public_data_leaf_hint.js'; export * from './public_data_read.js'; export * from './public_data_update_request.js'; export * from './public_data_write.js'; export * from './public_inner_call_request.js'; -export * from './public_validation_requests.js'; export * from './read_request.js'; export * from './read_request_hints/index.js'; export * from './recursive_proof.js'; @@ -100,7 +90,6 @@ export * from './scoped_key_validation_request_and_generator.js'; export * from './shared.js'; export * from './state_reference.js'; export * from './tree_leaf_read_request.js'; -export * from './tree_leaf_read_request_hint.js'; export * from './tree_snapshots.js'; export * from './trees/index.js'; export * from './tx_context.js'; diff --git a/yarn-project/circuits.js/src/structs/kernel/enqueued_call_data.ts b/yarn-project/circuits.js/src/structs/kernel/enqueued_call_data.ts deleted file mode 100644 index 86a500d2bc1..00000000000 --- a/yarn-project/circuits.js/src/structs/kernel/enqueued_call_data.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { BufferReader, serializeToBuffer } from '@aztec/foundation/serialize'; - -import { Proof } from '../proof.js'; -import { VMCircuitPublicInputs } from './vm_circuit_public_inputs.js'; - -/** - * Public calldata assembled from the kernel execution result and proof. - */ -export class EnqueuedCallData { - constructor(public readonly data: VMCircuitPublicInputs, public readonly proof: Proof) {} - - toBuffer() { - return serializeToBuffer(this.data, this.proof); - } - - static fromBuffer(buffer: BufferReader | Buffer) { - const reader = BufferReader.asReader(buffer); - return new EnqueuedCallData(reader.readObject(VMCircuitPublicInputs), reader.readObject(Proof)); - } -} diff --git a/yarn-project/circuits.js/src/structs/kernel/private_to_public_accumulated_data_builder.ts b/yarn-project/circuits.js/src/structs/kernel/private_to_public_accumulated_data_builder.ts index dd734a3e8ff..66733d24df0 100644 --- a/yarn-project/circuits.js/src/structs/kernel/private_to_public_accumulated_data_builder.ts +++ b/yarn-project/circuits.js/src/structs/kernel/private_to_public_accumulated_data_builder.ts @@ -17,8 +17,8 @@ import { PrivateToPublicAccumulatedData } from './private_to_public_accumulated_ /** * TESTS-ONLY CLASS - * Builder for PublicAccumulatedData, used to conveniently construct instances for testing, - * as PublicAccumulatedData is (or will shortly be) immutable. + * Builder for PrivateToPublicAccumulatedData, used to conveniently construct instances for testing, + * as PrivateToPublicAccumulatedData is (or will shortly be) immutable. * */ export class PrivateToPublicAccumulatedDataBuilder { @@ -28,7 +28,7 @@ export class PrivateToPublicAccumulatedDataBuilder { private noteEncryptedLogsHashes: LogHash[] = []; private encryptedLogsHashes: ScopedLogHash[] = []; private contractClassLogsHashes: ScopedLogHash[] = []; - private publicCallStack: PublicCallRequest[] = []; + private publicCallRequests: PublicCallRequest[] = []; pushNoteHash(newNoteHash: Fr) { this.noteHashes.push(newNoteHash); @@ -91,12 +91,12 @@ export class PrivateToPublicAccumulatedDataBuilder { } pushPublicCall(publicCall: PublicCallRequest) { - this.publicCallStack.push(publicCall); + this.publicCallRequests.push(publicCall); return this; } - withPublicCallStack(publicCallStack: PublicCallRequest[]) { - this.publicCallStack = publicCallStack; + withPublicCallRequests(publicCallRequests: PublicCallRequest[]) { + this.publicCallRequests = publicCallRequests; return this; } @@ -108,18 +108,7 @@ export class PrivateToPublicAccumulatedDataBuilder { padArrayEnd(this.noteEncryptedLogsHashes, LogHash.empty(), MAX_NOTE_ENCRYPTED_LOGS_PER_TX), padArrayEnd(this.encryptedLogsHashes, ScopedLogHash.empty(), MAX_ENCRYPTED_LOGS_PER_TX), padArrayEnd(this.contractClassLogsHashes, ScopedLogHash.empty(), MAX_CONTRACT_CLASS_LOGS_PER_TX), - padArrayEnd(this.publicCallStack, PublicCallRequest.empty(), MAX_ENQUEUED_CALLS_PER_TX), + padArrayEnd(this.publicCallRequests, PublicCallRequest.empty(), MAX_ENQUEUED_CALLS_PER_TX), ); } - - static fromPublicAccumulatedData(publicAccumulatedData: PrivateToPublicAccumulatedData) { - return new PrivateToPublicAccumulatedDataBuilder() - .withNoteHashes(publicAccumulatedData.noteHashes) - .withNullifiers(publicAccumulatedData.nullifiers) - .withL2ToL1Msgs(publicAccumulatedData.l2ToL1Msgs) - .withNoteEncryptedLogsHashes(publicAccumulatedData.noteEncryptedLogsHashes) - .withEncryptedLogsHashes(publicAccumulatedData.encryptedLogsHashes) - .withContractClassLogsHashes(publicAccumulatedData.contractClassLogsHashes) - .withPublicCallStack(publicAccumulatedData.publicCallRequests); - } } diff --git a/yarn-project/circuits.js/src/structs/kernel/public_accumulated_data.ts b/yarn-project/circuits.js/src/structs/kernel/public_accumulated_data.ts deleted file mode 100644 index 4a366853ebf..00000000000 --- a/yarn-project/circuits.js/src/structs/kernel/public_accumulated_data.ts +++ /dev/null @@ -1,321 +0,0 @@ -import { makeTuple } from '@aztec/foundation/array'; -import { arraySerializedSizeOfNonEmpty } from '@aztec/foundation/collection'; -import { type Fr } from '@aztec/foundation/fields'; -import { BufferReader, FieldReader, type Tuple, serializeToBuffer } from '@aztec/foundation/serialize'; - -import { inspect } from 'util'; - -import { - MAX_ENCRYPTED_LOGS_PER_TX, - MAX_ENQUEUED_CALLS_PER_TX, - MAX_L2_TO_L1_MSGS_PER_TX, - MAX_NOTE_ENCRYPTED_LOGS_PER_TX, - MAX_NOTE_HASHES_PER_TX, - MAX_NULLIFIERS_PER_TX, - MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, - MAX_UNENCRYPTED_LOGS_PER_TX, - NUM_PUBLIC_ACCUMULATED_DATA_ARRAYS, -} from '../../constants.gen.js'; -import { countAccumulatedItems } from '../../utils/index.js'; -import { Gas } from '../gas.js'; -import { ScopedL2ToL1Message } from '../l2_to_l1_message.js'; -import { LogHash, ScopedLogHash } from '../log_hash.js'; -import { ScopedNoteHash } from '../note_hash.js'; -import { Nullifier } from '../nullifier.js'; -import { PublicCallRequest } from '../public_call_request.js'; -import { PublicDataUpdateRequest } from '../public_data_update_request.js'; - -export class PublicAccumulatedData { - constructor( - /** - * The new note hashes made in this transaction. - */ - public readonly noteHashes: Tuple, - /** - * The new nullifiers made in this transaction. - */ - public readonly nullifiers: Tuple, - /** - * All the new L2 to L1 messages created in this transaction. - */ - public readonly l2ToL1Msgs: Tuple, - /** - * Accumulated encrypted note logs hashes from all the previous kernel iterations. - * Note: Truncated to 31 bytes to fit in Fr. - */ - public readonly noteEncryptedLogsHashes: Tuple, - /** - * Accumulated encrypted logs hashes from all the previous kernel iterations. - * Note: Truncated to 31 bytes to fit in Fr. - */ - public readonly encryptedLogsHashes: Tuple, - /** - * Accumulated unencrypted logs hashes from all the previous kernel iterations. - * Note: Truncated to 31 bytes to fit in Fr. - */ - public readonly unencryptedLogsHashes: Tuple, - /** - * All the public data update requests made in this transaction. - */ - public readonly publicDataUpdateRequests: Tuple< - PublicDataUpdateRequest, - typeof MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX - >, - /** - * Current public call stack. - */ - public readonly publicCallStack: Tuple, - - /** Gas used so far by the transaction. */ - public readonly gasUsed: Gas, - ) {} - - getSize() { - return ( - arraySerializedSizeOfNonEmpty(this.noteHashes) + - arraySerializedSizeOfNonEmpty(this.nullifiers) + - arraySerializedSizeOfNonEmpty(this.l2ToL1Msgs) + - arraySerializedSizeOfNonEmpty(this.noteEncryptedLogsHashes) + - arraySerializedSizeOfNonEmpty(this.encryptedLogsHashes) + - arraySerializedSizeOfNonEmpty(this.unencryptedLogsHashes) + - arraySerializedSizeOfNonEmpty(this.publicDataUpdateRequests) + - arraySerializedSizeOfNonEmpty(this.publicCallStack) + - this.gasUsed.toBuffer().length - ); - } - - toBuffer() { - return serializeToBuffer( - this.noteHashes, - this.nullifiers, - this.l2ToL1Msgs, - this.noteEncryptedLogsHashes, - this.encryptedLogsHashes, - this.unencryptedLogsHashes, - this.publicDataUpdateRequests, - this.publicCallStack, - this.gasUsed, - ); - } - - toString() { - return this.toBuffer().toString('hex'); - } - - isEmpty(): boolean { - return ( - this.noteHashes.every(x => x.isEmpty()) && - this.nullifiers.every(x => x.isEmpty()) && - this.l2ToL1Msgs.every(x => x.isEmpty()) && - this.noteEncryptedLogsHashes.every(x => x.isEmpty()) && - this.encryptedLogsHashes.every(x => x.isEmpty()) && - this.unencryptedLogsHashes.every(x => x.isEmpty()) && - this.publicDataUpdateRequests.every(x => x.isEmpty()) && - this.publicCallStack.every(x => x.isEmpty()) && - this.gasUsed.isEmpty() - ); - } - - [inspect.custom]() { - // print out the non-empty fields - return `PublicAccumulatedData { - noteHashes: [${this.noteHashes - .filter(x => !x.isEmpty()) - .map(h => inspect(h)) - .join(', ')}], - nullifiers: [${this.nullifiers - .filter(x => !x.isEmpty()) - .map(h => inspect(h)) - .join(', ')}], - l2ToL1Msgs: [${this.l2ToL1Msgs - .filter(x => !x.isEmpty()) - .map(h => inspect(h)) - .join(', ')}], - noteEncryptedLogsHashes: [${this.noteEncryptedLogsHashes - .filter(x => !x.isEmpty()) - .map(h => inspect(h)) - .join(', ')}], - encryptedLogsHashes: [${this.encryptedLogsHashes - .filter(x => !x.isEmpty()) - .map(h => inspect(h)) - .join(', ')}], - unencryptedLogsHashes: [${this.unencryptedLogsHashes - .filter(x => !x.isEmpty()) - .map(h => inspect(h)) - .join(', ')}], - publicDataUpdateRequests: [${this.publicDataUpdateRequests - .filter(x => !x.isEmpty()) - .map(h => inspect(h)) - .join(', ')}], - publicCallStack: [${this.publicCallStack - .filter(x => !x.isEmpty()) - .map(h => inspect(h)) - .join(', ')}], - gasUsed: [${inspect(this.gasUsed)}] -}`; - } - - /** - * Deserializes from a buffer or reader, corresponding to a write in cpp. - * @param buffer - Buffer or reader to read from. - * @returns Deserialized object. - */ - static fromBuffer(buffer: Buffer | BufferReader) { - const reader = BufferReader.asReader(buffer); - return new this( - reader.readArray(MAX_NOTE_HASHES_PER_TX, ScopedNoteHash), - reader.readArray(MAX_NULLIFIERS_PER_TX, Nullifier), - reader.readArray(MAX_L2_TO_L1_MSGS_PER_TX, ScopedL2ToL1Message), - reader.readArray(MAX_NOTE_ENCRYPTED_LOGS_PER_TX, LogHash), - reader.readArray(MAX_ENCRYPTED_LOGS_PER_TX, ScopedLogHash), - reader.readArray(MAX_UNENCRYPTED_LOGS_PER_TX, ScopedLogHash), - reader.readArray(MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, PublicDataUpdateRequest), - reader.readArray(MAX_ENQUEUED_CALLS_PER_TX, PublicCallRequest), - reader.readObject(Gas), - ); - } - - static fromFields(fields: Fr[] | FieldReader) { - const reader = FieldReader.asReader(fields); - return new this( - reader.readArray(MAX_NOTE_HASHES_PER_TX, ScopedNoteHash), - reader.readArray(MAX_NULLIFIERS_PER_TX, Nullifier), - reader.readArray(MAX_L2_TO_L1_MSGS_PER_TX, ScopedL2ToL1Message), - reader.readArray(MAX_NOTE_ENCRYPTED_LOGS_PER_TX, LogHash), - reader.readArray(MAX_ENCRYPTED_LOGS_PER_TX, ScopedLogHash), - reader.readArray(MAX_UNENCRYPTED_LOGS_PER_TX, ScopedLogHash), - reader.readArray(MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, PublicDataUpdateRequest), - reader.readArray(MAX_ENQUEUED_CALLS_PER_TX, PublicCallRequest), - reader.readObject(Gas), - ); - } - - /** - * Deserializes from a string, corresponding to a write in cpp. - * @param str - String to read from. - * @returns Deserialized object. - */ - static fromString(str: string) { - return this.fromBuffer(Buffer.from(str, 'hex')); - } - - static empty() { - return new this( - makeTuple(MAX_NOTE_HASHES_PER_TX, ScopedNoteHash.empty), - makeTuple(MAX_NULLIFIERS_PER_TX, Nullifier.empty), - makeTuple(MAX_L2_TO_L1_MSGS_PER_TX, ScopedL2ToL1Message.empty), - makeTuple(MAX_NOTE_ENCRYPTED_LOGS_PER_TX, LogHash.empty), - makeTuple(MAX_ENCRYPTED_LOGS_PER_TX, ScopedLogHash.empty), - makeTuple(MAX_UNENCRYPTED_LOGS_PER_TX, ScopedLogHash.empty), - makeTuple(MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, PublicDataUpdateRequest.empty), - makeTuple(MAX_ENQUEUED_CALLS_PER_TX, PublicCallRequest.empty), - Gas.empty(), - ); - } -} - -export class PublicAccumulatedDataArrayLengths { - constructor( - public readonly noteHashes: number, - public readonly nullifiers: number, - public readonly l2ToL1Msgs: number, - public readonly noteEncryptedLogsHashes: number, - public readonly encryptedLogsHashes: number, - public readonly unencryptedLogsHashes: number, - public readonly publicDataUpdateRequests: number, - public readonly publicCallStack: number, - ) {} - - static new(data: PublicAccumulatedData) { - return new PublicAccumulatedDataArrayLengths( - countAccumulatedItems(data.noteHashes), - countAccumulatedItems(data.nullifiers), - countAccumulatedItems(data.l2ToL1Msgs), - countAccumulatedItems(data.noteEncryptedLogsHashes), - countAccumulatedItems(data.encryptedLogsHashes), - countAccumulatedItems(data.unencryptedLogsHashes), - countAccumulatedItems(data.publicDataUpdateRequests), - countAccumulatedItems(data.publicCallStack), - ); - } - - getSize() { - return NUM_PUBLIC_ACCUMULATED_DATA_ARRAYS; - } - - toBuffer() { - return serializeToBuffer( - this.noteHashes, - this.nullifiers, - this.l2ToL1Msgs, - this.noteEncryptedLogsHashes, - this.encryptedLogsHashes, - this.unencryptedLogsHashes, - this.publicDataUpdateRequests, - this.publicCallStack, - ); - } - - toString() { - return this.toBuffer().toString('hex'); - } - - isEmpty(): boolean { - return ( - this.noteHashes == 0 && - this.nullifiers == 0 && - this.l2ToL1Msgs == 0 && - this.noteEncryptedLogsHashes == 0 && - this.encryptedLogsHashes == 0 && - this.unencryptedLogsHashes == 0 && - this.publicDataUpdateRequests == 0 && - this.publicCallStack == 0 - ); - } - - /** - * Deserializes from a buffer or reader, corresponding to a write in cpp. - * @param buffer - Buffer or reader to read from. - * @returns Deserialized object. - */ - static fromBuffer(buffer: Buffer | BufferReader) { - const reader = BufferReader.asReader(buffer); - return new this( - reader.readNumber(), - reader.readNumber(), - reader.readNumber(), - reader.readNumber(), - reader.readNumber(), - reader.readNumber(), - reader.readNumber(), - reader.readNumber(), - ); - } - - static fromFields(fields: Fr[] | FieldReader) { - const reader = FieldReader.asReader(fields); - return new this( - reader.readU32(), - reader.readU32(), - reader.readU32(), - reader.readU32(), - reader.readU32(), - reader.readU32(), - reader.readU32(), - reader.readU32(), - ); - } - - /** - * Deserializes from a string, corresponding to a write in cpp. - * @param str - String to read from. - * @returns Deserialized object. - */ - static fromString(str: string) { - return this.fromBuffer(Buffer.from(str, 'hex')); - } - - static empty() { - return new this(0, 0, 0, 0, 0, 0, 0, 0); - } -} diff --git a/yarn-project/circuits.js/src/structs/kernel/public_kernel_circuit_private_inputs.test.ts b/yarn-project/circuits.js/src/structs/kernel/public_kernel_circuit_private_inputs.test.ts deleted file mode 100644 index 9f783f1a305..00000000000 --- a/yarn-project/circuits.js/src/structs/kernel/public_kernel_circuit_private_inputs.test.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { makePublicKernelCircuitPrivateInputs } from '../../tests/factories.js'; -import { PublicKernelCircuitPrivateInputs } from './public_kernel_circuit_private_inputs.js'; - -describe('PublicKernelCircuitPrivateInputs', () => { - it('PublicKernelCircuitPrivateInputs after serialization and deserialization is equal to the original', () => { - const original = makePublicKernelCircuitPrivateInputs(123); - const serialized = PublicKernelCircuitPrivateInputs.fromBuffer(original.toBuffer()); - expect(original).toEqual(serialized); - }); - - it('PublicKernelCircuitPrivateInputs after clone is equal to the original', () => { - const original = makePublicKernelCircuitPrivateInputs(123); - const serialized = original.clone(); - expect(original).toEqual(serialized); - expect(original).not.toBe(serialized); - }); - - it('serializes to and deserializes from a string', () => { - const original = makePublicKernelCircuitPrivateInputs(123); - const serialized = PublicKernelCircuitPrivateInputs.fromString(original.toString()); - expect(original).toEqual(serialized); - }); -}); diff --git a/yarn-project/circuits.js/src/structs/kernel/public_kernel_circuit_private_inputs.ts b/yarn-project/circuits.js/src/structs/kernel/public_kernel_circuit_private_inputs.ts deleted file mode 100644 index 8d6cf987007..00000000000 --- a/yarn-project/circuits.js/src/structs/kernel/public_kernel_circuit_private_inputs.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { BufferReader, serializeToBuffer } from '@aztec/foundation/serialize'; - -import { EnqueuedCallData } from './enqueued_call_data.js'; -import { PublicKernelData } from './public_kernel_data.js'; - -/** - * Inputs to the public kernel circuit. - */ -export class PublicKernelCircuitPrivateInputs { - constructor( - /** - * Kernels are recursive and this is the data from the previous kernel. - */ - public readonly previousKernel: PublicKernelData, - /** - * Public calldata assembled from the execution result and proof. - */ - public readonly enqueuedCall: EnqueuedCallData, - ) {} - - /** - * Serializes the object to a buffer. - * @returns - Buffer representation of the object. - */ - toBuffer() { - return serializeToBuffer(this.previousKernel, this.enqueuedCall); - } - - /** - * Serializes the object to a hex string. - * @returns - Hex string representation of the object. - */ - toString() { - return this.toBuffer().toString('hex'); - } - - /** - * Deserializes the object from a buffer. - * @param buffer - Buffer to deserialize. - * @returns - Deserialized object. - */ - static fromBuffer(buffer: BufferReader | Buffer) { - const reader = BufferReader.asReader(buffer); - const previousKernel = reader.readObject(PublicKernelData); - const enqueuedCall = reader.readObject(EnqueuedCallData); - return new PublicKernelCircuitPrivateInputs(previousKernel, enqueuedCall); - } - - /** - * Deserializes the object from a hex string. - * @param str - Hex string to deserialize. - * @returns - Deserialized object. - */ - static fromString(str: string) { - return PublicKernelCircuitPrivateInputs.fromBuffer(Buffer.from(str, 'hex')); - } - - /** - * Clones the object. - * @returns - Cloned object. - */ - clone() { - return PublicKernelCircuitPrivateInputs.fromBuffer(this.toBuffer()); - } -} diff --git a/yarn-project/circuits.js/src/structs/kernel/public_kernel_circuit_public_inputs.test.ts b/yarn-project/circuits.js/src/structs/kernel/public_kernel_circuit_public_inputs.test.ts deleted file mode 100644 index 8242e119c55..00000000000 --- a/yarn-project/circuits.js/src/structs/kernel/public_kernel_circuit_public_inputs.test.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { makePublicKernelCircuitPublicInputs } from '../../tests/factories.js'; -import { PublicKernelCircuitPublicInputs } from './public_kernel_circuit_public_inputs.js'; - -describe('PublicKernelCircuitPublicInputs', () => { - it('PublicKernelCircuitPublicInputs after serialization and deserialization is equal to the original', () => { - const original = makePublicKernelCircuitPublicInputs(123); - const serialized = PublicKernelCircuitPublicInputs.fromBuffer(original.toBuffer()); - expect(original).toEqual(serialized); - }); - - it('PublicKernelCircuitPublicInputs after clone is equal to the original', () => { - const original = makePublicKernelCircuitPublicInputs(123); - const serialized = original.clone(); - expect(original).toEqual(serialized); - expect(original).not.toBe(serialized); - }); - - it('serializes to and deserializes from a string', () => { - const original = makePublicKernelCircuitPublicInputs(123); - const serialized = PublicKernelCircuitPublicInputs.fromString(original.toString()); - expect(original).toEqual(serialized); - }); -}); diff --git a/yarn-project/circuits.js/src/structs/kernel/public_kernel_circuit_public_inputs.ts b/yarn-project/circuits.js/src/structs/kernel/public_kernel_circuit_public_inputs.ts deleted file mode 100644 index 04214ea5b02..00000000000 --- a/yarn-project/circuits.js/src/structs/kernel/public_kernel_circuit_public_inputs.ts +++ /dev/null @@ -1,136 +0,0 @@ -import { AztecAddress } from '@aztec/foundation/aztec-address'; -import { type Fr } from '@aztec/foundation/fields'; -import { BufferReader, FieldReader, serializeToBuffer } from '@aztec/foundation/serialize'; - -import { inspect } from 'util'; - -import { PublicCallRequest } from '../public_call_request.js'; -import { PublicValidationRequests } from '../public_validation_requests.js'; -import { RevertCode } from '../revert_code.js'; -import { CombinedConstantData } from './combined_constant_data.js'; -import { PublicAccumulatedData } from './public_accumulated_data.js'; - -/** - * Outputs from the public kernel circuits. - * All Public kernels use this shape for outputs. - */ -export class PublicKernelCircuitPublicInputs { - constructor( - /** - * Data which is not modified by the circuits. - */ - public constants: CombinedConstantData, - /** - * Validation requests accumulated from public functions. - */ - public validationRequests: PublicValidationRequests, - /** - * Accumulated side effects and enqueued calls that are not revertible. - */ - public endNonRevertibleData: PublicAccumulatedData, - /** - * Data accumulated from both public and private circuits. - */ - public end: PublicAccumulatedData, - /** - * Counter of the last side effect. - */ - public endSideEffectCounter: number, - /** - * The call request for the public teardown function - */ - public publicTeardownCallRequest: PublicCallRequest, - /** - * The address of the fee payer for the transaction - */ - public feePayer: AztecAddress, - /** - * Indicates whether execution of the public circuit reverted. - */ - public revertCode: RevertCode, - ) {} - - toBuffer() { - return serializeToBuffer( - this.constants, - this.validationRequests, - this.endNonRevertibleData, - this.end, - this.endSideEffectCounter, - this.publicTeardownCallRequest, - this.feePayer, - this.revertCode, - ); - } - - clone() { - return PublicKernelCircuitPublicInputs.fromBuffer(this.toBuffer()); - } - - toString() { - return this.toBuffer().toString('hex'); - } - - static fromString(str: string) { - return PublicKernelCircuitPublicInputs.fromBuffer(Buffer.from(str, 'hex')); - } - - /** - * Deserializes from a buffer or reader, corresponding to a write in cpp. - * @param buffer - Buffer or reader to read from. - * @returns A new instance of PublicKernelCircuitPublicInputs. - */ - static fromBuffer(buffer: Buffer | BufferReader): PublicKernelCircuitPublicInputs { - const reader = BufferReader.asReader(buffer); - return new PublicKernelCircuitPublicInputs( - reader.readObject(CombinedConstantData), - reader.readObject(PublicValidationRequests), - reader.readObject(PublicAccumulatedData), - reader.readObject(PublicAccumulatedData), - reader.readNumber(), - reader.readObject(PublicCallRequest), - reader.readObject(AztecAddress), - reader.readObject(RevertCode), - ); - } - - static empty() { - return new PublicKernelCircuitPublicInputs( - CombinedConstantData.empty(), - PublicValidationRequests.empty(), - PublicAccumulatedData.empty(), - PublicAccumulatedData.empty(), - 0, - PublicCallRequest.empty(), - AztecAddress.ZERO, - RevertCode.OK, - ); - } - - static fromFields(fields: Fr[] | FieldReader): PublicKernelCircuitPublicInputs { - const reader = FieldReader.asReader(fields); - return new PublicKernelCircuitPublicInputs( - CombinedConstantData.fromFields(reader), - PublicValidationRequests.fromFields(reader), - PublicAccumulatedData.fromFields(reader), - PublicAccumulatedData.fromFields(reader), - reader.readU32(), - PublicCallRequest.fromFields(reader), - AztecAddress.fromFields(reader), - RevertCode.fromField(reader.readField()), - ); - } - - [inspect.custom]() { - return `PublicKernelCircuitPublicInputs { - constants: ${inspect(this.constants)}, - validationRequests: ${inspect(this.validationRequests)}, - endNonRevertibleData: ${inspect(this.endNonRevertibleData)}, - end: ${inspect(this.end)}, - endSideEffectCounter: ${this.endSideEffectCounter}, - publicTeardownCallRequest: ${inspect(this.publicTeardownCallRequest)}, - feePayer: ${this.feePayer}, - revertCode: ${this.revertCode}, - }`; - } -} diff --git a/yarn-project/circuits.js/src/structs/kernel/public_kernel_data.ts b/yarn-project/circuits.js/src/structs/kernel/public_kernel_data.ts deleted file mode 100644 index f771cc2207c..00000000000 --- a/yarn-project/circuits.js/src/structs/kernel/public_kernel_data.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { makeTuple } from '@aztec/foundation/array'; -import { Fr } from '@aztec/foundation/fields'; -import { BufferReader, type Tuple, serializeToBuffer } from '@aztec/foundation/serialize'; - -import { NESTED_RECURSIVE_PROOF_LENGTH, VK_TREE_HEIGHT } from '../../constants.gen.js'; -import { ClientIvcProof } from '../client_ivc_proof.js'; -import { RecursiveProof, makeEmptyRecursiveProof } from '../recursive_proof.js'; -import { type UInt32 } from '../shared.js'; -import { VerificationKeyData } from '../verification_key.js'; -import { PublicKernelCircuitPublicInputs } from './public_kernel_circuit_public_inputs.js'; - -/** - * Data of the previous public kernel iteration in the chain of kernels. - */ -export class PublicKernelData { - constructor( - /** - * Public inputs of the previous kernel. - */ - public publicInputs: PublicKernelCircuitPublicInputs, - /** - * Proof of the previous kernel. - */ - public proof: RecursiveProof, - /** - * Verification key of the previous kernel. - */ - public vk: VerificationKeyData, - /** - * Index of the previous kernel's vk in a tree of vks. - */ - public vkIndex: UInt32, - /** - * Sibling path of the previous kernel's vk in a tree of vks. - */ - public vkPath: Tuple, - - /** - * TODO(https://github.com/AztecProtocol/aztec-packages/issues/7369) this should be tube-proved for the first iteration and replace proof above - */ - public clientIvcProof: ClientIvcProof = ClientIvcProof.empty(), - ) {} - - static fromBuffer(buffer: Buffer | BufferReader): PublicKernelData { - const reader = BufferReader.asReader(buffer); - return new this( - reader.readObject(PublicKernelCircuitPublicInputs), - RecursiveProof.fromBuffer(reader, NESTED_RECURSIVE_PROOF_LENGTH), - reader.readObject(VerificationKeyData), - reader.readNumber(), - reader.readArray(VK_TREE_HEIGHT, Fr), - reader.readObject(ClientIvcProof), - ); - } - - static empty(): PublicKernelData { - return new this( - PublicKernelCircuitPublicInputs.empty(), - makeEmptyRecursiveProof(NESTED_RECURSIVE_PROOF_LENGTH), - VerificationKeyData.makeFakeHonk(), - 0, - makeTuple(VK_TREE_HEIGHT, Fr.zero), - ClientIvcProof.empty(), - ); - } - - /** - * Serialize this as a buffer. - * @returns The buffer. - */ - toBuffer() { - return serializeToBuffer(this.publicInputs, this.proof, this.vk, this.vkIndex, this.vkPath, this.clientIvcProof); - } -} diff --git a/yarn-project/circuits.js/src/structs/kernel/public_kernel_tail_circuit_private_inputs.test.ts b/yarn-project/circuits.js/src/structs/kernel/public_kernel_tail_circuit_private_inputs.test.ts deleted file mode 100644 index 3e2767cbda3..00000000000 --- a/yarn-project/circuits.js/src/structs/kernel/public_kernel_tail_circuit_private_inputs.test.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { makePublicKernelTailCircuitPrivateInputs } from '../../tests/factories.js'; -import { PublicKernelTailCircuitPrivateInputs } from './public_kernel_tail_circuit_private_inputs.js'; - -describe('PublicKernelTailCircuitPrivateInputs', () => { - it('PublicKernelTailCircuitPrivateInputs after serialization and deserialization is equal to the original', () => { - const original = makePublicKernelTailCircuitPrivateInputs(123); - const serialized = PublicKernelTailCircuitPrivateInputs.fromBuffer(original.toBuffer()); - expect(original).toEqual(serialized); - }); - - it('PublicKernelTailCircuitPrivateInputs after clone is equal to the original', () => { - const original = makePublicKernelTailCircuitPrivateInputs(123); - const serialized = original.clone(); - expect(original).toEqual(serialized); - expect(original).not.toBe(serialized); - }); - - it('serializes to string and back', () => { - const original = makePublicKernelTailCircuitPrivateInputs(123); - const str = original.toString(); - const deserialized = PublicKernelTailCircuitPrivateInputs.fromString(str); - expect(original).toEqual(deserialized); - }); -}); diff --git a/yarn-project/circuits.js/src/structs/kernel/public_kernel_tail_circuit_private_inputs.ts b/yarn-project/circuits.js/src/structs/kernel/public_kernel_tail_circuit_private_inputs.ts deleted file mode 100644 index c0bb9b4d82f..00000000000 --- a/yarn-project/circuits.js/src/structs/kernel/public_kernel_tail_circuit_private_inputs.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { BufferReader, type Tuple, serializeToBuffer } from '@aztec/foundation/serialize'; - -import { - L1_TO_L2_MSG_TREE_HEIGHT, - MAX_L1_TO_L2_MSG_READ_REQUESTS_PER_TX, - MAX_NOTE_HASH_READ_REQUESTS_PER_TX, - MAX_NULLIFIER_READ_REQUESTS_PER_TX, - MAX_PUBLIC_DATA_HINTS, - NOTE_HASH_TREE_HEIGHT, -} from '../../constants.gen.js'; -import { - type NullifierNonExistentReadRequestHints, - nullifierNonExistentReadRequestHintsFromBuffer, -} from '../non_existent_read_request_hints.js'; -import { PartialStateReference } from '../partial_state_reference.js'; -import { PublicDataLeafHint } from '../public_data_leaf_hint.js'; -import { type NullifierReadRequestHints, nullifierReadRequestHintsFromBuffer } from '../read_request_hints/index.js'; -import { TreeLeafReadRequestHint } from '../tree_leaf_read_request_hint.js'; -import { PublicKernelData } from './public_kernel_data.js'; - -export class PublicKernelTailCircuitPrivateInputs { - constructor( - /** - * Kernels are recursive and this is the data from the previous kernel. - */ - public readonly previousKernel: PublicKernelData, - public readonly noteHashReadRequestHints: Tuple< - TreeLeafReadRequestHint, - typeof MAX_NOTE_HASH_READ_REQUESTS_PER_TX - >, - /** - * Contains hints for the nullifier read requests to locate corresponding pending or settled nullifiers. - */ - public readonly nullifierReadRequestHints: NullifierReadRequestHints< - typeof MAX_NULLIFIER_READ_REQUESTS_PER_TX, - typeof MAX_NULLIFIER_READ_REQUESTS_PER_TX - >, - /** - * Contains hints for the nullifier non existent read requests. - */ - public readonly nullifierNonExistentReadRequestHints: NullifierNonExistentReadRequestHints, - public readonly l1ToL2MsgReadRequestHints: Tuple< - TreeLeafReadRequestHint, - typeof MAX_L1_TO_L2_MSG_READ_REQUESTS_PER_TX - >, - public readonly publicDataHints: Tuple, - public readonly startState: PartialStateReference, - ) {} - - toBuffer() { - return serializeToBuffer( - this.previousKernel, - this.noteHashReadRequestHints, - this.nullifierReadRequestHints, - this.nullifierNonExistentReadRequestHints, - this.l1ToL2MsgReadRequestHints, - this.publicDataHints, - this.startState, - ); - } - - toString() { - return this.toBuffer().toString('hex'); - } - - static fromString(str: string) { - return PublicKernelTailCircuitPrivateInputs.fromBuffer(Buffer.from(str, 'hex')); - } - - static fromBuffer(buffer: Buffer | BufferReader) { - const reader = BufferReader.asReader(buffer); - return new PublicKernelTailCircuitPrivateInputs( - reader.readObject(PublicKernelData), - reader.readArray(MAX_NOTE_HASH_READ_REQUESTS_PER_TX, { - fromBuffer: buf => TreeLeafReadRequestHint.fromBuffer(buf, NOTE_HASH_TREE_HEIGHT), - }), - nullifierReadRequestHintsFromBuffer( - reader, - MAX_NULLIFIER_READ_REQUESTS_PER_TX, - MAX_NULLIFIER_READ_REQUESTS_PER_TX, - ), - nullifierNonExistentReadRequestHintsFromBuffer(reader), - reader.readArray(MAX_L1_TO_L2_MSG_READ_REQUESTS_PER_TX, { - fromBuffer: buf => TreeLeafReadRequestHint.fromBuffer(buf, L1_TO_L2_MSG_TREE_HEIGHT), - }), - reader.readArray(MAX_PUBLIC_DATA_HINTS, PublicDataLeafHint), - reader.readObject(PartialStateReference), - ); - } - - clone() { - return PublicKernelTailCircuitPrivateInputs.fromBuffer(this.toBuffer()); - } -} diff --git a/yarn-project/circuits.js/src/structs/kernel/vm_circuit_public_inputs.ts b/yarn-project/circuits.js/src/structs/kernel/vm_circuit_public_inputs.ts deleted file mode 100644 index 7182981a3cb..00000000000 --- a/yarn-project/circuits.js/src/structs/kernel/vm_circuit_public_inputs.ts +++ /dev/null @@ -1,131 +0,0 @@ -import { makeTuple } from '@aztec/foundation/array'; -import { Fr } from '@aztec/foundation/fields'; -import { BufferReader, FieldReader, type Tuple, serializeToBuffer } from '@aztec/foundation/serialize'; - -import { inspect } from 'util'; - -import { MAX_ENQUEUED_CALLS_PER_TX } from '../../constants.gen.js'; -import { Gas } from '../gas.js'; -import { PublicCallRequest } from '../public_call_request.js'; -import { PublicInnerCallRequest } from '../public_inner_call_request.js'; -import { PublicValidationRequestArrayLengths, PublicValidationRequests } from '../public_validation_requests.js'; -import { CombinedConstantData } from './combined_constant_data.js'; -import { PublicAccumulatedData, PublicAccumulatedDataArrayLengths } from './public_accumulated_data.js'; - -/** - * Call stack item on a public call. - */ -export class VMCircuitPublicInputs { - constructor( - public constants: CombinedConstantData, - public callRequest: PublicCallRequest, - public publicCallStack: Tuple, - public previousValidationRequestArrayLengths: PublicValidationRequestArrayLengths, - public validationRequests: PublicValidationRequests, - public previousAccumulatedDataArrayLengths: PublicAccumulatedDataArrayLengths, - public accumulatedData: PublicAccumulatedData, - public startSideEffectCounter: number, - public endSideEffectCounter: number, - public startGasLeft: Gas, - public transactionFee: Fr, - public reverted: boolean, - ) {} - - toBuffer() { - return serializeToBuffer( - this.constants, - this.callRequest, - this.publicCallStack, - this.previousValidationRequestArrayLengths, - this.validationRequests, - this.previousAccumulatedDataArrayLengths, - this.accumulatedData, - this.startSideEffectCounter, - this.endSideEffectCounter, - this.startGasLeft, - this.transactionFee, - this.reverted, - ); - } - - clone() { - return VMCircuitPublicInputs.fromBuffer(this.toBuffer()); - } - - toString() { - return this.toBuffer().toString('hex'); - } - - static fromString(str: string) { - return VMCircuitPublicInputs.fromBuffer(Buffer.from(str, 'hex')); - } - - static fromBuffer(buffer: Buffer | BufferReader) { - const reader = BufferReader.asReader(buffer); - return new VMCircuitPublicInputs( - reader.readObject(CombinedConstantData), - reader.readObject(PublicCallRequest), - reader.readArray(MAX_ENQUEUED_CALLS_PER_TX, PublicInnerCallRequest), - reader.readObject(PublicValidationRequestArrayLengths), - reader.readObject(PublicValidationRequests), - reader.readObject(PublicAccumulatedDataArrayLengths), - reader.readObject(PublicAccumulatedData), - reader.readNumber(), - reader.readNumber(), - reader.readObject(Gas), - reader.readObject(Fr), - reader.readBoolean(), - ); - } - - static empty() { - return new VMCircuitPublicInputs( - CombinedConstantData.empty(), - PublicCallRequest.empty(), - makeTuple(MAX_ENQUEUED_CALLS_PER_TX, PublicInnerCallRequest.empty), - PublicValidationRequestArrayLengths.empty(), - PublicValidationRequests.empty(), - PublicAccumulatedDataArrayLengths.empty(), - PublicAccumulatedData.empty(), - 0, - 0, - Gas.empty(), - Fr.ZERO, - false, - ); - } - - static fromFields(fields: Fr[] | FieldReader) { - const reader = FieldReader.asReader(fields); - return new VMCircuitPublicInputs( - CombinedConstantData.fromFields(reader), - PublicCallRequest.fromFields(reader), - reader.readArray(MAX_ENQUEUED_CALLS_PER_TX, PublicInnerCallRequest), - PublicValidationRequestArrayLengths.fromFields(reader), - PublicValidationRequests.fromFields(reader), - PublicAccumulatedDataArrayLengths.fromFields(reader), - PublicAccumulatedData.fromFields(reader), - reader.readU32(), - reader.readU32(), - Gas.fromFields(reader), - reader.readField(), - reader.readBoolean(), - ); - } - - [inspect.custom]() { - return `VMCircuitPublicInputs { - constants: ${inspect(this.constants)}, - callRequest: ${inspect(this.callRequest)} - previousValidationRequestArrayLengths: ${inspect(this.previousValidationRequestArrayLengths)}, - validationRequests: ${inspect(this.validationRequests)}, - previousAccumulatedDataArrayLengths: ${inspect(this.previousAccumulatedDataArrayLengths)}, - accumulatedData: ${inspect(this.accumulatedData)}, - startSideEffectCounter: ${this.startSideEffectCounter}, - endSideEffectCounter: ${this.endSideEffectCounter}, - startGasLeft: ${this.startGasLeft}, - transactionFee: ${this.transactionFee}, - reverted: ${this.reverted}, - }`; - } -} diff --git a/yarn-project/circuits.js/src/structs/non_existent_read_request_hints.ts b/yarn-project/circuits.js/src/structs/non_existent_read_request_hints.ts deleted file mode 100644 index 2c675326077..00000000000 --- a/yarn-project/circuits.js/src/structs/non_existent_read_request_hints.ts +++ /dev/null @@ -1,161 +0,0 @@ -import { makeTuple } from '@aztec/foundation/array'; -import { BufferReader, type Tuple, serializeToBuffer } from '@aztec/foundation/serialize'; -import { type IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; - -import { - MAX_NULLIFIERS_PER_TX, - MAX_NULLIFIER_NON_EXISTENT_READ_REQUESTS_PER_TX, - NULLIFIER_TREE_HEIGHT, -} from '../constants.gen.js'; -import { MembershipWitness } from './membership_witness.js'; -import { Nullifier } from './nullifier.js'; -import { NullifierLeafPreimage } from './trees/index.js'; - -interface PendingValue { - toBuffer(): Buffer; -} - -export class NonMembershipHint { - constructor(public membershipWitness: MembershipWitness, public leafPreimage: LEAF_PREIMAGE) {} - - static empty( - treeHeight: TREE_HEIGHT, - makeEmptyLeafPreimage: () => LEAF_PREIMAGE, - ) { - return new NonMembershipHint(MembershipWitness.empty(treeHeight), makeEmptyLeafPreimage()); - } - - static fromBuffer( - buffer: Buffer | BufferReader, - treeHeight: TREE_HEIGHT, - leafPreimageFromBuffer: { fromBuffer: (buffer: BufferReader) => LEAF_PREIMAGE }, - ): NonMembershipHint { - const reader = BufferReader.asReader(buffer); - return new NonMembershipHint( - MembershipWitness.fromBuffer(reader, treeHeight), - reader.readObject(leafPreimageFromBuffer), - ); - } - - toBuffer() { - return serializeToBuffer(this.membershipWitness, this.leafPreimage); - } -} - -export class NonExistentReadRequestHints< - READ_REQUEST_LEN extends number, - TREE_HEIGHT extends number, - LEAF_PREIMAGE extends IndexedTreeLeafPreimage, - PENDING_VALUE_LEN extends number, - PENDING_VALUE extends PendingValue, -> { - constructor( - /** - * The hints for the low leaves of the read requests. - */ - public nonMembershipHints: Tuple, READ_REQUEST_LEN>, - /** - * Indices of the smallest pending values greater than the read requests. - */ - public nextPendingValueIndices: Tuple, - public sortedPendingValues: Tuple, - public sortedPendingValueHints: Tuple, - ) {} - - static fromBuffer< - READ_REQUEST_LEN extends number, - TREE_HEIGHT extends number, - LEAF_PREIMAGE extends IndexedTreeLeafPreimage, - PENDING_VALUE_LEN extends number, - PENDING_VALUE extends PendingValue, - >( - buffer: Buffer | BufferReader, - readRequestLen: READ_REQUEST_LEN, - treeHeight: TREE_HEIGHT, - leafPreimageFromBuffer: { fromBuffer: (buffer: BufferReader) => LEAF_PREIMAGE }, - pendingValueLen: PENDING_VALUE_LEN, - orderedValueFromBuffer: { fromBuffer: (buffer: BufferReader) => PENDING_VALUE }, - ): NonExistentReadRequestHints { - const reader = BufferReader.asReader(buffer); - return new NonExistentReadRequestHints( - reader.readArray(readRequestLen, { - fromBuffer: buf => NonMembershipHint.fromBuffer(buf, treeHeight, leafPreimageFromBuffer), - }), - reader.readNumbers(readRequestLen), - reader.readArray(pendingValueLen, orderedValueFromBuffer), - reader.readNumbers(pendingValueLen), - ); - } - - toBuffer() { - return serializeToBuffer( - this.nonMembershipHints, - this.nextPendingValueIndices, - this.sortedPendingValues, - this.sortedPendingValueHints, - ); - } -} - -export type NullifierNonExistentReadRequestHints = NonExistentReadRequestHints< - typeof MAX_NULLIFIER_NON_EXISTENT_READ_REQUESTS_PER_TX, - typeof NULLIFIER_TREE_HEIGHT, - NullifierLeafPreimage, - typeof MAX_NULLIFIERS_PER_TX, - Nullifier ->; - -export function nullifierNonExistentReadRequestHintsFromBuffer( - buffer: Buffer | BufferReader, -): NullifierNonExistentReadRequestHints { - return NonExistentReadRequestHints.fromBuffer( - buffer, - MAX_NULLIFIER_NON_EXISTENT_READ_REQUESTS_PER_TX, - NULLIFIER_TREE_HEIGHT, - NullifierLeafPreimage, - MAX_NULLIFIERS_PER_TX, - Nullifier, - ); -} - -export class NullifierNonExistentReadRequestHintsBuilder { - private hints: NullifierNonExistentReadRequestHints; - private readRequestIndex = 0; - - constructor( - sortedPendingNullifiers: Tuple, - sortedPendingNullifierIndexHints: Tuple, - ) { - this.hints = new NonExistentReadRequestHints( - makeTuple(MAX_NULLIFIER_NON_EXISTENT_READ_REQUESTS_PER_TX, () => - NonMembershipHint.empty(NULLIFIER_TREE_HEIGHT, NullifierLeafPreimage.empty), - ), - makeTuple(MAX_NULLIFIER_NON_EXISTENT_READ_REQUESTS_PER_TX, () => 0), - sortedPendingNullifiers, - sortedPendingNullifierIndexHints, - ); - } - - static empty() { - const emptySortedPendingNullifiers = makeTuple(MAX_NULLIFIERS_PER_TX, Nullifier.empty); - const emptySortedPendingNullifierIndexHints = makeTuple(MAX_NULLIFIERS_PER_TX, () => 0); - return new NullifierNonExistentReadRequestHintsBuilder( - emptySortedPendingNullifiers, - emptySortedPendingNullifierIndexHints, - ).toHints(); - } - - addHint( - membershipWitness: MembershipWitness, - lowLeafPreimage: NullifierLeafPreimage, - nextPendingValueIndex: number, - ) { - this.hints.nonMembershipHints[this.readRequestIndex] = new NonMembershipHint(membershipWitness, lowLeafPreimage); - this.hints.nextPendingValueIndices[this.readRequestIndex] = nextPendingValueIndex; - this.readRequestIndex++; - } - - toHints() { - return this.hints; - } -} diff --git a/yarn-project/circuits.js/src/structs/public_call_stack_item_compressed.ts b/yarn-project/circuits.js/src/structs/public_call_stack_item_compressed.ts index 8df09f6bf4a..64965ffd59e 100644 --- a/yarn-project/circuits.js/src/structs/public_call_stack_item_compressed.ts +++ b/yarn-project/circuits.js/src/structs/public_call_stack_item_compressed.ts @@ -8,6 +8,7 @@ import { CallContext } from './call_context.js'; import { Gas } from './gas.js'; import { RevertCode } from './revert_code.js'; +// TO BE REMOVED /** * Compressed call stack item on a public call. */ diff --git a/yarn-project/circuits.js/src/structs/public_circuit_public_inputs.test.ts b/yarn-project/circuits.js/src/structs/public_circuit_public_inputs.test.ts deleted file mode 100644 index 7d4f900e4f2..00000000000 --- a/yarn-project/circuits.js/src/structs/public_circuit_public_inputs.test.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { randomInt } from '@aztec/foundation/crypto'; -import { setupCustomSnapshotSerializers } from '@aztec/foundation/testing'; - -import { PUBLIC_CIRCUIT_PUBLIC_INPUTS_LENGTH } from '../constants.gen.js'; -import { makePublicCircuitPublicInputs } from '../tests/factories.js'; -import { PublicCircuitPublicInputs } from './public_circuit_public_inputs.js'; - -describe('PublicCircuitPublicInputs', () => { - setupCustomSnapshotSerializers(expect); - it('serializes to field array and deserializes it back', () => { - const expected = makePublicCircuitPublicInputs(randomInt(1000), undefined); - - const fieldArray = expected.toFields(); - const res = PublicCircuitPublicInputs.fromFields(fieldArray); - expect(res).toEqual(expected); - }); - - it(`initializes an empty PrivateCircuitPublicInputs`, () => { - const target = PublicCircuitPublicInputs.empty(); - expect(target.isEmpty()).toBe(true); - }); - - it('number of fields matches constant', () => { - const target = makePublicCircuitPublicInputs(327); - const fields = target.toFields(); - expect(fields.length).toBe(PUBLIC_CIRCUIT_PUBLIC_INPUTS_LENGTH); - }); -}); diff --git a/yarn-project/circuits.js/src/structs/public_circuit_public_inputs.ts b/yarn-project/circuits.js/src/structs/public_circuit_public_inputs.ts index b307b33f2a9..46437e0da40 100644 --- a/yarn-project/circuits.js/src/structs/public_circuit_public_inputs.ts +++ b/yarn-project/circuits.js/src/structs/public_circuit_public_inputs.ts @@ -10,6 +10,8 @@ import { } from '@aztec/foundation/serialize'; import { type FieldsOf } from '@aztec/foundation/types'; +import { inspect } from 'util'; + import { MAX_ENQUEUED_CALLS_PER_CALL, MAX_L1_TO_L2_MSG_READ_REQUESTS_PER_CALL, @@ -40,9 +42,8 @@ import { ReadRequest } from './read_request.js'; import { RevertCode } from './revert_code.js'; import { TreeLeafReadRequest } from './tree_leaf_read_request.js'; -/** - * Public inputs to a public circuit. - */ +// TO BE REMOVED +// This is currently the output of the AVM. It should be replaced by AvmCircuitPublicInputs eventually. export class PublicCircuitPublicInputs { constructor( /** @@ -325,4 +326,66 @@ export class PublicCircuitPublicInputs { reader.readField(), ); } + + [inspect.custom]() { + return `PublicCircuitPublicInputs { + callContext: ${inspect(this.callContext)}, + argsHash: ${inspect(this.argsHash)}, + returnsHash: ${inspect(this.returnsHash)}, + noteHashReadRequests: [${this.noteHashReadRequests + .filter(x => !x.isEmpty()) + .map(h => inspect(h)) + .join(', ')}]}, + nullifierReadRequests: [${this.nullifierReadRequests + .filter(x => !x.isEmpty()) + .map(h => inspect(h)) + .join(', ')}]}, + nullifierNonExistentReadRequests: [${this.nullifierNonExistentReadRequests + .filter(x => !x.isEmpty()) + .map(h => inspect(h)) + .join(', ')}]}, + l1ToL2MsgReadRequests: [${this.l1ToL2MsgReadRequests + .filter(x => !x.isEmpty()) + .map(h => inspect(h)) + .join(', ')}]}, + contractStorageUpdateRequests: [${this.contractStorageUpdateRequests + .filter(x => !x.isEmpty()) + .map(h => inspect(h)) + .join(', ')}]}, + contractStorageReads: [${this.contractStorageReads + .filter(x => !x.isEmpty()) + .map(h => inspect(h)) + .join(', ')}]}, + publicCallRequests: [${this.publicCallRequests + .filter(x => !x.isEmpty()) + .map(h => inspect(h)) + .join(', ')}]}, + noteHashes: [${this.noteHashes + .filter(x => !x.isEmpty()) + .map(h => inspect(h)) + .join(', ')}]}, + nullifiers: [${this.nullifiers + .filter(x => !x.isEmpty()) + .map(h => inspect(h)) + .join(', ')}]}, + l2ToL1Msgs: [${this.l2ToL1Msgs + .filter(x => !x.isEmpty()) + .map(h => inspect(h)) + .join(', ')}]}, + startSideEffectCounter: ${inspect(this.startSideEffectCounter)}, + endSideEffectCounter: ${inspect(this.endSideEffectCounter)}, + startSideEffectCounter: ${inspect(this.startSideEffectCounter)}, + unencryptedLogsHashes: [${this.unencryptedLogsHashes + .filter(x => !x.isEmpty()) + .map(h => inspect(h)) + .join(', ')}]}, + historicalHeader: ${inspect(this.historicalHeader)}, + globalVariables: ${inspect(this.globalVariables)}, + proverAddress: ${inspect(this.proverAddress)}, + revertCode: ${inspect(this.revertCode)}, + startGasLeft: ${inspect(this.startGasLeft)}, + endGasLeft: ${inspect(this.endGasLeft)}, + transactionFee: ${inspect(this.transactionFee)}, + }`; + } } diff --git a/yarn-project/circuits.js/src/structs/public_data_hint.ts b/yarn-project/circuits.js/src/structs/public_data_hint.ts index d1e4c70f381..a3e6b766725 100644 --- a/yarn-project/circuits.js/src/structs/public_data_hint.ts +++ b/yarn-project/circuits.js/src/structs/public_data_hint.ts @@ -9,7 +9,6 @@ export class PublicDataHint { constructor( public leafSlot: Fr, public value: Fr, - public overrideCounter: number, public membershipWitness: MembershipWitness, public leafPreimage: PublicDataTreeLeafPreimage, ) {} @@ -18,7 +17,6 @@ export class PublicDataHint { return new PublicDataHint( Fr.ZERO, Fr.ZERO, - 0, MembershipWitness.empty(PUBLIC_DATA_TREE_HEIGHT), PublicDataTreeLeafPreimage.empty(), ); @@ -29,19 +27,12 @@ export class PublicDataHint { return new PublicDataHint( reader.readObject(Fr), reader.readObject(Fr), - reader.readNumber(), MembershipWitness.fromBuffer(reader, PUBLIC_DATA_TREE_HEIGHT), reader.readObject(PublicDataTreeLeafPreimage), ); } toBuffer() { - return serializeToBuffer( - this.leafSlot, - this.value, - this.overrideCounter, - this.membershipWitness, - this.leafPreimage, - ); + return serializeToBuffer(this.leafSlot, this.value, this.membershipWitness, this.leafPreimage); } } diff --git a/yarn-project/circuits.js/src/structs/public_data_leaf_hint.ts b/yarn-project/circuits.js/src/structs/public_data_leaf_hint.ts deleted file mode 100644 index aba7c583e37..00000000000 --- a/yarn-project/circuits.js/src/structs/public_data_leaf_hint.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { BufferReader, serializeToBuffer } from '@aztec/foundation/serialize'; - -import { PUBLIC_DATA_TREE_HEIGHT } from '../constants.gen.js'; -import { MembershipWitness } from './membership_witness.js'; -import { PublicDataTreeLeafPreimage } from './trees/index.js'; - -export class PublicDataLeafHint { - constructor( - public preimage: PublicDataTreeLeafPreimage, - public membershipWitness: MembershipWitness, - ) {} - - static empty() { - return new PublicDataLeafHint(PublicDataTreeLeafPreimage.empty(), MembershipWitness.empty(PUBLIC_DATA_TREE_HEIGHT)); - } - - static fromBuffer(buffer: Buffer | BufferReader) { - const reader = BufferReader.asReader(buffer); - return new PublicDataLeafHint( - reader.readObject(PublicDataTreeLeafPreimage), - MembershipWitness.fromBuffer(reader, PUBLIC_DATA_TREE_HEIGHT), - ); - } - - toBuffer() { - return serializeToBuffer(this.preimage, this.membershipWitness); - } -} diff --git a/yarn-project/circuits.js/src/structs/public_data_update_request.ts b/yarn-project/circuits.js/src/structs/public_data_update_request.ts index d048a1100a4..ab1973fb081 100644 --- a/yarn-project/circuits.js/src/structs/public_data_update_request.ts +++ b/yarn-project/circuits.js/src/structs/public_data_update_request.ts @@ -1,8 +1,13 @@ +import { type AztecAddress } from '@aztec/foundation/aztec-address'; import { Fr } from '@aztec/foundation/fields'; import { BufferReader, FieldReader, serializeToBuffer } from '@aztec/foundation/serialize'; import { inspect } from 'util'; +import { computePublicDataTreeLeafSlot } from '../hash/hash.js'; +import { type ContractStorageUpdateRequest } from './contract_storage_update_request.js'; + +// TO BE REMOVED. /** * Write operations on the public data tree including the previous value. */ @@ -74,6 +79,12 @@ export class PublicDataUpdateRequest { return new PublicDataUpdateRequest(Fr.fromBuffer(reader), Fr.fromBuffer(reader), reader.readNumber()); } + static fromContractStorageUpdateRequest(contractAddress: AztecAddress, updateRequest: ContractStorageUpdateRequest) { + const leafSlot = computePublicDataTreeLeafSlot(contractAddress, updateRequest.storageSlot); + + return new PublicDataUpdateRequest(leafSlot, updateRequest.newValue, updateRequest.counter); + } + static empty() { return new PublicDataUpdateRequest(Fr.ZERO, Fr.ZERO, 0); } diff --git a/yarn-project/circuits.js/src/structs/public_inner_call_request.ts b/yarn-project/circuits.js/src/structs/public_inner_call_request.ts index 9dd2abd4de0..984b3b47c44 100644 --- a/yarn-project/circuits.js/src/structs/public_inner_call_request.ts +++ b/yarn-project/circuits.js/src/structs/public_inner_call_request.ts @@ -6,6 +6,7 @@ import { inspect } from 'util'; import { PublicCallStackItemCompressed } from './public_call_stack_item_compressed.js'; +// TO BE REMOVED /** * Represents a request to call a public function. */ diff --git a/yarn-project/circuits.js/src/structs/public_validation_requests.ts b/yarn-project/circuits.js/src/structs/public_validation_requests.ts deleted file mode 100644 index 4d45328914c..00000000000 --- a/yarn-project/circuits.js/src/structs/public_validation_requests.ts +++ /dev/null @@ -1,239 +0,0 @@ -import { makeTuple } from '@aztec/foundation/array'; -import { arraySerializedSizeOfNonEmpty } from '@aztec/foundation/collection'; -import { type Fr } from '@aztec/foundation/fields'; -import { BufferReader, FieldReader, type Tuple, serializeToBuffer } from '@aztec/foundation/serialize'; - -import { inspect } from 'util'; - -import { - MAX_L1_TO_L2_MSG_READ_REQUESTS_PER_TX, - MAX_NOTE_HASH_READ_REQUESTS_PER_TX, - MAX_NULLIFIER_NON_EXISTENT_READ_REQUESTS_PER_TX, - MAX_NULLIFIER_READ_REQUESTS_PER_TX, - MAX_PUBLIC_DATA_READS_PER_TX, - NUM_PUBLIC_VALIDATION_REQUEST_ARRAYS, -} from '../constants.gen.js'; -import { countAccumulatedItems } from '../utils/index.js'; -import { PublicDataRead } from './public_data_read.js'; -import { ScopedReadRequest } from './read_request.js'; -import { RollupValidationRequests } from './rollup_validation_requests.js'; -import { TreeLeafReadRequest } from './tree_leaf_read_request.js'; - -/** - * Validation requests accumulated during the execution of the transaction. - */ -export class PublicValidationRequests { - constructor( - /** - * Validation requests that cannot be fulfilled in the current context (private or public), and must be instead be - * forwarded to the rollup for it to take care of them. - */ - public forRollup: RollupValidationRequests, - public noteHashReadRequests: Tuple, - /** - * All the nullifier read requests made in this transaction. - */ - public nullifierReadRequests: Tuple, - /** - * The nullifier read requests made in this transaction. - */ - public nullifierNonExistentReadRequests: Tuple< - ScopedReadRequest, - typeof MAX_NULLIFIER_NON_EXISTENT_READ_REQUESTS_PER_TX - >, - public l1ToL2MsgReadRequests: Tuple, - /** - * All the public data reads made in this transaction. - */ - public publicDataReads: Tuple, - ) {} - - getSize() { - return ( - this.forRollup.getSize() + - arraySerializedSizeOfNonEmpty(this.noteHashReadRequests) + - arraySerializedSizeOfNonEmpty(this.nullifierReadRequests) + - arraySerializedSizeOfNonEmpty(this.nullifierNonExistentReadRequests) + - arraySerializedSizeOfNonEmpty(this.l1ToL2MsgReadRequests) + - arraySerializedSizeOfNonEmpty(this.publicDataReads) - ); - } - - toBuffer() { - return serializeToBuffer( - this.forRollup, - this.noteHashReadRequests, - this.nullifierReadRequests, - this.nullifierNonExistentReadRequests, - this.l1ToL2MsgReadRequests, - this.publicDataReads, - ); - } - - toString() { - return this.toBuffer().toString('hex'); - } - - static fromFields(fields: Fr[] | FieldReader) { - const reader = FieldReader.asReader(fields); - return new PublicValidationRequests( - reader.readObject(RollupValidationRequests), - reader.readArray(MAX_NOTE_HASH_READ_REQUESTS_PER_TX, TreeLeafReadRequest), - reader.readArray(MAX_NULLIFIER_READ_REQUESTS_PER_TX, ScopedReadRequest), - reader.readArray(MAX_NULLIFIER_NON_EXISTENT_READ_REQUESTS_PER_TX, ScopedReadRequest), - reader.readArray(MAX_L1_TO_L2_MSG_READ_REQUESTS_PER_TX, TreeLeafReadRequest), - reader.readArray(MAX_PUBLIC_DATA_READS_PER_TX, PublicDataRead), - ); - } - - /** - * Deserializes from a buffer or reader, corresponding to a write in cpp. - * @param buffer - Buffer or reader to read from. - * @returns Deserialized object. - */ - static fromBuffer(buffer: Buffer | BufferReader) { - const reader = BufferReader.asReader(buffer); - return new PublicValidationRequests( - reader.readObject(RollupValidationRequests), - reader.readArray(MAX_NOTE_HASH_READ_REQUESTS_PER_TX, TreeLeafReadRequest), - reader.readArray(MAX_NULLIFIER_READ_REQUESTS_PER_TX, ScopedReadRequest), - reader.readArray(MAX_NULLIFIER_NON_EXISTENT_READ_REQUESTS_PER_TX, ScopedReadRequest), - reader.readArray(MAX_L1_TO_L2_MSG_READ_REQUESTS_PER_TX, TreeLeafReadRequest), - reader.readArray(MAX_PUBLIC_DATA_READS_PER_TX, PublicDataRead), - ); - } - - /** - * Deserializes from a string, corresponding to a write in cpp. - * @param str - String to read from. - * @returns Deserialized object. - */ - static fromString(str: string) { - return PublicValidationRequests.fromBuffer(Buffer.from(str, 'hex')); - } - - static empty() { - return new PublicValidationRequests( - RollupValidationRequests.empty(), - makeTuple(MAX_NOTE_HASH_READ_REQUESTS_PER_TX, TreeLeafReadRequest.empty), - makeTuple(MAX_NULLIFIER_READ_REQUESTS_PER_TX, ScopedReadRequest.empty), - makeTuple(MAX_NULLIFIER_NON_EXISTENT_READ_REQUESTS_PER_TX, ScopedReadRequest.empty), - makeTuple(MAX_L1_TO_L2_MSG_READ_REQUESTS_PER_TX, TreeLeafReadRequest.empty), - makeTuple(MAX_PUBLIC_DATA_READS_PER_TX, PublicDataRead.empty), - ); - } - - [inspect.custom]() { - return `PublicValidationRequests { - forRollup: ${inspect(this.forRollup)}, - noteHashReadRequests: [${this.noteHashReadRequests - .filter(x => !x.isEmpty()) - .map(h => inspect(h)) - .join(', ')}], - nullifierReadRequests: [${this.nullifierReadRequests - .filter(x => !x.isEmpty()) - .map(h => inspect(h)) - .join(', ')}], - nullifierNonExistentReadRequests: [${this.nullifierNonExistentReadRequests - .filter(x => !x.isEmpty()) - .map(h => inspect(h)) - .join(', ')}], - l1ToL2MsgReadRequests: [${this.l1ToL2MsgReadRequests - .filter(x => !x.isEmpty()) - .map(h => inspect(h)) - .join(', ')}], - publicDataReads: [${this.publicDataReads - .filter(x => !x.isEmpty()) - .map(h => inspect(h)) - .join(', ')}] -}`; - } -} - -export class PublicValidationRequestArrayLengths { - constructor( - public noteHashReadRequests: number, - public nullifierReadRequests: number, - public nullifierNonExistentReadRequests: number, - public l1ToL2MsgReadRequests: number, - public publicDataReads: number, - ) {} - - static new(requests: PublicValidationRequests) { - return new PublicValidationRequestArrayLengths( - countAccumulatedItems(requests.noteHashReadRequests), - countAccumulatedItems(requests.nullifierReadRequests), - countAccumulatedItems(requests.nullifierNonExistentReadRequests), - countAccumulatedItems(requests.l1ToL2MsgReadRequests), - countAccumulatedItems(requests.publicDataReads), - ); - } - - getSize() { - return NUM_PUBLIC_VALIDATION_REQUEST_ARRAYS; - } - - toBuffer() { - return serializeToBuffer( - this.noteHashReadRequests, - this.nullifierReadRequests, - this.nullifierNonExistentReadRequests, - this.l1ToL2MsgReadRequests, - this.publicDataReads, - ); - } - - toString() { - return this.toBuffer().toString('hex'); - } - - static fromFields(fields: Fr[] | FieldReader) { - const reader = FieldReader.asReader(fields); - return new PublicValidationRequestArrayLengths( - reader.readU32(), - reader.readU32(), - reader.readU32(), - reader.readU32(), - reader.readU32(), - ); - } - - /** - * Deserializes from a buffer or reader, corresponding to a write in cpp. - * @param buffer - Buffer or reader to read from. - * @returns Deserialized object. - */ - static fromBuffer(buffer: Buffer | BufferReader) { - const reader = BufferReader.asReader(buffer); - return new PublicValidationRequestArrayLengths( - reader.readNumber(), - reader.readNumber(), - reader.readNumber(), - reader.readNumber(), - reader.readNumber(), - ); - } - - /** - * Deserializes from a string, corresponding to a write in cpp. - * @param str - String to read from. - * @returns Deserialized object. - */ - static fromString(str: string) { - return PublicValidationRequestArrayLengths.fromBuffer(Buffer.from(str, 'hex')); - } - - static empty() { - return new PublicValidationRequestArrayLengths(0, 0, 0, 0, 0); - } - - [inspect.custom]() { - return `PublicValidationRequestArrayLengths { - noteHashReadRequests: ${this.noteHashReadRequests}, - nullifierReadRequests: ${this.nullifierReadRequests}, - nullifierNonExistentReadRequests: ${this.nullifierNonExistentReadRequests}, - l1ToL2MsgReadRequests: ${this.l1ToL2MsgReadRequests}, - publicDataReads: ${this.publicDataReads} -}`; - } -} diff --git a/yarn-project/circuits.js/src/structs/rollup/base_rollup_hints.ts b/yarn-project/circuits.js/src/structs/rollup/base_rollup_hints.ts index 15364ffb636..d9176c82f6d 100644 --- a/yarn-project/circuits.js/src/structs/rollup/base_rollup_hints.ts +++ b/yarn-project/circuits.js/src/structs/rollup/base_rollup_hints.ts @@ -1,53 +1,98 @@ -import { makeTuple } from '@aztec/foundation/array'; -import { BufferReader, type Tuple, serializeToBuffer } from '@aztec/foundation/serialize'; +import { BufferReader, serializeToBuffer } from '@aztec/foundation/serialize'; import { type FieldsOf } from '@aztec/foundation/types'; -import { - ARCHIVE_HEIGHT, - MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, - PUBLIC_DATA_TREE_HEIGHT, -} from '../../constants.gen.js'; +import { ARCHIVE_HEIGHT } from '../../constants.gen.js'; import { MembershipWitness } from '../membership_witness.js'; import { PartialStateReference } from '../partial_state_reference.js'; import { PublicDataHint } from '../public_data_hint.js'; -import { type UInt32 } from '../shared.js'; -import { PublicDataTreeLeaf, PublicDataTreeLeafPreimage } from '../trees/index.js'; import { ConstantRollupData } from './constant_rollup_data.js'; -import { StateDiffHints } from './state_diff_hints.js'; +import { PrivateBaseStateDiffHints, PublicBaseStateDiffHints } from './state_diff_hints.js'; -export class BaseRollupHints { +export type BaseRollupHints = PrivateBaseRollupHints | PublicBaseRollupHints; + +export class PrivateBaseRollupHints { constructor( /** Partial state reference at the start of the rollup. */ public start: PartialStateReference, /** Hints used while proving state diff validity. */ - public stateDiffHints: StateDiffHints, + public stateDiffHints: PrivateBaseStateDiffHints, /** Public data read hint for accessing the balance of the fee payer. */ public feePayerFeeJuiceBalanceReadHint: PublicDataHint, - /** - * The public data writes to be inserted in the tree, sorted high slot to low slot. - */ - public sortedPublicDataWrites: Tuple, /** - * The indexes of the sorted public data writes to the original ones. - */ - public sortedPublicDataWritesIndexes: Tuple, - /** - * The public data writes which need to be updated to perform the batch insertion of the new public data writes. - * See `StandardIndexedTree.batchInsert` function for more details. + * Membership witnesses of blocks referred by each of the 2 kernels. */ - public lowPublicDataWritesPreimages: Tuple< - PublicDataTreeLeafPreimage, - typeof MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX - >, + public archiveRootMembershipWitness: MembershipWitness, /** - * Membership witnesses for the nullifiers which need to be updated to perform the batch insertion of the new - * nullifiers. + * Data which is not modified by the base rollup circuit. */ - public lowPublicDataWritesMembershipWitnesses: Tuple< - MembershipWitness, - typeof MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX - >, + public constants: ConstantRollupData, + ) {} + + static from(fields: FieldsOf): PrivateBaseRollupHints { + return new PrivateBaseRollupHints(...PrivateBaseRollupHints.getFields(fields)); + } + + static getFields(fields: FieldsOf) { + return [ + fields.start, + fields.stateDiffHints, + fields.feePayerFeeJuiceBalanceReadHint, + fields.archiveRootMembershipWitness, + fields.constants, + ] as const; + } + + /** + * Serializes the inputs to a buffer. + * @returns The inputs serialized to a buffer. + */ + toBuffer() { + return serializeToBuffer(...PrivateBaseRollupHints.getFields(this)); + } + + /** + * Serializes the inputs to a hex string. + * @returns The instance serialized to a hex string. + */ + toString() { + return this.toBuffer().toString('hex'); + } + + static fromBuffer(buffer: Buffer | BufferReader): PrivateBaseRollupHints { + const reader = BufferReader.asReader(buffer); + return new PrivateBaseRollupHints( + reader.readObject(PartialStateReference), + reader.readObject(PrivateBaseStateDiffHints), + reader.readObject(PublicDataHint), + MembershipWitness.fromBuffer(reader, ARCHIVE_HEIGHT), + reader.readObject(ConstantRollupData), + ); + } + + static fromString(str: string) { + return PrivateBaseRollupHints.fromBuffer(Buffer.from(str, 'hex')); + } + + static empty() { + return new PrivateBaseRollupHints( + PartialStateReference.empty(), + PrivateBaseStateDiffHints.empty(), + PublicDataHint.empty(), + MembershipWitness.empty(ARCHIVE_HEIGHT), + ConstantRollupData.empty(), + ); + } +} + +export class PublicBaseRollupHints { + constructor( + /** Partial state reference at the start of the rollup. */ + public start: PartialStateReference, + /** Hints used while proving state diff validity. */ + public stateDiffHints: PublicBaseStateDiffHints, + /** Public data read hint for accessing the balance of the fee payer. */ + public feePayerFeeJuiceBalanceReadHint: PublicDataHint, /** * Membership witnesses of blocks referred by each of the 2 kernels. */ @@ -58,19 +103,15 @@ export class BaseRollupHints { public constants: ConstantRollupData, ) {} - static from(fields: FieldsOf): BaseRollupHints { - return new BaseRollupHints(...BaseRollupHints.getFields(fields)); + static from(fields: FieldsOf): PublicBaseRollupHints { + return new PublicBaseRollupHints(...PublicBaseRollupHints.getFields(fields)); } - static getFields(fields: FieldsOf) { + static getFields(fields: FieldsOf) { return [ fields.start, fields.stateDiffHints, fields.feePayerFeeJuiceBalanceReadHint, - fields.sortedPublicDataWrites, - fields.sortedPublicDataWritesIndexes, - fields.lowPublicDataWritesPreimages, - fields.lowPublicDataWritesMembershipWitnesses, fields.archiveRootMembershipWitness, fields.constants, ] as const; @@ -81,7 +122,7 @@ export class BaseRollupHints { * @returns The inputs serialized to a buffer. */ toBuffer() { - return serializeToBuffer(...BaseRollupHints.getFields(this)); + return serializeToBuffer(...PublicBaseRollupHints.getFields(this)); } /** @@ -92,36 +133,26 @@ export class BaseRollupHints { return this.toBuffer().toString('hex'); } - static fromBuffer(buffer: Buffer | BufferReader): BaseRollupHints { + static fromBuffer(buffer: Buffer | BufferReader): PublicBaseRollupHints { const reader = BufferReader.asReader(buffer); - return new BaseRollupHints( + return new PublicBaseRollupHints( reader.readObject(PartialStateReference), - reader.readObject(StateDiffHints), + reader.readObject(PublicBaseStateDiffHints), reader.readObject(PublicDataHint), - reader.readArray(MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, PublicDataTreeLeaf), - reader.readNumbers(MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX), - reader.readArray(MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, PublicDataTreeLeafPreimage), - reader.readArray(MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, { - fromBuffer: buffer => MembershipWitness.fromBuffer(buffer, PUBLIC_DATA_TREE_HEIGHT), - }), MembershipWitness.fromBuffer(reader, ARCHIVE_HEIGHT), reader.readObject(ConstantRollupData), ); } static fromString(str: string) { - return BaseRollupHints.fromBuffer(Buffer.from(str, 'hex')); + return PublicBaseRollupHints.fromBuffer(Buffer.from(str, 'hex')); } static empty() { - return new BaseRollupHints( + return new PublicBaseRollupHints( PartialStateReference.empty(), - StateDiffHints.empty(), + PublicBaseStateDiffHints.empty(), PublicDataHint.empty(), - makeTuple(MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, PublicDataTreeLeaf.empty), - makeTuple(MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, () => 0), - makeTuple(MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, PublicDataTreeLeafPreimage.empty), - makeTuple(MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, () => MembershipWitness.empty(PUBLIC_DATA_TREE_HEIGHT)), MembershipWitness.empty(ARCHIVE_HEIGHT), ConstantRollupData.empty(), ); diff --git a/yarn-project/circuits.js/src/structs/rollup/private_base_rollup_inputs.ts b/yarn-project/circuits.js/src/structs/rollup/private_base_rollup_inputs.ts index 5cc93fdeb4e..1d354861e1a 100644 --- a/yarn-project/circuits.js/src/structs/rollup/private_base_rollup_inputs.ts +++ b/yarn-project/circuits.js/src/structs/rollup/private_base_rollup_inputs.ts @@ -2,11 +2,11 @@ import { hexSchemaFor } from '@aztec/foundation/schemas'; import { BufferReader, serializeToBuffer } from '@aztec/foundation/serialize'; import { type FieldsOf } from '@aztec/foundation/types'; -import { BaseRollupHints } from './base_rollup_hints.js'; +import { PrivateBaseRollupHints } from './base_rollup_hints.js'; import { PrivateTubeData } from './private_tube_data.js'; export class PrivateBaseRollupInputs { - constructor(public tubeData: PrivateTubeData, public hints: BaseRollupHints) {} + constructor(public tubeData: PrivateTubeData, public hints: PrivateBaseRollupHints) {} static from(fields: FieldsOf): PrivateBaseRollupInputs { return new PrivateBaseRollupInputs(...PrivateBaseRollupInputs.getFields(fields)); @@ -18,7 +18,7 @@ export class PrivateBaseRollupInputs { static fromBuffer(buffer: Buffer | BufferReader): PrivateBaseRollupInputs { const reader = BufferReader.asReader(buffer); - return new PrivateBaseRollupInputs(reader.readObject(PrivateTubeData), reader.readObject(BaseRollupHints)); + return new PrivateBaseRollupInputs(reader.readObject(PrivateTubeData), reader.readObject(PrivateBaseRollupHints)); } toBuffer() { @@ -34,7 +34,7 @@ export class PrivateBaseRollupInputs { } static empty() { - return new PrivateBaseRollupInputs(PrivateTubeData.empty(), BaseRollupHints.empty()); + return new PrivateBaseRollupInputs(PrivateTubeData.empty(), PrivateBaseRollupHints.empty()); } /** Returns a hex representation for JSON serialization. */ diff --git a/yarn-project/circuits.js/src/structs/rollup/public_base_rollup_inputs.ts b/yarn-project/circuits.js/src/structs/rollup/public_base_rollup_inputs.ts index b8ab34a167f..1fa11ed3688 100644 --- a/yarn-project/circuits.js/src/structs/rollup/public_base_rollup_inputs.ts +++ b/yarn-project/circuits.js/src/structs/rollup/public_base_rollup_inputs.ts @@ -3,11 +3,15 @@ import { BufferReader, serializeToBuffer } from '@aztec/foundation/serialize'; import { type FieldsOf } from '@aztec/foundation/types'; import { AvmProofData } from './avm_proof_data.js'; -import { BaseRollupHints } from './base_rollup_hints.js'; +import { PublicBaseRollupHints } from './base_rollup_hints.js'; import { PublicTubeData } from './public_tube_data.js'; export class PublicBaseRollupInputs { - constructor(public tubeData: PublicTubeData, public avmProofData: AvmProofData, public hints: BaseRollupHints) {} + constructor( + public tubeData: PublicTubeData, + public avmProofData: AvmProofData, + public hints: PublicBaseRollupHints, + ) {} static from(fields: FieldsOf): PublicBaseRollupInputs { return new PublicBaseRollupInputs(...PublicBaseRollupInputs.getFields(fields)); @@ -22,7 +26,7 @@ export class PublicBaseRollupInputs { return new PublicBaseRollupInputs( reader.readObject(PublicTubeData), reader.readObject(AvmProofData), - reader.readObject(BaseRollupHints), + reader.readObject(PublicBaseRollupHints), ); } @@ -38,7 +42,7 @@ export class PublicBaseRollupInputs { } static empty() { - return new PublicBaseRollupInputs(PublicTubeData.empty(), AvmProofData.empty(), BaseRollupHints.empty()); + return new PublicBaseRollupInputs(PublicTubeData.empty(), AvmProofData.empty(), PublicBaseRollupHints.empty()); } /** Returns a hex representation for JSON serialization. */ diff --git a/yarn-project/circuits.js/src/structs/rollup/state_diff_hints.test.ts b/yarn-project/circuits.js/src/structs/rollup/state_diff_hints.test.ts index b3c9a9f0ac6..a473a1375fa 100644 --- a/yarn-project/circuits.js/src/structs/rollup/state_diff_hints.test.ts +++ b/yarn-project/circuits.js/src/structs/rollup/state_diff_hints.test.ts @@ -1,11 +1,18 @@ -import { makeStateDiffHints } from '../../tests/factories.js'; -import { StateDiffHints } from './state_diff_hints.js'; +import { makePrivateBaseStateDiffHints, makePublicBaseStateDiffHints } from '../../tests/factories.js'; +import { PrivateBaseStateDiffHints, PublicBaseStateDiffHints } from './state_diff_hints.js'; describe('StateDiffHints', () => { - it('serializes to buffer and deserializes it back', () => { - const expected = makeStateDiffHints(); + it('serializes private hints to buffer and deserializes it back', () => { + const expected = makePrivateBaseStateDiffHints(); const buffer = expected.toBuffer(); - const res = StateDiffHints.fromBuffer(buffer); + const res = PrivateBaseStateDiffHints.fromBuffer(buffer); + expect(res).toEqual(expected); + }); + + it('serializes public hints to buffer and deserializes it back', () => { + const expected = makePublicBaseStateDiffHints(); + const buffer = expected.toBuffer(); + const res = PublicBaseStateDiffHints.fromBuffer(buffer); expect(res).toEqual(expected); }); }); diff --git a/yarn-project/circuits.js/src/structs/rollup/state_diff_hints.ts b/yarn-project/circuits.js/src/structs/rollup/state_diff_hints.ts index 288f5c5d85b..f1b2a5e986b 100644 --- a/yarn-project/circuits.js/src/structs/rollup/state_diff_hints.ts +++ b/yarn-project/circuits.js/src/structs/rollup/state_diff_hints.ts @@ -5,18 +5,19 @@ import { type FieldsOf } from '@aztec/foundation/types'; import { MAX_NULLIFIERS_PER_TX, + MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, NOTE_HASH_SUBTREE_SIBLING_PATH_LENGTH, NULLIFIER_SUBTREE_SIBLING_PATH_LENGTH, NULLIFIER_TREE_HEIGHT, - PUBLIC_DATA_SUBTREE_SIBLING_PATH_LENGTH, + PUBLIC_DATA_TREE_HEIGHT, } from '../../constants.gen.js'; import { MembershipWitness } from '../membership_witness.js'; -import { NullifierLeafPreimage } from '../trees/index.js'; +import { NullifierLeafPreimage, PublicDataTreeLeafPreimage } from '../trees/index.js'; /** - * Hints used while proving state diff validity. + * Hints used while proving state diff validity for the private base rollup. */ -export class StateDiffHints { +export class PrivateBaseStateDiffHints { constructor( /** * The nullifiers which need to be updated to perform the batch insertion of the new nullifiers. @@ -47,17 +48,26 @@ export class StateDiffHints { * Sibling path "pointing to" where the new nullifiers subtree should be inserted into the nullifier tree. */ public nullifierSubtreeSiblingPath: Tuple, + + /** + * Low leaf for the fee write in the public data tree. + */ + public feeWriteLowLeafPreimage: PublicDataTreeLeafPreimage, /** - * Sibling path "pointing to" where the new public data subtree should be inserted into the public data tree. + * Membership witness for the low leaf for the fee write in the public data tree. */ - public publicDataSiblingPath: Tuple, + public feeWriteLowLeafMembershipWitness: MembershipWitness, + /** + * Sibling path "pointing to" where the fee write should be inserted into the public data tree. + */ + public feeWriteSiblingPath: Tuple, ) {} - static from(fields: FieldsOf): StateDiffHints { - return new StateDiffHints(...StateDiffHints.getFields(fields)); + static from(fields: FieldsOf): PrivateBaseStateDiffHints { + return new PrivateBaseStateDiffHints(...PrivateBaseStateDiffHints.getFields(fields)); } - static getFields(fields: FieldsOf) { + static getFields(fields: FieldsOf) { return [ fields.nullifierPredecessorPreimages, fields.nullifierPredecessorMembershipWitnesses, @@ -65,7 +75,9 @@ export class StateDiffHints { fields.sortedNullifierIndexes, fields.noteHashSubtreeSiblingPath, fields.nullifierSubtreeSiblingPath, - fields.publicDataSiblingPath, + fields.feeWriteLowLeafPreimage, + fields.feeWriteLowLeafMembershipWitness, + fields.feeWriteSiblingPath, ] as const; } @@ -74,17 +86,17 @@ export class StateDiffHints { * @returns A buffer of the serialized state diff hints. */ toBuffer(): Buffer { - return serializeToBuffer(...StateDiffHints.getFields(this)); + return serializeToBuffer(...PrivateBaseStateDiffHints.getFields(this)); } /** * Deserializes the state diff hints from a buffer. * @param buffer - A buffer to deserialize from. - * @returns A new StateDiffHints instance. + * @returns A new PrivateBaseStateDiffHints instance. */ - static fromBuffer(buffer: Buffer | BufferReader): StateDiffHints { + static fromBuffer(buffer: Buffer | BufferReader): PrivateBaseStateDiffHints { const reader = BufferReader.asReader(buffer); - return new StateDiffHints( + return new PrivateBaseStateDiffHints( reader.readArray(MAX_NULLIFIERS_PER_TX, NullifierLeafPreimage), reader.readArray(MAX_NULLIFIERS_PER_TX, { fromBuffer: buffer => MembershipWitness.fromBuffer(buffer, NULLIFIER_TREE_HEIGHT), @@ -93,19 +105,149 @@ export class StateDiffHints { reader.readNumbers(MAX_NULLIFIERS_PER_TX), reader.readArray(NOTE_HASH_SUBTREE_SIBLING_PATH_LENGTH, Fr), reader.readArray(NULLIFIER_SUBTREE_SIBLING_PATH_LENGTH, Fr), - reader.readArray(PUBLIC_DATA_SUBTREE_SIBLING_PATH_LENGTH, Fr), + reader.readObject(PublicDataTreeLeafPreimage), + MembershipWitness.fromBuffer(reader, PUBLIC_DATA_TREE_HEIGHT), + reader.readArray(PUBLIC_DATA_TREE_HEIGHT, Fr), + ); + } + + static empty() { + return new PrivateBaseStateDiffHints( + makeTuple(MAX_NULLIFIERS_PER_TX, NullifierLeafPreimage.empty), + makeTuple(MAX_NULLIFIERS_PER_TX, () => MembershipWitness.empty(NULLIFIER_TREE_HEIGHT)), + makeTuple(MAX_NULLIFIERS_PER_TX, Fr.zero), + makeTuple(MAX_NULLIFIERS_PER_TX, () => 0), + makeTuple(NOTE_HASH_SUBTREE_SIBLING_PATH_LENGTH, Fr.zero), + makeTuple(NULLIFIER_SUBTREE_SIBLING_PATH_LENGTH, Fr.zero), + PublicDataTreeLeafPreimage.empty(), + MembershipWitness.empty(PUBLIC_DATA_TREE_HEIGHT), + makeTuple(PUBLIC_DATA_TREE_HEIGHT, Fr.zero), + ); + } +} + +export class PublicBaseStateDiffHints { + constructor( + /** + * The nullifiers which need to be updated to perform the batch insertion of the new nullifiers. + * See `StandardIndexedTree.batchInsert` function for more details. + */ + public nullifierPredecessorPreimages: Tuple, + /** + * Membership witnesses for the nullifiers which need to be updated to perform the batch insertion of the new + * nullifiers. + */ + public nullifierPredecessorMembershipWitnesses: Tuple< + MembershipWitness, + typeof MAX_NULLIFIERS_PER_TX + >, + /** + * The nullifiers to be inserted in the tree, sorted high to low. + */ + public sortedNullifiers: Tuple, + /** + * The indexes of the sorted nullifiers to the original ones. + */ + public sortedNullifierIndexes: Tuple, + /** + * Sibling path "pointing to" where the new note hash subtree should be inserted into the note hash tree. + */ + public noteHashSubtreeSiblingPath: Tuple, + /** + * Sibling path "pointing to" where the new nullifiers subtree should be inserted into the nullifier tree. + */ + public nullifierSubtreeSiblingPath: Tuple, + + /** + * Preimages of the low leaves for the public data writes + */ + public lowPublicDataWritesPreimages: Tuple< + PublicDataTreeLeafPreimage, + typeof MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX + >, + + /** + * Membership witnesses for the low leaves for the public data writes + */ + public lowPublicDataWritesMembershipWitnesses: Tuple< + MembershipWitness, + typeof MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX + >, + + /** + * Sibling paths "pointing to" where the new public data writes should be inserted (one by one) into the public data tree. + */ + public publicDataTreeSiblingPaths: Tuple< + Tuple, + typeof MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX + >, + ) {} + + static from(fields: FieldsOf): PublicBaseStateDiffHints { + return new PublicBaseStateDiffHints(...PublicBaseStateDiffHints.getFields(fields)); + } + + static getFields(fields: FieldsOf) { + return [ + fields.nullifierPredecessorPreimages, + fields.nullifierPredecessorMembershipWitnesses, + fields.sortedNullifiers, + fields.sortedNullifierIndexes, + fields.noteHashSubtreeSiblingPath, + fields.nullifierSubtreeSiblingPath, + fields.lowPublicDataWritesPreimages, + fields.lowPublicDataWritesMembershipWitnesses, + fields.publicDataTreeSiblingPaths, + ] as const; + } + + /** + * Serializes the state diff hints to a buffer. + * @returns A buffer of the serialized state diff hints. + */ + toBuffer(): Buffer { + return serializeToBuffer(...PublicBaseStateDiffHints.getFields(this)); + } + + /** + * Deserializes the state diff hints from a buffer. + * @param buffer - A buffer to deserialize from. + * @returns A new PublicBaseStateDiffHints instance. + */ + static fromBuffer(buffer: Buffer | BufferReader): PublicBaseStateDiffHints { + const reader = BufferReader.asReader(buffer); + return new PublicBaseStateDiffHints( + reader.readArray(MAX_NULLIFIERS_PER_TX, NullifierLeafPreimage), + reader.readArray(MAX_NULLIFIERS_PER_TX, { + fromBuffer: buffer => MembershipWitness.fromBuffer(buffer, NULLIFIER_TREE_HEIGHT), + }), + reader.readArray(MAX_NULLIFIERS_PER_TX, Fr), + reader.readNumbers(MAX_NULLIFIERS_PER_TX), + reader.readArray(NOTE_HASH_SUBTREE_SIBLING_PATH_LENGTH, Fr), + reader.readArray(NULLIFIER_SUBTREE_SIBLING_PATH_LENGTH, Fr), + reader.readArray(MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, PublicDataTreeLeafPreimage), + reader.readArray(MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, { + fromBuffer: buffer => MembershipWitness.fromBuffer(buffer, PUBLIC_DATA_TREE_HEIGHT), + }), + reader.readArray(MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, { + fromBuffer(reader) { + return BufferReader.asReader(reader).readArray(PUBLIC_DATA_TREE_HEIGHT, Fr); + }, + }), ); } static empty() { - return new StateDiffHints( + return new PublicBaseStateDiffHints( makeTuple(MAX_NULLIFIERS_PER_TX, NullifierLeafPreimage.empty), makeTuple(MAX_NULLIFIERS_PER_TX, () => MembershipWitness.empty(NULLIFIER_TREE_HEIGHT)), makeTuple(MAX_NULLIFIERS_PER_TX, Fr.zero), makeTuple(MAX_NULLIFIERS_PER_TX, () => 0), makeTuple(NOTE_HASH_SUBTREE_SIBLING_PATH_LENGTH, Fr.zero), makeTuple(NULLIFIER_SUBTREE_SIBLING_PATH_LENGTH, Fr.zero), - makeTuple(PUBLIC_DATA_SUBTREE_SIBLING_PATH_LENGTH, Fr.zero), + makeTuple(MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, PublicDataTreeLeafPreimage.empty), + makeTuple(MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, () => MembershipWitness.empty(PUBLIC_DATA_TREE_HEIGHT)), + makeTuple(MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, () => makeTuple(PUBLIC_DATA_TREE_HEIGHT, Fr.zero)), ); } } diff --git a/yarn-project/circuits.js/src/structs/tree_leaf_read_request_hint.ts b/yarn-project/circuits.js/src/structs/tree_leaf_read_request_hint.ts deleted file mode 100644 index e6b64deb6cb..00000000000 --- a/yarn-project/circuits.js/src/structs/tree_leaf_read_request_hint.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { assertMemberLength } from '@aztec/foundation/array'; -import { Fr } from '@aztec/foundation/fields'; -import { BufferReader, type Tuple, serializeToBuffer } from '@aztec/foundation/serialize'; - -/** - * Contains information which can be used to prove that a leaf is a member of a Merkle tree. - */ -export class TreeLeafReadRequestHint { - constructor( - /** - * Size of the sibling path (number of fields it contains). - */ - pathSize: N, - /** - * Sibling path of the leaf in the Merkle tree. - */ - public siblingPath: Tuple, - ) { - assertMemberLength(this, 'siblingPath', pathSize); - } - - toBuffer() { - return serializeToBuffer(this.siblingPath); - } - - public static empty(pathSize: N): TreeLeafReadRequestHint { - const arr = Array(pathSize) - .fill(0) - .map(() => Fr.ZERO) as Tuple; - return new TreeLeafReadRequestHint(pathSize, arr); - } - - static fromBuffer(buffer: Buffer | BufferReader, size: N): TreeLeafReadRequestHint { - const reader = BufferReader.asReader(buffer); - const siblingPath = reader.readArray(size, Fr); - return new TreeLeafReadRequestHint(size, siblingPath); - } -} diff --git a/yarn-project/circuits.js/src/tests/factories.ts b/yarn-project/circuits.js/src/tests/factories.ts index 9c448588c3b..3a2da20f3e9 100644 --- a/yarn-project/circuits.js/src/tests/factories.ts +++ b/yarn-project/circuits.js/src/tests/factories.ts @@ -43,7 +43,6 @@ import { KeyValidationRequest, KeyValidationRequestAndGenerator, L1_TO_L2_MSG_SUBTREE_SIBLING_PATH_LENGTH, - L1_TO_L2_MSG_TREE_HEIGHT, L2ToL1Message, LogHash, MAX_CONTRACT_CLASS_LOGS_PER_TX, @@ -52,8 +51,6 @@ import { MAX_ENQUEUED_CALLS_PER_CALL, MAX_ENQUEUED_CALLS_PER_TX, MAX_KEY_VALIDATION_REQUESTS_PER_CALL, - MAX_L1_TO_L2_MSG_READ_REQUESTS_PER_CALL, - MAX_L1_TO_L2_MSG_READ_REQUESTS_PER_TX, MAX_L2_TO_L1_MSGS_PER_CALL, MAX_L2_TO_L1_MSGS_PER_TX, MAX_NOTE_ENCRYPTED_LOGS_PER_CALL, @@ -61,28 +58,18 @@ import { MAX_NOTE_HASHES_PER_CALL, MAX_NOTE_HASHES_PER_TX, MAX_NOTE_HASH_READ_REQUESTS_PER_CALL, - MAX_NOTE_HASH_READ_REQUESTS_PER_TX, MAX_NULLIFIERS_PER_CALL, MAX_NULLIFIERS_PER_TX, - MAX_NULLIFIER_NON_EXISTENT_READ_REQUESTS_PER_CALL, - MAX_NULLIFIER_NON_EXISTENT_READ_REQUESTS_PER_TX, MAX_NULLIFIER_READ_REQUESTS_PER_CALL, - MAX_NULLIFIER_READ_REQUESTS_PER_TX, MAX_PRIVATE_CALL_STACK_LENGTH_PER_CALL, - MAX_PUBLIC_DATA_HINTS, - MAX_PUBLIC_DATA_READS_PER_CALL, - MAX_PUBLIC_DATA_READS_PER_TX, - MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_CALL, MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, - MAX_UNENCRYPTED_LOGS_PER_CALL, MAX_UNENCRYPTED_LOGS_PER_TX, MaxBlockNumber, MembershipWitness, MergeRollupInputs, NESTED_RECURSIVE_PROOF_LENGTH, NOTE_HASH_SUBTREE_SIBLING_PATH_LENGTH, - NOTE_HASH_TREE_HEIGHT, NULLIFIER_SUBTREE_SIBLING_PATH_LENGTH, NULLIFIER_TREE_HEIGHT, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, @@ -92,9 +79,6 @@ import { NoteLogHash, Nullifier, NullifierLeafPreimage, - NullifierNonExistentReadRequestHintsBuilder, - NullifierReadRequestHintsBuilder, - PUBLIC_DATA_SUBTREE_SIBLING_PATH_LENGTH, PUBLIC_DATA_TREE_HEIGHT, ParityPublicInputs, PartialPrivateTailPublicInputsForPublic, @@ -106,18 +90,12 @@ import { PrivateCircuitPublicInputs, PrivateKernelTailCircuitPublicInputs, Proof, - PublicAccumulatedData, PublicCallRequest, - PublicCallStackItemCompressed, PublicCircuitPublicInputs, PublicDataHint, PublicDataRead, PublicDataTreeLeaf, PublicDataTreeLeafPreimage, - PublicDataUpdateRequest, - PublicKernelCircuitPublicInputs, - PublicKernelData, - PublicKernelTailCircuitPrivateInputs, PublicKeys, RECURSIVE_PROOF_LENGTH, ReadRequest, @@ -128,8 +106,6 @@ import { RootRollupInputs, RootRollupPublicInputs, ScopedLogHash, - ScopedReadRequest, - StateDiffHints, StateReference, TUBE_PROOF_LENGTH, TxContext, @@ -161,31 +137,23 @@ import { AvmProofData, AvmPublicDataReadTreeHint, AvmPublicDataWriteTreeHint, - BaseRollupHints, CountedPublicCallRequest, - EnqueuedCallData, + PrivateBaseRollupHints, PrivateBaseRollupInputs, + PrivateBaseStateDiffHints, PrivateToAvmAccumulatedData, PrivateToAvmAccumulatedDataArrayLengths, PrivateToPublicAccumulatedData, PrivateToPublicKernelCircuitPublicInputs, PrivateTubeData, - PublicAccumulatedDataArrayLengths, + PublicBaseRollupHints, PublicBaseRollupInputs, - PublicDataLeafHint, + PublicBaseStateDiffHints, PublicDataWrite, - PublicInnerCallRequest, - PublicKernelCircuitPrivateInputs, PublicTubeData, - PublicValidationRequestArrayLengths, - PublicValidationRequests, ScopedL2ToL1Message, - ScopedNoteHash, - TreeLeafReadRequest, - TreeLeafReadRequestHint, TreeSnapshots, TxConstantData, - VMCircuitPublicInputs, VkWitnessData, } from '../structs/index.js'; import { KernelCircuitPublicInputs } from '../structs/kernel/kernel_circuit_public_inputs.js'; @@ -224,10 +192,6 @@ function makeNoteHash(seed: number) { return new NoteHash(fr(seed), seed + 1); } -function makeScopedNoteHash(seed: number) { - return new NoteHash(fr(seed), seed + 1).scope(makeAztecAddress(seed + 3)); -} - function makeNullifier(seed: number) { return new Nullifier(fr(seed), seed + 1, fr(seed + 2)); } @@ -262,18 +226,6 @@ function makeReadRequest(n: number): ReadRequest { return new ReadRequest(new Fr(BigInt(n)), n + 1); } -function makeScopedReadRequest(n: number): ScopedReadRequest { - return new ScopedReadRequest(makeReadRequest(n), AztecAddress.fromBigInt(BigInt(n + 2))); -} - -function makeTreeLeafReadRequest(seed: number) { - return new TreeLeafReadRequest(new Fr(seed), new Fr(seed + 1)); -} - -function makeTreeLeafReadRequestHint(seed: number, size: N) { - return new TreeLeafReadRequestHint(size, makeSiblingPath(seed, size)); -} - /** * Creates arbitrary KeyValidationRequest from the given seed. * @param seed - The seed to use for generating the KeyValidationRequest. @@ -292,23 +244,6 @@ function makeKeyValidationRequestAndGenerators(seed: number): KeyValidationReque return new KeyValidationRequestAndGenerator(makeKeyValidationRequests(seed), fr(seed + 4)); } -/** - * Creates arbitrary public data update request. - * @param seed - The seed to use for generating the public data update request. - * @returns A public data update request. - */ -export function makePublicDataUpdateRequest(seed = 1): PublicDataUpdateRequest { - return new PublicDataUpdateRequest(fr(seed), fr(seed + 1), seed + 2); -} - -/** - * Creates empty public data update request. - * @returns An empty public data update request. - */ -export function makeEmptyPublicDataUpdateRequest(): PublicDataUpdateRequest { - return new PublicDataUpdateRequest(fr(0), fr(0), 0); -} - function makePublicDataWrite(seed = 1) { return new PublicDataWrite(fr(seed), fr(seed + 1)); } @@ -348,21 +283,6 @@ export function makeContractStorageRead(seed = 1): ContractStorageRead { return new ContractStorageRead(fr(seed), fr(seed + 1), seed + 2); } -function makePublicValidationRequests(seed = 1) { - return new PublicValidationRequests( - makeRollupValidationRequests(seed), - makeTuple(MAX_NOTE_HASH_READ_REQUESTS_PER_TX, makeTreeLeafReadRequest, seed + 0x10), - makeTuple(MAX_NULLIFIER_READ_REQUESTS_PER_TX, makeScopedReadRequest, seed + 0x80), - makeTuple(MAX_NULLIFIER_NON_EXISTENT_READ_REQUESTS_PER_TX, makeScopedReadRequest, seed + 0x95), - makeTuple(MAX_L1_TO_L2_MSG_READ_REQUESTS_PER_TX, makeTreeLeafReadRequest, seed + 0x100), - makeTuple(MAX_PUBLIC_DATA_READS_PER_TX, makePublicDataRead, seed + 0xe00), - ); -} - -function makePublicValidationRequestArrayLengths(seed = 1) { - return new PublicValidationRequestArrayLengths(seed, seed + 1, seed + 2, seed + 3, seed + 4); -} - export function makeRollupValidationRequests(seed = 1) { return new RollupValidationRequests(new MaxBlockNumber(true, new Fr(seed + 0x31415))); } @@ -443,45 +363,6 @@ export function makeGas(seed = 1) { return new Gas(seed, seed + 1); } -/** - * Creates arbitrary accumulated data. - * @param seed - The seed to use for generating the accumulated data. - * @returns An accumulated data. - */ -function makePublicAccumulatedData(seed = 1, full = false): PublicAccumulatedData { - const tupleGenerator = full ? makeTuple : makeHalfFullTuple; - - return new PublicAccumulatedData( - tupleGenerator(MAX_NOTE_HASHES_PER_TX, makeScopedNoteHash, seed + 0x120, ScopedNoteHash.empty), - tupleGenerator(MAX_NULLIFIERS_PER_TX, makeNullifier, seed + 0x200, Nullifier.empty), - tupleGenerator(MAX_L2_TO_L1_MSGS_PER_TX, makeScopedL2ToL1Message, seed + 0x600, ScopedL2ToL1Message.empty), - tupleGenerator(MAX_NOTE_ENCRYPTED_LOGS_PER_TX, makeLogHash, seed + 0x700, LogHash.empty), // note encrypted logs hashes - tupleGenerator(MAX_ENCRYPTED_LOGS_PER_TX, makeScopedLogHash, seed + 0x800, ScopedLogHash.empty), // encrypted logs hashes - tupleGenerator(MAX_UNENCRYPTED_LOGS_PER_TX, makeScopedLogHash, seed + 0x900, ScopedLogHash.empty), // unencrypted logs hashes - tupleGenerator( - MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, - makePublicDataUpdateRequest, - seed + 0xd00, - PublicDataUpdateRequest.empty, - ), - tupleGenerator(MAX_ENQUEUED_CALLS_PER_TX, makePublicCallRequest, seed + 0x500, PublicCallRequest.empty), - makeGas(seed + 0x600), - ); -} - -function makePublicAccumulatedDataArrayLengths(seed = 1) { - return new PublicAccumulatedDataArrayLengths( - seed, - seed + 1, - seed + 2, - seed + 3, - seed + 4, - seed + 5, - seed + 6, - seed + 7, - ); -} - /** * Creates arbitrary call context. * @param seed - The seed to use for generating the call context. @@ -497,82 +378,6 @@ export function makeCallContext(seed = 0, overrides: Partial(NESTED_RECURSIVE_PROOF_LENGTH, seed + 0x80), - VerificationKeyData.makeFakeHonk(), - 0x42, - makeTuple(VK_TREE_HEIGHT, fr, 0x1000), - ); -} - /** * Makes arbitrary proof. * @param seed - The seed to use for generating/mocking the proof. @@ -746,19 +518,6 @@ function makePrivateCallRequest(seed = 1): PrivateCallRequest { return new PrivateCallRequest(makeCallContext(seed + 0x1), fr(seed + 0x3), fr(seed + 0x4), seed + 0x10, seed + 0x11); } -function makePublicCallStackItemCompressed(seed = 1): PublicCallStackItemCompressed { - const callContext = makeCallContext(seed); - return new PublicCallStackItemCompressed( - callContext.contractAddress, - callContext, - fr(seed + 0x20), - fr(seed + 0x30), - RevertCode.OK, - makeGas(seed + 0x40), - makeGas(seed + 0x50), - ); -} - export function makePublicCallRequest(seed = 1) { return new PublicCallRequest( makeAztecAddress(seed), @@ -773,48 +532,6 @@ function makeCountedPublicCallRequest(seed = 1) { return new CountedPublicCallRequest(makePublicCallRequest(seed), seed + 0x100); } -function makePublicInnerCallRequest(seed = 1): PublicInnerCallRequest { - return new PublicInnerCallRequest(makePublicCallStackItemCompressed(seed), seed + 0x60); -} - -function makeEnqueuedCallData(seed = 1) { - return new EnqueuedCallData(makeVMCircuitPublicInputs(seed), makeProof()); -} - -/** - * Makes arbitrary public kernel inputs. - * @param seed - The seed to use for generating the public kernel inputs. - * @returns Public kernel inputs. - */ -export function makePublicKernelCircuitPrivateInputs(seed = 1): PublicKernelCircuitPrivateInputs { - return new PublicKernelCircuitPrivateInputs(makePublicKernelData(seed), makeEnqueuedCallData(seed + 0x1000)); -} - -/** - * Makes arbitrary public kernel tail inputs. - * @param seed - The seed to use for generating the public kernel inputs. - * @returns Public kernel inputs. - */ -export function makePublicKernelTailCircuitPrivateInputs(seed = 1): PublicKernelTailCircuitPrivateInputs { - return new PublicKernelTailCircuitPrivateInputs( - makePublicKernelData(seed), - makeTuple( - MAX_NOTE_HASH_READ_REQUESTS_PER_TX, - s => makeTreeLeafReadRequestHint(s, NOTE_HASH_TREE_HEIGHT), - seed + 0x20, - ), - NullifierReadRequestHintsBuilder.empty(MAX_NULLIFIER_READ_REQUESTS_PER_TX, MAX_NULLIFIER_READ_REQUESTS_PER_TX), - NullifierNonExistentReadRequestHintsBuilder.empty(), - makeTuple( - MAX_L1_TO_L2_MSG_READ_REQUESTS_PER_TX, - s => makeTreeLeafReadRequestHint(s, L1_TO_L2_MSG_TREE_HEIGHT), - seed + 0x80, - ), - makeTuple(MAX_PUBLIC_DATA_HINTS, PublicDataLeafHint.empty), - makePartialStateReference(seed + 0x200), - ); -} - /** * Makes arbitrary tx request. * @param seed - The seed to use for generating the tx request. @@ -1260,11 +977,11 @@ export function makePublicDataTreeLeafPreimage(seed = 0): PublicDataTreeLeafPrei } /** - * Creates an instance of StateDiffHints with arbitrary values based on the provided seed. + * Creates an instance of PrivateBaseStateDiffHints with arbitrary values based on the provided seed. * @param seed - The seed to use for generating the hints. - * @returns A StateDiffHints object. + * @returns A PrivateBaseStateDiffHints object. */ -export function makeStateDiffHints(seed = 1): StateDiffHints { +export function makePrivateBaseStateDiffHints(seed = 1): PrivateBaseStateDiffHints { const nullifierPredecessorPreimages = makeTuple( MAX_NULLIFIERS_PER_TX, x => new NullifierLeafPreimage(fr(x), fr(x + 0x100), BigInt(x + 0x200)), @@ -1285,16 +1002,77 @@ export function makeStateDiffHints(seed = 1): StateDiffHints { const nullifierSubtreeSiblingPath = makeTuple(NULLIFIER_SUBTREE_SIBLING_PATH_LENGTH, fr, seed + 0x6000); - const publicDataSiblingPath = makeTuple(PUBLIC_DATA_SUBTREE_SIBLING_PATH_LENGTH, fr, 0x8000); + const feeWriteLowLeafPreimage = makePublicDataTreeLeafPreimage(seed + 0x7000); + const feeWriteLowLeafMembershipWitness = makeMembershipWitness(PUBLIC_DATA_TREE_HEIGHT, seed + 0x8000); + const feeWriteSiblingPath = makeTuple(PUBLIC_DATA_TREE_HEIGHT, fr, seed + 0x9000); - return new StateDiffHints( + return new PrivateBaseStateDiffHints( nullifierPredecessorPreimages, nullifierPredecessorMembershipWitnesses, sortedNullifiers, sortedNullifierIndexes, noteHashSubtreeSiblingPath, nullifierSubtreeSiblingPath, - publicDataSiblingPath, + feeWriteLowLeafPreimage, + feeWriteLowLeafMembershipWitness, + feeWriteSiblingPath, + ); +} + +/** + * Creates an instance of PublicBaseStateDiffHints with arbitrary values based on the provided seed. + * @param seed - The seed to use for generating the hints. + * @returns A PublicBaseStateDiffHints object. + */ +export function makePublicBaseStateDiffHints(seed = 1): PublicBaseStateDiffHints { + const nullifierPredecessorPreimages = makeTuple( + MAX_NULLIFIERS_PER_TX, + x => new NullifierLeafPreimage(fr(x), fr(x + 0x100), BigInt(x + 0x200)), + seed + 0x1000, + ); + + const nullifierPredecessorMembershipWitnesses = makeTuple( + MAX_NULLIFIERS_PER_TX, + x => makeMembershipWitness(NULLIFIER_TREE_HEIGHT, x), + seed + 0x2000, + ); + + const sortedNullifiers = makeTuple(MAX_NULLIFIERS_PER_TX, fr, seed + 0x3000); + + const sortedNullifierIndexes = makeTuple(MAX_NULLIFIERS_PER_TX, i => i, seed + 0x4000); + + const noteHashSubtreeSiblingPath = makeTuple(NOTE_HASH_SUBTREE_SIBLING_PATH_LENGTH, fr, seed + 0x5000); + + const nullifierSubtreeSiblingPath = makeTuple(NULLIFIER_SUBTREE_SIBLING_PATH_LENGTH, fr, seed + 0x6000); + + const lowPublicDataWritesPreimages = makeTuple( + MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, + makePublicDataTreeLeafPreimage, + seed + 0x7000, + ); + + const lowPublicDataWritesMembershipWitnesses = makeTuple( + MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, + i => makeMembershipWitness(PUBLIC_DATA_TREE_HEIGHT, i), + seed + 0x8000, + ); + + const publicDataTreeSiblingPaths = makeTuple( + MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, + i => makeTuple(PUBLIC_DATA_TREE_HEIGHT, fr, i), + seed + 0x9000, + ); + + return new PublicBaseStateDiffHints( + nullifierPredecessorPreimages, + nullifierPredecessorMembershipWitnesses, + sortedNullifiers, + sortedNullifierIndexes, + noteHashSubtreeSiblingPath, + nullifierSubtreeSiblingPath, + lowPublicDataWritesPreimages, + lowPublicDataWritesMembershipWitnesses, + publicDataTreeSiblingPaths, ); } @@ -1310,30 +1088,30 @@ function makePrivateTubeData(seed = 1, kernelPublicInputs?: KernelCircuitPublicI ); } -function makeBaseRollupHints(seed = 1) { +function makePrivateBaseRollupHints(seed = 1) { const start = makePartialStateReference(seed + 0x100); - const stateDiffHints = makeStateDiffHints(seed + 0x600); + const stateDiffHints = makePrivateBaseStateDiffHints(seed + 0x600); - const sortedPublicDataWrites = makeTuple( - MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, - makePublicDataTreeLeaf, - seed + 0x8000, - ); + const archiveRootMembershipWitness = makeMembershipWitness(ARCHIVE_HEIGHT, seed + 0x9000); + + const constants = makeConstantBaseRollupData(0x100); - const sortedPublicDataWritesIndexes = makeTuple(MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, i => i, 0); + const feePayerFeeJuiceBalanceReadHint = PublicDataHint.empty(); - const lowPublicDataWritesPreimages = makeTuple( - MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, - makePublicDataTreeLeafPreimage, - seed + 0x8200, - ); + return PrivateBaseRollupHints.from({ + start, + stateDiffHints, + archiveRootMembershipWitness, + constants, + feePayerFeeJuiceBalanceReadHint, + }); +} - const lowPublicDataWritesMembershipWitnesses = makeTuple( - MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, - i => makeMembershipWitness(PUBLIC_DATA_TREE_HEIGHT, i), - seed + 0x8400, - ); +function makePublicBaseRollupHints(seed = 1) { + const start = makePartialStateReference(seed + 0x100); + + const stateDiffHints = makePublicBaseStateDiffHints(seed + 0x600); const archiveRootMembershipWitness = makeMembershipWitness(ARCHIVE_HEIGHT, seed + 0x9000); @@ -1341,13 +1119,9 @@ function makeBaseRollupHints(seed = 1) { const feePayerFeeJuiceBalanceReadHint = PublicDataHint.empty(); - return BaseRollupHints.from({ + return PublicBaseRollupHints.from({ start, stateDiffHints, - sortedPublicDataWrites, - sortedPublicDataWritesIndexes, - lowPublicDataWritesPreimages, - lowPublicDataWritesMembershipWitnesses, archiveRootMembershipWitness, constants, feePayerFeeJuiceBalanceReadHint, @@ -1356,7 +1130,7 @@ function makeBaseRollupHints(seed = 1) { export function makePrivateBaseRollupInputs(seed = 0) { const tubeData = makePrivateTubeData(seed); - const hints = makeBaseRollupHints(seed + 0x100); + const hints = makePrivateBaseRollupHints(seed + 0x100); return PrivateBaseRollupInputs.from({ tubeData, @@ -1383,7 +1157,7 @@ function makeAvmProofData(seed = 1) { export function makePublicBaseRollupInputs(seed = 0) { const tubeData = makePublicTubeData(seed); const avmProofData = makeAvmProofData(seed + 0x100); - const hints = makeBaseRollupHints(seed + 0x100); + const hints = makePublicBaseRollupHints(seed + 0x200); return PublicBaseRollupInputs.from({ tubeData, @@ -1648,7 +1422,7 @@ export function makeAvmCircuitInputs(seed = 0, overrides: Partial new Fr(i), seed + 0x1000), - publicInputs: makePublicCircuitPublicInputs(seed + 0x2000), + publicInputs: PublicCircuitPublicInputs.empty(), avmHints: makeAvmExecutionHints(seed + 0x3000), output: makeAvmCircuitPublicInputs(seed + 0x4000), ...overrides, diff --git a/yarn-project/circuits.js/src/tests/fixtures.ts b/yarn-project/circuits.js/src/tests/fixtures.ts index ee8146a79fe..280c4240bb5 100644 --- a/yarn-project/circuits.js/src/tests/fixtures.ts +++ b/yarn-project/circuits.js/src/tests/fixtures.ts @@ -48,6 +48,6 @@ export function getSampleUnconstrainedFunctionBroadcastedEventPayload(): Buffer return Buffer.from(readFileSync(path).toString(), 'hex'); } -function getPathToFixture(name: string) { +export function getPathToFixture(name: string) { return resolve(dirname(fileURLToPath(import.meta.url)), `../../fixtures/${name}`); } diff --git a/yarn-project/cli-wallet/test/flows/create_account_pay_native.sh b/yarn-project/cli-wallet/test/flows/create_account_pay_native.sh index 57b4909a588..cd749fd9702 100755 --- a/yarn-project/cli-wallet/test/flows/create_account_pay_native.sh +++ b/yarn-project/cli-wallet/test/flows/create_account_pay_native.sh @@ -7,7 +7,7 @@ test_title "Create an account and deploy using native fee payment with bridging" echo warn //////////////////////////////////////////////////////////////// warn // WARNING: this test requires protocol contracts to be setup // -warn // > aztec setup-protocol-contracts // +warn // aztec setup-protocol-contracts // warn //////////////////////////////////////////////////////////////// echo diff --git a/yarn-project/cli-wallet/test/flows/private_transfer.sh b/yarn-project/cli-wallet/test/flows/private_transfer.sh new file mode 100755 index 00000000000..2d5e8a8e133 --- /dev/null +++ b/yarn-project/cli-wallet/test/flows/private_transfer.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -e +source ../utils/setup.sh + +test_title "Private transfer" + +MINT_AMOUNT=42 +TRANSFER_AMOUNT=21 + +source $TEST_FOLDER/token_utils/create_main_and_mint_to_private.sh $MINT_AMOUNT + +aztec-wallet create-account -a recipient + +aztec-wallet send transfer -ca token --args accounts:recipient $TRANSFER_AMOUNT -f main + +RESULT_MAIN=$(aztec-wallet simulate balance_of_private -ca token --args accounts:main -f main | grep "Simulation result:" | awk '{print $3}') +RESULT_RECIPIENT=$(aztec-wallet simulate balance_of_private -ca token --args accounts:recipient -f recipient | grep "Simulation result:" | awk '{print $3}') + +section "Main account private balance is ${RESULT_MAIN}, recipient account private balance is ${RESULT_RECIPIENT}" + +assert_eq ${RESULT_MAIN} ${RESULT_RECIPIENT} diff --git a/yarn-project/cli-wallet/test/flows/token_utils/create_main_and_mint_private.sh b/yarn-project/cli-wallet/test/flows/token_utils/create_main_and_mint_to_private.sh similarity index 99% rename from yarn-project/cli-wallet/test/flows/token_utils/create_main_and_mint_private.sh rename to yarn-project/cli-wallet/test/flows/token_utils/create_main_and_mint_to_private.sh index 6ae5143c52b..90307a0a166 100644 --- a/yarn-project/cli-wallet/test/flows/token_utils/create_main_and_mint_private.sh +++ b/yarn-project/cli-wallet/test/flows/token_utils/create_main_and_mint_to_private.sh @@ -4,7 +4,6 @@ aztec-wallet create-account -a main aztec-wallet deploy token_contract@Token --args accounts:main Test TST 18 -f main -a token aztec-wallet send mint_to_private -ca token --args accounts:main accounts:main $1 -f main - RESULT_MAIN=$(aztec-wallet simulate balance_of_private -ca token --args accounts:main -f main | grep "Simulation result:" | awk '{print $3}') if [ "${1}n" != "$RESULT_MAIN" ]; then diff --git a/yarn-project/cli-wallet/test/flows/token_utils/create_main_and_mint_public.sh b/yarn-project/cli-wallet/test/flows/token_utils/create_main_and_mint_to_public.sh similarity index 100% rename from yarn-project/cli-wallet/test/flows/token_utils/create_main_and_mint_public.sh rename to yarn-project/cli-wallet/test/flows/token_utils/create_main_and_mint_to_public.sh diff --git a/yarn-project/cli/src/utils/inspect.ts b/yarn-project/cli/src/utils/inspect.ts index 445e48c3bd1..855c38b9ef5 100644 --- a/yarn-project/cli/src/utils/inspect.ts +++ b/yarn-project/cli/src/utils/inspect.ts @@ -38,22 +38,23 @@ export async function inspectTx( log: LogFn, opts: { includeBlockInfo?: boolean; artifactMap?: ArtifactMap } = {}, ) { - const [receipt, effects, notes] = await Promise.all([ + const [receipt, effectsInBlock, notes] = await Promise.all([ pxe.getTxReceipt(txHash), pxe.getTxEffect(txHash), pxe.getIncomingNotes({ txHash, status: NoteStatus.ACTIVE_OR_NULLIFIED }), ]); // Base tx data log(`Tx ${txHash.toString()}`); - log(` Status: ${receipt.status} ${effects ? `(${effects.revertCode.getDescription()})` : ''}`); + log(` Status: ${receipt.status} ${effectsInBlock ? `(${effectsInBlock.data.revertCode.getDescription()})` : ''}`); if (receipt.error) { log(` Error: ${receipt.error}`); } - if (!effects) { + if (!effectsInBlock) { return; } + const effects = effectsInBlock.data; const artifactMap = opts?.artifactMap ?? (await getKnownArtifacts(pxe)); if (opts.includeBlockInfo) { diff --git a/yarn-project/end-to-end/scripts/docker-compose-no-sandbox.yml b/yarn-project/end-to-end/scripts/docker-compose-no-sandbox.yml index c11eca2414c..26aee913035 100644 --- a/yarn-project/end-to-end/scripts/docker-compose-no-sandbox.yml +++ b/yarn-project/end-to-end/scripts/docker-compose-no-sandbox.yml @@ -3,15 +3,9 @@ services: fork: image: aztecprotocol/foundry:25f24e677a6a32a62512ad4f561995589ac2c7dc-${ARCH_TAG:-amd64} pull_policy: always - entrypoint: > - sh -c ' - if [ -n "$FORK_BLOCK_NUMBER" ] && [ -n "$FORK_URL" ]; then - exec anvil --silent -p 8545 --host 0.0.0.0 --chain-id 31337 --fork-url "$FORK_URL" --fork-block-number "$FORK_BLOCK_NUMBER" - else - exec anvil --silent -p 8545 --host 0.0.0.0 --chain-id 31337 - fi' - expose: - - '8545' + entrypoint: 'anvil --silent -p 8545 --host 0.0.0.0 --chain-id 31337' + ports: + - 8545:8545 end-to-end: image: aztecprotocol/end-to-end:${AZTEC_DOCKER_TAG:-latest} diff --git a/yarn-project/end-to-end/scripts/docker-compose-wallet.yml b/yarn-project/end-to-end/scripts/docker-compose-wallet.yml index f896569f7ca..7aa1646aa72 100644 --- a/yarn-project/end-to-end/scripts/docker-compose-wallet.yml +++ b/yarn-project/end-to-end/scripts/docker-compose-wallet.yml @@ -3,15 +3,9 @@ services: fork: image: aztecprotocol/foundry:25f24e677a6a32a62512ad4f561995589ac2c7dc-${ARCH_TAG:-amd64} pull_policy: always - entrypoint: > - sh -c ' - if [ -n "$FORK_BLOCK_NUMBER" ] && [ -n "$FORK_URL" ]; then - exec anvil --silent -p 8545 --host 0.0.0.0 --chain-id 31337 --fork-url "$FORK_URL" --fork-block-number "$FORK_BLOCK_NUMBER" - else - exec anvil --silent -p 8545 --host 0.0.0.0 --chain-id 31337 - fi' - expose: - - '8545' + entrypoint: 'anvil --silent -p 8545 --host 0.0.0.0 --chain-id 31337' + ports: + - 8545:8545 sandbox: image: aztecprotocol/aztec:${AZTEC_DOCKER_TAG:-latest} diff --git a/yarn-project/end-to-end/scripts/docker-compose.yml b/yarn-project/end-to-end/scripts/docker-compose.yml index 8cc45e5510b..75b9cd0d36f 100644 --- a/yarn-project/end-to-end/scripts/docker-compose.yml +++ b/yarn-project/end-to-end/scripts/docker-compose.yml @@ -3,15 +3,9 @@ services: fork: image: aztecprotocol/foundry:25f24e677a6a32a62512ad4f561995589ac2c7dc-${ARCH_TAG:-amd64} pull_policy: always - entrypoint: > - sh -c ' - if [ -n "$FORK_BLOCK_NUMBER" ] && [ -n "$FORK_URL" ]; then - exec anvil --silent -p 8545 --host 0.0.0.0 --chain-id 31337 --fork-url "$FORK_URL" --fork-block-number "$FORK_BLOCK_NUMBER" - else - exec anvil --silent -p 8545 --host 0.0.0.0 --chain-id 31337 - fi' - expose: - - '8545' + entrypoint: 'anvil --silent -p 8545 --host 0.0.0.0 --chain-id 31337' + ports: + - 8545:8545 sandbox: image: aztecprotocol/aztec:${AZTEC_DOCKER_TAG:-latest} diff --git a/yarn-project/end-to-end/scripts/e2e_test_config.yml b/yarn-project/end-to-end/scripts/e2e_test_config.yml index 19a2f0003d5..b70bd9024d2 100644 --- a/yarn-project/end-to-end/scripts/e2e_test_config.yml +++ b/yarn-project/end-to-end/scripts/e2e_test_config.yml @@ -35,6 +35,7 @@ tests: e2e_devnet_smoke: {} docs_examples: use_compose: true + e2e_epochs: {} e2e_escrow_contract: {} e2e_fees_account_init: test_path: 'e2e_fees/account_init.test.ts' @@ -74,6 +75,8 @@ tests: env: HARDWARE_CONCURRENCY: '32' e2e_public_testnet: {} + e2e_pxe: + use_compose: true e2e_sandbox_example: use_compose: true e2e_state_vars: {} @@ -85,10 +88,12 @@ tests: e2e_p2p_upgrade_governance_proposer: test_path: 'e2e_p2p/upgrade_governance_proposer.test.ts' # https://github.com/AztecProtocol/aztec-packages/issues/9843 - # e2e_p2p_rediscovery: - # test_path: 'e2e_p2p/rediscovery.test.ts' + e2e_p2p_rediscovery: + test_path: 'e2e_p2p/rediscovery.test.ts' e2e_p2p_reqresp: test_path: 'e2e_p2p/reqresp.test.ts' + e2e_p2p_reex: + test_path: 'e2e_p2p/reex.test.ts' flakey_e2e_tests: test_path: './src/flakey' ignore_failures: true @@ -109,7 +114,6 @@ tests: test_path: 'guides/writing_an_account_contract.test.ts' integration_l1_publisher: use_compose: true - pxe: - use_compose: true - uniswap_trade_on_l1_from_l2: - use_compose: true + # https://github.com/AztecProtocol/aztec-packages/issues/10030 + # uniswap_trade_on_l1_from_l2: + # use_compose: true diff --git a/yarn-project/end-to-end/scripts/network_test.sh b/yarn-project/end-to-end/scripts/network_test.sh index 4f3aa699c53..6789c4a75b8 100755 --- a/yarn-project/end-to-end/scripts/network_test.sh +++ b/yarn-project/end-to-end/scripts/network_test.sh @@ -91,18 +91,18 @@ handle_network_shaping() { fi fi - echo "Deploying network shaping configuration..." - if ! helm upgrade --install network-shaping "$REPO/spartan/network-shaping/" \ + echo "Deploying Aztec Chaos Scenarios..." + if ! helm upgrade --install aztec-chaos-scenarios "$REPO/spartan/aztec-chaos-scenarios/" \ --namespace chaos-mesh \ - --values "$REPO/spartan/network-shaping/values/$CHAOS_VALUES" \ + --values "$REPO/spartan/aztec-chaos-scenarios/values/$CHAOS_VALUES" \ --set global.targetNamespace="$NAMESPACE" \ --wait \ --timeout=5m; then - echo "Error: failed to deploy network shaping configuration!" + echo "Error: failed to deploy Aztec Chaos Scenarios!" return 1 fi - echo "Network shaping configuration applied successfully" + echo "Aztec Chaos Scenarios applied successfully" return 0 fi return 0 @@ -163,10 +163,11 @@ fi docker run --rm --network=host \ -v ~/.kube:/root/.kube \ -e K8S=true \ + -e INSTANCE_NAME="spartan" \ -e SPARTAN_DIR="/usr/src/spartan" \ -e NAMESPACE="$NAMESPACE" \ -e HOST_PXE_PORT=$PXE_PORT \ - -e CONTAINER_PXE_PORT=8080 \ + -e CONTAINER_PXE_PORT=8081 \ -e HOST_ETHEREUM_PORT=$ANVIL_PORT \ -e CONTAINER_ETHEREUM_PORT=8545 \ -e DEBUG="aztec:*" \ diff --git a/yarn-project/end-to-end/src/benchmarks/bench_prover.test.ts b/yarn-project/end-to-end/src/benchmarks/bench_prover.test.ts index 04755b0b341..2934f662ff7 100644 --- a/yarn-project/end-to-end/src/benchmarks/bench_prover.test.ts +++ b/yarn-project/end-to-end/src/benchmarks/bench_prover.test.ts @@ -5,7 +5,7 @@ import { BBCircuitVerifier } from '@aztec/bb-prover'; import { CompleteAddress, Fq, Fr, GasSettings } from '@aztec/circuits.js'; import { FPCContract, FeeJuiceContract, TestContract, TokenContract } from '@aztec/noir-contracts.js'; import { ProtocolContractAddress } from '@aztec/protocol-contracts'; -import { type PXEService, PXEServiceConfig, createPXEService } from '@aztec/pxe'; +import { type PXEService, type PXEServiceConfig, createPXEService } from '@aztec/pxe'; import { jest } from '@jest/globals'; diff --git a/yarn-project/end-to-end/src/benchmarks/utils.ts b/yarn-project/end-to-end/src/benchmarks/utils.ts index c8de5dc0ec3..4d2a78bdaad 100644 --- a/yarn-project/end-to-end/src/benchmarks/utils.ts +++ b/yarn-project/end-to-end/src/benchmarks/utils.ts @@ -2,9 +2,8 @@ import { type AztecNodeConfig, type AztecNodeService } from '@aztec/aztec-node'; import { type AztecNode, BatchCall, INITIAL_L2_BLOCK_NUM, type SentTx, retryUntil, sleep } from '@aztec/aztec.js'; import { times } from '@aztec/foundation/collection'; import { randomInt } from '@aztec/foundation/crypto'; -import { DataStoreConfig } from '@aztec/kv-store/config'; import { BenchmarkingContract } from '@aztec/noir-contracts.js/Benchmarking'; -import { type PXEService, PXEServiceConfig, createPXEService } from '@aztec/pxe'; +import { type PXEService, type PXEServiceConfig, createPXEService } from '@aztec/pxe'; import { mkdirpSync } from 'fs-extra'; import { globSync } from 'glob'; diff --git a/yarn-project/end-to-end/src/composed/pxe.test.ts b/yarn-project/end-to-end/src/composed/e2e_pxe.test.ts similarity index 87% rename from yarn-project/end-to-end/src/composed/pxe.test.ts rename to yarn-project/end-to-end/src/composed/e2e_pxe.test.ts index 6e972b8eb6e..89bcae3be1a 100644 --- a/yarn-project/end-to-end/src/composed/pxe.test.ts +++ b/yarn-project/end-to-end/src/composed/e2e_pxe.test.ts @@ -9,4 +9,4 @@ const setupEnv = async () => { return pxe; }; -pxeTestSuite('pxe', setupEnv); +pxeTestSuite('e2e_pxe', setupEnv); diff --git a/yarn-project/end-to-end/src/composed/e2e_sandbox_example.test.ts b/yarn-project/end-to-end/src/composed/e2e_sandbox_example.test.ts index 9522bbf30ba..b2ed9d69432 100644 --- a/yarn-project/end-to-end/src/composed/e2e_sandbox_example.test.ts +++ b/yarn-project/end-to-end/src/composed/e2e_sandbox_example.test.ts @@ -5,10 +5,10 @@ import { Fr, GrumpkinScalar, type PXE, createDebugLogger, createPXEClient, waitF import { format } from 'util'; +// docs:end:imports import { deployToken, mintTokensToPrivate } from '../fixtures/token_utils.js'; const { PXE_URL = 'http://localhost:8080' } = process.env; -// docs:end:imports describe('e2e_sandbox_example', () => { it('sandbox example works', async () => { diff --git a/yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts b/yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts index f1339b66836..d2775d2eda4 100644 --- a/yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts +++ b/yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts @@ -393,10 +393,12 @@ describe('L1Publisher integration', () => { abi: RollupAbi, functionName: 'propose', args: [ - `0x${block.header.toBuffer().toString('hex')}`, - `0x${block.archive.root.toBuffer().toString('hex')}`, - `0x${block.header.hash().toBuffer().toString('hex')}`, - [], + { + header: `0x${block.header.toBuffer().toString('hex')}`, + archive: `0x${block.archive.root.toBuffer().toString('hex')}`, + blockHash: `0x${block.header.hash().toBuffer().toString('hex')}`, + txHashes: [], + }, [], `0x${block.body.toBuffer().toString('hex')}`, ], @@ -490,10 +492,12 @@ describe('L1Publisher integration', () => { abi: RollupAbi, functionName: 'propose', args: [ - `0x${block.header.toBuffer().toString('hex')}`, - `0x${block.archive.root.toBuffer().toString('hex')}`, - `0x${block.header.hash().toBuffer().toString('hex')}`, - [], + { + header: `0x${block.header.toBuffer().toString('hex')}`, + archive: `0x${block.archive.root.toBuffer().toString('hex')}`, + blockHash: `0x${block.header.hash().toBuffer().toString('hex')}`, + txHashes: [], + }, [], `0x${block.body.toBuffer().toString('hex')}`, ], diff --git a/yarn-project/end-to-end/src/e2e_avm_simulator.test.ts b/yarn-project/end-to-end/src/e2e_avm_simulator.test.ts index 00a1c15d585..449a14eaf1b 100644 --- a/yarn-project/end-to-end/src/e2e_avm_simulator.test.ts +++ b/yarn-project/end-to-end/src/e2e_avm_simulator.test.ts @@ -155,6 +155,20 @@ describe('e2e_avm_simulator', () => { }); describe('Nested calls', () => { + it('Top-level call to non-existent contract reverts', async () => { + // The nested call reverts (returns failure), but the caller doesn't HAVE to rethrow. + const tx = await avmContract.methods.nested_call_to_nothing_recovers().send().wait(); + expect(tx.status).toEqual(TxStatus.SUCCESS); + }); + it('Nested call to non-existent contract reverts & rethrows by default', async () => { + // The nested call reverts and by default caller rethrows + await expect(avmContract.methods.nested_call_to_nothing().send().wait()).rejects.toThrow(/No bytecode/); + }); + it('Nested CALL instruction to non-existent contract returns failure, but caller can recover', async () => { + // The nested call reverts (returns failure), but the caller doesn't HAVE to rethrow. + const tx = await avmContract.methods.nested_call_to_nothing_recovers().send().wait(); + expect(tx.status).toEqual(TxStatus.SUCCESS); + }); it('Should NOT be able to emit the same unsiloed nullifier from the same contract', async () => { const nullifier = new Fr(1); await expect( diff --git a/yarn-project/end-to-end/src/e2e_block_building.test.ts b/yarn-project/end-to-end/src/e2e_block_building.test.ts index 88bfd6a1b23..41a27b70a92 100644 --- a/yarn-project/end-to-end/src/e2e_block_building.test.ts +++ b/yarn-project/end-to-end/src/e2e_block_building.test.ts @@ -1,5 +1,4 @@ import { getSchnorrAccount } from '@aztec/accounts/schnorr'; -import { createAccount } from '@aztec/accounts/testing'; import { type AztecAddress, type AztecNode, @@ -23,7 +22,6 @@ import { poseidon2HashWithSeparator } from '@aztec/foundation/crypto'; import { StatefulTestContract, StatefulTestContractArtifact } from '@aztec/noir-contracts.js'; import { TestContract } from '@aztec/noir-contracts.js/Test'; import { TokenContract } from '@aztec/noir-contracts.js/Token'; -import { createPXEService, getPXEServiceConfig } from '@aztec/pxe'; import 'jest-extended'; @@ -439,49 +437,63 @@ describe('e2e_block_building', () => { await cheatCodes.rollup.advanceToNextEpoch(); await cheatCodes.rollup.markAsProven(); - // Send a tx to the contract that updates the public data tree, this should take the first slot + // Send a tx to the contract that creates a note. This tx will be reorgd but re-included, + // since it is being built against a proven block number. logger.info('Sending initial tx'); - const tx1 = await contract.methods.increment_public_value(ownerAddress, 20).send().wait(); + const tx1 = await contract.methods.create_note(ownerAddress, ownerAddress, 20).send().wait(); expect(tx1.blockNumber).toEqual(initialBlockNumber + 1); - expect(await contract.methods.get_public_value(ownerAddress).simulate()).toEqual(20n); + expect(await contract.methods.summed_values(ownerAddress).simulate()).toEqual(21n); + + // And send a second one, which won't be re-included. + logger.info('Sending second tx'); + const tx2 = await contract.methods.create_note(ownerAddress, ownerAddress, 30).send().wait(); + expect(tx2.blockNumber).toEqual(initialBlockNumber + 2); + expect(await contract.methods.summed_values(ownerAddress).simulate()).toEqual(51n); // Now move to a new epoch and past the proof claim window to cause a reorg logger.info('Advancing past the proof claim window'); await cheatCodes.rollup.advanceToNextEpoch(); await cheatCodes.rollup.advanceSlots(aztecEpochProofClaimWindowInL2Slots + 1); // off-by-one? - // Wait a bit before spawning a new pxe - await sleep(2000); + // Wait until the sequencer kicks out tx1 + logger.info(`Waiting for node to prune tx1`); + await retryUntil( + async () => (await aztecNode.getTxReceipt(tx1.txHash)).status === TxStatus.PENDING, + 'wait for pruning', + 15, + 1, + ); - // tx1 is valid because it was build against a proven block number - // the sequencer will bring it back on chain + // And wait until it is brought back tx1 + logger.info(`Waiting for node to re-include tx1`); await retryUntil( async () => (await aztecNode.getTxReceipt(tx1.txHash)).status === TxStatus.SUCCESS, 'wait for re-inclusion', - 60, + 15, 1, ); + // Tx1 should have been mined in a block with the same number but different hash now const newTx1Receipt = await aztecNode.getTxReceipt(tx1.txHash); expect(newTx1Receipt.blockNumber).toEqual(tx1.blockNumber); expect(newTx1Receipt.blockHash).not.toEqual(tx1.blockHash); - // Send another tx which should be mined a block that is built on the reorg'd chain - // We need to send it from a new pxe since pxe doesn't detect reorgs (yet) - logger.info(`Creating new PXE service`); - const pxeServiceConfig = { ...getPXEServiceConfig() }; - const newPxe = await createPXEService(aztecNode, pxeServiceConfig); - const newWallet = await createAccount(newPxe); + // PXE should have cleared out the 30-note from tx2, but reapplied the 20-note from tx1 + expect(await contract.methods.summed_values(ownerAddress).simulate()).toEqual(21n); - // TODO: Contract.at should automatically register the instance in the pxe - logger.info(`Registering contract at ${contract.address} in new pxe`); - await newPxe.registerContract({ instance: contract.instance, artifact: StatefulTestContractArtifact }); - const contractFromNewPxe = await StatefulTestContract.at(contract.address, newWallet); + // PXE should be synced to the block number on the new chain + await retryUntil( + async () => (await pxe.getSyncStatus()).blocks === newTx1Receipt.blockNumber, + 'wait for pxe block header sync', + 15, + 1, + ); + // And we should be able to send a new tx on the new chain logger.info('Sending new tx on reorgd chain'); - const tx2 = await contractFromNewPxe.methods.increment_public_value(ownerAddress, 10).send().wait(); - expect(await contractFromNewPxe.methods.get_public_value(ownerAddress).simulate()).toEqual(30n); - expect(tx2.blockNumber).toEqual(initialBlockNumber + 3); + const tx3 = await contract.methods.create_note(ownerAddress, ownerAddress, 10).send().wait(); + expect(await contract.methods.summed_values(ownerAddress).simulate()).toEqual(31n); + expect(tx3.blockNumber).toBeGreaterThanOrEqual(newTx1Receipt.blockNumber! + 1); }); }); }); diff --git a/yarn-project/end-to-end/src/e2e_deploy_contract/contract_class_registration.test.ts b/yarn-project/end-to-end/src/e2e_deploy_contract/contract_class_registration.test.ts index 9908dfecbec..ccde3cdf980 100644 --- a/yarn-project/end-to-end/src/e2e_deploy_contract/contract_class_registration.test.ts +++ b/yarn-project/end-to-end/src/e2e_deploy_contract/contract_class_registration.test.ts @@ -5,8 +5,10 @@ import { type ContractClassWithId, type ContractInstanceWithAddress, type DebugLogger, + type FieldsOf, Fr, type PXE, + type TxReceipt, TxStatus, type Wallet, getContractClassFromArtifact, @@ -18,10 +20,10 @@ import { deployInstance, registerContractClass, } from '@aztec/aztec.js/deployment'; -import { type ContractClassIdPreimage, PublicKeys } from '@aztec/circuits.js'; +import { type ContractClassIdPreimage, PublicKeys, computeContractClassId } from '@aztec/circuits.js'; import { FunctionSelector, FunctionType } from '@aztec/foundation/abi'; import { writeTestData } from '@aztec/foundation/testing'; -import { StatefulTestContract } from '@aztec/noir-contracts.js'; +import { StatefulTestContract, TokenContractArtifact } from '@aztec/noir-contracts.js'; import { TestContract } from '@aztec/noir-contracts.js/Test'; import { DUPLICATE_NULLIFIER_ERROR } from '../fixtures/fixtures.js'; @@ -43,56 +45,74 @@ describe('e2e_deploy_contract contract class registration', () => { let artifact: ContractArtifact; let contractClass: ContractClassWithId & ContractClassIdPreimage; + let registrationTxReceipt: FieldsOf; beforeAll(async () => { artifact = StatefulTestContract.artifact; - await registerContractClass(wallet, artifact).then(c => c.send().wait()); + registrationTxReceipt = await registerContractClass(wallet, artifact).then(c => c.send().wait()); contractClass = getContractClassFromArtifact(artifact); - }); - it('registers the contract class on the node', async () => { - const registeredClass = await aztecNode.getContractClass(contractClass.id); - expect(registeredClass).toBeDefined(); - expect(registeredClass!.artifactHash.toString()).toEqual(contractClass.artifactHash.toString()); - expect(registeredClass!.privateFunctionsRoot.toString()).toEqual(contractClass.privateFunctionsRoot.toString()); - expect(registeredClass!.packedBytecode.toString('hex')).toEqual(contractClass.packedBytecode.toString('hex')); - expect(registeredClass!.publicFunctions).toEqual(contractClass.publicFunctions); - expect(registeredClass!.privateFunctions).toEqual([]); + // TODO(#10007) Remove this call. Node should get the bytecode from the event broadcast. + expect(await aztecNode.getContractClass(contractClass.id)).toBeUndefined(); + await aztecNode.addContractClass({ ...contractClass, privateFunctions: [], unconstrainedFunctions: [] }); }); - it('broadcasts a private function', async () => { - const constructorArtifact = artifact.functions.find(fn => fn.name == 'constructor'); - if (!constructorArtifact) { - // If this gets thrown you've probably modified the StatefulTestContract to no longer include constructor. - // If that's the case you should update this test to use a private function which fits into the bytecode size - // limit. - throw new Error('No constructor found in the StatefulTestContract artifact. Does it still exist?'); - } - const selector = FunctionSelector.fromNameAndParameters(constructorArtifact.name, constructorArtifact.parameters); - - const tx = await (await broadcastPrivateFunction(wallet, artifact, selector)).send().wait(); - const logs = await pxe.getContractClassLogs({ txHash: tx.txHash }); - const logData = logs.logs[0].log.data; - writeTestData('yarn-project/circuits.js/fixtures/PrivateFunctionBroadcastedEventData.hex', logData); - - const fetchedClass = await aztecNode.getContractClass(contractClass.id); - const fetchedFunction = fetchedClass!.privateFunctions[0]!; - expect(fetchedFunction).toBeDefined(); - expect(fetchedFunction.selector).toEqual(selector); - }); + describe('registering a contract class', () => { + // TODO(#10007) Remove this test. We should always broadcast public bytecode. + it('bypasses broadcast if exceeds bytecode limit for event size', async () => { + const logs = await aztecNode.getContractClassLogs({ txHash: registrationTxReceipt.txHash }); + expect(logs.logs.length).toEqual(0); + }); + + // TODO(#10007) Remove this test as well. + it('starts archiver with pre-registered common contracts', async () => { + const classId = computeContractClassId(getContractClassFromArtifact(TokenContractArtifact)); + expect(await aztecNode.getContractClass(classId)).not.toBeUndefined(); + }); - it('broadcasts an unconstrained function', async () => { - const functionArtifact = artifact.functions.find(fn => fn.functionType === FunctionType.UNCONSTRAINED)!; - const selector = FunctionSelector.fromNameAndParameters(functionArtifact); - const tx = await (await broadcastUnconstrainedFunction(wallet, artifact, selector)).send().wait(); - const logs = await pxe.getContractClassLogs({ txHash: tx.txHash }); - const logData = logs.logs[0].log.data; - writeTestData('yarn-project/circuits.js/fixtures/UnconstrainedFunctionBroadcastedEventData.hex', logData); - - const fetchedClass = await aztecNode.getContractClass(contractClass.id); - const fetchedFunction = fetchedClass!.unconstrainedFunctions[0]!; - expect(fetchedFunction).toBeDefined(); - expect(fetchedFunction.selector).toEqual(selector); + it('registers the contract class on the node', async () => { + const registeredClass = await aztecNode.getContractClass(contractClass.id); + expect(registeredClass).toBeDefined(); + expect(registeredClass!.artifactHash.toString()).toEqual(contractClass.artifactHash.toString()); + expect(registeredClass!.privateFunctionsRoot.toString()).toEqual(contractClass.privateFunctionsRoot.toString()); + expect(registeredClass!.packedBytecode.toString('hex')).toEqual(contractClass.packedBytecode.toString('hex')); + expect(registeredClass!.publicFunctions).toEqual(contractClass.publicFunctions); + expect(registeredClass!.privateFunctions).toEqual([]); + }); + + it('broadcasts a private function', async () => { + const constructorArtifact = artifact.functions.find(fn => fn.name == 'constructor'); + if (!constructorArtifact) { + // If this gets thrown you've probably modified the StatefulTestContract to no longer include constructor. + // If that's the case you should update this test to use a private function which fits into the bytecode size limit. + throw new Error('No constructor found in the StatefulTestContract artifact. Does it still exist?'); + } + const selector = FunctionSelector.fromNameAndParameters(constructorArtifact.name, constructorArtifact.parameters); + + const tx = await (await broadcastPrivateFunction(wallet, artifact, selector)).send().wait(); + const logs = await pxe.getContractClassLogs({ txHash: tx.txHash }); + const logData = logs.logs[0].log.data; + writeTestData('yarn-project/circuits.js/fixtures/PrivateFunctionBroadcastedEventData.hex', logData); + + const fetchedClass = await aztecNode.getContractClass(contractClass.id); + const fetchedFunction = fetchedClass!.privateFunctions[0]!; + expect(fetchedFunction).toBeDefined(); + expect(fetchedFunction.selector).toEqual(selector); + }); + + it('broadcasts an unconstrained function', async () => { + const functionArtifact = artifact.functions.find(fn => fn.functionType === FunctionType.UNCONSTRAINED)!; + const selector = FunctionSelector.fromNameAndParameters(functionArtifact); + const tx = await (await broadcastUnconstrainedFunction(wallet, artifact, selector)).send().wait(); + const logs = await pxe.getContractClassLogs({ txHash: tx.txHash }); + const logData = logs.logs[0].log.data; + writeTestData('yarn-project/circuits.js/fixtures/UnconstrainedFunctionBroadcastedEventData.hex', logData); + + const fetchedClass = await aztecNode.getContractClass(contractClass.id); + const fetchedFunction = fetchedClass!.unconstrainedFunctions[0]!; + expect(fetchedFunction).toBeDefined(); + expect(fetchedFunction.selector).toEqual(selector); + }); }); const testDeployingAnInstance = (how: string, deployFn: (toDeploy: ContractInstanceWithAddress) => Promise) => @@ -257,13 +277,21 @@ describe('e2e_deploy_contract contract class registration', () => { }); describe('error scenarios in deployment', () => { - it('refuses to call a public function on an undeployed contract', async () => { + it('app logic call to an undeployed contract reverts, but can be included is not dropped', async () => { const whom = wallet.getAddress(); const outgoingViewer = whom; const instance = await t.registerContract(wallet, StatefulTestContract, { initArgs: [whom, outgoingViewer, 42] }); - await expect( - instance.methods.increment_public_value_no_init_check(whom, 10).send({ skipPublicSimulation: true }).wait(), - ).rejects.toThrow(/dropped/); + // Confirm that the tx reverts with the expected message + await expect(instance.methods.increment_public_value_no_init_check(whom, 10).send().wait()).rejects.toThrow( + /No bytecode/, + ); + // This time, don't throw on revert and confirm that the tx is included + // despite reverting in app logic because of the call to a non-existent contract + const tx = await instance.methods + .increment_public_value_no_init_check(whom, 10) + .send({ skipPublicSimulation: true }) + .wait({ dontThrowOnRevert: true }); + expect(tx.status).toEqual(TxStatus.APP_LOGIC_REVERTED); }); it('refuses to deploy an instance from a different deployer', () => { diff --git a/yarn-project/end-to-end/src/e2e_deploy_contract/legacy.test.ts b/yarn-project/end-to-end/src/e2e_deploy_contract/legacy.test.ts index e712e0e9ebb..a1f6442a81f 100644 --- a/yarn-project/end-to-end/src/e2e_deploy_contract/legacy.test.ts +++ b/yarn-project/end-to-end/src/e2e_deploy_contract/legacy.test.ts @@ -84,7 +84,8 @@ describe('e2e_deploy_contract legacy', () => { await expect(deployer.deploy().send({ contractAddressSalt }).wait()).rejects.toThrow(/dropped/); }); - it('should not deploy a contract which failed the public part of the execution', async () => { + // TODO(#10007): Reenable this test. + it.skip('should not deploy a contract which failed the public part of the execution', async () => { // This test requires at least another good transaction to go through in the same block as the bad one. const artifact = TokenContractArtifact; const initArgs = ['TokenName', 'TKN', 18] as const; diff --git a/yarn-project/end-to-end/src/e2e_epochs.test.ts b/yarn-project/end-to-end/src/e2e_epochs.test.ts new file mode 100644 index 00000000000..3ac4e07afd8 --- /dev/null +++ b/yarn-project/end-to-end/src/e2e_epochs.test.ts @@ -0,0 +1,146 @@ +import { type EpochConstants, getTimestampRangeForEpoch } from '@aztec/archiver/epoch'; +import { type DebugLogger, retryUntil } from '@aztec/aztec.js'; +import { RollupContract } from '@aztec/ethereum/contracts'; +import { type Delayer, waitUntilL1Timestamp } from '@aztec/ethereum/test'; + +import { type PublicClient } from 'viem'; + +import { type EndToEndContext, setup } from './fixtures/utils.js'; + +// Tests building of epochs using fast block times and short epochs. +// Spawns an aztec node and a prover node with fake proofs. +// Sequencer is allowed to build empty blocks. +describe('e2e_epochs', () => { + let context: EndToEndContext; + let l1Client: PublicClient; + let rollup: RollupContract; + let constants: EpochConstants; + let logger: DebugLogger; + let proverDelayer: Delayer; + let sequencerDelayer: Delayer; + + let l2BlockNumber: number = 0; + let l2ProvenBlockNumber: number = 0; + let l1BlockNumber: number; + let handle: NodeJS.Timeout; + + const EPOCH_DURATION = 4; + const L1_BLOCK_TIME = 5; + const L2_SLOT_DURATION_IN_L1_BLOCKS = 2; + + beforeAll(async () => { + // Set up system without any account nor protocol contracts + // and with faster block times and shorter epochs. + context = await setup(0, { + assumeProvenThrough: undefined, + skipProtocolContracts: true, + salt: 1, + aztecEpochDuration: EPOCH_DURATION, + aztecSlotDuration: L1_BLOCK_TIME * L2_SLOT_DURATION_IN_L1_BLOCKS, + ethereumSlotDuration: L1_BLOCK_TIME, + aztecEpochProofClaimWindowInL2Slots: EPOCH_DURATION / 2, + minTxsPerBlock: 0, + realProofs: false, + startProverNode: true, + }); + + logger = context.logger; + l1Client = context.deployL1ContractsValues.publicClient; + rollup = RollupContract.getFromConfig(context.config); + + // Loop that tracks L1 and L2 block numbers and logs whenever there's a new one. + // We could refactor this out to an utility if we want to use this in other tests. + handle = setInterval(async () => { + const newL1BlockNumber = Number(await l1Client.getBlockNumber({ cacheTime: 0 })); + if (l1BlockNumber === newL1BlockNumber) { + return; + } + const block = await l1Client.getBlock({ blockNumber: BigInt(newL1BlockNumber), includeTransactions: false }); + const timestamp = block.timestamp; + l1BlockNumber = newL1BlockNumber; + + let msg = `L1 block ${newL1BlockNumber} mined at ${timestamp}`; + + const newL2BlockNumber = Number(await rollup.getBlockNumber()); + if (l2BlockNumber !== newL2BlockNumber) { + const epochNumber = await rollup.getEpochNumber(BigInt(newL2BlockNumber)); + msg += ` with new L2 block ${newL2BlockNumber} for epoch ${epochNumber}`; + l2BlockNumber = newL2BlockNumber; + } + + const newL2ProvenBlockNumber = Number(await rollup.getProvenBlockNumber()); + if (l2ProvenBlockNumber !== newL2ProvenBlockNumber) { + const epochNumber = await rollup.getEpochNumber(BigInt(newL2ProvenBlockNumber)); + msg += ` with proof up to L2 block ${newL2ProvenBlockNumber} for epoch ${epochNumber}`; + l2ProvenBlockNumber = newL2ProvenBlockNumber; + } + logger.info(msg); + }, 200); + + // The "as any" cast sucks, but it saves us from having to define test-only types for the provernode + // and sequencer that are exactly like the real ones but with the publisher exposed. We should + // do it if we see the this pattern popping up in more places. + proverDelayer = (context.proverNode as any).publisher.delayer; + sequencerDelayer = (context.sequencer as any).sequencer.publisher.delayer; + expect(proverDelayer).toBeDefined(); + expect(sequencerDelayer).toBeDefined(); + + // Constants used for time calculation + constants = { + epochDuration: EPOCH_DURATION, + slotDuration: L1_BLOCK_TIME * L2_SLOT_DURATION_IN_L1_BLOCKS, + l1GenesisBlock: await rollup.getL1StartBlock(), + l1GenesisTime: await rollup.getL1GenesisTime(), + }; + + logger.info(`L2 genesis at L1 block ${constants.l1GenesisBlock} (timestamp ${constants.l1GenesisTime})`); + }); + + afterAll(async () => { + clearInterval(handle); + await context.teardown(); + }); + + /** Waits until the epoch begins (ie until the immediately previous L1 block is mined). */ + const waitUntilEpochStarts = async (epoch: number) => { + const [start] = getTimestampRangeForEpoch(BigInt(epoch), constants); + logger.info(`Waiting until L1 timestamp ${start} is reached as the start of epoch ${epoch}`); + await waitUntilL1Timestamp(l1Client, start - BigInt(L1_BLOCK_TIME)); + return start; + }; + + /** Waits until the given L2 block number is mined. */ + const waitUntilL2BlockNumber = async (target: number) => { + await retryUntil(() => Promise.resolve(target === l2BlockNumber), `Wait until L2 block ${l2BlockNumber}`, 60, 0.1); + }; + + it('does not allow submitting proof after epoch end', async () => { + await waitUntilEpochStarts(1); + const blockNumberAtEndOfEpoch0 = Number(await rollup.getBlockNumber()); + logger.info(`Starting epoch 1 after L2 block ${blockNumberAtEndOfEpoch0}`); + + // Hold off prover tx until end of next epoch! + const [epoch2Start] = getTimestampRangeForEpoch(2n, constants); + proverDelayer.pauseNextTxUntilTimestamp(epoch2Start); + logger.info(`Delayed prover tx until epoch 2 starts at ${epoch2Start}`); + + // Wait until the last block of epoch 1 is published and then hold off the sequencer + await waitUntilL2BlockNumber(blockNumberAtEndOfEpoch0 + EPOCH_DURATION); + sequencerDelayer.pauseNextTxUntilTimestamp(epoch2Start + BigInt(L1_BLOCK_TIME)); + + // Next sequencer to publish a block should trigger a rollback to block 1 + await waitUntilL1Timestamp(l1Client, epoch2Start + BigInt(L1_BLOCK_TIME)); + expect(await rollup.getBlockNumber()).toEqual(1n); + expect(await rollup.getSlotNumber()).toEqual(8n); + + // The prover tx should have been rejected, and mined strictly before the one that triggered the rollback + const lastProverTxHash = proverDelayer.getTxs().at(-1); + const lastProverTxReceipt = await l1Client.getTransactionReceipt({ hash: lastProverTxHash! }); + expect(lastProverTxReceipt.status).toEqual('reverted'); + + const lastL2BlockTxHash = sequencerDelayer.getTxs().at(-1); + const lastL2BlockTxReceipt = await l1Client.getTransactionReceipt({ hash: lastL2BlockTxHash! }); + expect(lastL2BlockTxReceipt.status).toEqual('success'); + expect(lastL2BlockTxReceipt.blockNumber).toBeGreaterThan(lastProverTxReceipt!.blockNumber); + }); +}); diff --git a/yarn-project/end-to-end/src/e2e_event_logs.test.ts b/yarn-project/end-to-end/src/e2e_event_logs.test.ts index ab8d4f88785..162d53d6eb0 100644 --- a/yarn-project/end-to-end/src/e2e_event_logs.test.ts +++ b/yarn-project/end-to-end/src/e2e_event_logs.test.ts @@ -48,7 +48,7 @@ describe('Logs', () => { const txEffect = await node.getTxEffect(tx.txHash); - const encryptedLogs = txEffect!.encryptedLogs.unrollLogs(); + const encryptedLogs = txEffect!.data.encryptedLogs.unrollLogs(); expect(encryptedLogs.length).toBe(3); const decryptedEvent0 = L1EventPayload.decryptAsIncoming(encryptedLogs[0], wallets[0].getEncryptionSecret())!; diff --git a/yarn-project/end-to-end/src/e2e_l1_with_wall_time.test.ts b/yarn-project/end-to-end/src/e2e_l1_with_wall_time.test.ts index 15abecfb446..01a6d9f96ea 100644 --- a/yarn-project/end-to-end/src/e2e_l1_with_wall_time.test.ts +++ b/yarn-project/end-to-end/src/e2e_l1_with_wall_time.test.ts @@ -20,7 +20,7 @@ describe('e2e_l1_with_wall_time', () => { ({ teardown, logger, pxe } = await setup(0, { initialValidators, - l1BlockTime: ethereumSlotDuration, + ethereumSlotDuration, salt: 420, })); }); diff --git a/yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts b/yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts index 5e8e5d50e94..de2c603e404 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts @@ -3,7 +3,7 @@ import { sleep } from '@aztec/aztec.js'; import fs from 'fs'; -import { METRICS_PORT } from '../fixtures/fixtures.js'; +import { shouldCollectMetrics } from '../fixtures/fixtures.js'; import { type NodeContext, createNodes } from '../fixtures/setup_p2p_test.js'; import { P2PNetworkTest, WAIT_FOR_TX_TIMEOUT } from './p2p_network.js'; import { createPXEServiceAndSubmitTransactions } from './shared.js'; @@ -24,8 +24,8 @@ describe('e2e_p2p_network', () => { testName: 'e2e_p2p_network', numberOfNodes: NUM_NODES, basePort: BOOT_NODE_UDP_PORT, - // Uncomment to collect metrics - run in aztec-packages `docker compose --profile metrics up` - // metricsPort: METRICS_PORT, + // To collect metrics - run in aztec-packages `docker compose --profile metrics up` and set COLLECT_METRICS=true + metricsPort: shouldCollectMetrics(), }); await t.applyBaseSnapshots(); await t.setup(); @@ -44,6 +44,9 @@ describe('e2e_p2p_network', () => { if (!t.bootstrapNodeEnr) { throw new Error('Bootstrap node ENR is not available'); } + + t.ctx.aztecNodeConfig.validatorReexecute = true; + // create our network of nodes and submit txs into each of them // the number of txs per node and the number of txs per rollup // should be set so that the only way for rollups to be built @@ -57,7 +60,8 @@ describe('e2e_p2p_network', () => { NUM_NODES, BOOT_NODE_UDP_PORT, DATA_DIR, - METRICS_PORT, + // To collect metrics - run in aztec-packages `docker compose --profile metrics up` and set COLLECT_METRICS=true + shouldCollectMetrics(), ); // wait a bit for peers to discover each other diff --git a/yarn-project/end-to-end/src/e2e_p2p/p2p_network.ts b/yarn-project/end-to-end/src/e2e_p2p/p2p_network.ts index 529e2776294..95d263156e6 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/p2p_network.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/p2p_network.ts @@ -1,9 +1,11 @@ +import { getSchnorrAccount } from '@aztec/accounts/schnorr'; import { type AztecNodeConfig, type AztecNodeService } from '@aztec/aztec-node'; -import { EthCheatCodes } from '@aztec/aztec.js'; +import { type AccountWalletWithSecretKey, EthCheatCodes } from '@aztec/aztec.js'; import { EthAddress } from '@aztec/circuits.js'; import { getL1ContractsConfigEnvVars } from '@aztec/ethereum'; import { type DebugLogger, createDebugLogger } from '@aztec/foundation/log'; import { RollupAbi } from '@aztec/l1-artifacts'; +import { SpamContract } from '@aztec/noir-contracts.js'; import { type BootstrapNode } from '@aztec/p2p'; import { createBootstrapNodeFromPrivateKey } from '@aztec/p2p/mocks'; @@ -12,11 +14,17 @@ import { getContract } from 'viem'; import { privateKeyToAccount } from 'viem/accounts'; import { + PRIVATE_KEYS_START_INDEX, createValidatorConfig, generateNodePrivateKeys, generatePeerIdPrivateKeys, } from '../fixtures/setup_p2p_test.js'; -import { type ISnapshotManager, type SubsystemsContext, createSnapshotManager } from '../fixtures/snapshot_manager.js'; +import { + type ISnapshotManager, + type SubsystemsContext, + addAccounts, + createSnapshotManager, +} from '../fixtures/snapshot_manager.js'; import { getPrivateKeyFromIndex } from '../fixtures/utils.js'; import { getEndToEndTestTelemetryClient } from '../fixtures/with_telemetry_utils.js'; @@ -33,10 +41,15 @@ export class P2PNetworkTest { public ctx!: SubsystemsContext; public nodePrivateKeys: `0x${string}`[] = []; + public nodePublicKeys: string[] = []; public peerIdPrivateKeys: string[] = []; public bootstrapNodeEnr: string = ''; + // The re-execution test needs a wallet and a spam contract + public wallet?: AccountWalletWithSecretKey; + public spamContract?: SpamContract; + constructor( testName: string, public bootstrapNode: BootstrapNode, @@ -51,7 +64,8 @@ export class P2PNetworkTest { // Set up the base account and node private keys for the initial network deployment this.baseAccount = privateKeyToAccount(`0x${getPrivateKeyFromIndex(0)!.toString('hex')}`); - this.nodePrivateKeys = generateNodePrivateKeys(1, numberOfNodes); + this.nodePrivateKeys = generateNodePrivateKeys(PRIVATE_KEYS_START_INDEX, numberOfNodes); + this.nodePublicKeys = this.nodePrivateKeys.map(privateKey => privateKeyToAccount(privateKey).address); this.peerIdPrivateKeys = generatePeerIdPrivateKeys(numberOfNodes); this.bootstrapNodeEnr = bootstrapNode.getENR().encodeTxt(); @@ -60,7 +74,7 @@ export class P2PNetworkTest { this.snapshotManager = createSnapshotManager(`e2e_p2p_network/${testName}`, process.env.E2E_DATA_PATH, { ...initialValidatorConfig, - l1BlockTime: l1ContractsConfig.ethereumSlotDuration, + ethereumSlotDuration: l1ContractsConfig.ethereumSlotDuration, salt: 420, initialValidators, metricsPort: metricsPort, @@ -105,19 +119,18 @@ export class P2PNetworkTest { client: deployL1ContractsValues.walletClient, }); + this.logger.verbose(`Adding ${this.numberOfNodes} validators`); + const txHashes: `0x${string}`[] = []; for (let i = 0; i < this.numberOfNodes; i++) { const account = privateKeyToAccount(this.nodePrivateKeys[i]!); + this.logger.debug(`Adding ${account.address} as validator`); const txHash = await rollup.write.addValidator([account.address]); txHashes.push(txHash); + this.logger.debug(`Adding ${account.address} as validator`); } - // Remove the setup validator - const initialValidatorAddress = privateKeyToAccount(`0x${getPrivateKeyFromIndex(0)!.toString('hex')}`).address; - const txHash = await rollup.write.removeValidator([initialValidatorAddress]); - txHashes.push(txHash); - // Wait for all the transactions adding validators to be mined await Promise.all( txHashes.map(txHash => @@ -150,12 +163,85 @@ export class P2PNetworkTest { }); } + async setupAccount() { + await this.snapshotManager.snapshot( + 'setup-account', + addAccounts(1, this.logger, false), + async ({ accountKeys }, ctx) => { + const accountManagers = accountKeys.map(ak => getSchnorrAccount(ctx.pxe, ak[0], ak[1], 1)); + await Promise.all(accountManagers.map(a => a.register())); + const wallets = await Promise.all(accountManagers.map(a => a.getWallet())); + this.wallet = wallets[0]; + }, + ); + } + + async deploySpamContract() { + await this.snapshotManager.snapshot( + 'add-spam-contract', + async () => { + if (!this.wallet) { + throw new Error('Call snapshot t.setupAccount before deploying account contract'); + } + + const spamContract = await SpamContract.deploy(this.wallet).send().deployed(); + return { contractAddress: spamContract.address }; + }, + async ({ contractAddress }) => { + if (!this.wallet) { + throw new Error('Call snapshot t.setupAccount before deploying account contract'); + } + this.spamContract = await SpamContract.at(contractAddress, this.wallet); + }, + ); + } + + async removeInitialNode() { + await this.snapshotManager.snapshot( + 'remove-inital-validator', + async ({ deployL1ContractsValues, aztecNodeConfig }) => { + const rollup = getContract({ + address: deployL1ContractsValues.l1ContractAddresses.rollupAddress.toString(), + abi: RollupAbi, + client: deployL1ContractsValues.walletClient, + }); + + // Remove the setup validator + const initialValidatorAddress = privateKeyToAccount(`0x${getPrivateKeyFromIndex(0)!.toString('hex')}`).address; + const txHash = await rollup.write.removeValidator([initialValidatorAddress]); + + await deployL1ContractsValues.publicClient.waitForTransactionReceipt({ + hash: txHash, + }); + + //@note Now we jump ahead to the next epoch such that the validator committee is picked + // INTERVAL MINING: If we are using anvil interval mining this will NOT progress the time! + // Which means that the validator set will still be empty! So anyone can propose. + const slotsInEpoch = await rollup.read.EPOCH_DURATION(); + const timestamp = await rollup.read.getTimestampForSlot([slotsInEpoch]); + const cheatCodes = new EthCheatCodes(aztecNodeConfig.l1RpcUrl); + try { + await cheatCodes.warp(Number(timestamp)); + } catch (err) { + this.logger.debug('Warp failed, time already satisfied'); + } + + // Send and await a tx to make sure we mine a block for the warp to correctly progress. + await deployL1ContractsValues.publicClient.waitForTransactionReceipt({ + hash: await deployL1ContractsValues.walletClient.sendTransaction({ + to: this.baseAccount.address, + value: 1n, + account: this.baseAccount, + }), + }); + + await this.ctx.aztecNode.stop(); + }, + ); + } + async setup() { this.ctx = await this.snapshotManager.setup(); - - // TODO(md): make it such that the test can set these up - this.ctx.aztecNodeConfig.minTxsPerBlock = 4; - this.ctx.aztecNodeConfig.maxTxsPerBlock = 4; } async stopNodes(nodes: AztecNodeService[]) { diff --git a/yarn-project/end-to-end/src/e2e_p2p/rediscovery.test.ts b/yarn-project/end-to-end/src/e2e_p2p/rediscovery.test.ts index b1596b24dcb..bf3879d248c 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/rediscovery.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/rediscovery.test.ts @@ -26,6 +26,9 @@ describe('e2e_p2p_rediscovery', () => { }); await t.applyBaseSnapshots(); await t.setup(); + + // We remove the initial node such that it will no longer attempt to build blocks / be in the sequencing set + await t.removeInitialNode(); }); afterEach(async () => { @@ -61,7 +64,7 @@ describe('e2e_p2p_rediscovery', () => { const node = nodes[i]; await node.stop(); t.logger.info(`Node ${i} stopped`); - await sleep(1200); + await sleep(2500); const newNode = await createNode( t.ctx.aztecNodeConfig, diff --git a/yarn-project/end-to-end/src/e2e_p2p/reex.test.ts b/yarn-project/end-to-end/src/e2e_p2p/reex.test.ts new file mode 100644 index 00000000000..631f2910596 --- /dev/null +++ b/yarn-project/end-to-end/src/e2e_p2p/reex.test.ts @@ -0,0 +1,130 @@ +import { type AztecNodeService } from '@aztec/aztec-node'; +import { type SentTx, sleep } from '@aztec/aztec.js'; + +/* eslint-disable-next-line no-restricted-imports */ +import { BlockProposal, SignatureDomainSeperator, getHashedSignaturePayload } from '@aztec/circuit-types'; + +import { beforeAll, describe, it, jest } from '@jest/globals'; +import fs from 'fs'; + +import { createNodes } from '../fixtures/setup_p2p_test.js'; +import { P2PNetworkTest } from './p2p_network.js'; +import { submitComplexTxsTo } from './shared.js'; + +const NUM_NODES = 4; +const NUM_TXS_PER_NODE = 1; +const BOOT_NODE_UDP_PORT = 41000; + +const DATA_DIR = './data/re-ex'; + +describe('e2e_p2p_reex', () => { + let t: P2PNetworkTest; + let nodes: AztecNodeService[]; + + beforeAll(async () => { + nodes = []; + + t = await P2PNetworkTest.create({ + testName: 'e2e_p2p_reex', + numberOfNodes: NUM_NODES, + basePort: BOOT_NODE_UDP_PORT, + }); + + t.logger.verbose('Setup account'); + await t.setupAccount(); + + t.logger.verbose('Deploy spam contract'); + await t.deploySpamContract(); + + t.logger.verbose('Apply base snapshots'); + await t.applyBaseSnapshots(); + + t.logger.verbose('Setup nodes'); + await t.setup(); + }); + + afterAll(async () => { + // shutdown all nodes. + await t.stopNodes(nodes); + await t.teardown(); + for (let i = 0; i < NUM_NODES; i++) { + fs.rmSync(`${DATA_DIR}-${i}`, { recursive: true, force: true }); + } + }); + + it('validators should re-execute transactions before attesting', async () => { + // create the bootstrap node for the network + if (!t.bootstrapNodeEnr) { + throw new Error('Bootstrap node ENR is not available'); + } + + t.ctx.aztecNodeConfig.validatorReexecute = true; + + nodes = await createNodes( + t.ctx.aztecNodeConfig, + t.peerIdPrivateKeys, + t.bootstrapNodeEnr, + NUM_NODES, + BOOT_NODE_UDP_PORT, + ); + + // Hook into the node and intercept re-execution logic, ensuring that it was infact called + const reExecutionSpies = []; + for (const node of nodes) { + // Make sure the nodes submit faulty proposals, in this case a faulty proposal is one where we remove one of the transactions + // Such that the calculated archive will be different! + jest.spyOn((node as any).p2pClient, 'broadcastProposal').mockImplementation(async (...args: unknown[]) => { + // We remove one of the transactions, therefore the block root will be different! + const proposal = args[0] as BlockProposal; + const { txHashes } = proposal.payload; + + // We need to mutate the proposal, so we cast to any + (proposal.payload as any).txHashes = txHashes.slice(0, txHashes.length - 1); + + // We sign over the proposal using the node's signing key + // Abusing javascript to access the nodes signing key + const signer = (node as any).sequencer.sequencer.validatorClient.validationService.keyStore; + const newProposal = new BlockProposal( + proposal.payload, + await signer.signMessage(getHashedSignaturePayload(proposal.payload, SignatureDomainSeperator.blockProposal)), + ); + + return (node as any).p2pClient.p2pService.propagate(newProposal); + }); + + // Store re-execution spys node -> sequencer Client -> seqeuncer -> validator + const spy = jest.spyOn((node as any).sequencer.sequencer.validatorClient, 'reExecuteTransactions'); + reExecutionSpies.push(spy); + } + + // wait a bit for peers to discover each other + await sleep(4000); + + nodes.forEach(node => { + node.getSequencer()?.updateSequencerConfig({ + minTxsPerBlock: NUM_TXS_PER_NODE, + maxTxsPerBlock: NUM_TXS_PER_NODE, + }); + }); + const txs = await submitComplexTxsTo(t.logger, t.spamContract!, NUM_TXS_PER_NODE); + + // We ensure that the transactions are NOT mined + try { + await Promise.all( + txs.map(async (tx: SentTx, i: number) => { + t.logger.info(`Waiting for tx ${i}: ${await tx.getTxHash()} to be mined`); + return tx.wait(); + }), + ); + } catch (e) { + t.logger.info('Failed to mine all txs, as planned'); + } + + // Expect that all of the re-execution attempts failed with an invalid root + for (const spy of reExecutionSpies) { + for (const result of spy.mock.results) { + await expect(result.value).rejects.toThrow('Validator Error: Re-execution state mismatch'); + } + } + }); +}); diff --git a/yarn-project/end-to-end/src/e2e_p2p/reqresp.test.ts b/yarn-project/end-to-end/src/e2e_p2p/reqresp.test.ts index 9de73adca98..1f0d20c04d3 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/reqresp.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/reqresp.test.ts @@ -1,15 +1,17 @@ import { type AztecNodeService } from '@aztec/aztec-node'; import { sleep } from '@aztec/aztec.js'; +import { RollupAbi } from '@aztec/l1-artifacts'; import { jest } from '@jest/globals'; import fs from 'fs'; +import { getContract } from 'viem'; import { type NodeContext, createNodes } from '../fixtures/setup_p2p_test.js'; import { P2PNetworkTest, WAIT_FOR_TX_TIMEOUT } from './p2p_network.js'; import { createPXEServiceAndSubmitTransactions } from './shared.js'; // Don't set this to a higher value than 9 because each node will use a different L1 publisher account and anvil seeds -const NUM_NODES = 4; +const NUM_NODES = 6; const NUM_TXS_PER_NODE = 2; const BOOT_NODE_UDP_PORT = 40800; @@ -27,6 +29,7 @@ describe('e2e_p2p_reqresp_tx', () => { }); await t.applyBaseSnapshots(); await t.setup(); + await t.removeInitialNode(); }); afterEach(async () => { @@ -37,10 +40,6 @@ describe('e2e_p2p_reqresp_tx', () => { } }); - // NOTE: If this test fails in a PR where the shuffling algorithm is changed, then it is failing as the node with - // the mocked p2p layer is being picked as the sequencer, and it does not have any transactions in it's mempool. - // If this is the case, then we should update the test to switch off the mempool of a different node. - // adjust `nodeToTurnOffTxGossip` in the test below. it('should produce an attestation by requesting tx data over the p2p network', async () => { /** * Birds eye overview of the test @@ -73,12 +72,15 @@ describe('e2e_p2p_reqresp_tx', () => { // wait a bit for peers to discover each other await sleep(4000); - t.logger.info('Turning off tx gossip'); + const { proposerIndexes, nodesToTurnOffTxGossip } = await getProposerIndexes(); + + t.logger.info(`Turning off tx gossip for nodes: ${nodesToTurnOffTxGossip}`); + t.logger.info(`Sending txs to proposer nodes: ${proposerIndexes}`); + // Replace the p2p node implementation of some of the nodes with a spy such that it does not store transactions that are gossiped to it // Original implementation of `processTxFromPeer` will store received transactions in the tx pool. - // We have chosen nodes 0,3 as they do not get chosen to be the sequencer in this test. - const nodeToTurnOffTxGossip = [0, 3]; - for (const nodeIndex of nodeToTurnOffTxGossip) { + // We chose the first 2 nodes that will be the proposers for the next few slots + for (const nodeIndex of nodesToTurnOffTxGossip) { jest .spyOn((nodes[nodeIndex] as any).p2pClient.p2pService, 'processTxFromPeer') .mockImplementation((): Promise => { @@ -87,10 +89,9 @@ describe('e2e_p2p_reqresp_tx', () => { } t.logger.info('Submitting transactions'); - // Only submit transactions to the first two nodes, so that we avoid our sequencer with a mocked p2p layer being picked to produce a block. - // If the shuffling algorithm changes, then this will need to be updated. - for (let i = 1; i < 3; i++) { - const context = await createPXEServiceAndSubmitTransactions(t.logger, nodes[i], NUM_TXS_PER_NODE); + + for (const nodeIndex of proposerIndexes.slice(0, 2)) { + const context = await createPXEServiceAndSubmitTransactions(t.logger, nodes[nodeIndex], NUM_TXS_PER_NODE); contexts.push(context); } @@ -107,4 +108,34 @@ describe('e2e_p2p_reqresp_tx', () => { ); t.logger.info('All transactions mined'); }); + + /** + * Get the indexes in the nodes array that will produce the next few blocks + */ + async function getProposerIndexes() { + // Get the nodes for the next set of slots + const rollupContract = getContract({ + address: t.ctx.deployL1ContractsValues.l1ContractAddresses.rollupAddress.toString(), + abi: RollupAbi, + client: t.ctx.deployL1ContractsValues.publicClient, + }); + + const currentTime = await t.ctx.cheatCodes.eth.timestamp(); + const slotDuration = await rollupContract.read.SLOT_DURATION(); + + const proposers = []; + + for (let i = 0; i < 3; i++) { + const nextSlot = BigInt(currentTime) + BigInt(i) * BigInt(slotDuration); + const proposer = await rollupContract.read.getProposerAt([nextSlot]); + proposers.push(proposer); + } + + // Get the indexes of the nodes that are responsible for the next two slots + const proposerIndexes = proposers.map(proposer => t.nodePublicKeys.indexOf(proposer)); + const nodesToTurnOffTxGossip = Array.from({ length: NUM_NODES }, (_, i) => i).filter( + i => !proposerIndexes.includes(i), + ); + return { proposerIndexes, nodesToTurnOffTxGossip }; + } }); diff --git a/yarn-project/end-to-end/src/e2e_p2p/shared.ts b/yarn-project/end-to-end/src/e2e_p2p/shared.ts index 9b787eaa564..d1c35dfdb66 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/shared.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/shared.ts @@ -1,12 +1,37 @@ import { getSchnorrAccount } from '@aztec/accounts/schnorr'; import { type AztecNodeService } from '@aztec/aztec-node'; -import { type DebugLogger } from '@aztec/aztec.js'; +import { type DebugLogger, type SentTx } from '@aztec/aztec.js'; import { CompleteAddress, TxStatus } from '@aztec/aztec.js'; import { Fr, GrumpkinScalar } from '@aztec/foundation/fields'; +import { type SpamContract } from '@aztec/noir-contracts.js'; import { type PXEService, createPXEService, getPXEServiceConfig as getRpcConfig } from '@aztec/pxe'; import { type NodeContext } from '../fixtures/setup_p2p_test.js'; +// submits a set of transactions to the provided Private eXecution Environment (PXE) +export const submitComplexTxsTo = async (logger: DebugLogger, spamContract: SpamContract, numTxs: number) => { + const txs: SentTx[] = []; + + const seed = 1234n; + const spamCount = 15; + for (let i = 0; i < numTxs; i++) { + const tx = spamContract.methods.spam(seed + BigInt(i * spamCount), spamCount, false).send(); + const txHash = await tx.getTxHash(); + + logger.info(`Tx sent with hash ${txHash}`); + const receipt = await tx.getReceipt(); + expect(receipt).toEqual( + expect.objectContaining({ + status: TxStatus.PENDING, + error: '', + }), + ); + logger.info(`Receipt received for ${txHash}`); + txs.push(tx); + } + return txs; +}; + // creates an instance of the PXE and submit a given number of transactions to it. export const createPXEServiceAndSubmitTransactions = async ( logger: DebugLogger, diff --git a/yarn-project/end-to-end/src/e2e_synching.test.ts b/yarn-project/end-to-end/src/e2e_synching.test.ts index 743c780bda8..37c9237ab0c 100644 --- a/yarn-project/end-to-end/src/e2e_synching.test.ts +++ b/yarn-project/end-to-end/src/e2e_synching.test.ts @@ -11,7 +11,6 @@ * * To run the Setup run with the `AZTEC_GENERATE_TEST_DATA=1` flag. Without * this flag, we will run in execution. - * * There is functionality to store the `stats` of a sync, but currently we * will simply be writing it to the log instead. * @@ -419,10 +418,7 @@ describe('e2e_synching', () => { async (opts: Partial, variant: TestVariant) => { // All the blocks have been "re-played" and we are now to simply get a new node up to speed const timer = new Timer(); - const freshNode = await AztecNodeService.createAndSync( - { ...opts.config!, disableValidator: true }, - new NoopTelemetryClient(), - ); + const freshNode = await AztecNodeService.createAndSync({ ...opts.config!, disableValidator: true }); const syncTime = timer.s(); const blockNumber = await freshNode.getBlockNumber(); @@ -468,7 +464,7 @@ describe('e2e_synching', () => { ); await watcher.start(); - const aztecNode = await AztecNodeService.createAndSync(opts.config!, new NoopTelemetryClient()); + const aztecNode = await AztecNodeService.createAndSync(opts.config!); const sequencer = aztecNode.getSequencer(); const { pxe } = await setupPXEService(aztecNode!); @@ -579,7 +575,7 @@ describe('e2e_synching', () => { const pendingBlockNumber = await rollup.read.getPendingBlockNumber(); await rollup.write.setAssumeProvenThroughBlockNumber([pendingBlockNumber - BigInt(variant.blockCount) / 2n]); - const aztecNode = await AztecNodeService.createAndSync(opts.config!, new NoopTelemetryClient()); + const aztecNode = await AztecNodeService.createAndSync(opts.config!); const sequencer = aztecNode.getSequencer(); const blockBeforePrune = await aztecNode.getBlockNumber(); @@ -660,7 +656,7 @@ describe('e2e_synching', () => { await watcher.start(); // The sync here could likely be avoided by using the node we just synched. - const aztecNode = await AztecNodeService.createAndSync(opts.config!, new NoopTelemetryClient()); + const aztecNode = await AztecNodeService.createAndSync(opts.config!); const sequencer = aztecNode.getSequencer(); const { pxe } = await setupPXEService(aztecNode!); diff --git a/yarn-project/end-to-end/src/e2e_token_contract/token_contract_test.ts b/yarn-project/end-to-end/src/e2e_token_contract/token_contract_test.ts index ccaea95bc3f..711e3fa92d7 100644 --- a/yarn-project/end-to-end/src/e2e_token_contract/token_contract_test.ts +++ b/yarn-project/end-to-end/src/e2e_token_contract/token_contract_test.ts @@ -46,14 +46,6 @@ export class TokenContractTest { const accountManagers = accountKeys.map(ak => getSchnorrAccount(pxe, ak[0], ak[1], 1)); this.wallets = await Promise.all(accountManagers.map(a => a.getWallet())); this.accounts = await pxe.getRegisteredAccounts(); - // // Add every wallet the contacts of every other wallet. This way, they can send notes to each other and discover them - // await Promise.all( - // this.wallets.map(w => { - // const otherWallets = this.wallets.filter(ow => ow.getAddress() !== w.getAddress()); - // return Promise.all(otherWallets.map(ow => w.registerContact(ow.getAddress()))); - // }), - // ); - // this.wallets.forEach((w, i) => this.logger.verbose(`Wallet ${i} address: ${w.getAddress()}`)); }); await this.snapshotManager.snapshot( diff --git a/yarn-project/end-to-end/src/fixtures/fixtures.ts b/yarn-project/end-to-end/src/fixtures/fixtures.ts index c9ee5dd9601..ccaa85f3516 100644 --- a/yarn-project/end-to-end/src/fixtures/fixtures.ts +++ b/yarn-project/end-to-end/src/fixtures/fixtures.ts @@ -1,5 +1,12 @@ export const METRICS_PORT = 4318; +export const shouldCollectMetrics = () => { + if (process.env.COLLECT_METRICS) { + return METRICS_PORT; + } + return undefined; +}; + export const MNEMONIC = 'test test test test test test test test test test test junk'; export const privateKey = Buffer.from('ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80', 'hex'); export const privateKey2 = Buffer.from('59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d', 'hex'); diff --git a/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts b/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts index 46994fbfd7f..7dac778d50f 100644 --- a/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts +++ b/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts @@ -12,6 +12,10 @@ import { generatePrivateKey } from 'viem/accounts'; import { getPrivateKeyFromIndex } from './utils.js'; import { getEndToEndTestTelemetryClient } from './with_telemetry_utils.js'; +// Setup snapshots will create a node with index 0, so all of our loops here +// need to start from 1 to avoid running validators with the same key +export const PRIVATE_KEYS_START_INDEX = 1; + export interface NodeContext { node: AztecNodeService; pxeService: PXEService; @@ -52,11 +56,19 @@ export function createNodes( ): Promise { const nodePromises = []; for (let i = 0; i < numNodes; i++) { - // We run on ports from the bootnode upwards if a port if provided, otherwise we get a random port + // We run on ports from the bootnode upwards const port = bootNodePort + i + 1; const dataDir = dataDirectory ? `${dataDirectory}-${i}` : undefined; - const nodePromise = createNode(config, peerIdPrivateKeys[i], port, bootstrapNodeEnr, i, dataDir, metricsPort); + const nodePromise = createNode( + config, + peerIdPrivateKeys[i], + port, + bootstrapNodeEnr, + i + PRIVATE_KEYS_START_INDEX, + dataDir, + metricsPort, + ); nodePromises.push(nodePromise); } return Promise.all(nodePromises); @@ -83,11 +95,10 @@ export async function createNode( const telemetryClient = await getEndToEndTestTelemetryClient(metricsPort, /*serviceName*/ `node:${tcpPort}`); - return await AztecNodeService.createAndSync( - validatorConfig, - telemetryClient, - createDebugLogger(`aztec:node-${tcpPort}`), - ); + return await AztecNodeService.createAndSync(validatorConfig, { + telemetry: telemetryClient, + logger: createDebugLogger(`aztec:node-${tcpPort}`), + }); } export async function createValidatorConfig( @@ -95,7 +106,7 @@ export async function createValidatorConfig( bootstrapNodeEnr?: string, port?: number, peerIdPrivateKey?: string, - accountIndex: number = 0, + accountIndex: number = 1, dataDirectory?: string, ) { peerIdPrivateKey = peerIdPrivateKey ?? generatePeerIdPrivateKey(); @@ -114,8 +125,6 @@ export async function createValidatorConfig( tcpListenAddress: `0.0.0.0:${port}`, tcpAnnounceAddress: `127.0.0.1:${port}`, udpAnnounceAddress: `127.0.0.1:${port}`, - minTxsPerBlock: config.minTxsPerBlock, - maxTxsPerBlock: config.maxTxsPerBlock, p2pEnabled: true, blockCheckIntervalMS: 1000, transactionProtocol: '', diff --git a/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts b/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts index ee16764f278..3a85d0ff79b 100644 --- a/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts +++ b/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts @@ -1,10 +1,8 @@ import { SchnorrAccountContractArtifact, getSchnorrAccount } from '@aztec/accounts/schnorr'; -import { type Archiver, createArchiver } from '@aztec/archiver'; import { type AztecNodeConfig, AztecNodeService, getConfigEnvVars } from '@aztec/aztec-node'; import { AnvilTestWatcher, type AztecAddress, - type AztecNode, BatchCall, CheatCodes, type CompleteAddress, @@ -18,18 +16,17 @@ import { } from '@aztec/aztec.js'; import { deployInstance, registerContractClass } from '@aztec/aztec.js/deployment'; import { type DeployL1ContractsArgs, createL1Clients, getL1ContractsConfigEnvVars, l1Artifacts } from '@aztec/ethereum'; +import { startAnvil } from '@aztec/ethereum/test'; import { asyncMap } from '@aztec/foundation/async-map'; import { type Logger, createDebugLogger } from '@aztec/foundation/log'; import { resolver, reviver } from '@aztec/foundation/serialize'; -import { type ProverNode, type ProverNodeConfig, createProverNode } from '@aztec/prover-node'; +import { type ProverNode } from '@aztec/prover-node'; import { type PXEService, createPXEService, getPXEServiceConfig } from '@aztec/pxe'; -import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { createAndStartTelemetryClient, getConfigEnvVars as getTelemetryConfig } from '@aztec/telemetry-client/start'; -import { type Anvil, createAnvil } from '@viem/anvil'; +import { type Anvil } from '@viem/anvil'; import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs'; import { copySync, removeSync } from 'fs-extra/esm'; -import getPort from 'get-port'; import { join } from 'path'; import { type Hex, getContract } from 'viem'; import { mnemonicToAccount } from 'viem/accounts'; @@ -38,7 +35,7 @@ import { MNEMONIC } from './fixtures.js'; import { getACVMConfig } from './get_acvm_config.js'; import { getBBConfig } from './get_bb_config.js'; import { setupL1Contracts } from './setup_l1_contracts.js'; -import { type SetupOptions, getPrivateKeyFromIndex, startAnvil } from './utils.js'; +import { type SetupOptions, createAndSyncProverNode, getPrivateKeyFromIndex } from './utils.js'; import { getEndToEndTestTelemetryClient } from './with_telemetry_utils.js'; export type SubsystemsContext = { @@ -252,47 +249,6 @@ async function teardown(context: SubsystemsContext | undefined) { await context.watcher.stop(); } -async function createAndSyncProverNode( - proverNodePrivateKey: `0x${string}`, - aztecNodeConfig: AztecNodeConfig, - aztecNode: AztecNode, -) { - // Disable stopping the aztec node as the prover coordination test will kill it otherwise - // This is only required when stopping the prover node for testing - const aztecNodeWithoutStop = { - addEpochProofQuote: aztecNode.addEpochProofQuote.bind(aztecNode), - getTxByHash: aztecNode.getTxByHash.bind(aztecNode), - stop: () => Promise.resolve(), - }; - - // Creating temp store and archiver for simulated prover node - const archiverConfig = { ...aztecNodeConfig, dataDirectory: undefined }; - const archiver = await createArchiver(archiverConfig, new NoopTelemetryClient(), { blockUntilSync: true }); - - // Prover node config is for simulated proofs - const proverConfig: ProverNodeConfig = { - ...aztecNodeConfig, - proverCoordinationNodeUrl: undefined, - dataDirectory: undefined, - proverId: new Fr(42), - realProofs: false, - proverAgentConcurrency: 2, - publisherPrivateKey: proverNodePrivateKey, - proverNodeMaxPendingJobs: 10, - proverNodePollingIntervalMs: 200, - quoteProviderBasisPointFee: 100, - quoteProviderBondAmount: 1000n, - proverMinimumEscrowAmount: 1000n, - proverTargetEscrowAmount: 2000n, - }; - const proverNode = await createProverNode(proverConfig, { - aztecNodeTxProvider: aztecNodeWithoutStop, - archiver: archiver as Archiver, - }); - await proverNode.start(); - return proverNode; -} - /** * Initializes a fresh set of subsystems. * If given a statePath, the state will be written to the path. @@ -304,6 +260,7 @@ async function setupFromFresh( opts: SetupOptions = {}, deployL1ContractsArgs: Partial = { assumeProvenThrough: Number.MAX_SAFE_INTEGER, + initialValidators: [], }, ): Promise { logger.verbose(`Initializing state...`); @@ -315,7 +272,7 @@ async function setupFromFresh( // Start anvil. We go via a wrapper script to ensure if the parent dies, anvil dies. logger.verbose('Starting anvil...'); - const res = await startAnvil(opts.l1BlockTime); + const res = await startAnvil(opts.ethereumSlotDuration); const anvil = res.anvil; aztecNodeConfig.l1RpcUrl = res.rpcUrl; @@ -325,8 +282,8 @@ async function setupFromFresh( const publisherPrivKeyRaw = hdAccount.getHdKey().privateKey; const publisherPrivKey = publisherPrivKeyRaw === null ? null : Buffer.from(publisherPrivKeyRaw); - const validatorPrivKey = getPrivateKeyFromIndex(1); - const proverNodePrivateKey = getPrivateKeyFromIndex(2); + const validatorPrivKey = getPrivateKeyFromIndex(0); + const proverNodePrivateKey = getPrivateKeyFromIndex(0); aztecNodeConfig.publisherPrivateKey = `0x${publisherPrivKey!.toString('hex')}`; aztecNodeConfig.validatorPrivateKey = `0x${validatorPrivKey!.toString('hex')}`; @@ -391,7 +348,7 @@ async function setupFromFresh( const telemetry = await getEndToEndTestTelemetryClient(opts.metricsPort, /*serviceName*/ 'basenode'); logger.verbose('Creating and synching an aztec node...'); - const aztecNode = await AztecNodeService.createAndSync(aztecNodeConfig, telemetry); + const aztecNode = await AztecNodeService.createAndSync(aztecNodeConfig, { telemetry }); let proverNode: ProverNode | undefined = undefined; if (opts.startProverNode) { @@ -434,7 +391,6 @@ async function setupFromFresh( async function setupFromState(statePath: string, logger: Logger): Promise { logger.verbose(`Initializing with saved state at ${statePath}...`); - // Load config. // TODO: For some reason this is currently the union of a bunch of subsystems. That needs fixing. const aztecNodeConfig: AztecNodeConfig & SetupOptions = JSON.parse( readFileSync(`${statePath}/aztec_node_config.json`, 'utf-8'), @@ -443,10 +399,8 @@ async function setupFromState(statePath: string, logger: Logger): Promise provenTx.send())); await Promise.all(txs.map(tx => tx.wait({ interval: 0.1, proven: waitUntilProven }))); diff --git a/yarn-project/end-to-end/src/fixtures/token_utils.ts b/yarn-project/end-to-end/src/fixtures/token_utils.ts index 6769aae9daa..f623bcf3d5d 100644 --- a/yarn-project/end-to-end/src/fixtures/token_utils.ts +++ b/yarn-project/end-to-end/src/fixtures/token_utils.ts @@ -1,3 +1,4 @@ +// docs:start:token_utils import { type AztecAddress, type DebugLogger, type Wallet } from '@aztec/aztec.js'; import { TokenContract } from '@aztec/noir-contracts.js'; @@ -27,6 +28,7 @@ export async function mintTokensToPrivate( const from = minterWallet.getAddress(); // we are setting from to minter here because of TODO(#9887) await tokenAsMinter.methods.mint_to_private(from, recipient, amount).send().wait(); } +// docs:end:token_utils export async function expectTokenBalance( wallet: Wallet, diff --git a/yarn-project/end-to-end/src/fixtures/utils.ts b/yarn-project/end-to-end/src/fixtures/utils.ts index b47671ad72d..85666248f53 100644 --- a/yarn-project/end-to-end/src/fixtures/utils.ts +++ b/yarn-project/end-to-end/src/fixtures/utils.ts @@ -1,5 +1,6 @@ import { SchnorrAccountContractArtifact } from '@aztec/accounts/schnorr'; import { createAccounts, getDeployedTestAccountsWallets } from '@aztec/accounts/testing'; +import { type Archiver, createArchiver } from '@aztec/archiver'; import { type AztecNodeConfig, AztecNodeService, getConfigEnvVars } from '@aztec/aztec-node'; import { type AccountWalletWithSecretKey, @@ -29,23 +30,32 @@ import { import { deployInstance, registerContractClass } from '@aztec/aztec.js/deployment'; import { DefaultMultiCallEntrypoint } from '@aztec/aztec.js/entrypoint'; import { type BBNativePrivateKernelProver } from '@aztec/bb-prover'; -import { type EthAddress, GasSettings, getContractClassFromArtifact } from '@aztec/circuits.js'; -import { NULL_KEY, getL1ContractsConfigEnvVars, isAnvilTestChain, l1Artifacts } from '@aztec/ethereum'; -import { makeBackoff, retry, retryUntil } from '@aztec/foundation/retry'; +import { type EthAddress, Fr, GasSettings, getContractClassFromArtifact } from '@aztec/circuits.js'; +import { + type DeployL1ContractsArgs, + NULL_KEY, + getL1ContractsConfigEnvVars, + isAnvilTestChain, + l1Artifacts, +} from '@aztec/ethereum'; +import { startAnvil } from '@aztec/ethereum/test'; +import { retryUntil } from '@aztec/foundation/retry'; import { FeeJuiceContract } from '@aztec/noir-contracts.js/FeeJuice'; import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types'; import { ProtocolContractAddress, protocolContractTreeRoot } from '@aztec/protocol-contracts'; +import { type ProverNode, type ProverNodeConfig, createProverNode } from '@aztec/prover-node'; import { PXEService, type PXEServiceConfig, createPXEService, getPXEServiceConfig } from '@aztec/pxe'; -import { type SequencerClient } from '@aztec/sequencer-client'; +import { type SequencerClient, TestL1Publisher } from '@aztec/sequencer-client'; +import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { createAndStartTelemetryClient, getConfigEnvVars as getTelemetryConfig } from '@aztec/telemetry-client/start'; -import { type Anvil, createAnvil } from '@viem/anvil'; -import getPort from 'get-port'; +import { type Anvil } from '@viem/anvil'; import * as path from 'path'; import { type Account, type Chain, type HDAccount, + type Hex, type HttpTransport, type PrivateKeyAccount, createPublicClient, @@ -62,6 +72,7 @@ import { getBBConfig } from './get_bb_config.js'; import { isMetricsLoggingRequested, setupMetricsLogger } from './logging.js'; export { deployAndInitializeTokenAndBridgeContracts } from '../shared/cross_chain_test_harness.js'; +export { startAnvil }; const { PXE_URL = '' } = process.env; @@ -87,13 +98,7 @@ export const setupL1Contracts = async ( l1RpcUrl: string, account: HDAccount | PrivateKeyAccount, logger: DebugLogger, - args: { - salt?: number; - initialValidators?: EthAddress[]; - assumeProvenThrough?: number; - } = { - assumeProvenThrough: Number.MAX_SAFE_INTEGER, - }, + args: Partial = {}, chain: Chain = foundry, ) => { const l1Data = await deployL1Contracts(l1RpcUrl, account, chain, logger, { @@ -104,6 +109,7 @@ export const setupL1Contracts = async ( initialValidators: args.initialValidators, assumeProvenThrough: args.assumeProvenThrough, ...getL1ContractsConfigEnvVars(), + ...args, }); return l1Data; @@ -213,7 +219,7 @@ async function setupWithRemoteEnvironment( return { aztecNode, sequencer: undefined, - prover: undefined, + proverNode: undefined, pxe: pxeClient, deployL1ContractsValues, accounts: await pxeClient!.getRegisteredAccounts(), @@ -241,8 +247,6 @@ export type SetupOptions = { salt?: number; /** An initial set of validators */ initialValidators?: EthAddress[]; - /** Anvil block time (interval) */ - l1BlockTime?: number; /** Anvil Start time */ l1StartTime?: number; /** The anvil time where we should at the earliest be seeing L2 blocks */ @@ -259,6 +263,8 @@ export type SetupOptions = { export type EndToEndContext = { /** The Aztec Node service or client a connected to it. */ aztecNode: AztecNode; + /** The prover node service (only set if startProverNode is true) */ + proverNode: ProverNode | undefined; /** A client to the sequencer service (undefined if connected to remote environment) */ sequencer: SequencerClient | undefined; /** The Private eXecution Environment (PXE). */ @@ -310,7 +316,7 @@ export async function setup( ); } - const res = await startAnvil(opts.l1BlockTime); + const res = await startAnvil(opts.ethereumSlotDuration); anvil = res.anvil; config.l1RpcUrl = res.rpcUrl; } @@ -355,14 +361,7 @@ export async function setup( } const deployL1ContractsValues = - opts.deployL1ContractsValues ?? - (await setupL1Contracts( - config.l1RpcUrl, - publisherHdAccount!, - logger, - { salt: opts.salt, initialValidators: opts.initialValidators, assumeProvenThrough: opts.assumeProvenThrough }, - chain, - )); + opts.deployL1ContractsValues ?? (await setupL1Contracts(config.l1RpcUrl, publisherHdAccount!, logger, opts, chain)); config.l1Contracts = deployL1ContractsValues.l1ContractAddresses; @@ -419,11 +418,19 @@ export async function setup( config.l1PublishRetryIntervalMS = 100; const telemetry = await telemetryPromise; - const aztecNode = await AztecNodeService.createAndSync(config, telemetry); + const publisher = new TestL1Publisher(config, telemetry); + const aztecNode = await AztecNodeService.createAndSync(config, { telemetry, publisher }); const sequencer = aztecNode.getSequencer(); - logger.verbose('Creating a pxe...'); + let proverNode: ProverNode | undefined = undefined; + if (opts.startProverNode) { + logger.verbose('Creating and syncing a simulated prover node...'); + const proverNodePrivateKey = getPrivateKeyFromIndex(2); + const proverNodePrivateKeyHex: Hex = `0x${proverNodePrivateKey!.toString('hex')}`; + proverNode = await createAndSyncProverNode(proverNodePrivateKeyHex, config, aztecNode); + } + logger.verbose('Creating a pxe...'); const { pxe } = await setupPXEService(aztecNode!, pxeOpts, logger); if (!config.skipProtocolContracts) { @@ -456,6 +463,7 @@ export async function setup( return { aztecNode, + proverNode, pxe, deployL1ContractsValues, config, @@ -479,37 +487,6 @@ export function getL1WalletClient(rpcUrl: string, index: number) { }); } -/** - * Ensures there's a running Anvil instance and returns the RPC URL. - * @returns - */ -export async function startAnvil(l1BlockTime?: number): Promise<{ anvil: Anvil; rpcUrl: string }> { - let rpcUrl: string | undefined = undefined; - - // Start anvil. - // We go via a wrapper script to ensure if the parent dies, anvil dies. - const anvil = await retry( - async () => { - const ethereumHostPort = await getPort(); - rpcUrl = `http://127.0.0.1:${ethereumHostPort}`; - const anvil = createAnvil({ - anvilBinary: './scripts/anvil_kill_wrapper.sh', - port: ethereumHostPort, - blockTime: l1BlockTime, - }); - await anvil.start(); - return anvil; - }, - 'Start anvil', - makeBackoff([5, 5, 5]), - ); - - if (!rpcUrl) { - throw new Error('Failed to start anvil'); - } - - return { anvil, rpcUrl }; -} /** * Registers the contract class used for test accounts and publicly deploys the instances requested. * Use this when you need to make a public call to an account contract, such as for requesting a public authwit. @@ -699,3 +676,49 @@ export async function waitForProvenChain(node: AztecNode, targetBlock?: number, intervalSec, ); } + +export async function createAndSyncProverNode( + proverNodePrivateKey: `0x${string}`, + aztecNodeConfig: AztecNodeConfig, + aztecNode: AztecNode, +) { + // Disable stopping the aztec node as the prover coordination test will kill it otherwise + // This is only required when stopping the prover node for testing + const aztecNodeWithoutStop = { + addEpochProofQuote: aztecNode.addEpochProofQuote.bind(aztecNode), + getTxByHash: aztecNode.getTxByHash.bind(aztecNode), + stop: () => Promise.resolve(), + }; + + // Creating temp store and archiver for simulated prover node + const archiverConfig = { ...aztecNodeConfig, dataDirectory: undefined }; + const archiver = await createArchiver(archiverConfig, new NoopTelemetryClient(), { blockUntilSync: true }); + + // Prover node config is for simulated proofs + const proverConfig: ProverNodeConfig = { + ...aztecNodeConfig, + proverCoordinationNodeUrl: undefined, + dataDirectory: undefined, + proverId: new Fr(42), + realProofs: false, + proverAgentConcurrency: 2, + publisherPrivateKey: proverNodePrivateKey, + proverNodeMaxPendingJobs: 10, + proverNodePollingIntervalMs: 200, + quoteProviderBasisPointFee: 100, + quoteProviderBondAmount: 1000n, + proverMinimumEscrowAmount: 1000n, + proverTargetEscrowAmount: 2000n, + }; + + // Use testing l1 publisher + const publisher = new TestL1Publisher(proverConfig, new NoopTelemetryClient()); + + const proverNode = await createProverNode(proverConfig, { + aztecNodeTxProvider: aztecNodeWithoutStop, + archiver: archiver as Archiver, + publisher, + }); + await proverNode.start(); + return proverNode; +} diff --git a/yarn-project/end-to-end/src/prover-coordination/e2e_prover_coordination.test.ts b/yarn-project/end-to-end/src/prover-coordination/e2e_prover_coordination.test.ts index a6ea11a011b..665abe523dc 100644 --- a/yarn-project/end-to-end/src/prover-coordination/e2e_prover_coordination.test.ts +++ b/yarn-project/end-to-end/src/prover-coordination/e2e_prover_coordination.test.ts @@ -359,6 +359,12 @@ describe('e2e_prover_coordination', () => { }); it('Can claim proving rights after a prune', async () => { + await logState(); + + const tips = await ctx.cheatCodes.rollup.getTips(); + + let currentPending = tips.pending; + let currentProven = tips.proven; // Here we are creating a proof quote for epoch 0 const quoteForEpoch0 = await makeEpochProofQuote({ epochToProve: 0n, @@ -373,9 +379,11 @@ describe('e2e_prover_coordination', () => { // Build a block in epoch 1, we should see the quote for epoch 0 submitted earlier published to L1 await contract.methods.create_note(recipient, recipient, 10).send().wait(); + currentPending++; + // Verify that we can claim the current epoch await expectProofClaimOnL1({ ...quoteForEpoch0.payload, proposer: publisherAddress }); - await expectTips({ pending: 3n, proven: 0n }); + await expectTips({ pending: currentPending, proven: currentProven }); // Now go to epoch 1 await advanceToNextEpoch(); @@ -384,19 +392,23 @@ describe('e2e_prover_coordination', () => { const epoch0BlockNumber = await getPendingBlockNumber(); await rollupContract.write.setAssumeProvenThroughBlockNumber([BigInt(epoch0BlockNumber)]); + currentProven = epoch0BlockNumber; + // Go to epoch 2 await advanceToNextEpoch(); // Progress epochs with a block in each until we hit a reorg // Note tips are block numbers, not slots - await expectTips({ pending: 3n, proven: 3n }); + await expectTips({ pending: currentPending, proven: currentProven }); const tx2BeforeReorg = await contract.methods.create_note(recipient, recipient, 10).send().wait(); - await expectTips({ pending: 4n, proven: 3n }); + currentPending++; + await expectTips({ pending: currentPending, proven: currentProven }); // Go to epoch 3 await advanceToNextEpoch(); const tx3BeforeReorg = await contract.methods.create_note(recipient, recipient, 10).send().wait(); - await expectTips({ pending: 5n, proven: 3n }); + currentPending++; + await expectTips({ pending: currentPending, proven: currentProven }); // Go to epoch 4 !!! REORG !!! ay caramba !!! await advanceToNextEpoch(); @@ -430,12 +442,15 @@ describe('e2e_prover_coordination', () => { const newWallet = await createAccount(newPxe); const newWalletAddress = newWallet.getAddress(); - // The chain will prune back to block 3 + // after the re-org the pending chain has moved on by 2 blocks + currentPending = currentProven + 2n; + + // The chain will prune back to the proven block number // then include the txs from the pruned epochs that are still valid - // bringing us back to block 4 (same number, different hash) + // bringing us back to block proven + 1 (same number, different hash) // creating a new account will produce another block - // so we expect 5 blocks in the pending chain here! - await expectTips({ pending: 5n, proven: 3n }); + // so we expect proven + 2 blocks in the pending chain here! + await expectTips({ pending: currentPending, proven: currentProven }); // Submit proof claim for the new epoch const quoteForEpoch4 = await makeEpochProofQuote({ @@ -453,7 +468,8 @@ describe('e2e_prover_coordination', () => { logger.info('Sending new tx on reorged chain'); await contractFromNewPxe.methods.create_note(newWalletAddress, newWalletAddress, 10).send().wait(); - await expectTips({ pending: 6n, proven: 3n }); + currentPending++; + await expectTips({ pending: currentPending, proven: currentProven }); // Expect the proof claim to be accepted for the chain after the reorg await expectProofClaimOnL1({ ...quoteForEpoch4.payload, proposer: publisherAddress }); diff --git a/yarn-project/end-to-end/src/spartan/4epochs.test.ts b/yarn-project/end-to-end/src/spartan/4epochs.test.ts index 354da3335f0..feef5c9f243 100644 --- a/yarn-project/end-to-end/src/spartan/4epochs.test.ts +++ b/yarn-project/end-to-end/src/spartan/4epochs.test.ts @@ -6,8 +6,8 @@ import { TokenContract } from '@aztec/noir-contracts.js'; import { jest } from '@jest/globals'; import { RollupCheatCodes } from '../../../aztec.js/src/utils/cheat_codes.js'; -import { getConfig, isK8sConfig, startPortForward } from './k8_utils.js'; import { type TestWallets, setupTestWalletsWithTokens } from './setup_test_wallets.js'; +import { getConfig, isK8sConfig, startPortForward } from './utils.js'; const config = getConfig(process.env); @@ -30,13 +30,13 @@ describe('token transfer test', () => { beforeAll(async () => { if (isK8sConfig(config)) { await startPortForward({ - resource: 'svc/spartan-aztec-network-pxe', + resource: `svc/${config.INSTANCE_NAME}-aztec-network-pxe`, namespace: config.NAMESPACE, containerPort: config.CONTAINER_PXE_PORT, hostPort: config.HOST_PXE_PORT, }); await startPortForward({ - resource: 'svc/spartan-aztec-network-ethereum', + resource: `svc/${config.INSTANCE_NAME}-aztec-network-ethereum`, namespace: config.NAMESPACE, containerPort: config.CONTAINER_ETHEREUM_PORT, hostPort: config.HOST_ETHEREUM_PORT, diff --git a/yarn-project/end-to-end/src/spartan/gating-passive.test.ts b/yarn-project/end-to-end/src/spartan/gating-passive.test.ts new file mode 100644 index 00000000000..8623027e7de --- /dev/null +++ b/yarn-project/end-to-end/src/spartan/gating-passive.test.ts @@ -0,0 +1,116 @@ +import { EthCheatCodes, createCompatibleClient, sleep } from '@aztec/aztec.js'; +import { createDebugLogger } from '@aztec/foundation/log'; + +import { expect, jest } from '@jest/globals'; + +import { RollupCheatCodes } from '../../../aztec.js/src/utils/cheat_codes.js'; +import { + applyBootNodeFailure, + applyNetworkShaping, + applyValidatorKill, + awaitL2BlockNumber, + enableValidatorDynamicBootNode, + getConfig, + isK8sConfig, + restartBot, + startPortForward, +} from './utils.js'; + +const config = getConfig(process.env); +if (!isK8sConfig(config)) { + throw new Error('This test must be run in a k8s environment'); +} +const { + NAMESPACE, + HOST_PXE_PORT, + HOST_ETHEREUM_PORT, + CONTAINER_PXE_PORT, + CONTAINER_ETHEREUM_PORT, + SPARTAN_DIR, + INSTANCE_NAME, +} = config; +const debugLogger = createDebugLogger('aztec:spartan-test:reorg'); + +describe('a test that passively observes the network in the presence of network chaos', () => { + jest.setTimeout(60 * 60 * 1000); // 60 minutes + + const ETHEREUM_HOST = `http://127.0.0.1:${HOST_ETHEREUM_PORT}`; + const PXE_URL = `http://127.0.0.1:${HOST_PXE_PORT}`; + // 50% is the max that we expect to miss + const MAX_MISSED_SLOT_PERCENT = 0.5; + + it('survives network chaos', async () => { + await startPortForward({ + resource: `svc/${config.INSTANCE_NAME}-aztec-network-pxe`, + namespace: NAMESPACE, + containerPort: CONTAINER_PXE_PORT, + hostPort: HOST_PXE_PORT, + }); + await startPortForward({ + resource: `svc/${config.INSTANCE_NAME}-aztec-network-ethereum`, + namespace: NAMESPACE, + containerPort: CONTAINER_ETHEREUM_PORT, + hostPort: HOST_ETHEREUM_PORT, + }); + const client = await createCompatibleClient(PXE_URL, debugLogger); + const ethCheatCodes = new EthCheatCodes(ETHEREUM_HOST); + const rollupCheatCodes = new RollupCheatCodes( + ethCheatCodes, + await client.getNodeInfo().then(n => n.l1ContractAddresses), + ); + const { epochDuration, slotDuration } = await rollupCheatCodes.getConfig(); + + // make it so the validator will use its peers to bootstrap + await enableValidatorDynamicBootNode(INSTANCE_NAME, NAMESPACE, SPARTAN_DIR, debugLogger); + + // restart the bot to ensure that it's not affected by the previous test + await restartBot(NAMESPACE, debugLogger); + + // wait for the chain to build at least 1 epoch's worth of blocks + // note, don't forget that normally an epoch doesn't need epochDuration worth of blocks, + // but here we do double duty: + // we want a handful of blocks, and we want to pass the epoch boundary + await awaitL2BlockNumber(rollupCheatCodes, epochDuration, 60 * 5, debugLogger); + + let deploymentOutput: string = ''; + deploymentOutput = await applyNetworkShaping({ + valuesFile: 'network-requirements.yaml', + namespace: NAMESPACE, + spartanDir: SPARTAN_DIR, + logger: debugLogger, + }); + debugLogger.info(deploymentOutput); + deploymentOutput = await applyBootNodeFailure({ + durationSeconds: 60 * 60 * 24, + namespace: NAMESPACE, + spartanDir: SPARTAN_DIR, + logger: debugLogger, + }); + debugLogger.info(deploymentOutput); + await restartBot(NAMESPACE, debugLogger); + + const rounds = 3; + for (let i = 0; i < rounds; i++) { + debugLogger.info(`Round ${i + 1}/${rounds}`); + deploymentOutput = await applyValidatorKill({ + namespace: NAMESPACE, + spartanDir: SPARTAN_DIR, + logger: debugLogger, + }); + debugLogger.info(deploymentOutput); + debugLogger.info(`Waiting for 1 epoch to pass`); + const controlTips = await rollupCheatCodes.getTips(); + await sleep(Number(epochDuration * slotDuration) * 1000); + const newTips = await rollupCheatCodes.getTips(); + + const expectedPending = + controlTips.pending + BigInt(Math.floor((1 - MAX_MISSED_SLOT_PERCENT) * Number(epochDuration))); + expect(newTips.pending).toBeGreaterThan(expectedPending); + // calculate the percentage of slots missed + const perfectPending = controlTips.pending + BigInt(Math.floor(Number(epochDuration))); + const missedSlots = Number(perfectPending) - Number(newTips.pending); + const missedSlotsPercentage = (missedSlots / Number(epochDuration)) * 100; + debugLogger.info(`Missed ${missedSlots} slots, ${missedSlotsPercentage.toFixed(2)}%`); + } + }); +}); diff --git a/yarn-project/end-to-end/src/spartan/proving.test.ts b/yarn-project/end-to-end/src/spartan/proving.test.ts index 11def1f3e21..8681f17601c 100644 --- a/yarn-project/end-to-end/src/spartan/proving.test.ts +++ b/yarn-project/end-to-end/src/spartan/proving.test.ts @@ -4,7 +4,7 @@ import { createDebugLogger } from '@aztec/foundation/log'; import { jest } from '@jest/globals'; import { type ChildProcess } from 'child_process'; -import { getConfig, isK8sConfig, startPortForward } from './k8_utils.js'; +import { getConfig, isK8sConfig, startPortForward } from './utils.js'; jest.setTimeout(2_400_000); // 40 minutes @@ -19,7 +19,7 @@ describe('proving test', () => { let PXE_URL; if (isK8sConfig(config)) { proc = await startPortForward({ - resource: 'svc/spartan-aztec-network-pxe', + resource: `svc/${config.INSTANCE_NAME}-aztec-network-pxe`, namespace: config.NAMESPACE, containerPort: config.CONTAINER_PXE_PORT, hostPort: config.HOST_PXE_PORT, diff --git a/yarn-project/end-to-end/src/spartan/reorg.test.ts b/yarn-project/end-to-end/src/spartan/reorg.test.ts index 046f51a1357..c315fe05def 100644 --- a/yarn-project/end-to-end/src/spartan/reorg.test.ts +++ b/yarn-project/end-to-end/src/spartan/reorg.test.ts @@ -4,15 +4,15 @@ import { createDebugLogger } from '@aztec/foundation/log'; import { expect, jest } from '@jest/globals'; import { RollupCheatCodes } from '../../../aztec.js/src/utils/cheat_codes.js'; +import { type TestWallets, performTransfers, setupTestWalletsWithTokens } from './setup_test_wallets.js'; import { - applyKillProvers, + applyProverFailure, deleteResourceByLabel, getConfig, isK8sConfig, startPortForward, waitForResourceByLabel, -} from './k8_utils.js'; -import { type TestWallets, performTransfers, setupTestWalletsWithTokens } from './setup_test_wallets.js'; +} from './utils.js'; const config = getConfig(process.env); if (!isK8sConfig(config)) { @@ -47,13 +47,13 @@ describe('reorg test', () => { it('survives a reorg', async () => { await startPortForward({ - resource: 'svc/spartan-aztec-network-pxe', + resource: `svc/${config.INSTANCE_NAME}-aztec-network-pxe`, namespace: NAMESPACE, containerPort: CONTAINER_PXE_PORT, hostPort: HOST_PXE_PORT, }); await startPortForward({ - resource: 'svc/spartan-aztec-network-ethereum', + resource: `svc/${config.INSTANCE_NAME}-aztec-network-ethereum`, namespace: NAMESPACE, containerPort: CONTAINER_ETHEREUM_PORT, hostPort: HOST_ETHEREUM_PORT, @@ -78,10 +78,11 @@ describe('reorg test', () => { const { pending: preReorgPending, proven: preReorgProven } = await rollupCheatCodes.getTips(); // kill the provers - const stdout = await applyKillProvers({ + const stdout = await applyProverFailure({ namespace: NAMESPACE, spartanDir: SPARTAN_DIR, durationSeconds: Number(epochDuration * slotDuration) * 2, + logger: debugLogger, }); debugLogger.info(stdout); @@ -98,7 +99,7 @@ describe('reorg test', () => { await waitForResourceByLabel({ resource: 'pods', namespace: NAMESPACE, label: 'app=pxe' }); await sleep(30 * 1000); await startPortForward({ - resource: 'svc/spartan-aztec-network-pxe', + resource: `svc/${config.INSTANCE_NAME}-aztec-network-pxe`, namespace: NAMESPACE, containerPort: CONTAINER_PXE_PORT, hostPort: HOST_PXE_PORT, diff --git a/yarn-project/end-to-end/src/spartan/smoke.test.ts b/yarn-project/end-to-end/src/spartan/smoke.test.ts index 4eafdd9d4ef..dc47f4f97f8 100644 --- a/yarn-project/end-to-end/src/spartan/smoke.test.ts +++ b/yarn-project/end-to-end/src/spartan/smoke.test.ts @@ -5,7 +5,7 @@ import { RollupAbi } from '@aztec/l1-artifacts'; import { createPublicClient, getAddress, getContract, http } from 'viem'; import { foundry } from 'viem/chains'; -import { getConfig, isK8sConfig, startPortForward } from './k8_utils.js'; +import { getConfig, isK8sConfig, startPortForward } from './utils.js'; const config = getConfig(process.env); @@ -18,7 +18,7 @@ describe('smoke test', () => { let PXE_URL; if (isK8sConfig(config)) { await startPortForward({ - resource: 'svc/spartan-aztec-network-pxe', + resource: `svc/${config.INSTANCE_NAME}-aztec-network-pxe`, namespace: config.NAMESPACE, containerPort: config.CONTAINER_PXE_PORT, hostPort: config.HOST_PXE_PORT, diff --git a/yarn-project/end-to-end/src/spartan/transfer.test.ts b/yarn-project/end-to-end/src/spartan/transfer.test.ts index a073247cae3..a1a9d7aea9a 100644 --- a/yarn-project/end-to-end/src/spartan/transfer.test.ts +++ b/yarn-project/end-to-end/src/spartan/transfer.test.ts @@ -4,8 +4,8 @@ import { TokenContract } from '@aztec/noir-contracts.js'; import { jest } from '@jest/globals'; -import { getConfig, isK8sConfig, startPortForward } from './k8_utils.js'; import { type TestWallets, setupTestWalletsWithTokens } from './setup_test_wallets.js'; +import { getConfig, isK8sConfig, startPortForward } from './utils.js'; const config = getConfig(process.env); @@ -23,7 +23,7 @@ describe('token transfer test', () => { let PXE_URL; if (isK8sConfig(config)) { await startPortForward({ - resource: 'svc/spartan-aztec-network-pxe', + resource: `svc/${config.INSTANCE_NAME}-aztec-network-pxe`, namespace: config.NAMESPACE, containerPort: config.CONTAINER_PXE_PORT, hostPort: config.HOST_PXE_PORT, diff --git a/yarn-project/end-to-end/src/spartan/k8_utils.ts b/yarn-project/end-to-end/src/spartan/utils.ts similarity index 54% rename from yarn-project/end-to-end/src/spartan/k8_utils.ts rename to yarn-project/end-to-end/src/spartan/utils.ts index 9fd2b81a827..1e5fd64145b 100644 --- a/yarn-project/end-to-end/src/spartan/k8_utils.ts +++ b/yarn-project/end-to-end/src/spartan/utils.ts @@ -1,15 +1,19 @@ -import { createDebugLogger } from '@aztec/aztec.js'; +import { createDebugLogger, sleep } from '@aztec/aztec.js'; +import type { Logger } from '@aztec/foundation/log'; import { exec, spawn } from 'child_process'; import path from 'path'; import { promisify } from 'util'; import { z } from 'zod'; +import type { RollupCheatCodes } from '../../../aztec.js/src/utils/cheat_codes.js'; + const execAsync = promisify(exec); const logger = createDebugLogger('k8s-utils'); const k8sConfigSchema = z.object({ + INSTANCE_NAME: z.string().min(1, 'INSTANCE_NAME env variable must be set'), NAMESPACE: z.string().min(1, 'NAMESPACE env variable must be set'), HOST_PXE_PORT: z.coerce.number().min(1, 'HOST_PXE_PORT env variable must be set'), CONTAINER_PXE_PORT: z.coerce.number().default(8080), @@ -92,12 +96,16 @@ export async function deleteResourceByName({ resource, namespace, name, + force = false, }: { resource: string; namespace: string; name: string; + force?: boolean; }) { - const command = `kubectl delete ${resource} ${name} -n ${namespace} --ignore-not-found=true --wait=true`; + const command = `kubectl delete ${resource} ${name} -n ${namespace} --ignore-not-found=true --wait=true ${ + force ? '--force' : '' + }`; logger.info(`command: ${command}`); const { stdout } = await execAsync(command); return stdout; @@ -141,12 +149,43 @@ export function getChartDir(spartanDir: string, chartName: string) { return path.join(spartanDir.trim(), chartName); } -function valuesToArgs(values: Record) { +function valuesToArgs(values: Record) { return Object.entries(values) .map(([key, value]) => `--set ${key}=${value}`) .join(' '); } +function createHelmCommand({ + instanceName, + helmChartDir, + namespace, + valuesFile, + timeout, + values, + reuseValues = false, +}: { + instanceName: string; + helmChartDir: string; + namespace: string; + valuesFile: string | undefined; + timeout: string; + values: Record; + reuseValues?: boolean; +}) { + const valuesFileArgs = valuesFile ? `--values ${helmChartDir}/values/${valuesFile}` : ''; + const reuseValuesArgs = reuseValues ? '--reuse-values' : ''; + return `helm upgrade --install ${instanceName} ${helmChartDir} --namespace ${namespace} ${valuesFileArgs} ${reuseValuesArgs} --wait --timeout=${timeout} ${valuesToArgs( + values, + )}`; +} + +async function execHelmCommand(args: Parameters[0]) { + const helmCommand = createHelmCommand(args); + logger.info(`helm command: ${helmCommand}`); + const { stdout } = await execAsync(helmCommand); + return stdout; +} + /** * Installs a Helm chart with the given parameters. * @param instanceName - The name of the Helm chart instance. @@ -160,7 +199,7 @@ function valuesToArgs(values: Record) { * * Example usage: * ```typescript - * const stdout = await installChaosMeshChart({ instanceName: 'force-reorg', targetNamespace: 'smoke', valuesFile: 'kill-provers.yaml'}); + * const stdout = await installChaosMeshChart({ instanceName: 'force-reorg', targetNamespace: 'smoke', valuesFile: 'prover-failure.yaml'}); * console.log(stdout); * ``` */ @@ -173,6 +212,7 @@ export async function installChaosMeshChart({ timeout = '5m', clean = true, values = {}, + logger, }: { instanceName: string; targetNamespace: string; @@ -181,42 +221,164 @@ export async function installChaosMeshChart({ chaosMeshNamespace?: string; timeout?: string; clean?: boolean; - values?: Record; + values?: Record; + logger: Logger; }) { if (clean) { // uninstall the helm chart if it exists + logger.info(`Uninstalling helm chart ${instanceName}`); await execAsync(`helm uninstall ${instanceName} --namespace ${chaosMeshNamespace} --wait --ignore-not-found`); // and delete the podchaos resource - await deleteResourceByName({ + const deleteArgs = { resource: 'podchaos', namespace: chaosMeshNamespace, name: `${targetNamespace}-${instanceName}`, + }; + logger.info(`Deleting podchaos resource`); + await deleteResourceByName(deleteArgs).catch(e => { + logger.error(`Error deleting podchaos resource: ${e}`); + logger.info(`Force deleting podchaos resource`); + return deleteResourceByName({ ...deleteArgs, force: true }); }); } - const helmCommand = `helm upgrade --install ${instanceName} ${helmChartDir} --namespace ${chaosMeshNamespace} --values ${helmChartDir}/values/${valuesFile} --wait --timeout=${timeout} --set global.targetNamespace=${targetNamespace} ${valuesToArgs( - values, - )}`; - const { stdout } = await execAsync(helmCommand); - return stdout; + return execHelmCommand({ + instanceName, + helmChartDir, + namespace: chaosMeshNamespace, + valuesFile, + timeout, + values: { ...values, 'global.targetNamespace': targetNamespace }, + }); } -export function applyKillProvers({ +export function applyProverFailure({ namespace, spartanDir, durationSeconds, + logger, }: { namespace: string; spartanDir: string; durationSeconds: number; + logger: Logger; }) { return installChaosMeshChart({ - instanceName: 'kill-provers', + instanceName: 'prover-failure', targetNamespace: namespace, - valuesFile: 'kill-provers.yaml', - helmChartDir: getChartDir(spartanDir, 'network-shaping'), + valuesFile: 'prover-failure.yaml', + helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'), values: { - 'networkShaping.conditions.killProvers.duration': `${durationSeconds}s`, + 'proverFailure.duration': `${durationSeconds}s`, }, + logger, }); } + +export function applyBootNodeFailure({ + namespace, + spartanDir, + durationSeconds, + logger, +}: { + namespace: string; + spartanDir: string; + durationSeconds: number; + logger: Logger; +}) { + return installChaosMeshChart({ + instanceName: 'boot-node-failure', + targetNamespace: namespace, + valuesFile: 'boot-node-failure.yaml', + helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'), + values: { + 'bootNodeFailure.duration': `${durationSeconds}s`, + }, + logger, + }); +} + +export function applyValidatorKill({ + namespace, + spartanDir, + logger, +}: { + namespace: string; + spartanDir: string; + logger: Logger; +}) { + return installChaosMeshChart({ + instanceName: 'validator-kill', + targetNamespace: namespace, + valuesFile: 'validator-kill.yaml', + helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'), + logger, + }); +} + +export function applyNetworkShaping({ + valuesFile, + namespace, + spartanDir, + logger, +}: { + valuesFile: string; + namespace: string; + spartanDir: string; + logger: Logger; +}) { + return installChaosMeshChart({ + instanceName: 'network-shaping', + targetNamespace: namespace, + valuesFile, + helmChartDir: getChartDir(spartanDir, 'aztec-chaos-scenarios'), + logger, + }); +} + +export async function awaitL2BlockNumber( + rollupCheatCodes: RollupCheatCodes, + blockNumber: bigint, + timeoutSeconds: number, + logger: Logger, +) { + logger.info(`Waiting for L2 Block ${blockNumber}`); + let tips = await rollupCheatCodes.getTips(); + const endTime = Date.now() + timeoutSeconds * 1000; + while (tips.pending < blockNumber && Date.now() < endTime) { + logger.info(`At L2 Block ${tips.pending}`); + await sleep(1000); + tips = await rollupCheatCodes.getTips(); + } + logger.info(`Reached L2 Block ${tips.pending}`); +} + +export async function restartBot(namespace: string, logger: Logger) { + logger.info(`Restarting bot`); + await deleteResourceByLabel({ resource: 'pods', namespace, label: 'app=bot' }); + await sleep(10 * 1000); + await waitForResourceByLabel({ resource: 'pods', namespace, label: 'app=bot' }); + logger.info(`Bot restarted`); +} + +export async function enableValidatorDynamicBootNode( + instanceName: string, + namespace: string, + spartanDir: string, + logger: Logger, +) { + logger.info(`Enabling validator dynamic boot node`); + await execHelmCommand({ + instanceName, + namespace, + helmChartDir: getChartDir(spartanDir, 'aztec-network'), + values: { + 'validator.dynamicBootNode': 'true', + }, + valuesFile: undefined, + timeout: '10m', + reuseValues: true, + }); + + logger.info(`Validator dynamic boot node enabled`); +} diff --git a/yarn-project/ethereum/package.json b/yarn-project/ethereum/package.json index afee02eac0a..887ad01645d 100644 --- a/yarn-project/ethereum/package.json +++ b/yarn-project/ethereum/package.json @@ -2,7 +2,11 @@ "name": "@aztec/ethereum", "version": "0.1.0", "type": "module", - "exports": "./dest/index.js", + "exports": { + ".": "./dest/index.js", + "./test": "./dest/test/index.js", + "./contracts": "./dest/contracts/index.js" + }, "typedocOptions": { "entryPoints": [ "./src/index.ts" @@ -26,7 +30,9 @@ "dependencies": { "@aztec/foundation": "workspace:^", "@aztec/l1-artifacts": "workspace:^", + "@viem/anvil": "^0.0.10", "dotenv": "^16.0.3", + "get-port": "^7.1.0", "tslib": "^2.4.0", "viem": "^2.7.15", "zod": "^3.23.8" diff --git a/yarn-project/end-to-end/scripts/anvil_kill_wrapper.sh b/yarn-project/ethereum/scripts/anvil_kill_wrapper.sh similarity index 100% rename from yarn-project/end-to-end/scripts/anvil_kill_wrapper.sh rename to yarn-project/ethereum/scripts/anvil_kill_wrapper.sh diff --git a/yarn-project/ethereum/src/contracts/index.ts b/yarn-project/ethereum/src/contracts/index.ts new file mode 100644 index 00000000000..f35e118a5a1 --- /dev/null +++ b/yarn-project/ethereum/src/contracts/index.ts @@ -0,0 +1 @@ +export * from './rollup.js'; diff --git a/yarn-project/ethereum/src/contracts/rollup.ts b/yarn-project/ethereum/src/contracts/rollup.ts new file mode 100644 index 00000000000..98e3ac29fe2 --- /dev/null +++ b/yarn-project/ethereum/src/contracts/rollup.ts @@ -0,0 +1,60 @@ +import { memoize } from '@aztec/foundation/decorators'; +import { RollupAbi } from '@aztec/l1-artifacts'; + +import { + type Chain, + type GetContractReturnType, + type Hex, + type HttpTransport, + type PublicClient, + createPublicClient, + getContract, + http, +} from 'viem'; + +import { createEthereumChain } from '../ethereum_chain.js'; +import { type L1ReaderConfig } from '../l1_reader.js'; + +export class RollupContract { + private readonly rollup: GetContractReturnType>; + + constructor(client: PublicClient, address: Hex) { + this.rollup = getContract({ address, abi: RollupAbi, client }); + } + + @memoize + getL1StartBlock() { + return this.rollup.read.L1_BLOCK_AT_GENESIS(); + } + + @memoize + getL1GenesisTime() { + return this.rollup.read.GENESIS_TIME(); + } + + getBlockNumber() { + return this.rollup.read.getPendingBlockNumber(); + } + + getProvenBlockNumber() { + return this.rollup.read.getProvenBlockNumber(); + } + + getSlotNumber() { + return this.rollup.read.getCurrentSlot(); + } + + async getEpochNumber(blockNumber?: bigint) { + blockNumber ??= await this.getBlockNumber(); + return this.rollup.read.getEpochForBlock([BigInt(blockNumber)]); + } + + static getFromConfig(config: L1ReaderConfig) { + const client = createPublicClient({ + transport: http(config.l1RpcUrl), + chain: createEthereumChain(config.l1RpcUrl, config.l1ChainId).chainInfo, + }); + const address = config.l1Contracts.rollupAddress.toString(); + return new RollupContract(client, address); + } +} diff --git a/yarn-project/ethereum/src/index.ts b/yarn-project/ethereum/src/index.ts index 80bc84bcbc1..30a990db651 100644 --- a/yarn-project/ethereum/src/index.ts +++ b/yarn-project/ethereum/src/index.ts @@ -5,3 +5,4 @@ export * from './l1_reader.js'; export * from './ethereum_chain.js'; export * from './utils.js'; export * from './config.js'; +export * from './types.js'; diff --git a/yarn-project/ethereum/src/test/index.ts b/yarn-project/ethereum/src/test/index.ts new file mode 100644 index 00000000000..e6e7d745ad6 --- /dev/null +++ b/yarn-project/ethereum/src/test/index.ts @@ -0,0 +1,2 @@ +export * from './start_anvil.js'; +export * from './tx_delayer.js'; diff --git a/yarn-project/ethereum/src/test/start_anvil.test.ts b/yarn-project/ethereum/src/test/start_anvil.test.ts new file mode 100644 index 00000000000..8efdff4452b --- /dev/null +++ b/yarn-project/ethereum/src/test/start_anvil.test.ts @@ -0,0 +1,16 @@ +import { createPublicClient, http } from 'viem'; + +import { startAnvil } from './start_anvil.js'; + +describe('start_anvil', () => { + it('starts anvil on a free port', async () => { + const { anvil, rpcUrl } = await startAnvil(); + const publicClient = createPublicClient({ transport: http(rpcUrl) }); + const chainId = await publicClient.getChainId(); + expect(chainId).toEqual(31337); + expect(anvil.status).toEqual('listening'); + + await anvil.stop(); + expect(anvil.status).toEqual('idle'); + }); +}); diff --git a/yarn-project/ethereum/src/test/start_anvil.ts b/yarn-project/ethereum/src/test/start_anvil.ts new file mode 100644 index 00000000000..b8c287681b3 --- /dev/null +++ b/yarn-project/ethereum/src/test/start_anvil.ts @@ -0,0 +1,39 @@ +import { makeBackoff, retry } from '@aztec/foundation/retry'; +import { fileURLToPath } from '@aztec/foundation/url'; + +import { type Anvil, createAnvil } from '@viem/anvil'; +import getPort from 'get-port'; +import { dirname, resolve } from 'path'; + +/** + * Ensures there's a running Anvil instance and returns the RPC URL. + */ +export async function startAnvil(l1BlockTime?: number): Promise<{ anvil: Anvil; rpcUrl: string }> { + let ethereumHostPort: number | undefined; + + const anvilBinary = resolve(dirname(fileURLToPath(import.meta.url)), '../../', 'scripts/anvil_kill_wrapper.sh'); + + // Start anvil. + // We go via a wrapper script to ensure if the parent dies, anvil dies. + const anvil = await retry( + async () => { + ethereumHostPort = await getPort(); + const anvil = createAnvil({ + anvilBinary, + port: ethereumHostPort, + blockTime: l1BlockTime, + }); + await anvil.start(); + return anvil; + }, + 'Start anvil', + makeBackoff([5, 5, 5]), + ); + + if (!ethereumHostPort) { + throw new Error('Failed to start anvil'); + } + + const rpcUrl = `http://127.0.0.1:${ethereumHostPort}`; + return { anvil, rpcUrl }; +} diff --git a/yarn-project/ethereum/src/test/tx_delayer.test.ts b/yarn-project/ethereum/src/test/tx_delayer.test.ts new file mode 100644 index 00000000000..f85bcd453cf --- /dev/null +++ b/yarn-project/ethereum/src/test/tx_delayer.test.ts @@ -0,0 +1,97 @@ +import { type DebugLogger, createDebugLogger } from '@aztec/foundation/log'; +import { TestERC20Abi, TestERC20Bytecode } from '@aztec/l1-artifacts'; + +import { type Anvil } from '@viem/anvil'; +import { type PrivateKeyAccount, createWalletClient, getContract, http, publicActions } from 'viem'; +import { privateKeyToAccount } from 'viem/accounts'; +import { foundry } from 'viem/chains'; + +import { type ViemClient } from '../types.js'; +import { startAnvil } from './start_anvil.js'; +import { type Delayer, withDelayer } from './tx_delayer.js'; + +describe('tx_delayer', () => { + let anvil: Anvil; + let rpcUrl: string; + let logger: DebugLogger; + let account: PrivateKeyAccount; + let client: ViemClient; + let delayer: Delayer; + + const ETHEREUM_SLOT_DURATION = 2; + + beforeAll(async () => { + ({ anvil, rpcUrl } = await startAnvil(ETHEREUM_SLOT_DURATION)); + logger = createDebugLogger('aztec:ethereum:test:tx_delayer'); + }); + + beforeEach(() => { + const transport = http(rpcUrl); + account = privateKeyToAccount('0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80'); + ({ client, delayer } = withDelayer( + createWalletClient({ transport, chain: foundry, account }).extend(publicActions), + { ethereumSlotDuration: ETHEREUM_SLOT_DURATION }, + )); + }); + + const receiptNotFound = expect.objectContaining({ name: 'TransactionReceiptNotFoundError' }); + + it('sends a regular tx', async () => { + const blockNumber = await client.getBlockNumber({ cacheTime: 0 }); + const hash = await client.sendTransaction({ to: account.address }); + const receipt = await client.waitForTransactionReceipt({ hash }); + expect(receipt).toBeDefined(); + expect(receipt.blockNumber).toEqual(blockNumber + 1n); + }); + + it('delays a transaction until a given L1 block number', async () => { + const blockNumber = await client.getBlockNumber({ cacheTime: 0 }); + delayer.pauseNextTxUntilBlock(blockNumber + 3n); + logger.info(`Pausing next tx until block ${blockNumber + 3n}`); + + const delayedTxHash = await client.sendTransaction({ to: account.address }); + await expect(client.getTransactionReceipt({ hash: delayedTxHash })).rejects.toThrow(receiptNotFound); + + logger.info(`Delayed tx sent. Awaiting receipt.`); + const delayedTxReceipt = await client.waitForTransactionReceipt({ hash: delayedTxHash }); + expect(delayedTxReceipt.blockNumber).toEqual(blockNumber + 3n); + }, 20000); + + it('delays a transaction until a given L1 timestamp', async () => { + const block = await client.getBlock({ includeTransactions: false }); + const timestamp = block.timestamp; + delayer.pauseNextTxUntilTimestamp(timestamp + 6n); + logger.info(`Pausing next tx until timestamp ${timestamp + 6n}`); + + const delayedTxHash = await client.sendTransaction({ to: account.address }); + await expect(client.getTransactionReceipt({ hash: delayedTxHash })).rejects.toThrow(receiptNotFound); + + logger.info(`Delayed tx sent. Awaiting receipt.`); + const delayedTxReceipt = await client.waitForTransactionReceipt({ hash: delayedTxHash }); + expect(delayedTxReceipt.blockNumber).toEqual(block.number + 3n); + }, 20000); + + it('delays a tx sent through a contract', async () => { + const deployTxHash = await client.deployContract({ abi: TestERC20Abi, bytecode: TestERC20Bytecode, args: [] }); + const { contractAddress, blockNumber } = await client.waitForTransactionReceipt({ + hash: deployTxHash, + pollingInterval: 100, + }); + logger.info(`Deployed contract at ${contractAddress} on block ${blockNumber}`); + + delayer.pauseNextTxUntilBlock(blockNumber + 3n); + logger.info(`Pausing next tx until block ${blockNumber + 3n}`); + + const contract = getContract({ address: contractAddress!, abi: TestERC20Abi, client }); + const delayedTxHash = await contract.write.mint([account.address, 100n]); + await expect(client.getTransactionReceipt({ hash: delayedTxHash })).rejects.toThrow(receiptNotFound); + + logger.info(`Delayed tx sent. Awaiting receipt.`); + const delayedTxReceipt = await client.waitForTransactionReceipt({ hash: delayedTxHash }); + expect(delayedTxReceipt.blockNumber).toEqual(blockNumber + 3n); + }, 20000); + + afterAll(async () => { + await anvil.stop(); + }); +}); diff --git a/yarn-project/ethereum/src/test/tx_delayer.ts b/yarn-project/ethereum/src/test/tx_delayer.ts new file mode 100644 index 00000000000..220823692e1 --- /dev/null +++ b/yarn-project/ethereum/src/test/tx_delayer.ts @@ -0,0 +1,150 @@ +import { type DebugLogger, createDebugLogger } from '@aztec/foundation/log'; +import { retryUntil } from '@aztec/foundation/retry'; + +import { inspect } from 'util'; +import { + type Client, + type Hex, + type PublicClient, + type WalletClient, + keccak256, + publicActions, + walletActions, +} from 'viem'; + +export function waitUntilBlock(client: T, blockNumber: number | bigint, logger?: DebugLogger) { + const publicClient = + 'getBlockNumber' in client && typeof client.getBlockNumber === 'function' + ? (client as unknown as PublicClient) + : client.extend(publicActions); + + return retryUntil( + async () => { + const currentBlockNumber = await publicClient.getBlockNumber({ cacheTime: 0 }); + logger?.debug(`Block number is ${currentBlockNumber} (waiting until ${blockNumber})`); + return currentBlockNumber >= BigInt(blockNumber); + }, + `Wait until L1 block ${blockNumber}`, + 60, + 0.1, + ); +} + +export function waitUntilL1Timestamp(client: T, timestamp: number | bigint, logger?: DebugLogger) { + const publicClient = + 'getBlockNumber' in client && typeof client.getBlockNumber === 'function' + ? (client as unknown as PublicClient) + : client.extend(publicActions); + + let lastBlock: bigint | undefined = undefined; + return retryUntil( + async () => { + const currentBlockNumber = await publicClient.getBlockNumber({ cacheTime: 0 }); + if (currentBlockNumber === lastBlock) { + return false; + } + lastBlock = currentBlockNumber; + const currentBlock = await publicClient.getBlock({ includeTransactions: false, blockNumber: currentBlockNumber }); + const currentTs = currentBlock.timestamp; + logger?.debug(`Block timstamp is ${currentTs} (waiting until ${timestamp})`); + return currentTs >= BigInt(timestamp); + }, + `Wait until L1 timestamp ${timestamp}`, + 60, + 0.1, + ); +} + +export interface Delayer { + /** Returns the list of all txs (not just the delayed ones) sent through the attached client. */ + getTxs(): Hex[]; + /** Delays the next tx to be sent so it lands on the given L1 block number. */ + pauseNextTxUntilBlock(l1BlockNumber: number | bigint | undefined): void; + /** Delays the next tx to be sent so it lands on the given timestamp. */ + pauseNextTxUntilTimestamp(l1Timestamp: number | bigint | undefined): void; +} + +class DelayerImpl implements Delayer { + constructor(opts: { ethereumSlotDuration: bigint | number }) { + this.ethereumSlotDuration = BigInt(opts.ethereumSlotDuration); + } + + public ethereumSlotDuration: bigint; + public nextWait: { l1Timestamp: bigint } | { l1BlockNumber: bigint } | undefined = undefined; + public txs: Hex[] = []; + + getTxs() { + return this.txs; + } + + pauseNextTxUntilBlock(l1BlockNumber: number | bigint) { + this.nextWait = { l1BlockNumber: BigInt(l1BlockNumber) }; + } + + pauseNextTxUntilTimestamp(l1Timestamp: number | bigint) { + this.nextWait = { l1Timestamp: BigInt(l1Timestamp) }; + } +} + +/** + * Returns a new client (without modifying the one passed in) with an injected tx delayer. + * The delayer can be used to hold off the next tx to be sent until a given block number. + */ +export function withDelayer( + client: T, + opts: { ethereumSlotDuration: bigint | number }, +): { client: T; delayer: Delayer } { + const logger = createDebugLogger('aztec:ethereum:tx_delayer'); + const delayer = new DelayerImpl(opts); + const extended = client + // Tweak sendRawTransaction so it uses the delay defined in the delayer. + // Note that this will only work with local accounts (ie accounts for which we have the private key). + // Transactions signed by the node will not be delayed since they use sendTransaction directly, + // but we do not use them in our codebase at all. + .extend(client => ({ + async sendRawTransaction(...args) { + if (delayer.nextWait !== undefined) { + const waitUntil = delayer.nextWait; + delayer.nextWait = undefined; + + const publicClient = client as unknown as PublicClient; + const wait = + 'l1BlockNumber' in waitUntil + ? waitUntilBlock(publicClient, waitUntil.l1BlockNumber - 1n, logger) + : waitUntilL1Timestamp(publicClient, waitUntil.l1Timestamp - delayer.ethereumSlotDuration, logger); + + // Compute the tx hash manually so we emulate sendRawTransaction response + const { serializedTransaction } = args[0]; + const txHash = keccak256(serializedTransaction); + logger.info(`Delaying tx ${txHash} until ${inspect(waitUntil)}`); + + // Do not await here so we can return the tx hash immediately as if it had been sent on the spot. + // Instead, delay it so it lands on the desired block number or timestamp, assuming anvil will + // mine it immediately. + void wait + .then(async () => { + const txHash = await client.sendRawTransaction(...args); + logger.info(`Sent previously delayed tx ${txHash} to land on ${inspect(waitUntil)}`); + delayer.txs.push(txHash); + }) + .catch(err => logger.error(`Error sending tx after delay`, err)); + + return Promise.resolve(txHash); + } else { + const txHash = await client.sendRawTransaction(...args); + logger.debug(`Sent tx immediately ${txHash}`); + delayer.txs.push(txHash); + return txHash; + } + }, + })) + // Re-extend with sendTransaction so it uses the modified sendRawTransaction. + .extend(client => ({ sendTransaction: walletActions(client).sendTransaction })) + // And with the actions that depend on the modified sendTransaction + .extend(client => ({ + writeContract: walletActions(client).writeContract, + deployContract: walletActions(client).deployContract, + })) as T; + + return { client: extended, delayer }; +} diff --git a/yarn-project/ethereum/src/types.ts b/yarn-project/ethereum/src/types.ts new file mode 100644 index 00000000000..2dfb9dc591c --- /dev/null +++ b/yarn-project/ethereum/src/types.ts @@ -0,0 +1,22 @@ +import { + type Chain, + type Client, + type HttpTransport, + type PrivateKeyAccount, + type PublicActions, + type PublicRpcSchema, + type WalletActions, + type WalletRpcSchema, +} from 'viem'; + +/** + * Type for a viem wallet and public client using a local private key. + * Created as: `createWalletClient({ account: privateKeyToAccount(key), transport: http(url), chain }).extend(publicActions)` + */ +export type ViemClient = Client< + HttpTransport, + Chain, + PrivateKeyAccount, + [...PublicRpcSchema, ...WalletRpcSchema], + PublicActions & WalletActions +>; diff --git a/yarn-project/foundation/src/abi/abi.ts b/yarn-project/foundation/src/abi/abi.ts index b58d5cc5527..2f13d6ab67a 100644 --- a/yarn-project/foundation/src/abi/abi.ts +++ b/yarn-project/foundation/src/abi/abi.ts @@ -358,7 +358,18 @@ export const ContractArtifactSchema: ZodFor = z.object({ aztecNrVersion: z.string().optional(), functions: z.array(FunctionArtifactSchema), outputs: z.object({ - structs: z.record(z.array(AbiTypeSchema)), + structs: z.record(z.array(AbiTypeSchema)).transform(structs => { + for (const [key, value] of Object.entries(structs)) { + // We are manually ordering events and functions in the abi by path. + // The path ordering is arbitrary, and only needed to ensure deterministic order. + // These are the only arrays in the artifact with arbitrary order, and hence the only ones + // we need to sort. + if (key === 'events' || key === 'functions') { + structs[key] = (value as StructType[]).sort((a, b) => (a.path > b.path ? -1 : 1)); + } + } + return structs; + }), globals: z.record(z.array(AbiValueSchema)), }), storageLayout: z.record(z.object({ slot: schemas.Fr })), diff --git a/yarn-project/foundation/src/abi/note_selector.ts b/yarn-project/foundation/src/abi/note_selector.ts index 392399f7ee1..8fdcaa945c3 100644 --- a/yarn-project/foundation/src/abi/note_selector.ts +++ b/yarn-project/foundation/src/abi/note_selector.ts @@ -27,7 +27,7 @@ export class NoteSelector extends Selector { } static fromString(buf: string) { - const withoutPrefix = buf.replace(/^0x/i, ''); + const withoutPrefix = buf.replace(/^0x/i, '').slice(-8); const buffer = Buffer.from(withoutPrefix, 'hex'); return NoteSelector.fromBuffer(buffer); } diff --git a/yarn-project/foundation/src/aztec-address/aztec-address.test.ts b/yarn-project/foundation/src/aztec-address/aztec-address.test.ts new file mode 100644 index 00000000000..d7009692275 --- /dev/null +++ b/yarn-project/foundation/src/aztec-address/aztec-address.test.ts @@ -0,0 +1,50 @@ +import { Fr } from '../fields/fields.js'; +import { AztecAddress } from './index.js'; + +describe('aztec-address', () => { + describe('isValid', () => { + it('returns true for a valid address', () => { + // The point (5, 21888242871839275195798879923479812031525119486506890092185616889232283231735) is on the + // Grumpkin curve. + const address = new AztecAddress(new Fr(5)); + expect(address.isValid()).toEqual(true); + }); + + it('returns false for an invalid address', () => { + // No point on the Grumpkin curve has an x coordinate equal to 6. + const address = new AztecAddress(new Fr(6)); + expect(address.isValid()).toEqual(false); + }); + }); + + describe('random', () => { + it('always returns a valid address', () => { + for (let i = 0; i < 100; ++i) { + const address = AztecAddress.random(); + expect(address.isValid()).toEqual(true); + } + }); + + it('returns a different address on each call', () => { + const set = new Set(); + for (let i = 0; i < 100; ++i) { + set.add(AztecAddress.random()); + } + + expect(set.size).toEqual(100); + }); + }); + + describe('toAddressPoint', () => { + it("reconstructs an address's point", () => { + const address = AztecAddress.random(); + const point = address.toAddressPoint(); + expect(point.isOnGrumpkin()).toEqual(true); + }); + + it('throws for an invalid address', () => { + const address = new AztecAddress(new Fr(6)); + expect(() => address.toAddressPoint()).toThrow('The given x-coordinate is not on the Grumpkin curve'); + }); + }); +}); diff --git a/yarn-project/foundation/src/aztec-address/index.ts b/yarn-project/foundation/src/aztec-address/index.ts index fb64a0ecd25..d58fd9b0d77 100644 --- a/yarn-project/foundation/src/aztec-address/index.ts +++ b/yarn-project/foundation/src/aztec-address/index.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/no-unsafe-declaration-merging */ import { inspect } from 'util'; -import { Fr, fromBuffer } from '../fields/index.js'; +import { Fr, Point, fromBuffer } from '../fields/index.js'; import { type BufferReader, FieldReader } from '../serialize/index.js'; import { TypeRegistry } from '../serialize/type_registry.js'; import { hexToBuffer } from '../string/index.js'; @@ -12,20 +12,22 @@ export interface AztecAddress { _branding: 'AztecAddress'; } /** - * AztecAddress represents a 32-byte address in the Aztec Protocol. - * It provides methods to create, manipulate, and compare addresses. - * The maximum value of an address is determined by the field modulus and all instances of AztecAddress. - * It should have a value less than or equal to this max value. - * This class also provides helper functions to convert addresses from strings, buffers, and other formats. + * AztecAddress represents a 32-byte address in the Aztec Protocol. It provides methods to create, manipulate, and + * compare addresses, as well as conversion to and from strings, buffers, and other formats. + * Addresses are the x coordinate of a point in the Grumpkin curve, and therefore their maximum is determined by the + * field modulus. An address with a value that is not the x coordinate of a point in the curve is a called an 'invalid + * address'. These addresses have a greatly reduced feature set, as they cannot own secrets nor have messages encrypted + * to them, making them quite useless. We need to be able to represent them however as they can be encountered in the + * wild. */ export class AztecAddress { - private value: Fr; + private xCoord: Fr; constructor(buffer: Buffer | Fr) { if ('length' in buffer && buffer.length !== 32) { throw new Error(`Invalid AztecAddress length ${buffer.length}.`); } - this.value = new Fr(buffer); + this.xCoord = new Fr(buffer); } [inspect.custom]() { @@ -69,36 +71,65 @@ export class AztecAddress { return new AztecAddress(hexToBuffer(buf)); } + /** + * @returns a random valid address (i.e. one that can be encrypted to). + */ static random() { - return new AztecAddress(Fr.random()); + // About half of random field elements result in invalid addresses, so we loop until we get a valid one. + while (true) { + const candidate = new AztecAddress(Fr.random()); + if (candidate.isValid()) { + return candidate; + } + } } get size() { - return this.value.size; + return this.xCoord.size; } equals(other: AztecAddress) { - return this.value.equals(other.value); + return this.xCoord.equals(other.xCoord); } isZero() { - return this.value.isZero(); + return this.xCoord.isZero(); + } + + /** + * @returns true if the address is valid. Invalid addresses cannot receive encrypted messages. + */ + isValid() { + // An address is a field value (Fr), which for some purposes is assumed to be the x coordinate of a point in the + // Grumpkin curve (notably in order to encrypt to it). An address that is not the x coordinate of such a point is + // called an 'invalid' address. + // + // For Grumpkin, y^2 = x^3 − 17 . There exist values x ∈ Fr for which no y satisfies this equation. This means that + // given such an x and t = x^3 − 17, then sqrt(t) does not exist in Fr. + return Point.YFromX(this.xCoord) !== null; + } + + /** + * @returns the Point from which the address is derived. Throws if the address is invalid. + */ + toAddressPoint() { + return Point.fromXAndSign(this.xCoord, true); } toBuffer() { - return this.value.toBuffer(); + return this.xCoord.toBuffer(); } toBigInt() { - return this.value.toBigInt(); + return this.xCoord.toBigInt(); } toField() { - return this.value; + return this.xCoord; } toString() { - return this.value.toString(); + return this.xCoord.toString(); } toJSON() { diff --git a/yarn-project/foundation/src/config/env_var.ts b/yarn-project/foundation/src/config/env_var.ts index c62e2a03c1d..4a2015ddd0f 100644 --- a/yarn-project/foundation/src/config/env_var.ts +++ b/yarn-project/foundation/src/config/env_var.ts @@ -53,6 +53,7 @@ export type EnvVar = | 'L1_CHAIN_ID' | 'L1_PRIVATE_KEY' | 'L2_QUEUE_SIZE' + | 'LOG_ELAPSED_TIME' | 'LOG_JSON' | 'LOG_LEVEL' | 'MNEMONIC' @@ -143,6 +144,7 @@ export type EnvVar = | 'VALIDATOR_ATTESTATIONS_WAIT_TIMEOUT_MS' | 'VALIDATOR_DISABLED' | 'VALIDATOR_PRIVATE_KEY' + | 'VALIDATOR_REEXECUTE' | 'VERSION' | 'WS_BLOCK_CHECK_INTERVAL_MS' | 'WS_PROVEN_BLOCKS_ONLY' diff --git a/yarn-project/foundation/src/fields/point.test.ts b/yarn-project/foundation/src/fields/point.test.ts index 1b4d333eb77..f2b4813683a 100644 --- a/yarn-project/foundation/src/fields/point.test.ts +++ b/yarn-project/foundation/src/fields/point.test.ts @@ -4,6 +4,24 @@ import { Fr } from './fields.js'; import { Point } from './point.js'; describe('Point', () => { + describe('random', () => { + it('always returns a valid point', () => { + for (let i = 0; i < 100; ++i) { + const point = Point.random(); + expect(point.isOnGrumpkin()).toEqual(true); + } + }); + + it('returns a different points on each call', () => { + const set = new Set(); + for (let i = 0; i < 100; ++i) { + set.add(Point.random()); + } + + expect(set.size).toEqual(100); + }); + }); + it('converts to and from x and sign of y coordinate', () => { const p = new Point( new Fr(0x30426e64aee30e998c13c8ceecda3a77807dbead52bc2f3bf0eae851b4b710c1n), @@ -17,10 +35,6 @@ describe('Point', () => { expect(p.equals(p2)).toBeTruthy(); }); - it('creates a valid random point', () => { - expect(Point.random().isOnGrumpkin()).toBeTruthy(); - }); - it('converts to and from buffer', () => { const p = Point.random(); const p2 = Point.fromBuffer(p.toBuffer()); diff --git a/yarn-project/foundation/src/fields/point.ts b/yarn-project/foundation/src/fields/point.ts index 94b4394c2cd..bfe1adcb3e7 100644 --- a/yarn-project/foundation/src/fields/point.ts +++ b/yarn-project/foundation/src/fields/point.ts @@ -117,14 +117,8 @@ export class Point { * @returns The point as an array of 2 fields */ static fromXAndSign(x: Fr, sign: boolean) { - // Calculate y^2 = x^3 - 17 - const ySquared = x.square().mul(x).sub(new Fr(17)); - - // Calculate the square root of ySquared - const y = ySquared.sqrt(); - - // If y is null, the x-coordinate is not on the curve - if (y === null) { + const y = Point.YFromX(x); + if (y == null) { throw new NotOnCurveError(x); } @@ -138,6 +132,18 @@ export class Point { return new this(x, finalY, false); } + /** + * @returns + */ + static YFromX(x: Fr): Fr | null { + // Calculate y^2 = x^3 - 17 (i.e. the Grumpkin curve equation) + const ySquared = x.square().mul(x).sub(new Fr(17)); + + // y is then simply the square root. Note however that not all square roots exist in the field: if sqrt returns null + // then there is no point in the curve with this x coordinate. + return ySquared.sqrt(); + } + /** * Returns the x coordinate and the sign of the y coordinate. * @dev The y sign can be determined by checking if the y coordinate is greater than half of the modulus. @@ -267,10 +273,10 @@ export class Point { return true; } - // p.y * p.y == p.x * p.x * p.x - 17 - const A = new Fr(17); + // The Grumpkin equation is y^2 = x^3 - 17. We could use `YFromX` and then compare to `this.y`, but this would + // involve computing the square root of y, of which there are two possible valid values. This method is also faster. const lhs = this.y.square(); - const rhs = this.x.square().mul(this.x).sub(A); + const rhs = this.x.mul(this.x).mul(this.x).sub(new Fr(17)); return lhs.equals(rhs); } } diff --git a/yarn-project/foundation/src/log/logger.ts b/yarn-project/foundation/src/log/logger.ts index f771af9851c..2f5954f6eb0 100644 --- a/yarn-project/foundation/src/log/logger.ts +++ b/yarn-project/foundation/src/log/logger.ts @@ -12,14 +12,22 @@ export type LogLevel = (typeof LogLevels)[number]; function getLogLevel() { const envLogLevel = process.env.LOG_LEVEL?.toLowerCase() as LogLevel; - const defaultNonTestLogLevel = - process.env.DEBUG === undefined || process.env.DEBUG === '' ? ('info' as const) : ('debug' as const); - const defaultLogLevel = process.env.NODE_ENV === 'test' ? ('silent' as const) : defaultNonTestLogLevel; + let defaultLogLevel: LogLevel = 'info'; + if (process.env.DEBUG) { + // if we set DEBUG to a non-empty string, use debug as default + defaultLogLevel = 'debug'; + } else if (process.env.NODE_ENV === 'test') { + // otherwise, be silent in tests as these are frequently ran en-masse + defaultLogLevel = 'silent'; + } return LogLevels.includes(envLogLevel) ? envLogLevel : defaultLogLevel; } export let currentLevel = getLogLevel(); +const logElapsedTime = ['1', 'true'].includes(process.env.LOG_ELAPSED_TIME ?? ''); +const firstTimestamp: number = Date.now(); + function filterNegativePatterns(debugString: string): string { return debugString .split(',') @@ -141,7 +149,12 @@ function logWithDebug(debug: debug.Debugger, level: LogLevel, msg: string, data? msg = data ? `${msg} ${fmtLogData(data)}` : msg; if (debug.enabled && LogLevels.indexOf(level) <= LogLevels.indexOf(currentLevel)) { - debug('[%s] %s', level.toUpperCase(), msg); + if (logElapsedTime) { + const ts = ((Date.now() - firstTimestamp) / 1000).toFixed(3); + debug('%ss [%s] %s', ts, level.toUpperCase(), msg); + } else { + debug('[%s] %s', level.toUpperCase(), msg); + } } } diff --git a/yarn-project/foundation/src/string/index.ts b/yarn-project/foundation/src/string/index.ts index 0d1fd95cb07..250b3c02581 100644 --- a/yarn-project/foundation/src/string/index.ts +++ b/yarn-project/foundation/src/string/index.ts @@ -13,3 +13,11 @@ export function isHex(str: string): boolean { export function hexToBuffer(str: string): Buffer { return Buffer.from(withoutHexPrefix(str), 'hex'); } + +export function pluralize(str: string, count: number | bigint, plural?: string): string { + return count === 1 || count === 1n ? str : plural ?? `${str}s`; +} + +export function count(count: number | bigint, str: string, plural?: string): string { + return `${count} ${pluralize(str, count, plural)}`; +} diff --git a/yarn-project/ivc-integration/package.json b/yarn-project/ivc-integration/package.json index c9fc4dae62e..54c7f409712 100644 --- a/yarn-project/ivc-integration/package.json +++ b/yarn-project/ivc-integration/package.json @@ -11,7 +11,7 @@ "./package.local.json" ], "scripts": { - "build": "yarn clean && yarn generate && rm -rf dest && webpack && tsc -b", + "build": "yarn clean && yarn generate && tsc -b && rm -rf dest && webpack", "clean": "rm -rf ./dest .tsbuildinfo src/types artifacts", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", diff --git a/yarn-project/ivc-integration/package.local.json b/yarn-project/ivc-integration/package.local.json index ecdfdf5167a..f3ab16fc679 100644 --- a/yarn-project/ivc-integration/package.local.json +++ b/yarn-project/ivc-integration/package.local.json @@ -1,6 +1,6 @@ { "scripts": { - "build": "yarn clean && yarn generate && rm -rf dest && webpack && tsc -b", + "build": "yarn clean && yarn generate && tsc -b && rm -rf dest && webpack", "clean": "rm -rf ./dest .tsbuildinfo src/types artifacts", "test:non-browser": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --testPathIgnorePatterns=browser --passWithNoTests ", "test:browser": "./run_browser_tests.sh", diff --git a/yarn-project/ivc-integration/src/avm_integration.test.ts b/yarn-project/ivc-integration/src/avm_integration.test.ts index 967fd92bc1c..63ed8fbaa5b 100644 --- a/yarn-project/ivc-integration/src/avm_integration.test.ts +++ b/yarn-project/ivc-integration/src/avm_integration.test.ts @@ -1,21 +1,4 @@ -import { - type BBSuccess, - BB_RESULT, - generateAvmProof, - generateProof, - getPublicInputs, - verifyProof, -} from '@aztec/bb-prover'; -import { - AvmCircuitInputs, - AvmCircuitPublicInputs, - AztecAddress, - Gas, - GlobalVariables, - type PublicFunction, - PublicKeys, - SerializableContractInstance, -} from '@aztec/circuits.js'; +import { type BBSuccess, BB_RESULT, generateAvmProof, generateProof, verifyProof } from '@aztec/bb-prover'; import { AVM_PROOF_LENGTH_IN_FIELDS, AVM_PUBLIC_COLUMN_MAX_SIZE, @@ -23,33 +6,20 @@ import { AVM_VERIFICATION_KEY_LENGTH_IN_FIELDS, PUBLIC_CIRCUIT_PUBLIC_INPUTS_LENGTH, } from '@aztec/circuits.js/constants'; -import { makeContractClassPublic, makeContractInstanceFromClassId } from '@aztec/circuits.js/testing'; -import { Fr, Point } from '@aztec/foundation/fields'; +import { Fr } from '@aztec/foundation/fields'; import { createDebugLogger } from '@aztec/foundation/log'; import { BufferReader } from '@aztec/foundation/serialize'; -import { openTmpStore } from '@aztec/kv-store/utils'; import { type FixedLengthArray } from '@aztec/noir-protocol-circuits-types/types'; -import { AvmSimulator, PublicSideEffectTrace, type WorldStateDB } from '@aztec/simulator'; -import { - getAvmTestContractBytecode, - getAvmTestContractFunctionSelector, - initContext, - initExecutionEnvironment, - initPersistableStateManager, - resolveAvmTestContractAssertionMessage, -} from '@aztec/simulator/avm/fixtures'; -import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; -import { MerkleTrees } from '@aztec/world-state'; +import { simulateAvmTestContractGenerateCircuitInputs } from '@aztec/simulator/public/fixtures'; import { jest } from '@jest/globals'; import fs from 'fs/promises'; -import { mock } from 'jest-mock-extended'; import { tmpdir } from 'node:os'; import os from 'os'; import path from 'path'; import { fileURLToPath } from 'url'; -import { MockPublicKernelCircuit, witnessGenMockPublicKernelCircuit } from './index.js'; +import { MockPublicBaseCircuit, witnessGenMockPublicBaseCircuit } from './index.js'; // Auto-generated types from noir are not in camel case. /* eslint-disable camelcase */ @@ -75,7 +45,7 @@ describe('AVM Integration', () => { const provingResult = await generateProof( bbBinaryPath, bbWorkingDirectory, - 'mock-public-kernel', + 'mock-public-base', Buffer.from(bytecode, 'base64'), recursive, witnessFileName, @@ -127,7 +97,7 @@ describe('AVM Integration', () => { const vk = vkReader.readArray(AVM_VERIFICATION_KEY_LENGTH_IN_FIELDS, Fr); expect(vk.length).toBe(AVM_VERIFICATION_KEY_LENGTH_IN_FIELDS); - const witGenResult = await witnessGenMockPublicKernelCircuit({ + const witGenResult = await witnessGenMockPublicBaseCircuit({ verification_key: vk.map(x => x.toString()) as FixedLengthArray< string, typeof AVM_VERIFICATION_KEY_LENGTH_IN_FIELDS @@ -139,7 +109,7 @@ describe('AVM Integration', () => { >, }); - await createHonkProof(witGenResult.witness, MockPublicKernelCircuit.bytecode); + await createHonkProof(witGenResult.witness, MockPublicBaseCircuit.bytecode); const verifyResult = await verifyProof( bbBinaryPath, @@ -153,108 +123,22 @@ describe('AVM Integration', () => { }); }); -// Helper - -const proveAvmTestContract = async ( - functionName: string, - calldata: Fr[] = [], - assertionErrString?: string, -): Promise => { - const worldStateDB = mock(); - const startSideEffectCounter = 0; - const functionSelector = getAvmTestContractFunctionSelector(functionName); - calldata = [functionSelector.toField(), ...calldata]; - const globals = GlobalVariables.empty(); - - // Top level contract call - const bytecode = getAvmTestContractBytecode('public_dispatch'); - const fnSelector = getAvmTestContractFunctionSelector('public_dispatch'); - const publicFn: PublicFunction = { bytecode, selector: fnSelector }; - const contractClass = makeContractClassPublic(0, publicFn); - const contractInstance = makeContractInstanceFromClassId(contractClass.id); - - const instanceGet = new SerializableContractInstance({ - version: 1, - salt: new Fr(0x123), - deployer: AztecAddress.fromNumber(0x456), - contractClassId: new Fr(0x789), - initializationHash: new Fr(0x101112), - publicKeys: new PublicKeys( - new Point(new Fr(0x131415), new Fr(0x161718), false), - new Point(new Fr(0x192021), new Fr(0x222324), false), - new Point(new Fr(0x252627), new Fr(0x282930), false), - new Point(new Fr(0x313233), new Fr(0x343536), false), - ), - }).withAddress(contractInstance.address); - - worldStateDB.getContractInstance - .mockResolvedValueOnce(contractInstance) - .mockResolvedValueOnce(instanceGet) // test gets deployer - .mockResolvedValueOnce(instanceGet) // test gets class id - .mockResolvedValueOnce(instanceGet) // test gets init hash - .mockResolvedValue(contractInstance); - - worldStateDB.getContractClass.mockResolvedValue(contractClass); - - const storageValue = new Fr(5); - worldStateDB.storageRead.mockResolvedValue(storageValue); +async function proveAvmTestContract(functionName: string, calldata: Fr[] = []): Promise { + const avmCircuitInputs = await simulateAvmTestContractGenerateCircuitInputs(functionName, calldata); - const trace = new PublicSideEffectTrace(startSideEffectCounter); - const telemetry = new NoopTelemetryClient(); - const merkleTrees = await (await MerkleTrees.new(openTmpStore(), telemetry)).fork(); - worldStateDB.getMerkleInterface.mockReturnValue(merkleTrees); - const persistableState = initPersistableStateManager({ worldStateDB, trace, merkleTrees, doMerkleOperations: true }); - const environment = initExecutionEnvironment({ - functionSelector, - calldata, - globals, - address: contractInstance.address, - }); - const context = initContext({ env: environment, persistableState }); - - worldStateDB.getBytecode.mockResolvedValue(bytecode); - - const startGas = new Gas(context.machineState.gasLeft.daGas, context.machineState.gasLeft.l2Gas); + const internalLogger = createDebugLogger('aztec:avm-proving-test'); + const logger = (msg: string, _data?: any) => internalLogger.verbose(msg); - // Use a simple contract that emits a side effect // The paths for the barretenberg binary and the write path are hardcoded for now. const bbPath = path.resolve('../../barretenberg/cpp/build/bin/bb'); const bbWorkingDirectory = await fs.mkdtemp(path.join(tmpdir(), 'bb-')); - // First we simulate (though it's not needed in this simple case). - const simulator = new AvmSimulator(context); - const avmResult = await simulator.execute(); - - if (assertionErrString == undefined) { - expect(avmResult.reverted).toBe(false); - } else { - // Explicit revert when an assertion failed. - expect(avmResult.reverted).toBe(true); - expect(avmResult.revertReason).toBeDefined(); - expect(resolveAvmTestContractAssertionMessage(functionName, avmResult.revertReason!, avmResult.output)).toContain( - assertionErrString, - ); - } - - const pxResult = trace.toPublicFunctionCallResult( - environment, - startGas, - /*endGasLeft=*/ Gas.from(context.machineState.gasLeft), - /*bytecode=*/ simulator.getBytecode()!, - avmResult, - functionName, - ); - - const avmCircuitInputs = new AvmCircuitInputs( - functionName, - /*calldata=*/ context.environment.calldata, - /*publicInputs=*/ getPublicInputs(pxResult), - /*avmHints=*/ pxResult.avmCircuitHints, - AvmCircuitPublicInputs.empty(), - ); // Then we prove. - const proofRes = await generateAvmProof(bbPath, bbWorkingDirectory, avmCircuitInputs, logger.info); + const proofRes = await generateAvmProof(bbPath, bbWorkingDirectory, avmCircuitInputs, logger); + if (proofRes.status === BB_RESULT.FAILURE) { + internalLogger.error(`Proof generation failed: ${proofRes.reason}`); + } expect(proofRes.status).toEqual(BB_RESULT.SUCCESS); return proofRes as BBSuccess; -}; +} diff --git a/yarn-project/ivc-integration/src/index.ts b/yarn-project/ivc-integration/src/index.ts index 17b6a95888f..93234564864 100644 --- a/yarn-project/ivc-integration/src/index.ts +++ b/yarn-project/ivc-integration/src/index.ts @@ -9,7 +9,7 @@ import MockPrivateKernelInitCircuit from '../artifacts/mock_private_kernel_init. import MockPrivateKernelInnerCircuit from '../artifacts/mock_private_kernel_inner.json' assert { type: 'json' }; import MockPrivateKernelResetCircuit from '../artifacts/mock_private_kernel_reset.json' assert { type: 'json' }; import MockPrivateKernelTailCircuit from '../artifacts/mock_private_kernel_tail.json' assert { type: 'json' }; -import MockPublicKernelCircuit from '../artifacts/mock_public_kernel.json' assert { type: 'json' }; +import MockPublicBaseCircuit from '../artifacts/mock_public_base.json' assert { type: 'json' }; import type { AppCreatorInputType, AppPublicInputs, @@ -19,7 +19,7 @@ import type { MockPrivateKernelInnerInputType, MockPrivateKernelResetInputType, MockPrivateKernelTailInputType, - MockPublicKernelInputType, + MockPublicBaseInputType, PrivateKernelPublicInputs, u8, } from './types/index.js'; @@ -32,7 +32,7 @@ export { MockPrivateKernelInnerCircuit, MockPrivateKernelResetCircuit, MockPrivateKernelTailCircuit, - MockPublicKernelCircuit, + MockPublicBaseCircuit, }; createDebug.enable('*'); @@ -117,10 +117,8 @@ export async function witnessGenMockPrivateKernelTailCircuit( }; } -export async function witnessGenMockPublicKernelCircuit( - args: MockPublicKernelInputType, -): Promise> { - const program = new Noir(MockPublicKernelCircuit); +export async function witnessGenMockPublicBaseCircuit(args: MockPublicBaseInputType): Promise> { + const program = new Noir(MockPublicBaseCircuit); const { witness, returnValue } = await program.execute(args, foreignCallHandler); return { witness, diff --git a/yarn-project/ivc-integration/src/scripts/generate_ts_from_abi.ts b/yarn-project/ivc-integration/src/scripts/generate_ts_from_abi.ts index d020c891fa5..80545c2587a 100644 --- a/yarn-project/ivc-integration/src/scripts/generate_ts_from_abi.ts +++ b/yarn-project/ivc-integration/src/scripts/generate_ts_from_abi.ts @@ -14,7 +14,7 @@ const circuits = [ 'mock_private_kernel_inner', 'mock_private_kernel_reset', 'mock_private_kernel_tail', - 'mock_public_kernel', + 'mock_public_base', ]; const main = async () => { diff --git a/yarn-project/ivc-integration/src/serve.ts b/yarn-project/ivc-integration/src/serve.ts index 0910e64aae2..b16a125a05c 100644 --- a/yarn-project/ivc-integration/src/serve.ts +++ b/yarn-project/ivc-integration/src/serve.ts @@ -5,14 +5,88 @@ import { generate3FunctionTestingIVCStack, proveAndVerifyBrowser } from './index createDebug.enable('*'); const logger = createDebug('aztec:ivc-test'); +/* eslint-disable no-console */ + +// Function to set up the output element and redirect all console output +function setupConsoleOutput() { + const container = document.createElement('div'); + container.style.marginBottom = '10px'; + document.body.appendChild(container); + + const copyButton = document.createElement('button'); + copyButton.innerText = 'Copy Logs to Clipboard'; + copyButton.style.marginBottom = '10px'; + copyButton.addEventListener('click', () => { + const logContent = logContainer.textContent || ''; // Get text content of log container + navigator.clipboard + .writeText(logContent) + .then(() => { + alert('Logs copied to clipboard!'); + }) + .catch(err => { + console.error('Failed to copy logs:', err); + }); + }); + container.appendChild(copyButton); + + const logContainer = document.createElement('pre'); + logContainer.id = 'logOutput'; + logContainer.style.border = '1px solid #ccc'; + logContainer.style.padding = '10px'; + logContainer.style.maxHeight = '400px'; + logContainer.style.overflowY = 'auto'; + container.appendChild(logContainer); + + // Helper to append messages to logContainer + function addLogMessage(message: string) { + logContainer.textContent += message + '\n'; + logContainer.scrollTop = logContainer.scrollHeight; // Auto-scroll to the bottom + } + + // Override console methods to output clean logs + const originalLog = console.log; + const originalDebug = console.debug; + + console.log = (...args: any[]) => { + const message = args + .map(arg => + typeof arg === 'string' + ? arg + .replace(/%c/g, '') + .replace(/color:.*?(;|$)/g, '') + .trim() + : arg, + ) + .join(' '); + originalLog.apply(console, args); // Keep original behavior + addLogMessage(message); + }; + + console.debug = (...args: any[]) => { + const message = args + .map(arg => + typeof arg === 'string' + ? arg + .replace(/%c/g, '') + .replace(/color:.*?(;|$)/g, '') + .trim() + : arg, + ) + .join(' '); + originalDebug.apply(console, args); // Keep original behavior + addLogMessage(message); + }; +} + (window as any).proveAndVerifyBrowser = proveAndVerifyBrowser; document.addEventListener('DOMContentLoaded', function () { + setupConsoleOutput(); // Initialize console output capture + const button = document.createElement('button'); button.innerText = 'Run Test'; button.addEventListener('click', async () => { logger(`generating circuit and witness...`); - []; const [bytecodes, witnessStack] = await generate3FunctionTestingIVCStack(); logger(`done. proving and verifying...`); const verified = await proveAndVerifyBrowser(bytecodes, witnessStack); diff --git a/yarn-project/kv-store/package.json b/yarn-project/kv-store/package.json index 870afec3c63..bacc49e1a38 100644 --- a/yarn-project/kv-store/package.json +++ b/yarn-project/kv-store/package.json @@ -6,6 +6,7 @@ ".": "./dest/interfaces/index.js", "./lmdb": "./dest/lmdb/index.js", "./utils": "./dest/utils.js", + "./stores": "./dest/stores/index.js", "./config": "./dest/config.js" }, "scripts": { @@ -56,11 +57,13 @@ ] }, "dependencies": { + "@aztec/circuit-types": "workspace:^", "@aztec/ethereum": "workspace:^", "@aztec/foundation": "workspace:^", "lmdb": "^3.0.6" }, "devDependencies": { + "@aztec/circuits.js": "workspace:^", "@jest/globals": "^29.5.0", "@types/jest": "^29.5.0", "@types/node": "^18.7.23", diff --git a/yarn-project/kv-store/src/lmdb/store.ts b/yarn-project/kv-store/src/lmdb/store.ts index 91b1020a289..3e43972f088 100644 --- a/yarn-project/kv-store/src/lmdb/store.ts +++ b/yarn-project/kv-store/src/lmdb/store.ts @@ -58,14 +58,14 @@ export class AztecLmdbStore implements AztecKVStore { */ static open( path?: string, - mapSizeKb?: number, + mapSizeKb = 1 * 1024 * 1024, // defaults to 1 GB map size ephemeral: boolean = false, log = createDebugLogger('aztec:kv-store:lmdb'), ): AztecLmdbStore { if (path) { mkdirSync(path, { recursive: true }); } - const mapSize = mapSizeKb === undefined ? undefined : 1024 * mapSizeKb; + const mapSize = 1024 * mapSizeKb; log.debug(`Opening LMDB database at ${path || 'temporary location'} with map size ${mapSize}`); const rootDb = open({ path, noSync: ephemeral, mapSize }); return new AztecLmdbStore(rootDb, ephemeral, path); diff --git a/yarn-project/kv-store/src/stores/index.ts b/yarn-project/kv-store/src/stores/index.ts new file mode 100644 index 00000000000..c279b4ba628 --- /dev/null +++ b/yarn-project/kv-store/src/stores/index.ts @@ -0,0 +1 @@ +export * from './l2_tips_store.js'; diff --git a/yarn-project/kv-store/src/stores/l2_tips_store.test.ts b/yarn-project/kv-store/src/stores/l2_tips_store.test.ts new file mode 100644 index 00000000000..2b820aaf432 --- /dev/null +++ b/yarn-project/kv-store/src/stores/l2_tips_store.test.ts @@ -0,0 +1,71 @@ +import { type L2Block } from '@aztec/circuit-types'; +import { Fr, type Header } from '@aztec/circuits.js'; +import { times } from '@aztec/foundation/collection'; +import { type AztecKVStore } from '@aztec/kv-store'; +import { openTmpStore } from '@aztec/kv-store/utils'; + +import { L2TipsStore } from './l2_tips_store.js'; + +describe('L2TipsStore', () => { + let kvStore: AztecKVStore; + let tipsStore: L2TipsStore; + + beforeEach(() => { + kvStore = openTmpStore(true); + tipsStore = new L2TipsStore(kvStore, 'test'); + }); + + const makeBlock = (number: number): L2Block => + ({ number, header: { hash: () => new Fr(number) } as Header } as L2Block); + + const makeTip = (number: number) => ({ number, hash: number === 0 ? undefined : new Fr(number).toString() }); + + const makeTips = (latest: number, proven: number, finalized: number) => ({ + latest: makeTip(latest), + proven: makeTip(proven), + finalized: makeTip(finalized), + }); + + it('returns zero if no tips are stored', async () => { + const tips = await tipsStore.getL2Tips(); + expect(tips).toEqual(makeTips(0, 0, 0)); + }); + + it('stores chain tips', async () => { + await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks: times(20, i => makeBlock(i + 1)) }); + + await tipsStore.handleBlockStreamEvent({ type: 'chain-finalized', blockNumber: 5 }); + await tipsStore.handleBlockStreamEvent({ type: 'chain-proven', blockNumber: 8 }); + await tipsStore.handleBlockStreamEvent({ type: 'chain-pruned', blockNumber: 10 }); + + const tips = await tipsStore.getL2Tips(); + expect(tips).toEqual(makeTips(10, 8, 5)); + }); + + it('sets latest tip from blocks added', async () => { + await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks: times(3, i => makeBlock(i + 1)) }); + + const tips = await tipsStore.getL2Tips(); + expect(tips).toEqual(makeTips(3, 0, 0)); + + expect(await tipsStore.getL2BlockHash(1)).toEqual(new Fr(1).toString()); + expect(await tipsStore.getL2BlockHash(2)).toEqual(new Fr(2).toString()); + expect(await tipsStore.getL2BlockHash(3)).toEqual(new Fr(3).toString()); + }); + + it('clears block hashes when setting finalized chain', async () => { + await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks: times(5, i => makeBlock(i + 1)) }); + await tipsStore.handleBlockStreamEvent({ type: 'chain-proven', blockNumber: 3 }); + await tipsStore.handleBlockStreamEvent({ type: 'chain-finalized', blockNumber: 3 }); + + const tips = await tipsStore.getL2Tips(); + expect(tips).toEqual(makeTips(5, 3, 3)); + + expect(await tipsStore.getL2BlockHash(1)).toBeUndefined(); + expect(await tipsStore.getL2BlockHash(2)).toBeUndefined(); + + expect(await tipsStore.getL2BlockHash(3)).toEqual(new Fr(3).toString()); + expect(await tipsStore.getL2BlockHash(4)).toEqual(new Fr(4).toString()); + expect(await tipsStore.getL2BlockHash(5)).toEqual(new Fr(5).toString()); + }); +}); diff --git a/yarn-project/kv-store/src/stores/l2_tips_store.ts b/yarn-project/kv-store/src/stores/l2_tips_store.ts new file mode 100644 index 00000000000..8141d804ce0 --- /dev/null +++ b/yarn-project/kv-store/src/stores/l2_tips_store.ts @@ -0,0 +1,69 @@ +import { + type L2BlockId, + type L2BlockStreamEvent, + type L2BlockStreamEventHandler, + type L2BlockStreamLocalDataProvider, + type L2BlockTag, + type L2Tips, +} from '@aztec/circuit-types'; + +import { type AztecMap } from '../interfaces/map.js'; +import { type AztecKVStore } from '../interfaces/store.js'; + +/** Stores currently synced L2 tips and unfinalized block hashes. */ +export class L2TipsStore implements L2BlockStreamEventHandler, L2BlockStreamLocalDataProvider { + private readonly l2TipsStore: AztecMap; + private readonly l2BlockHashesStore: AztecMap; + + constructor(store: AztecKVStore, namespace: string) { + this.l2TipsStore = store.openMap([namespace, 'l2_tips'].join('_')); + this.l2BlockHashesStore = store.openMap([namespace, 'l2_block_hashes'].join('_')); + } + + public getL2BlockHash(number: number): Promise { + return Promise.resolve(this.l2BlockHashesStore.get(number)); + } + + public getL2Tips(): Promise { + return Promise.resolve({ + latest: this.getL2Tip('latest'), + finalized: this.getL2Tip('finalized'), + proven: this.getL2Tip('proven'), + }); + } + + private getL2Tip(tag: L2BlockTag): L2BlockId { + const blockNumber = this.l2TipsStore.get(tag); + if (blockNumber === undefined || blockNumber === 0) { + return { number: 0, hash: undefined }; + } + const blockHash = this.l2BlockHashesStore.get(blockNumber); + if (!blockHash) { + throw new Error(`Block hash not found for block number ${blockNumber}`); + } + return { number: blockNumber, hash: blockHash }; + } + + public async handleBlockStreamEvent(event: L2BlockStreamEvent): Promise { + switch (event.type) { + case 'blocks-added': + await this.l2TipsStore.set('latest', event.blocks.at(-1)!.number); + for (const block of event.blocks) { + await this.l2BlockHashesStore.set(block.number, block.header.hash().toString()); + } + break; + case 'chain-pruned': + await this.l2TipsStore.set('latest', event.blockNumber); + break; + case 'chain-proven': + await this.l2TipsStore.set('proven', event.blockNumber); + break; + case 'chain-finalized': + await this.l2TipsStore.set('finalized', event.blockNumber); + for (const key of this.l2BlockHashesStore.keys({ end: event.blockNumber })) { + await this.l2BlockHashesStore.delete(key); + } + break; + } + } +} diff --git a/yarn-project/kv-store/src/utils.ts b/yarn-project/kv-store/src/utils.ts index b8637975e8a..0344e2be200 100644 --- a/yarn-project/kv-store/src/utils.ts +++ b/yarn-project/kv-store/src/utils.ts @@ -62,7 +62,7 @@ async function initStoreForRollup( * @param ephemeral - true if the store should only exist in memory and not automatically be flushed to disk. Optional * @returns A new store */ -export function openTmpStore(ephemeral: boolean = false): AztecKVStore { - const mapSize = 1024 * 1024 * 1024 * 10; // 10 GB map size +export function openTmpStore(ephemeral: boolean = false): AztecLmdbStore { + const mapSize = 1024 * 1024 * 10; // 10 GB map size return AztecLmdbStore.open(undefined, mapSize, ephemeral); } diff --git a/yarn-project/kv-store/tsconfig.json b/yarn-project/kv-store/tsconfig.json index 18fc3bcf3f2..bd860591ecb 100644 --- a/yarn-project/kv-store/tsconfig.json +++ b/yarn-project/kv-store/tsconfig.json @@ -7,10 +7,16 @@ }, "references": [ { - "path": "../foundation" + "path": "../circuit-types" }, { "path": "../ethereum" + }, + { + "path": "../foundation" + }, + { + "path": "../circuits.js" } ], "include": ["src"] diff --git a/yarn-project/noir-protocol-circuits-types/src/artifacts.ts b/yarn-project/noir-protocol-circuits-types/src/artifacts.ts index 2befc9ad5fe..d0339d17050 100644 --- a/yarn-project/noir-protocol-circuits-types/src/artifacts.ts +++ b/yarn-project/noir-protocol-circuits-types/src/artifacts.ts @@ -14,8 +14,6 @@ import PrivateKernelTailJson from '../artifacts/private_kernel_tail.json' assert import PrivateKernelTailSimulatedJson from '../artifacts/private_kernel_tail_simulated.json' assert { type: 'json' }; import PrivateKernelTailToPublicJson from '../artifacts/private_kernel_tail_to_public.json' assert { type: 'json' }; import PrivateKernelTailToPublicSimulatedJson from '../artifacts/private_kernel_tail_to_public_simulated.json' assert { type: 'json' }; -import PublicKernelMergeSimulatedJson from '../artifacts/public_kernel_merge_simulated.json' assert { type: 'json' }; -import PublicKernelTailSimulatedJson from '../artifacts/public_kernel_tail_simulated.json' assert { type: 'json' }; import PrivateBaseRollupJson from '../artifacts/rollup_base_private.json' assert { type: 'json' }; import PrivateBaseRollupSimulatedJson from '../artifacts/rollup_base_private_simulated.json' assert { type: 'json' }; import PublicBaseRollupJson from '../artifacts/rollup_base_public.json' assert { type: 'json' }; @@ -31,10 +29,6 @@ import { type PrivateResetArtifact, } from './private_kernel_reset_data.js'; -// To be deprecated. -export const SimulatedPublicKernelMergeArtifact = PublicKernelMergeSimulatedJson as NoirCompiledCircuit; -export const SimulatedPublicKernelTailArtifact = PublicKernelTailSimulatedJson as NoirCompiledCircuit; - // These are all circuits that should generate proofs with the `recursive` flag. export type ServerProtocolArtifact = | 'EmptyNestedArtifact' diff --git a/yarn-project/noir-protocol-circuits-types/src/index.ts b/yarn-project/noir-protocol-circuits-types/src/index.ts index c7744134bfd..82b3e206384 100644 --- a/yarn-project/noir-protocol-circuits-types/src/index.ts +++ b/yarn-project/noir-protocol-circuits-types/src/index.ts @@ -19,9 +19,6 @@ import { type PrivateKernelTailCircuitPrivateInputs, type PrivateKernelTailCircuitPublicInputs, type PublicBaseRollupInputs, - type PublicKernelCircuitPrivateInputs, - type PublicKernelCircuitPublicInputs, - type PublicKernelTailCircuitPrivateInputs, type RootParityInputs, type RootRollupInputs, type RootRollupPublicInputs, @@ -38,8 +35,6 @@ import { ClientCircuitArtifacts, ServerCircuitArtifacts, SimulatedClientCircuitArtifacts, - SimulatedPublicKernelMergeArtifact, - SimulatedPublicKernelTailArtifact, SimulatedServerCircuitArtifacts, } from './artifacts.js'; import { type PrivateResetArtifact } from './private_kernel_reset_data.js'; @@ -65,9 +60,6 @@ import { mapPrivateKernelTailCircuitPublicInputsForPublicFromNoir, mapPrivateKernelTailCircuitPublicInputsForRollupFromNoir, mapPublicBaseRollupInputsToNoir, - mapPublicKernelCircuitPrivateInputsToNoir, - mapPublicKernelCircuitPublicInputsFromNoir, - mapPublicKernelTailCircuitPrivateInputsToNoir, mapRootParityInputsToNoir, mapRootRollupInputsToNoir, mapRootRollupPublicInputsFromNoir, @@ -82,8 +74,6 @@ import { type PrivateKernelResetReturnType, type PrivateKernelTailReturnType, type PrivateKernelTailToPublicReturnType, - type PublicKernelMergeSimulatedReturnType, - type PublicKernelTailSimulatedReturnType, type RollupBasePrivateReturnType, type RollupBasePublicReturnType, type RollupBlockMergeReturnType, @@ -541,32 +531,6 @@ export function convertRootRollupInputsToWitnessMap(inputs: RootRollupInputs): W return initialWitnessMap; } -/** - * Converts the inputs of the public merge circuit into a witness map - * @param inputs - The public kernel inputs. - * @returns The witness map - */ -export function convertSimulatedPublicMergeInputsToWitnessMap(inputs: PublicKernelCircuitPrivateInputs): WitnessMap { - const mapped = mapPublicKernelCircuitPrivateInputsToNoir(inputs); - const initialWitnessMap = abiEncode(SimulatedPublicKernelMergeArtifact.abi, { - input: mapped as any, - }); - return initialWitnessMap; -} - -/** - * Converts the inputs of the public tail circuit into a witness map - * @param inputs - The public kernel inputs. - * @returns The witness map - */ -export function convertSimulatedPublicTailInputsToWitnessMap(inputs: PublicKernelTailCircuitPrivateInputs): WitnessMap { - const mapped = mapPublicKernelTailCircuitPrivateInputsToNoir(inputs); - const initialWitnessMap = abiEncode(SimulatedPublicKernelTailArtifact.abi, { - input: mapped as any, - }); - return initialWitnessMap; -} - export function convertPrivateKernelEmptyOutputsFromWitnessMap(outputs: WitnessMap): KernelCircuitPublicInputs { const decodedInputs: DecodedInputs = abiDecode(ServerCircuitArtifacts.PrivateKernelEmptyArtifact.abi, outputs); const returnType = decodedInputs.return_value as PrivateKernelEmptyReturnType; @@ -760,36 +724,6 @@ export function convertRootParityOutputsFromWitnessMap(outputs: WitnessMap): Par return mapParityPublicInputsFromNoir(returnType); } -/** - * Converts the outputs of the public merge circuit from a witness map. - * @param outputs - The public kernel outputs as a witness map. - * @returns The public inputs. - */ -export function convertSimulatedPublicMergeOutputFromWitnessMap(outputs: WitnessMap): PublicKernelCircuitPublicInputs { - // Decode the witness map into two fields, the return values and the inputs - const decodedInputs: DecodedInputs = abiDecode(SimulatedPublicKernelMergeArtifact.abi, outputs); - - // Cast the inputs as the return type - const returnType = decodedInputs.return_value as PublicKernelMergeSimulatedReturnType; - - return mapPublicKernelCircuitPublicInputsFromNoir(returnType); -} - -/** - * Converts the outputs of the public tail circuit from a witness map. - * @param outputs - The public kernel outputs as a witness map. - * @returns The public inputs. - */ -export function convertSimulatedPublicTailOutputFromWitnessMap(outputs: WitnessMap): KernelCircuitPublicInputs { - // Decode the witness map into two fields, the return values and the inputs - const decodedInputs: DecodedInputs = abiDecode(SimulatedPublicKernelTailArtifact.abi, outputs); - - // Cast the inputs as the return type - const returnType = decodedInputs.return_value as PublicKernelTailSimulatedReturnType; - - return mapKernelCircuitPublicInputsFromNoir(returnType); -} - function fromACVMField(field: string): Fr { return Fr.fromBuffer(Buffer.from(field.slice(2), 'hex')); } diff --git a/yarn-project/noir-protocol-circuits-types/src/scripts/generate_ts_from_abi.ts b/yarn-project/noir-protocol-circuits-types/src/scripts/generate_ts_from_abi.ts index ea26faf2102..a6b6ceae2ea 100644 --- a/yarn-project/noir-protocol-circuits-types/src/scripts/generate_ts_from_abi.ts +++ b/yarn-project/noir-protocol-circuits-types/src/scripts/generate_ts_from_abi.ts @@ -15,8 +15,6 @@ const circuits = [ 'private_kernel_reset', 'private_kernel_tail', 'private_kernel_tail_to_public', - 'public_kernel_merge_simulated', - 'public_kernel_tail_simulated', 'rollup_base_private', 'rollup_base_public', 'rollup_merge', diff --git a/yarn-project/noir-protocol-circuits-types/src/type_conversion.ts b/yarn-project/noir-protocol-circuits-types/src/type_conversion.ts index 0bfc86ba335..27391eb641f 100644 --- a/yarn-project/noir-protocol-circuits-types/src/type_conversion.ts +++ b/yarn-project/noir-protocol-circuits-types/src/type_conversion.ts @@ -22,7 +22,6 @@ import { type EmptyBlockRootRollupInputs, type EmptyNestedData, EncryptedLogHash, - type EnqueuedCallData, EthAddress, FeeRecipient, Fr, @@ -39,23 +38,19 @@ import { type KeyValidationHint, KeyValidationRequest, KeyValidationRequestAndGenerator, - type L1_TO_L2_MSG_TREE_HEIGHT, L2ToL1Message, LogHash, MAX_CONTRACT_CLASS_LOGS_PER_TX, MAX_ENCRYPTED_LOGS_PER_TX, MAX_ENQUEUED_CALLS_PER_TX, MAX_KEY_VALIDATION_REQUESTS_PER_TX, - MAX_L1_TO_L2_MSG_READ_REQUESTS_PER_TX, MAX_L2_TO_L1_MSGS_PER_TX, MAX_NOTE_ENCRYPTED_LOGS_PER_TX, MAX_NOTE_HASHES_PER_TX, MAX_NOTE_HASH_READ_REQUESTS_PER_TX, MAX_NULLIFIERS_PER_TX, - MAX_NULLIFIER_NON_EXISTENT_READ_REQUESTS_PER_TX, MAX_NULLIFIER_READ_REQUESTS_PER_TX, MAX_PRIVATE_CALL_STACK_LENGTH_PER_TX, - MAX_PUBLIC_DATA_READS_PER_TX, MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, MAX_UNENCRYPTED_LOGS_PER_TX, MaxBlockNumber, @@ -65,13 +60,11 @@ import { type NOTE_HASH_TREE_HEIGHT, type NULLIFIER_TREE_HEIGHT, NUM_BYTES_PER_SHA256, - type NonMembershipHint, NoteHash, type NoteHashReadRequestHints, NoteLogHash, Nullifier, type NullifierLeafPreimage, - type NullifierNonExistentReadRequestHints, type NullifierReadRequestHints, OptionalNumber, type PUBLIC_DATA_TREE_HEIGHT, @@ -85,6 +78,7 @@ import { type PreviousRollupData, PrivateAccumulatedData, type PrivateBaseRollupInputs, + type PrivateBaseStateDiffHints, type PrivateCallData, PrivateCallRequest, type PrivateCircuitPublicInputs, @@ -99,27 +93,14 @@ import { type PrivateToPublicKernelCircuitPublicInputs, type PrivateTubeData, PrivateValidationRequests, - PublicAccumulatedData, - PublicAccumulatedDataArrayLengths, type PublicBaseRollupInputs, + type PublicBaseStateDiffHints, PublicCallRequest, - PublicCallStackItemCompressed, type PublicDataHint, - type PublicDataLeafHint, - PublicDataRead, - type PublicDataTreeLeaf, type PublicDataTreeLeafPreimage, - PublicDataUpdateRequest, PublicDataWrite, - PublicInnerCallRequest, - type PublicKernelCircuitPrivateInputs, - PublicKernelCircuitPublicInputs, - type PublicKernelData, - type PublicKernelTailCircuitPrivateInputs, type PublicKeys, type PublicTubeData, - PublicValidationRequestArrayLengths, - PublicValidationRequests, type RECURSIVE_PROOF_LENGTH, ReadRequest, type ReadRequestStatus, @@ -138,17 +119,13 @@ import { ScopedNullifier, ScopedReadRequest, type SettledReadHint, - type StateDiffHints, StateReference, type TUBE_PROOF_LENGTH, type TransientDataIndexHint, - TreeLeafReadRequest, - type TreeLeafReadRequestHint, type TreeSnapshots, TxConstantData, TxContext, type TxRequest, - VMCircuitPublicInputs, type VerificationKeyAsFields, type VkWitnessData, } from '@aztec/circuits.js'; @@ -174,7 +151,6 @@ import type { EmptyBlockRootRollupInputs as EmptyBlockRootRollupInputsNoir, EmptyNestedCircuitPublicInputs as EmptyNestedDataNoir, EncryptedLogHash as EncryptedLogHashNoir, - EnqueuedCallData as EnqueuedCallDataNoir, FeeRecipient as FeeRecipientNoir, Field, FixedLengthArray, @@ -206,8 +182,6 @@ import type { NoteLogHash as NoteLogHashNoir, NullifierLeafPreimage as NullifierLeafPreimageNoir, Nullifier as NullifierNoir, - NullifierNonExistentReadRequestHints as NullifierNonExistentReadRequestHintsNoir, - NullifierNonMembershipHint as NullifierNonMembershipHintNoir, NullifierReadRequestHints as NullifierReadRequestHintsNoir, NullifierSettledReadHint as NullifierSettledReadHintNoir, Option as OptionalNumberNoir, @@ -219,6 +193,7 @@ import type { PreviousRollupData as PreviousRollupDataNoir, PrivateAccumulatedData as PrivateAccumulatedDataNoir, PrivateBaseRollupInputs as PrivateBaseRollupInputsNoir, + PrivateBaseStateDiffHints as PrivateBaseStateDiffHintsNoir, PrivateCallDataWithoutPublicInputs as PrivateCallDataWithoutPublicInputsNoir, PrivateCallRequest as PrivateCallRequestNoir, PrivateCircuitPublicInputs as PrivateCircuitPublicInputsNoir, @@ -232,27 +207,14 @@ import type { PrivateToPublicKernelCircuitPublicInputs as PrivateToPublicKernelCircuitPublicInputsNoir, PrivateTubeData as PrivateTubeDataNoir, PrivateValidationRequests as PrivateValidationRequestsNoir, - PublicAccumulatedDataArrayLengths as PublicAccumulatedDataArrayLengthsNoir, - PublicAccumulatedData as PublicAccumulatedDataNoir, PublicBaseRollupInputs as PublicBaseRollupInputsNoir, + PublicBaseStateDiffHints as PublicBaseStateDiffHintsNoir, PublicCallRequest as PublicCallRequestNoir, - PublicCallStackItemCompressed as PublicCallStackItemCompressedNoir, PublicDataHint as PublicDataHintNoir, - PublicDataLeafHint as PublicDataLeafHintNoir, - PublicDataRead as PublicDataReadNoir, - PublicDataTreeLeaf as PublicDataTreeLeafNoir, PublicDataTreeLeafPreimage as PublicDataTreeLeafPreimageNoir, - PublicDataUpdateRequest as PublicDataUpdateRequestNoir, PublicDataWrite as PublicDataWriteNoir, - PublicInnerCallRequest as PublicInnerCallRequestNoir, - PublicKernelCircuitPublicInputs as PublicKernelCircuitPublicInputsNoir, - PublicKernelData as PublicKernelDataNoir, - PublicKernelMergeCircuitPrivateInputs as PublicKernelMergeCircuitPrivateInputsNoir, - PublicKernelTailCircuitPrivateInputs as PublicKernelTailCircuitPrivateInputsNoir, PublicKeys as PublicKeysNoir, PublicTubeData as PublicTubeDataNoir, - PublicValidationRequestArrayLengths as PublicValidationRequestArrayLengthsNoir, - PublicValidationRequests as PublicValidationRequestsNoir, ReadRequest as ReadRequestNoir, ReadRequestStatus as ReadRequestStatusNoir, RollupValidationRequests as RollupValidationRequestsNoir, @@ -267,16 +229,12 @@ import type { ScopedNoteHash as ScopedNoteHashNoir, ScopedNullifier as ScopedNullifierNoir, ScopedReadRequest as ScopedReadRequestNoir, - StateDiffHints as StateDiffHintsNoir, StateReference as StateReferenceNoir, TransientDataIndexHint as TransientDataIndexHintNoir, - TreeLeafReadRequestHint as TreeLeafReadRequestHintNoir, - TreeLeafReadRequest as TreeLeafReadRequestNoir, TreeSnapshots as TreeSnapshotsNoir, TxConstantData as TxConstantDataNoir, TxContext as TxContextNoir, TxRequest as TxRequestNoir, - VMCircuitPublicInputs as VMCircuitPublicInputsNoir, VerificationKey as VerificationKeyNoir, VkData as VkDataNoir, } from './types/index.js'; @@ -564,32 +522,6 @@ function mapPrivateCallRequestToNoir(callRequest: PrivateCallRequest): PrivateCa }; } -function mapPublicCallStackItemCompressedFromNoir(data: PublicCallStackItemCompressedNoir) { - return new PublicCallStackItemCompressed( - mapAztecAddressFromNoir(data.contract_address), - mapCallContextFromNoir(data.call_context), - mapFieldFromNoir(data.args_hash), - mapFieldFromNoir(data.returns_hash), - mapRevertCodeFromNoir(data.revert_code), - mapGasFromNoir(data.start_gas_left), - mapGasFromNoir(data.end_gas_left), - ); -} - -function mapPublicCallStackItemCompressedToNoir( - data: PublicCallStackItemCompressed, -): PublicCallStackItemCompressedNoir { - return { - contract_address: mapAztecAddressToNoir(data.contractAddress), - call_context: mapCallContextToNoir(data.callContext), - args_hash: mapFieldToNoir(data.argsHash), - returns_hash: mapFieldToNoir(data.returnsHash), - revert_code: mapRevertCodeToNoir(data.revertCode), - start_gas_left: mapGasToNoir(data.startGasLeft), - end_gas_left: mapGasToNoir(data.endGasLeft), - }; -} - function mapPublicCallRequestFromNoir(request: PublicCallRequestNoir) { return new PublicCallRequest( mapAztecAddressFromNoir(request.msg_sender), @@ -621,20 +553,6 @@ function mapCountedPublicCallRequestToNoir(request: CountedPublicCallRequest): C }; } -function mapPublicInnerCallRequestFromNoir(request: PublicInnerCallRequestNoir) { - return new PublicInnerCallRequest( - mapPublicCallStackItemCompressedFromNoir(request.item), - mapNumberFromNoir(request.counter), - ); -} - -function mapPublicInnerCallRequestToNoir(request: PublicInnerCallRequest): PublicInnerCallRequestNoir { - return { - item: mapPublicCallStackItemCompressedToNoir(request.item), - counter: mapNumberToNoir(request.counter), - }; -} - function mapNoteHashToNoir(noteHash: NoteHash): NoteHashNoir { return { value: mapFieldToNoir(noteHash.value), @@ -860,17 +778,6 @@ export function mapScopedReadRequestFromNoir(scoped: ScopedReadRequestNoir): Sco ); } -function mapTreeLeafReadRequestToNoir(readRequest: TreeLeafReadRequest): TreeLeafReadRequestNoir { - return { - value: mapFieldToNoir(readRequest.value), - leaf_index: mapFieldToNoir(readRequest.leafIndex), - }; -} - -function mapTreeLeafReadRequestFromNoir(readRequest: TreeLeafReadRequestNoir) { - return new TreeLeafReadRequest(mapFieldFromNoir(readRequest.value), mapFieldFromNoir(readRequest.leaf_index)); -} - /** * Maps a KeyValidationRequest to a noir KeyValidationRequest. * @param request - The KeyValidationRequest. @@ -1079,36 +986,6 @@ export function mapSha256HashToNoir(hash: Buffer): Field { return mapFieldToNoir(toTruncField(hash)); } -/** - * Maps public data update request from noir to the parsed type. - * @param publicDataUpdateRequest - The noir public data update request. - * @returns The parsed public data update request. - */ -export function mapPublicDataUpdateRequestFromNoir( - publicDataUpdateRequest: PublicDataUpdateRequestNoir, -): PublicDataUpdateRequest { - return new PublicDataUpdateRequest( - mapFieldFromNoir(publicDataUpdateRequest.leaf_slot), - mapFieldFromNoir(publicDataUpdateRequest.new_value), - mapNumberFromNoir(publicDataUpdateRequest.counter), - ); -} - -/** - * Maps public data update request to noir public data update request. - * @param publicDataUpdateRequest - The public data update request. - * @returns The noir public data update request. - */ -export function mapPublicDataUpdateRequestToNoir( - publicDataUpdateRequest: PublicDataUpdateRequest, -): PublicDataUpdateRequestNoir { - return { - leaf_slot: mapFieldToNoir(publicDataUpdateRequest.leafSlot), - new_value: mapFieldToNoir(publicDataUpdateRequest.newValue), - counter: mapNumberToNoir(publicDataUpdateRequest.sideEffectCounter), - }; -} - function mapPublicDataWriteFromNoir(write: PublicDataWriteNoir) { return new PublicDataWrite(mapFieldFromNoir(write.leaf_slot), mapFieldFromNoir(write.value)); } @@ -1120,32 +997,6 @@ function mapPublicDataWriteToNoir(write: PublicDataWrite): PublicDataWriteNoir { }; } -/** - * Maps public data read from noir to the parsed type. - * @param publicDataRead - The noir public data read. - * @returns The parsed public data read. - */ -export function mapPublicDataReadFromNoir(publicDataRead: PublicDataReadNoir): PublicDataRead { - return new PublicDataRead( - mapFieldFromNoir(publicDataRead.leaf_slot), - mapFieldFromNoir(publicDataRead.value), - mapNumberFromNoir(publicDataRead.counter), - ); -} - -/** - * Maps public data read to noir public data read. - * @param publicDataRead - The public data read. - * @returns The noir public data read. - */ -export function mapPublicDataReadToNoir(publicDataRead: PublicDataRead): PublicDataReadNoir { - return { - leaf_slot: mapFieldToNoir(publicDataRead.leafSlot), - value: mapFieldToNoir(publicDataRead.value), - counter: mapNumberToNoir(publicDataRead.counter), - }; -} - function mapReadRequestStatusToNoir(readRequestStatus: ReadRequestStatus): ReadRequestStatusNoir { return { state: mapNumberToNoir(readRequestStatus.state), @@ -1160,14 +1011,6 @@ function mapPendingReadHintToNoir(hint: PendingReadHint): PendingReadHintNoir { }; } -function mapTreeLeafReadRequestHintToNoir( - hint: TreeLeafReadRequestHint, -): TreeLeafReadRequestHintNoir { - return { - sibling_path: mapTuple(hint.siblingPath, mapFieldToNoir) as FixedLengthArray, - }; -} - function mapNoteHashSettledReadHintToNoir( hint: SettledReadHint, ): NoteHashSettledReadHintNoir { @@ -1221,43 +1064,15 @@ function mapNullifierReadRequestHintsToNoir, -): NullifierNonMembershipHintNoir { - return { - low_leaf_preimage: mapNullifierLeafPreimageToNoir(hint.leafPreimage), - membership_witness: mapMembershipWitnessToNoir(hint.membershipWitness), - }; -} - -function mapNullifierNonExistentReadRequestHintsToNoir( - hints: NullifierNonExistentReadRequestHints, -): NullifierNonExistentReadRequestHintsNoir { - return { - non_membership_hints: mapTuple(hints.nonMembershipHints, mapNullifierNonMembershipHintToNoir), - sorted_pending_values: mapTuple(hints.sortedPendingValues, mapNullifierToNoir), - sorted_pending_value_index_hints: mapTuple(hints.sortedPendingValueHints, mapNumberToNoir), - next_pending_value_indices: mapTuple(hints.nextPendingValueIndices, mapNumberToNoir), - }; -} - function mapPublicDataHintToNoir(hint: PublicDataHint): PublicDataHintNoir { return { leaf_slot: mapFieldToNoir(hint.leafSlot), value: mapFieldToNoir(hint.value), - override_counter: mapNumberToNoir(hint.overrideCounter), membership_witness: mapMembershipWitnessToNoir(hint.membershipWitness), leaf_preimage: mapPublicDataTreePreimageToNoir(hint.leafPreimage), }; } -function mapPublicDataLeafHintToNoir(hint: PublicDataLeafHint): PublicDataLeafHintNoir { - return { - preimage: mapPublicDataTreePreimageToNoir(hint.preimage), - membership_witness: mapMembershipWitnessToNoir(hint.membershipWitness), - }; -} - function mapOptionalNumberToNoir(option: OptionalNumber): OptionalNumberNoir { return { _is_some: option.isSome, @@ -1304,69 +1119,6 @@ function mapPrivateValidationRequestsFromNoir(requests: PrivateValidationRequest ); } -function mapPublicValidationRequestsToNoir(requests: PublicValidationRequests): PublicValidationRequestsNoir { - return { - for_rollup: mapRollupValidationRequestsToNoir(requests.forRollup), - note_hash_read_requests: mapTuple(requests.noteHashReadRequests, mapTreeLeafReadRequestToNoir), - nullifier_read_requests: mapTuple(requests.nullifierReadRequests, mapScopedReadRequestToNoir), - nullifier_non_existent_read_requests: mapTuple( - requests.nullifierNonExistentReadRequests, - mapScopedReadRequestToNoir, - ), - l1_to_l2_msg_read_requests: mapTuple(requests.l1ToL2MsgReadRequests, mapTreeLeafReadRequestToNoir), - public_data_reads: mapTuple(requests.publicDataReads, mapPublicDataReadToNoir), - }; -} - -function mapPublicValidationRequestsFromNoir(requests: PublicValidationRequestsNoir): PublicValidationRequests { - return new PublicValidationRequests( - mapRollupValidationRequestsFromNoir(requests.for_rollup), - mapTupleFromNoir( - requests.note_hash_read_requests, - MAX_NOTE_HASH_READ_REQUESTS_PER_TX, - mapTreeLeafReadRequestFromNoir, - ), - mapTupleFromNoir( - requests.nullifier_read_requests, - MAX_NULLIFIER_READ_REQUESTS_PER_TX, - mapScopedReadRequestFromNoir, - ), - mapTupleFromNoir( - requests.nullifier_non_existent_read_requests, - MAX_NULLIFIER_NON_EXISTENT_READ_REQUESTS_PER_TX, - mapScopedReadRequestFromNoir, - ), - mapTupleFromNoir( - requests.l1_to_l2_msg_read_requests, - MAX_L1_TO_L2_MSG_READ_REQUESTS_PER_TX, - mapTreeLeafReadRequestFromNoir, - ), - mapTupleFromNoir(requests.public_data_reads, MAX_PUBLIC_DATA_READS_PER_TX, mapPublicDataReadFromNoir), - ); -} - -function mapPublicValidationRequestArrayLengthsFromNoir(lengths: PublicValidationRequestArrayLengthsNoir) { - return new PublicValidationRequestArrayLengths( - mapNumberFromNoir(lengths.note_hash_read_requests), - mapNumberFromNoir(lengths.nullifier_read_requests), - mapNumberFromNoir(lengths.nullifier_non_existent_read_requests), - mapNumberFromNoir(lengths.l1_to_l2_msg_read_requests), - mapNumberFromNoir(lengths.public_data_reads), - ); -} - -function mapPublicValidationRequestArrayLengthsToNoir( - lengths: PublicValidationRequestArrayLengths, -): PublicValidationRequestArrayLengthsNoir { - return { - note_hash_read_requests: mapNumberToNoir(lengths.noteHashReadRequests), - nullifier_read_requests: mapNumberToNoir(lengths.nullifierReadRequests), - nullifier_non_existent_read_requests: mapNumberToNoir(lengths.nullifierNonExistentReadRequests), - l1_to_l2_msg_read_requests: mapNumberToNoir(lengths.l1ToL2MsgReadRequests), - public_data_reads: mapNumberToNoir(lengths.publicDataReads), - }; -} - export function mapPrivateAccumulatedDataFromNoir( privateAccumulatedData: PrivateAccumulatedDataNoir, ): PrivateAccumulatedData { @@ -1415,81 +1167,6 @@ export function mapPrivateAccumulatedDataToNoir(data: PrivateAccumulatedData): P }; } -export function mapPublicAccumulatedDataFromNoir( - publicAccumulatedData: PublicAccumulatedDataNoir, -): PublicAccumulatedData { - return new PublicAccumulatedData( - mapTupleFromNoir(publicAccumulatedData.note_hashes, MAX_NOTE_HASHES_PER_TX, mapScopedNoteHashFromNoir), - mapTupleFromNoir(publicAccumulatedData.nullifiers, MAX_NULLIFIERS_PER_TX, mapNullifierFromNoir), - mapTupleFromNoir(publicAccumulatedData.l2_to_l1_msgs, MAX_L2_TO_L1_MSGS_PER_TX, mapScopedL2ToL1MessageFromNoir), - mapTupleFromNoir( - publicAccumulatedData.note_encrypted_logs_hashes, - MAX_NOTE_ENCRYPTED_LOGS_PER_TX, - mapLogHashFromNoir, - ), - mapTupleFromNoir(publicAccumulatedData.encrypted_logs_hashes, MAX_ENCRYPTED_LOGS_PER_TX, mapScopedLogHashFromNoir), - mapTupleFromNoir( - publicAccumulatedData.unencrypted_logs_hashes, - MAX_UNENCRYPTED_LOGS_PER_TX, - mapScopedLogHashFromNoir, - ), - mapTupleFromNoir( - publicAccumulatedData.public_data_update_requests, - MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, - mapPublicDataUpdateRequestFromNoir, - ), - mapTupleFromNoir(publicAccumulatedData.public_call_stack, MAX_ENQUEUED_CALLS_PER_TX, mapPublicCallRequestFromNoir), - mapGasFromNoir(publicAccumulatedData.gas_used), - ); -} - -export function mapPublicAccumulatedDataToNoir( - publicAccumulatedData: PublicAccumulatedData, -): PublicAccumulatedDataNoir { - return { - note_hashes: mapTuple(publicAccumulatedData.noteHashes, mapScopedNoteHashToNoir), - nullifiers: mapTuple(publicAccumulatedData.nullifiers, mapNullifierToNoir), - l2_to_l1_msgs: mapTuple(publicAccumulatedData.l2ToL1Msgs, mapScopedL2ToL1MessageToNoir), - note_encrypted_logs_hashes: mapTuple(publicAccumulatedData.noteEncryptedLogsHashes, mapLogHashToNoir), - encrypted_logs_hashes: mapTuple(publicAccumulatedData.encryptedLogsHashes, mapScopedLogHashToNoir), - unencrypted_logs_hashes: mapTuple(publicAccumulatedData.unencryptedLogsHashes, mapScopedLogHashToNoir), - public_data_update_requests: mapTuple( - publicAccumulatedData.publicDataUpdateRequests, - mapPublicDataUpdateRequestToNoir, - ), - public_call_stack: mapTuple(publicAccumulatedData.publicCallStack, mapPublicCallRequestToNoir), - gas_used: mapGasToNoir(publicAccumulatedData.gasUsed), - }; -} - -function mapPublicAccumulatedDataArrayLengthsFromNoir(lengths: PublicAccumulatedDataArrayLengthsNoir) { - return new PublicAccumulatedDataArrayLengths( - mapNumberFromNoir(lengths.note_hashes), - mapNumberFromNoir(lengths.nullifiers), - mapNumberFromNoir(lengths.l2_to_l1_msgs), - mapNumberFromNoir(lengths.note_encrypted_logs_hashes), - mapNumberFromNoir(lengths.encrypted_logs_hashes), - mapNumberFromNoir(lengths.unencrypted_logs_hashes), - mapNumberFromNoir(lengths.public_data_update_requests), - mapNumberFromNoir(lengths.public_call_stack), - ); -} - -function mapPublicAccumulatedDataArrayLengthsToNoir( - lengths: PublicAccumulatedDataArrayLengths, -): PublicAccumulatedDataArrayLengthsNoir { - return { - note_hashes: mapNumberToNoir(lengths.noteHashes), - nullifiers: mapNumberToNoir(lengths.nullifiers), - l2_to_l1_msgs: mapNumberToNoir(lengths.l2ToL1Msgs), - note_encrypted_logs_hashes: mapNumberToNoir(lengths.noteEncryptedLogsHashes), - encrypted_logs_hashes: mapNumberToNoir(lengths.encryptedLogsHashes), - unencrypted_logs_hashes: mapNumberToNoir(lengths.unencryptedLogsHashes), - public_data_update_requests: mapNumberToNoir(lengths.publicDataUpdateRequests), - public_call_stack: mapNumberToNoir(lengths.publicCallStack), - }; -} - export function mapGasFromNoir(gasUsed: GasNoir): Gas { return Gas.from({ daGas: mapNumberFromNoir(gasUsed.da_gas), @@ -1686,21 +1363,6 @@ function mapCombinedConstantDataToNoir(combinedConstantData: CombinedConstantDat }; } -export function mapPublicKernelCircuitPublicInputsToNoir( - inputs: PublicKernelCircuitPublicInputs, -): PublicKernelCircuitPublicInputsNoir { - return { - constants: mapCombinedConstantDataToNoir(inputs.constants), - validation_requests: mapPublicValidationRequestsToNoir(inputs.validationRequests), - end: mapPublicAccumulatedDataToNoir(inputs.end), - end_non_revertible: mapPublicAccumulatedDataToNoir(inputs.endNonRevertibleData), - end_side_effect_counter: mapNumberToNoir(inputs.endSideEffectCounter), - public_teardown_call_request: mapPublicCallRequestToNoir(inputs.publicTeardownCallRequest), - fee_payer: mapAztecAddressToNoir(inputs.feePayer), - revert_code: mapRevertCodeToNoir(inputs.revertCode), - }; -} - export function mapPrivateToPublicKernelCircuitPublicInputsToNoir( inputs: PrivateToPublicKernelCircuitPublicInputs, ): PrivateToPublicKernelCircuitPublicInputsNoir { @@ -1739,21 +1401,6 @@ export function mapKernelCircuitPublicInputsToNoir(inputs: KernelCircuitPublicIn }; } -/** - * Maps a public kernel inner data to a noir public kernel data. - * @param publicKernelData - The public kernel inner data. - * @returns The noir public kernel data. - */ -function mapPublicKernelDataToNoir(publicKernelData: PublicKernelData): PublicKernelDataNoir { - return { - public_inputs: mapPublicKernelCircuitPublicInputsToNoir(publicKernelData.publicInputs), - proof: mapRecursiveProofToNoir(publicKernelData.proof), - vk: mapVerificationKeyToNoir(publicKernelData.vk.keyAsFields, HONK_VERIFICATION_KEY_LENGTH_IN_FIELDS), - vk_index: mapFieldToNoir(new Fr(publicKernelData.vkIndex)), - vk_path: mapTuple(publicKernelData.vkPath, mapFieldToNoir), - }; -} - export function mapVerificationKeyToNoir( key: VerificationKeyAsFields, length: N, @@ -1893,52 +1540,6 @@ export function mapPrivateKernelResetHintsToNoir< }; } -export function mapPublicKernelCircuitPrivateInputsToNoir( - inputs: PublicKernelCircuitPrivateInputs, -): PublicKernelMergeCircuitPrivateInputsNoir { - return { - previous_kernel: mapPublicKernelDataToNoir(inputs.previousKernel), - enqueued_call: mapEnqueuedCallDataToNoir(inputs.enqueuedCall), - }; -} - -export function mapPublicKernelTailCircuitPrivateInputsToNoir( - inputs: PublicKernelTailCircuitPrivateInputs, -): PublicKernelTailCircuitPrivateInputsNoir { - return { - previous_kernel: mapPublicKernelDataToNoir(inputs.previousKernel), - note_hash_read_request_hints: mapTuple( - inputs.noteHashReadRequestHints, - (hint: TreeLeafReadRequestHint) => mapTreeLeafReadRequestHintToNoir(hint), - ), - nullifier_read_request_hints: mapNullifierReadRequestHintsToNoir(inputs.nullifierReadRequestHints), - nullifier_non_existent_read_request_hints: mapNullifierNonExistentReadRequestHintsToNoir( - inputs.nullifierNonExistentReadRequestHints, - ), - l1_to_l2_msg_read_request_hints: mapTuple( - inputs.l1ToL2MsgReadRequestHints, - (hint: TreeLeafReadRequestHint) => mapTreeLeafReadRequestHintToNoir(hint), - ), - public_data_hints: mapTuple(inputs.publicDataHints, mapPublicDataLeafHintToNoir), - start_state: mapPartialStateReferenceToNoir(inputs.startState), - }; -} - -export function mapPublicKernelCircuitPublicInputsFromNoir( - inputs: PublicKernelCircuitPublicInputsNoir, -): PublicKernelCircuitPublicInputs { - return new PublicKernelCircuitPublicInputs( - mapCombinedConstantDataFromNoir(inputs.constants), - mapPublicValidationRequestsFromNoir(inputs.validation_requests), - mapPublicAccumulatedDataFromNoir(inputs.end_non_revertible), - mapPublicAccumulatedDataFromNoir(inputs.end), - mapNumberFromNoir(inputs.end_side_effect_counter), - mapPublicCallRequestFromNoir(inputs.public_teardown_call_request), - mapAztecAddressFromNoir(inputs.fee_payer), - mapRevertCodeFromNoir(inputs.revert_code), - ); -} - /** * Maps global variables to the noir type. * @param globalVariables - The global variables. @@ -2068,51 +1669,6 @@ export function mapBlockRootOrBlockMergePublicInputsToNoir( }; } -function mapVMCircuitPublicInputsToNoir(inputs: VMCircuitPublicInputs): VMCircuitPublicInputsNoir { - return { - constants: mapCombinedConstantDataToNoir(inputs.constants), - call_request: mapPublicCallRequestToNoir(inputs.callRequest), - public_call_stack: mapTuple(inputs.publicCallStack, mapPublicInnerCallRequestToNoir), - previous_validation_request_array_lengths: mapPublicValidationRequestArrayLengthsToNoir( - inputs.previousValidationRequestArrayLengths, - ), - validation_requests: mapPublicValidationRequestsToNoir(inputs.validationRequests), - previous_accumulated_data_array_lengths: mapPublicAccumulatedDataArrayLengthsToNoir( - inputs.previousAccumulatedDataArrayLengths, - ), - accumulated_data: mapPublicAccumulatedDataToNoir(inputs.accumulatedData), - start_side_effect_counter: mapNumberToNoir(inputs.startSideEffectCounter), - end_side_effect_counter: mapNumberToNoir(inputs.endSideEffectCounter), - start_gas_left: mapGasToNoir(inputs.startGasLeft), - transaction_fee: mapFieldToNoir(inputs.transactionFee), - reverted: inputs.reverted, - }; -} - -export function mapVMCircuitPublicInputsFromNoir(inputs: VMCircuitPublicInputsNoir) { - return new VMCircuitPublicInputs( - mapCombinedConstantDataFromNoir(inputs.constants), - mapPublicCallRequestFromNoir(inputs.call_request), - mapTupleFromNoir(inputs.public_call_stack, MAX_ENQUEUED_CALLS_PER_TX, mapPublicInnerCallRequestFromNoir), - mapPublicValidationRequestArrayLengthsFromNoir(inputs.previous_validation_request_array_lengths), - mapPublicValidationRequestsFromNoir(inputs.validation_requests), - mapPublicAccumulatedDataArrayLengthsFromNoir(inputs.previous_accumulated_data_array_lengths), - mapPublicAccumulatedDataFromNoir(inputs.accumulated_data), - mapNumberFromNoir(inputs.start_side_effect_counter), - mapNumberFromNoir(inputs.end_side_effect_counter), - mapGasFromNoir(inputs.start_gas_left), - mapFieldFromNoir(inputs.transaction_fee), - inputs.reverted, - ); -} - -function mapEnqueuedCallDataToNoir(enqueuedCallData: EnqueuedCallData): EnqueuedCallDataNoir { - return { - data: mapVMCircuitPublicInputsToNoir(enqueuedCallData.data), - proof: {}, - }; -} - function mapAvmCircuitPublicInputsToNoir(inputs: AvmCircuitPublicInputs): AvmCircuitPublicInputsNoir { return { global_variables: mapGlobalVariablesToNoir(inputs.globalVariables), @@ -2539,16 +2095,6 @@ function mapMembershipWitnessToNoir(witness: MembershipWitness }; } -/** - * Maps a leaf of the public data tree to noir. - */ -export function mapPublicDataTreeLeafToNoir(leaf: PublicDataTreeLeaf): PublicDataTreeLeafNoir { - return { - slot: mapFieldToNoir(leaf.slot), - value: mapFieldToNoir(leaf.value), - }; -} - /** * Maps a leaf preimage of the public data tree to noir. */ @@ -2577,11 +2123,11 @@ export function mapPartialStateReferenceToNoir( } /** - * Maps state diff hints to a noir state diff hints. + * Maps private base state diff hints to a noir state diff hints. * @param hints - The state diff hints. * @returns The noir state diff hints. */ -export function mapStateDiffHintsToNoir(hints: StateDiffHints): StateDiffHintsNoir { +export function mapPrivateBaseStateDiffHintsToNoir(hints: PrivateBaseStateDiffHints): PrivateBaseStateDiffHintsNoir { return { nullifier_predecessor_preimages: mapTuple(hints.nullifierPredecessorPreimages, mapNullifierLeafPreimageToNoir), nullifier_predecessor_membership_witnesses: mapTuple( @@ -2592,7 +2138,34 @@ export function mapStateDiffHintsToNoir(hints: StateDiffHints): StateDiffHintsNo sorted_nullifier_indexes: mapTuple(hints.sortedNullifierIndexes, (index: number) => mapNumberToNoir(index)), note_hash_subtree_sibling_path: mapTuple(hints.noteHashSubtreeSiblingPath, mapFieldToNoir), nullifier_subtree_sibling_path: mapTuple(hints.nullifierSubtreeSiblingPath, mapFieldToNoir), - public_data_sibling_path: mapTuple(hints.publicDataSiblingPath, mapFieldToNoir), + fee_write_low_leaf_preimage: mapPublicDataTreePreimageToNoir(hints.feeWriteLowLeafPreimage), + fee_write_low_leaf_membership_witness: mapMembershipWitnessToNoir(hints.feeWriteLowLeafMembershipWitness), + fee_write_sibling_path: mapTuple(hints.feeWriteSiblingPath, mapFieldToNoir), + }; +} + +/** + * Maps public base state diff hints to a noir state diff hints. + * @param hints - The state diff hints. + * @returns The noir state diff hints. + */ +export function mapPublicBaseStateDiffHintsToNoir(hints: PublicBaseStateDiffHints): PublicBaseStateDiffHintsNoir { + return { + nullifier_predecessor_preimages: mapTuple(hints.nullifierPredecessorPreimages, mapNullifierLeafPreimageToNoir), + nullifier_predecessor_membership_witnesses: mapTuple( + hints.nullifierPredecessorMembershipWitnesses, + (witness: MembershipWitness) => mapMembershipWitnessToNoir(witness), + ), + sorted_nullifiers: mapTuple(hints.sortedNullifiers, mapFieldToNoir), + sorted_nullifier_indexes: mapTuple(hints.sortedNullifierIndexes, (index: number) => mapNumberToNoir(index)), + note_hash_subtree_sibling_path: mapTuple(hints.noteHashSubtreeSiblingPath, mapFieldToNoir), + nullifier_subtree_sibling_path: mapTuple(hints.nullifierSubtreeSiblingPath, mapFieldToNoir), + low_public_data_writes_preimages: mapTuple(hints.lowPublicDataWritesPreimages, mapPublicDataTreePreimageToNoir), + low_public_data_writes_witnesses: mapTuple( + hints.lowPublicDataWritesMembershipWitnesses, + (witness: MembershipWitness) => mapMembershipWitnessToNoir(witness), + ), + public_data_tree_sibling_paths: mapTuple(hints.publicDataTreeSiblingPaths, path => mapTuple(path, mapFieldToNoir)), }; } @@ -2637,21 +2210,7 @@ export function mapPrivateBaseRollupInputsToNoir(inputs: PrivateBaseRollupInputs tube_data: mapPrivateTubeDataToNoir(inputs.tubeData), start: mapPartialStateReferenceToNoir(inputs.hints.start), - state_diff_hints: mapStateDiffHintsToNoir(inputs.hints.stateDiffHints), - - sorted_public_data_writes: mapTuple(inputs.hints.sortedPublicDataWrites, mapPublicDataTreeLeafToNoir), - - sorted_public_data_writes_indexes: mapTuple(inputs.hints.sortedPublicDataWritesIndexes, mapNumberToNoir), - - low_public_data_writes_preimages: mapTuple( - inputs.hints.lowPublicDataWritesPreimages, - mapPublicDataTreePreimageToNoir, - ), - - low_public_data_writes_witnesses: mapTuple( - inputs.hints.lowPublicDataWritesMembershipWitnesses, - (witness: MembershipWitness) => mapMembershipWitnessToNoir(witness), - ), + state_diff_hints: mapPrivateBaseStateDiffHintsToNoir(inputs.hints.stateDiffHints), archive_root_membership_witness: mapMembershipWitnessToNoir(inputs.hints.archiveRootMembershipWitness), constants: mapConstantRollupDataToNoir(inputs.hints.constants), @@ -2681,21 +2240,7 @@ export function mapPublicBaseRollupInputsToNoir(inputs: PublicBaseRollupInputs): avm_proof_data: mapAvmProofDataToNoir(inputs.avmProofData), start: mapPartialStateReferenceToNoir(inputs.hints.start), - state_diff_hints: mapStateDiffHintsToNoir(inputs.hints.stateDiffHints), - - sorted_public_data_writes: mapTuple(inputs.hints.sortedPublicDataWrites, mapPublicDataTreeLeafToNoir), - - sorted_public_data_writes_indexes: mapTuple(inputs.hints.sortedPublicDataWritesIndexes, mapNumberToNoir), - - low_public_data_writes_preimages: mapTuple( - inputs.hints.lowPublicDataWritesPreimages, - mapPublicDataTreePreimageToNoir, - ), - - low_public_data_writes_witnesses: mapTuple( - inputs.hints.lowPublicDataWritesMembershipWitnesses, - (witness: MembershipWitness) => mapMembershipWitnessToNoir(witness), - ), + state_diff_hints: mapPublicBaseStateDiffHintsToNoir(inputs.hints.stateDiffHints), archive_root_membership_witness: mapMembershipWitnessToNoir(inputs.hints.archiveRootMembershipWitness), constants: mapConstantRollupDataToNoir(inputs.hints.constants), diff --git a/yarn-project/p2p/src/mem_pools/attestation_pool/memory_attestation_pool.test.ts b/yarn-project/p2p/src/mem_pools/attestation_pool/memory_attestation_pool.test.ts index 4675af7bebf..b8bb71f30ce 100644 --- a/yarn-project/p2p/src/mem_pools/attestation_pool/memory_attestation_pool.test.ts +++ b/yarn-project/p2p/src/mem_pools/attestation_pool/memory_attestation_pool.test.ts @@ -1,19 +1,19 @@ -import { type BlockAttestation } from '@aztec/circuit-types'; +import { type BlockAttestation, TxHash } from '@aztec/circuit-types'; +import { Secp256k1Signer } from '@aztec/foundation/crypto'; import { Fr } from '@aztec/foundation/fields'; import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { type MockProxy, mock } from 'jest-mock-extended'; -import { type PrivateKeyAccount } from 'viem'; import { type PoolInstrumentation } from '../instrumentation.js'; import { InMemoryAttestationPool } from './memory_attestation_pool.js'; -import { generateAccount, mockAttestation } from './mocks.js'; +import { mockAttestation } from './mocks.js'; const NUMBER_OF_SIGNERS_PER_TEST = 4; describe('MemoryAttestationPool', () => { let ap: InMemoryAttestationPool; - let signers: PrivateKeyAccount[]; + let signers: Secp256k1Signer[]; const telemetry = new NoopTelemetryClient(); // Check that metrics are recorded correctly @@ -23,7 +23,7 @@ describe('MemoryAttestationPool', () => { // Use noop telemetry client while testing. ap = new InMemoryAttestationPool(telemetry); - signers = Array.from({ length: NUMBER_OF_SIGNERS_PER_TEST }, generateAccount); + signers = Array.from({ length: NUMBER_OF_SIGNERS_PER_TEST }, () => Secp256k1Signer.random()); metricsMock = mock>(); // Can i overwrite this like this?? @@ -33,7 +33,7 @@ describe('MemoryAttestationPool', () => { it('should add attestations to pool', async () => { const slotNumber = 420; const archive = Fr.random(); - const attestations = await Promise.all(signers.map(signer => mockAttestation(signer, slotNumber, archive))); + const attestations = signers.map(signer => mockAttestation(signer, slotNumber, archive)); await ap.addAttestations(attestations); @@ -54,9 +54,30 @@ describe('MemoryAttestationPool', () => { expect(retreivedAttestationsAfterDelete.length).toBe(0); }); + it('Should handle duplicate proposals in a slot', async () => { + const slotNumber = 420; + const archive = Fr.random(); + const txs = [0, 1, 2, 3, 4, 5].map(() => TxHash.random()); + + // Use the same signer for all attestations + const attestations: BlockAttestation[] = []; + const signer = signers[0]; + for (let i = 0; i < NUMBER_OF_SIGNERS_PER_TEST; i++) { + attestations.push(mockAttestation(signer, slotNumber, archive, txs)); + } + + await ap.addAttestations(attestations); + + const retreivedAttestations = await ap.getAttestationsForSlot(BigInt(slotNumber), archive.toString()); + expect(retreivedAttestations.length).toBe(1); + expect(retreivedAttestations[0]).toEqual(attestations[0]); + expect(retreivedAttestations[0].payload.txHashes).toEqual(txs); + expect(retreivedAttestations[0].getSender().toString()).toEqual(signer.address.toString()); + }); + it('Should store attestations by differing slot', async () => { const slotNumbers = [1, 2, 3, 4]; - const attestations = await Promise.all(signers.map((signer, i) => mockAttestation(signer, slotNumbers[i]))); + const attestations = signers.map((signer, i) => mockAttestation(signer, slotNumbers[i])); await ap.addAttestations(attestations); @@ -74,9 +95,7 @@ describe('MemoryAttestationPool', () => { it('Should store attestations by differing slot and archive', async () => { const slotNumbers = [1, 2, 3, 4]; const archives = [Fr.random(), Fr.random(), Fr.random(), Fr.random()]; - const attestations = await Promise.all( - signers.map((signer, i) => mockAttestation(signer, slotNumbers[i], archives[i])), - ); + const attestations = signers.map((signer, i) => mockAttestation(signer, slotNumbers[i], archives[i])); await ap.addAttestations(attestations); @@ -94,7 +113,7 @@ describe('MemoryAttestationPool', () => { it('Should delete attestations', async () => { const slotNumber = 420; const archive = Fr.random(); - const attestations = await Promise.all(signers.map(signer => mockAttestation(signer, slotNumber, archive))); + const attestations = signers.map(signer => mockAttestation(signer, slotNumber, archive)); const proposalId = attestations[0].archive.toString(); await ap.addAttestations(attestations); @@ -134,7 +153,7 @@ describe('MemoryAttestationPool', () => { it('Should blanket delete attestations per slot and proposal', async () => { const slotNumber = 420; const archive = Fr.random(); - const attestations = await Promise.all(signers.map(signer => mockAttestation(signer, slotNumber, archive))); + const attestations = signers.map(signer => mockAttestation(signer, slotNumber, archive)); const proposalId = attestations[0].archive.toString(); await ap.addAttestations(attestations); diff --git a/yarn-project/p2p/src/mem_pools/attestation_pool/mocks.ts b/yarn-project/p2p/src/mem_pools/attestation_pool/mocks.ts index 97bf92329b8..8a1b60a9ffd 100644 --- a/yarn-project/p2p/src/mem_pools/attestation_pool/mocks.ts +++ b/yarn-project/p2p/src/mem_pools/attestation_pool/mocks.ts @@ -1,9 +1,14 @@ -import { BlockAttestation, ConsensusPayload, SignatureDomainSeperator, TxHash } from '@aztec/circuit-types'; +import { + BlockAttestation, + ConsensusPayload, + SignatureDomainSeperator, + TxHash, + getHashedSignaturePayloadEthSignedMessage, +} from '@aztec/circuit-types'; import { makeHeader } from '@aztec/circuits.js/testing'; -import { Signature } from '@aztec/foundation/eth-signature'; +import { type Secp256k1Signer } from '@aztec/foundation/crypto'; import { Fr } from '@aztec/foundation/fields'; -import { type PrivateKeyAccount } from 'viem'; import { generatePrivateKey, privateKeyToAccount } from 'viem/accounts'; /** Generate Account @@ -22,22 +27,18 @@ export const generateAccount = () => { * @param slot The slot number the attestation is for * @returns A Block Attestation */ -export const mockAttestation = async ( - signer: PrivateKeyAccount, +export const mockAttestation = ( + signer: Secp256k1Signer, slot: number = 0, archive: Fr = Fr.random(), -): Promise => { + txs: TxHash[] = [0, 1, 2, 3, 4, 5].map(() => TxHash.random()), +): BlockAttestation => { // Use arbitrary numbers for all other than slot const header = makeHeader(1, 2, slot); - const txs = [0, 1, 2, 3, 4, 5].map(() => TxHash.random()); - const payload = new ConsensusPayload(header, archive, txs); - const message: `0x${string}` = `0x${payload - .getPayloadToSign(SignatureDomainSeperator.blockAttestation) - .toString('hex')}`; - const sigString = await signer.signMessage({ message }); + const hash = getHashedSignaturePayloadEthSignedMessage(payload, SignatureDomainSeperator.blockAttestation); + const signature = signer.sign(hash); - const signature = Signature.from0xString(sigString); return new BlockAttestation(payload, signature); }; diff --git a/yarn-project/p2p/src/service/data_store.test.ts b/yarn-project/p2p/src/service/data_store.test.ts index 1b3b3da60a8..f46ecf32525 100644 --- a/yarn-project/p2p/src/service/data_store.test.ts +++ b/yarn-project/p2p/src/service/data_store.test.ts @@ -23,19 +23,18 @@ describe('AztecDatastore with AztecLmdbStore', () => { let datastore: AztecDatastore; let aztecStore: AztecLmdbStore; - beforeAll(() => { + beforeEach(() => { aztecStore = AztecLmdbStore.open(); + datastore = new AztecDatastore(aztecStore); }); - beforeEach(async () => { - datastore = new AztecDatastore(aztecStore); - await aztecStore.clear(); + afterEach(async () => { + await aztecStore.delete(); }); it('should store and retrieve an item', async () => { const key = new Key('testKey'); const value = new Uint8Array([1, 2, 3]); - await datastore.put(key, value); const retrieved = datastore.get(key); diff --git a/yarn-project/prover-client/package.json b/yarn-project/prover-client/package.json index 07a91d15bd8..97480297dbf 100644 --- a/yarn-project/prover-client/package.json +++ b/yarn-project/prover-client/package.json @@ -26,7 +26,7 @@ "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", "bb": "node --no-warnings ./dest/bb/index.js", - "test": "LOG_LEVEL=${LOG_LEVEL:-silent} DEBUG_COLORS=1 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --testTimeout=1500000 --forceExit", + "test": "DEBUG_COLORS=1 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --testTimeout=1500000 --forceExit", "test:debug": "LOG_LEVEL=debug DEBUG_COLORS=1 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --testTimeout=1500000 --forceExit --testNamePattern prover/bb_prover/parity" }, "jest": { diff --git a/yarn-project/prover-client/package.local.json b/yarn-project/prover-client/package.local.json index 87474270493..bc11a5330d0 100644 --- a/yarn-project/prover-client/package.local.json +++ b/yarn-project/prover-client/package.local.json @@ -1,5 +1,5 @@ { "scripts": { - "test": "LOG_LEVEL=${LOG_LEVEL:-silent} DEBUG_COLORS=1 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --testTimeout=1500000 --forceExit" + "test": "DEBUG_COLORS=1 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --testTimeout=1500000 --forceExit" } } diff --git a/yarn-project/prover-client/src/mocks/fixtures.ts b/yarn-project/prover-client/src/mocks/fixtures.ts index a3b507b13b8..34b7cee5935 100644 --- a/yarn-project/prover-client/src/mocks/fixtures.ts +++ b/yarn-project/prover-client/src/mocks/fixtures.ts @@ -13,10 +13,7 @@ import { GlobalVariables, MAX_NOTE_HASHES_PER_TX, MAX_NULLIFIERS_PER_TX, - MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, NULLIFIER_TREE_HEIGHT, - PUBLIC_DATA_SUBTREE_HEIGHT, - PublicDataWrite, } from '@aztec/circuits.js'; import { padArrayEnd } from '@aztec/foundation/collection'; import { randomBytes } from '@aztec/foundation/crypto'; @@ -114,12 +111,8 @@ export const updateExpectedTreesFromTxs = async (db: MerkleTreeWriteOperations, for (const tx of txs) { await db.batchInsert( MerkleTreeId.PUBLIC_DATA_TREE, - padArrayEnd( - tx.txEffect.publicDataWrites, - PublicDataWrite.empty(), - MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, - ).map(write => write.toBuffer()), - PUBLIC_DATA_SUBTREE_HEIGHT, + tx.txEffect.publicDataWrites.map(write => write.toBuffer()), + 0, ); } }; diff --git a/yarn-project/prover-client/src/mocks/test_context.ts b/yarn-project/prover-client/src/mocks/test_context.ts index f04621c16f2..ebecd07801a 100644 --- a/yarn-project/prover-client/src/mocks/test_context.ts +++ b/yarn-project/prover-client/src/mocks/test_context.ts @@ -13,11 +13,8 @@ import { type Fr } from '@aztec/foundation/fields'; import { type DebugLogger } from '@aztec/foundation/log'; import { openTmpStore } from '@aztec/kv-store/utils'; import { - type PublicExecutionResult, - PublicExecutionResultBuilder, - type PublicExecutor, PublicProcessor, - RealPublicKernelCircuitSimulator, + PublicTxSimulator, type SimulationProvider, WASMSimulator, type WorldStateDB, @@ -26,10 +23,12 @@ import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { MerkleTrees } from '@aztec/world-state'; import { NativeWorldStateService } from '@aztec/world-state/native'; +import { jest } from '@jest/globals'; import * as fs from 'fs/promises'; import { type MockProxy, mock } from 'jest-mock-extended'; import { TestCircuitProver } from '../../../bb-prover/src/test/test_circuit_prover.js'; +import { AvmFinalizedCallResult } from '../../../simulator/src/avm/avm_contract_call_result.js'; import { type AvmPersistableStateManager } from '../../../simulator/src/avm/journal/journal.js'; import { ProvingOrchestrator } from '../orchestrator/index.js'; import { MemoryProvingQueue } from '../prover-agent/memory-proving-queue.js'; @@ -38,7 +37,7 @@ import { getEnvironmentConfig, getSimulationProvider, makeGlobals } from './fixt export class TestContext { constructor( - public publicExecutor: MockProxy, + public publicTxSimulator: PublicTxSimulator, public worldStateDB: MockProxy, public publicProcessor: PublicProcessor, public simulationProvider: SimulationProvider, @@ -67,9 +66,7 @@ export class TestContext { const directoriesToCleanup: string[] = []; const globalVariables = makeGlobals(blockNumber); - const publicExecutor = mock(); const worldStateDB = mock(); - const publicKernel = new RealPublicKernelCircuitSimulator(new WASMSimulator()); const telemetry = new NoopTelemetryClient(); // Separated dbs for public processor and prover - see public_processor for context @@ -85,14 +82,15 @@ export class TestContext { publicDb = await ws.getLatest(); proverDb = await ws.getLatest(); } + worldStateDB.getMerkleInterface.mockReturnValue(publicDb); - const processor = PublicProcessor.create( + const publicTxSimulator = new PublicTxSimulator(publicDb, worldStateDB, telemetry, globalVariables); + const processor = new PublicProcessor( publicDb, - publicExecutor, - publicKernel, globalVariables, Header.empty(), worldStateDB, + publicTxSimulator, telemetry, ); @@ -127,7 +125,7 @@ export class TestContext { agent.start(queue); return new this( - publicExecutor, + publicTxSimulator, worldStateDB, processor, simulationProvider, @@ -157,25 +155,24 @@ export class TestContext { ) { const defaultExecutorImplementation = ( _stateManager: AvmPersistableStateManager, - execution: PublicExecutionRequest, - _globalVariables: GlobalVariables, + executionRequest: PublicExecutionRequest, allocatedGas: Gas, - _transactionFee?: Fr, + _transactionFee: Fr, + _fnName: string, ) => { for (const tx of txs) { const allCalls = tx.publicTeardownFunctionCall.isEmpty() ? tx.enqueuedPublicFunctionCalls : [...tx.enqueuedPublicFunctionCalls, tx.publicTeardownFunctionCall]; for (const request of allCalls) { - if (execution.callContext.equals(request.callContext)) { - const result = PublicExecutionResultBuilder.empty().build({ - endGasLeft: allocatedGas, - }); - return Promise.resolve(result); + if (executionRequest.callContext.equals(request.callContext)) { + return Promise.resolve( + new AvmFinalizedCallResult(/*reverted=*/ false, /*output=*/ [], /*gasLeft=*/ allocatedGas), + ); } } } - throw new Error(`Unexpected execution request: ${execution}`); + throw new Error(`Unexpected execution request: ${executionRequest}`); }; return await this.processPublicFunctionsWithMockExecutorImplementation( txs, @@ -186,21 +183,36 @@ export class TestContext { ); } - public async processPublicFunctionsWithMockExecutorImplementation( + private async processPublicFunctionsWithMockExecutorImplementation( txs: Tx[], maxTransactions: number, txHandler?: ProcessedTxHandler, txValidator?: TxValidator, executorMock?: ( stateManager: AvmPersistableStateManager, - execution: PublicExecutionRequest, - globalVariables: GlobalVariables, + executionRequest: PublicExecutionRequest, allocatedGas: Gas, - transactionFee?: Fr, - ) => Promise, + transactionFee: Fr, + fnName: string, + ) => Promise, ) { + // Mock the internal private function. Borrowed from https://stackoverflow.com/a/71033167 + const simulateInternal: jest.SpiedFunction< + ( + stateManager: AvmPersistableStateManager, + executionResult: any, + allocatedGas: Gas, + transactionFee: any, + fnName: any, + ) => Promise + > = jest.spyOn( + this.publicTxSimulator as unknown as { + simulateEnqueuedCallInternal: PublicTxSimulator['simulateEnqueuedCallInternal']; + }, + 'simulateEnqueuedCallInternal', + ); if (executorMock) { - this.publicExecutor.simulate.mockImplementation(executorMock); + simulateInternal.mockImplementation(executorMock); } return await this.publicProcessor.process(txs, maxTransactions, txHandler, txValidator); } diff --git a/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts b/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts index 75044301ba5..121d13a00d2 100644 --- a/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts +++ b/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts @@ -10,7 +10,6 @@ import { ARCHIVE_HEIGHT, AppendOnlyTreeSnapshot, type BaseOrMergeRollupPublicInputs, - BaseRollupHints, BlockMergeRollupInputs, type BlockRootOrBlockMergePublicInputs, ConstantRollupData, @@ -32,25 +31,25 @@ import { NULLIFIER_TREE_HEIGHT, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, NullifierLeafPreimage, - PUBLIC_DATA_SUBTREE_HEIGHT, - PUBLIC_DATA_SUBTREE_SIBLING_PATH_LENGTH, PUBLIC_DATA_TREE_HEIGHT, type ParityPublicInputs, PartialStateReference, PreviousRollupBlockData, PreviousRollupData, + PrivateBaseRollupHints, + PrivateBaseStateDiffHints, + PublicBaseRollupHints, + PublicBaseStateDiffHints, PublicDataHint, PublicDataTreeLeaf, - type PublicDataTreeLeafPreimage, - PublicDataWrite, + PublicDataTreeLeafPreimage, type RecursiveProof, RootRollupInputs, - StateDiffHints, StateReference, VK_TREE_HEIGHT, type VerificationKeyAsFields, } from '@aztec/circuits.js'; -import { assertPermutation, makeTuple } from '@aztec/foundation/array'; +import { makeTuple } from '@aztec/foundation/array'; import { padArrayEnd } from '@aztec/foundation/collection'; import { sha256Trunc } from '@aztec/foundation/crypto'; import { type DebugLogger } from '@aztec/foundation/log'; @@ -58,7 +57,7 @@ import { type Tuple, assertLength, toFriendlyJSON } from '@aztec/foundation/seri import { computeUnbalancedMerkleRoot } from '@aztec/foundation/trees'; import { getVKIndex, getVKSiblingPath, getVKTreeRoot } from '@aztec/noir-protocol-circuits-types'; import { protocolContractTreeRoot } from '@aztec/protocol-contracts'; -import { HintsBuilder, computeFeePayerBalanceLeafSlot } from '@aztec/simulator'; +import { computeFeePayerBalanceLeafSlot } from '@aztec/simulator'; import { type MerkleTreeReadOperations } from '@aztec/world-state'; import { inspect } from 'util'; @@ -98,11 +97,10 @@ export async function buildBaseRollupHints( // Create data hint for reading fee payer initial balance in Fee Juice // If no fee payer is set, read hint should be empty - const hintsBuilder = new HintsBuilder(db); const leafSlot = computeFeePayerBalanceLeafSlot(tx.data.feePayer); const feePayerFeeJuiceBalanceReadHint = tx.data.feePayer.isZero() ? PublicDataHint.empty() - : await hintsBuilder.getPublicDataHint(leafSlot.toBigInt()); + : await getPublicDataHint(db, leafSlot.toBigInt()); // Update the note hash trees with the new items being inserted to get the new roots // that will be used by the next iteration of the base rollup circuit, skipping the empty ones @@ -124,6 +122,7 @@ export async function buildBaseRollupHints( padArrayEnd(tx.txEffect.nullifiers, Fr.ZERO, MAX_NULLIFIERS_PER_TX).map(n => n.toBuffer()), NULLIFIER_SUBTREE_HEIGHT, ); + if (nullifierWitnessLeaves === undefined) { throw new Error(`Could not craft nullifier batch insertion proofs`); } @@ -140,45 +139,129 @@ export async function buildBaseRollupHints( i < nullifierSubtreeSiblingPathArray.length ? nullifierSubtreeSiblingPathArray[i] : Fr.ZERO, ); - const publicDataSiblingPath = txPublicDataUpdateRequestInfo.newPublicDataSubtreeSiblingPath; + if (tx.avmProvingRequest) { + // Build public base rollup hints + const stateDiffHints = PublicBaseStateDiffHints.from({ + nullifierPredecessorPreimages: makeTuple(MAX_NULLIFIERS_PER_TX, i => + i < nullifierWitnessLeaves.length + ? (nullifierWitnessLeaves[i].leafPreimage as NullifierLeafPreimage) + : NullifierLeafPreimage.empty(), + ), + nullifierPredecessorMembershipWitnesses: makeTuple(MAX_NULLIFIERS_PER_TX, i => + i < nullifierPredecessorMembershipWitnessesWithoutPadding.length + ? nullifierPredecessorMembershipWitnessesWithoutPadding[i] + : makeEmptyMembershipWitness(NULLIFIER_TREE_HEIGHT), + ), + sortedNullifiers: makeTuple(MAX_NULLIFIERS_PER_TX, i => Fr.fromBuffer(sortednullifiers[i])), + sortedNullifierIndexes: makeTuple(MAX_NULLIFIERS_PER_TX, i => sortedNewLeavesIndexes[i]), + noteHashSubtreeSiblingPath, + nullifierSubtreeSiblingPath, + lowPublicDataWritesPreimages: padArrayEnd( + txPublicDataUpdateRequestInfo.lowPublicDataWritesPreimages, + PublicDataTreeLeafPreimage.empty(), + MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, + ), + lowPublicDataWritesMembershipWitnesses: padArrayEnd( + txPublicDataUpdateRequestInfo.lowPublicDataWritesMembershipWitnesses, + MembershipWitness.empty(PUBLIC_DATA_TREE_HEIGHT), + MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, + ), + publicDataTreeSiblingPaths: padArrayEnd( + txPublicDataUpdateRequestInfo.publicDataWritesSiblingPaths, + makeTuple(PUBLIC_DATA_TREE_HEIGHT, () => Fr.ZERO), + MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, + ), + }); + + const blockHash = tx.constants.historicalHeader.hash(); + const archiveRootMembershipWitness = await getMembershipWitnessFor( + blockHash, + MerkleTreeId.ARCHIVE, + ARCHIVE_HEIGHT, + db, + ); - const stateDiffHints = StateDiffHints.from({ - nullifierPredecessorPreimages: makeTuple(MAX_NULLIFIERS_PER_TX, i => - i < nullifierWitnessLeaves.length - ? (nullifierWitnessLeaves[i].leafPreimage as NullifierLeafPreimage) - : NullifierLeafPreimage.empty(), - ), - nullifierPredecessorMembershipWitnesses: makeTuple(MAX_NULLIFIERS_PER_TX, i => - i < nullifierPredecessorMembershipWitnessesWithoutPadding.length - ? nullifierPredecessorMembershipWitnessesWithoutPadding[i] - : makeEmptyMembershipWitness(NULLIFIER_TREE_HEIGHT), - ), - sortedNullifiers: makeTuple(MAX_NULLIFIERS_PER_TX, i => Fr.fromBuffer(sortednullifiers[i])), - sortedNullifierIndexes: makeTuple(MAX_NULLIFIERS_PER_TX, i => sortedNewLeavesIndexes[i]), - noteHashSubtreeSiblingPath, - nullifierSubtreeSiblingPath, - publicDataSiblingPath, - }); + return PublicBaseRollupHints.from({ + start, + stateDiffHints, + feePayerFeeJuiceBalanceReadHint: feePayerFeeJuiceBalanceReadHint, + archiveRootMembershipWitness, + constants, + }); + } else { + if ( + txPublicDataUpdateRequestInfo.lowPublicDataWritesMembershipWitnesses.length > 1 || + txPublicDataUpdateRequestInfo.lowPublicDataWritesPreimages.length > 1 || + txPublicDataUpdateRequestInfo.publicDataWritesSiblingPaths.length > 1 + ) { + throw new Error(`More than one public data write in a private only tx`); + } + + const feeWriteLowLeafPreimage = + txPublicDataUpdateRequestInfo.lowPublicDataWritesPreimages[0] || PublicDataTreeLeafPreimage.empty(); + const feeWriteLowLeafMembershipWitness = + txPublicDataUpdateRequestInfo.lowPublicDataWritesMembershipWitnesses[0] || + MembershipWitness.empty(PUBLIC_DATA_TREE_HEIGHT); + const feeWriteSiblingPath = + txPublicDataUpdateRequestInfo.publicDataWritesSiblingPaths[0] || + makeTuple(PUBLIC_DATA_TREE_HEIGHT, () => Fr.ZERO); + + const stateDiffHints = PrivateBaseStateDiffHints.from({ + nullifierPredecessorPreimages: makeTuple(MAX_NULLIFIERS_PER_TX, i => + i < nullifierWitnessLeaves.length + ? (nullifierWitnessLeaves[i].leafPreimage as NullifierLeafPreimage) + : NullifierLeafPreimage.empty(), + ), + nullifierPredecessorMembershipWitnesses: makeTuple(MAX_NULLIFIERS_PER_TX, i => + i < nullifierPredecessorMembershipWitnessesWithoutPadding.length + ? nullifierPredecessorMembershipWitnessesWithoutPadding[i] + : makeEmptyMembershipWitness(NULLIFIER_TREE_HEIGHT), + ), + sortedNullifiers: makeTuple(MAX_NULLIFIERS_PER_TX, i => Fr.fromBuffer(sortednullifiers[i])), + sortedNullifierIndexes: makeTuple(MAX_NULLIFIERS_PER_TX, i => sortedNewLeavesIndexes[i]), + noteHashSubtreeSiblingPath, + nullifierSubtreeSiblingPath, + feeWriteLowLeafPreimage, + feeWriteLowLeafMembershipWitness, + feeWriteSiblingPath, + }); + + const blockHash = tx.constants.historicalHeader.hash(); + const archiveRootMembershipWitness = await getMembershipWitnessFor( + blockHash, + MerkleTreeId.ARCHIVE, + ARCHIVE_HEIGHT, + db, + ); - const blockHash = tx.constants.historicalHeader.hash(); - const archiveRootMembershipWitness = await getMembershipWitnessFor( - blockHash, - MerkleTreeId.ARCHIVE, - ARCHIVE_HEIGHT, - db, - ); + return PrivateBaseRollupHints.from({ + start, + stateDiffHints, + feePayerFeeJuiceBalanceReadHint: feePayerFeeJuiceBalanceReadHint, + archiveRootMembershipWitness, + constants, + }); + } +} - return BaseRollupHints.from({ - start, - stateDiffHints, - feePayerFeeJuiceBalanceReadHint: feePayerFeeJuiceBalanceReadHint, - sortedPublicDataWrites: txPublicDataUpdateRequestInfo.sortedPublicDataWrites, - sortedPublicDataWritesIndexes: txPublicDataUpdateRequestInfo.sortedPublicDataWritesIndexes, - lowPublicDataWritesPreimages: txPublicDataUpdateRequestInfo.lowPublicDataWritesPreimages, - lowPublicDataWritesMembershipWitnesses: txPublicDataUpdateRequestInfo.lowPublicDataWritesMembershipWitnesses, - archiveRootMembershipWitness, - constants, - }); +async function getPublicDataHint(db: MerkleTreeWriteOperations, leafSlot: bigint) { + const { index } = (await db.getPreviousValueIndex(MerkleTreeId.PUBLIC_DATA_TREE, leafSlot)) ?? {}; + if (index === undefined) { + throw new Error(`Cannot find the previous value index for public data ${leafSlot}.`); + } + + const siblingPath = await db.getSiblingPath(MerkleTreeId.PUBLIC_DATA_TREE, index); + const membershipWitness = new MembershipWitness(PUBLIC_DATA_TREE_HEIGHT, index, siblingPath.toTuple()); + + const leafPreimage = (await db.getLeafPreimage(MerkleTreeId.PUBLIC_DATA_TREE, index)) as PublicDataTreeLeafPreimage; + if (!leafPreimage) { + throw new Error(`Cannot find the leaf preimage for public data tree at index ${index}.`); + } + + const exists = leafPreimage.slot.toBigInt() === leafSlot; + const value = exists ? leafPreimage.value : Fr.ZERO; + + return new PublicDataHint(new Fr(leafSlot), value, membershipWitness, leafPreimage); } export function createMergeRollupInputs( @@ -397,67 +480,50 @@ export function makeEmptyMembershipWitness(height: N) { } async function processPublicDataUpdateRequests(tx: ProcessedTx, db: MerkleTreeWriteOperations) { - const allPublicDataUpdateRequests = padArrayEnd( - tx.txEffect.publicDataWrites, - PublicDataWrite.empty(), - MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, - ); - - const allPublicDataWrites = allPublicDataUpdateRequests.map( + const allPublicDataWrites = tx.txEffect.publicDataWrites.map( ({ leafSlot, value }) => new PublicDataTreeLeaf(leafSlot, value), ); - const { lowLeavesWitnessData, newSubtreeSiblingPath, sortedNewLeaves, sortedNewLeavesIndexes } = await db.batchInsert( - MerkleTreeId.PUBLIC_DATA_TREE, - allPublicDataWrites.map(x => x.toBuffer()), - // TODO(#3675) remove oldValue from update requests - PUBLIC_DATA_SUBTREE_HEIGHT, - ); - - if (lowLeavesWitnessData === undefined) { - throw new Error(`Could not craft public data batch insertion proofs`); - } - - const sortedPublicDataWrites = makeTuple(MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, i => { - return PublicDataTreeLeaf.fromBuffer(sortedNewLeaves[i]); - }); - - const sortedPublicDataWritesIndexes = makeTuple(MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, i => { - return sortedNewLeavesIndexes[i]; - }); - const subtreeSiblingPathAsFields = newSubtreeSiblingPath.toFields(); - const newPublicDataSubtreeSiblingPath = makeTuple(PUBLIC_DATA_SUBTREE_SIBLING_PATH_LENGTH, i => { - return subtreeSiblingPathAsFields[i]; - }); + const lowPublicDataWritesPreimages = []; + const lowPublicDataWritesMembershipWitnesses = []; + const publicDataWritesSiblingPaths = []; + + for (const write of allPublicDataWrites) { + if (write.isEmpty()) { + throw new Error(`Empty public data write in tx: ${toFriendlyJSON(tx)}`); + } + + // TODO(Alvaro) write a specialized function for this? Internally add_or_update_value uses batch insertion anyway + const { lowLeavesWitnessData, newSubtreeSiblingPath } = await db.batchInsert( + MerkleTreeId.PUBLIC_DATA_TREE, + [write.toBuffer()], + // TODO(#3675) remove oldValue from update requests + 0, + ); - const lowPublicDataWritesMembershipWitnesses: Tuple< - MembershipWitness, - typeof MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX - > = makeTuple(MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, i => { - const witness = lowLeavesWitnessData[i]; - return MembershipWitness.fromBufferArray( - witness.index, - assertLength(witness.siblingPath.toBufferArray(), PUBLIC_DATA_TREE_HEIGHT), + if (lowLeavesWitnessData === undefined) { + throw new Error(`Could not craft public data batch insertion proofs`); + } + + const [lowLeafWitness] = lowLeavesWitnessData; + lowPublicDataWritesPreimages.push(lowLeafWitness.leafPreimage as PublicDataTreeLeafPreimage); + lowPublicDataWritesMembershipWitnesses.push( + MembershipWitness.fromBufferArray( + lowLeafWitness.index, + assertLength(lowLeafWitness.siblingPath.toBufferArray(), PUBLIC_DATA_TREE_HEIGHT), + ), ); - }); - const lowPublicDataWritesPreimages: Tuple< - PublicDataTreeLeafPreimage, - typeof MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX - > = makeTuple(MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, i => { - return lowLeavesWitnessData[i].leafPreimage as PublicDataTreeLeafPreimage; - }); + const insertionSiblingPath = newSubtreeSiblingPath.toFields(); + assertLength(insertionSiblingPath, PUBLIC_DATA_TREE_HEIGHT); - // validate that the sortedPublicDataWrites and sortedPublicDataWritesIndexes are in the correct order - // otherwise it will just fail in the circuit - assertPermutation(allPublicDataWrites, sortedPublicDataWrites, sortedPublicDataWritesIndexes, (a, b) => a.equals(b)); + publicDataWritesSiblingPaths.push(insertionSiblingPath as Tuple); + } return { lowPublicDataWritesPreimages, lowPublicDataWritesMembershipWitnesses, - newPublicDataSubtreeSiblingPath, - sortedPublicDataWrites, - sortedPublicDataWritesIndexes, + publicDataWritesSiblingPaths, }; } diff --git a/yarn-project/prover-client/src/orchestrator/tx-proving-state.ts b/yarn-project/prover-client/src/orchestrator/tx-proving-state.ts index 00dde60f9b6..d3bcec2b7d5 100644 --- a/yarn-project/prover-client/src/orchestrator/tx-proving-state.ts +++ b/yarn-project/prover-client/src/orchestrator/tx-proving-state.ts @@ -13,8 +13,10 @@ import { AvmProofData, type BaseRollupHints, Fr, + PrivateBaseRollupHints, PrivateBaseRollupInputs, PrivateTubeData, + PublicBaseRollupHints, PublicBaseRollupInputs, PublicTubeData, type TUBE_PROOF_LENGTH, @@ -66,6 +68,9 @@ export class TxProvingState { const vkData = this.getTubeVkData(); const tubeData = new PrivateTubeData(this.processedTx.data.toKernelCircuitPublicInputs(), this.tube.proof, vkData); + if (!(this.baseRollupHints instanceof PrivateBaseRollupHints)) { + throw new Error('Mismatched base rollup hints, expected private base rollup hints'); + } return new PrivateBaseRollupInputs(tubeData, this.baseRollupHints); } @@ -92,6 +97,10 @@ export class TxProvingState { this.getAvmVkData(), ); + if (!(this.baseRollupHints instanceof PublicBaseRollupHints)) { + throw new Error('Mismatched base rollup hints, expected public base rollup hints'); + } + return new PublicBaseRollupInputs(tubeData, avmProofData, this.baseRollupHints); } diff --git a/yarn-project/prover-client/src/proving_broker/proving_agent.test.ts b/yarn-project/prover-client/src/proving_broker/proving_agent.test.ts new file mode 100644 index 00000000000..9a2c7db1da9 --- /dev/null +++ b/yarn-project/prover-client/src/proving_broker/proving_agent.test.ts @@ -0,0 +1,226 @@ +import { + ProvingError, + ProvingRequestType, + type PublicInputsAndRecursiveProof, + type V2ProvingJob, + type V2ProvingJobId, + makePublicInputsAndRecursiveProof, +} from '@aztec/circuit-types'; +import { + type ParityPublicInputs, + RECURSIVE_PROOF_LENGTH, + VerificationKeyData, + makeRecursiveProof, +} from '@aztec/circuits.js'; +import { makeBaseParityInputs, makeParityPublicInputs } from '@aztec/circuits.js/testing'; +import { randomBytes } from '@aztec/foundation/crypto'; +import { AbortError } from '@aztec/foundation/error'; +import { promiseWithResolvers } from '@aztec/foundation/promise'; + +import { jest } from '@jest/globals'; + +import { MockProver } from '../test/mock_prover.js'; +import { ProvingAgent } from './proving_agent.js'; +import { type ProvingJobConsumer } from './proving_broker_interface.js'; + +describe('ProvingAgent', () => { + let prover: MockProver; + let jobSource: jest.Mocked; + let agent: ProvingAgent; + const agentPollIntervalMs = 1000; + + beforeEach(() => { + jest.useFakeTimers(); + + prover = new MockProver(); + jobSource = { + getProvingJob: jest.fn(), + reportProvingJobProgress: jest.fn(), + reportProvingJobError: jest.fn(), + reportProvingJobSuccess: jest.fn(), + }; + agent = new ProvingAgent(jobSource, prover, [ProvingRequestType.BASE_PARITY]); + }); + + afterEach(async () => { + await agent.stop(); + }); + + it('polls for jobs passing the permitted list of proofs', () => { + agent.start(); + expect(jobSource.getProvingJob).toHaveBeenCalledWith({ allowList: [ProvingRequestType.BASE_PARITY] }); + }); + + it('only takes a single job from the source at a time', async () => { + expect(jobSource.getProvingJob).not.toHaveBeenCalled(); + + // simulate the proof taking a long time + const { promise, resolve } = + promiseWithResolvers>(); + jest.spyOn(prover, 'getBaseParityProof').mockReturnValueOnce(promise); + + const jobResponse = makeBaseParityJob(); + jobSource.getProvingJob.mockResolvedValueOnce(jobResponse); + agent.start(); + + await jest.advanceTimersByTimeAsync(agentPollIntervalMs); + expect(jobSource.getProvingJob).toHaveBeenCalledTimes(1); + + await jest.advanceTimersByTimeAsync(agentPollIntervalMs); + expect(jobSource.getProvingJob).toHaveBeenCalledTimes(1); + + await jest.advanceTimersByTimeAsync(agentPollIntervalMs); + expect(jobSource.getProvingJob).toHaveBeenCalledTimes(1); + + // let's resolve the proof + const result = makePublicInputsAndRecursiveProof( + makeParityPublicInputs(), + makeRecursiveProof(RECURSIVE_PROOF_LENGTH), + VerificationKeyData.makeFakeHonk(), + ); + resolve(result); + + await jest.advanceTimersByTimeAsync(agentPollIntervalMs); + expect(jobSource.getProvingJob).toHaveBeenCalledTimes(2); + }); + + it('reports success to the job source', async () => { + const jobResponse = makeBaseParityJob(); + const result = makeBaseParityResult(); + jest.spyOn(prover, 'getBaseParityProof').mockResolvedValueOnce(result.value); + + jobSource.getProvingJob.mockResolvedValueOnce(jobResponse); + agent.start(); + + await jest.advanceTimersByTimeAsync(agentPollIntervalMs); + expect(jobSource.reportProvingJobSuccess).toHaveBeenCalledWith(jobResponse.job.id, result); + }); + + it('reports errors to the job source', async () => { + const jobResponse = makeBaseParityJob(); + jest.spyOn(prover, 'getBaseParityProof').mockRejectedValueOnce(new Error('test error')); + + jobSource.getProvingJob.mockResolvedValueOnce(jobResponse); + agent.start(); + + await jest.advanceTimersByTimeAsync(agentPollIntervalMs); + expect(jobSource.reportProvingJobError).toHaveBeenCalledWith(jobResponse.job.id, new Error('test error'), false); + }); + + it('sets the retry flag on when reporting an error', async () => { + const jobResponse = makeBaseParityJob(); + const err = new ProvingError('test error', undefined, true); + jest.spyOn(prover, 'getBaseParityProof').mockRejectedValueOnce(err); + + jobSource.getProvingJob.mockResolvedValueOnce(jobResponse); + agent.start(); + + await jest.advanceTimersByTimeAsync(agentPollIntervalMs); + expect(jobSource.reportProvingJobError).toHaveBeenCalledWith(jobResponse.job.id, err, true); + }); + + it('reports jobs in progress to the job source', async () => { + const jobResponse = makeBaseParityJob(); + const { promise, resolve } = + promiseWithResolvers>(); + jest.spyOn(prover, 'getBaseParityProof').mockReturnValueOnce(promise); + + jobSource.getProvingJob.mockResolvedValueOnce(jobResponse); + agent.start(); + + await jest.advanceTimersByTimeAsync(agentPollIntervalMs); + expect(jobSource.reportProvingJobProgress).toHaveBeenCalledWith(jobResponse.job.id, jobResponse.time, { + allowList: [ProvingRequestType.BASE_PARITY], + }); + + await jest.advanceTimersByTimeAsync(agentPollIntervalMs); + expect(jobSource.reportProvingJobProgress).toHaveBeenCalledWith(jobResponse.job.id, jobResponse.time, { + allowList: [ProvingRequestType.BASE_PARITY], + }); + + resolve(makeBaseParityResult().value); + }); + + it('abandons jobs if told so by the source', async () => { + const firstJobResponse = makeBaseParityJob(); + let firstProofAborted = false; + const firstProof = + promiseWithResolvers>(); + + // simulate a long running proving job that can be aborted + jest.spyOn(prover, 'getBaseParityProof').mockImplementationOnce((_, signal) => { + signal?.addEventListener('abort', () => { + firstProof.reject(new AbortError('test abort')); + firstProofAborted = true; + }); + return firstProof.promise; + }); + + jobSource.getProvingJob.mockResolvedValueOnce(firstJobResponse); + agent.start(); + + // now the agent should be happily proving and reporting progress + await jest.advanceTimersByTimeAsync(agentPollIntervalMs); + expect(jobSource.reportProvingJobProgress).toHaveBeenCalledTimes(1); + expect(jobSource.reportProvingJobProgress).toHaveBeenCalledWith(firstJobResponse.job.id, firstJobResponse.time, { + allowList: [ProvingRequestType.BASE_PARITY], + }); + + await jest.advanceTimersByTimeAsync(agentPollIntervalMs); + expect(jobSource.reportProvingJobProgress).toHaveBeenCalledTimes(2); + + // now let's simulate the job source cancelling the job and giving the agent something else to do + // this should cause the agent to abort the current job and start the new one + const secondJobResponse = makeBaseParityJob(); + jobSource.reportProvingJobProgress.mockResolvedValueOnce(secondJobResponse); + + const secondProof = + promiseWithResolvers>(); + jest.spyOn(prover, 'getBaseParityProof').mockReturnValueOnce(secondProof.promise); + + await jest.advanceTimersByTimeAsync(agentPollIntervalMs); + expect(jobSource.reportProvingJobProgress).toHaveBeenCalledTimes(3); + expect(jobSource.reportProvingJobProgress).toHaveBeenLastCalledWith( + firstJobResponse.job.id, + firstJobResponse.time, + { + allowList: [ProvingRequestType.BASE_PARITY], + }, + ); + expect(firstProofAborted).toBe(true); + + // agent should have switched now + await jest.advanceTimersByTimeAsync(agentPollIntervalMs); + expect(jobSource.reportProvingJobProgress).toHaveBeenCalledTimes(4); + expect(jobSource.reportProvingJobProgress).toHaveBeenLastCalledWith( + secondJobResponse.job.id, + secondJobResponse.time, + { + allowList: [ProvingRequestType.BASE_PARITY], + }, + ); + + secondProof.resolve(makeBaseParityResult().value); + }); + + function makeBaseParityJob(): { job: V2ProvingJob; time: number } { + const time = jest.now(); + const job: V2ProvingJob = { + id: randomBytes(8).toString('hex') as V2ProvingJobId, + blockNumber: 1, + type: ProvingRequestType.BASE_PARITY, + inputs: makeBaseParityInputs(), + }; + + return { job, time }; + } + + function makeBaseParityResult() { + const value = makePublicInputsAndRecursiveProof( + makeParityPublicInputs(), + makeRecursiveProof(RECURSIVE_PROOF_LENGTH), + VerificationKeyData.makeFakeHonk(), + ); + return { type: ProvingRequestType.BASE_PARITY, value }; + } +}); diff --git a/yarn-project/prover-client/src/proving_broker/proving_agent.ts b/yarn-project/prover-client/src/proving_broker/proving_agent.ts new file mode 100644 index 00000000000..5ee86900e0d --- /dev/null +++ b/yarn-project/prover-client/src/proving_broker/proving_agent.ts @@ -0,0 +1,90 @@ +import { + ProvingError, + type ProvingRequestType, + type ServerCircuitProver, + type V2ProvingJob, +} from '@aztec/circuit-types'; +import { createDebugLogger } from '@aztec/foundation/log'; +import { RunningPromise } from '@aztec/foundation/running-promise'; + +import { type ProvingJobConsumer } from './proving_broker_interface.js'; +import { ProvingJobController, ProvingJobStatus } from './proving_job_controller.js'; + +/** + * A helper class that encapsulates a circuit prover and connects it to a job source. + */ +export class ProvingAgent { + private currentJobController?: ProvingJobController; + private runningPromise: RunningPromise; + + constructor( + /** The source of proving jobs */ + private jobSource: ProvingJobConsumer, + /** The prover implementation to defer jobs to */ + private circuitProver: ServerCircuitProver, + /** Optional list of allowed proof types to build */ + private proofAllowList?: Array, + /** How long to wait between jobs */ + private pollIntervalMs = 1000, + private log = createDebugLogger('aztec:proving-broker:proving-agent'), + ) { + this.runningPromise = new RunningPromise(this.safeWork, this.pollIntervalMs); + } + + public setCircuitProver(circuitProver: ServerCircuitProver): void { + this.circuitProver = circuitProver; + } + + public isRunning(): boolean { + return this.runningPromise?.isRunning() ?? false; + } + + public start(): void { + this.runningPromise.start(); + } + + public async stop(): Promise { + this.currentJobController?.abort(); + await this.runningPromise.stop(); + } + + private safeWork = async () => { + try { + // every tick we need to + // (1) either do a heartbeat, telling the broker that we're working + // (2) get a new job + // If during (1) the broker returns a new job that means we can cancel the current job and start the new one + let maybeJob: { job: V2ProvingJob; time: number } | undefined; + if (this.currentJobController?.getStatus() === ProvingJobStatus.PROVING) { + maybeJob = await this.jobSource.reportProvingJobProgress( + this.currentJobController.getJobId(), + this.currentJobController.getStartedAt(), + { allowList: this.proofAllowList }, + ); + } else { + maybeJob = await this.jobSource.getProvingJob({ allowList: this.proofAllowList }); + } + + if (!maybeJob) { + return; + } + + if (this.currentJobController?.getStatus() === ProvingJobStatus.PROVING) { + this.currentJobController?.abort(); + } + + const { job, time } = maybeJob; + this.currentJobController = new ProvingJobController(job, time, this.circuitProver, (err, result) => { + if (err) { + const retry = err.name === ProvingError.NAME ? (err as ProvingError).retry : false; + return this.jobSource.reportProvingJobError(job.id, err, retry); + } else if (result) { + return this.jobSource.reportProvingJobSuccess(job.id, result); + } + }); + this.currentJobController.start(); + } catch (err) { + this.log.error(`Error in ProvingAgent: ${String(err)}`); + } + }; +} diff --git a/yarn-project/prover-client/src/proving_broker/proving_broker.test.ts b/yarn-project/prover-client/src/proving_broker/proving_broker.test.ts index c992f18c87f..fef79bfb99f 100644 --- a/yarn-project/prover-client/src/proving_broker/proving_broker.test.ts +++ b/yarn-project/prover-client/src/proving_broker/proving_broker.test.ts @@ -1,5 +1,6 @@ import { ProvingRequestType, + type V2ProofOutput, type V2ProvingJob, type V2ProvingJobId, makePublicInputsAndRecursiveProof, @@ -13,28 +14,41 @@ import { makeRootParityInputs, } from '@aztec/circuits.js/testing'; import { randomBytes } from '@aztec/foundation/crypto'; +import { openTmpStore } from '@aztec/kv-store/utils'; import { jest } from '@jest/globals'; import { ProvingBroker } from './proving_broker.js'; -import { InMemoryDatabase } from './proving_broker_database.js'; +import { type ProvingJobDatabase } from './proving_job_database.js'; +import { InMemoryDatabase } from './proving_job_database/memory.js'; +import { PersistedProvingJobDatabase } from './proving_job_database/persisted.js'; beforeAll(() => { jest.useFakeTimers(); }); -describe('ProvingBroker', () => { - let database: InMemoryDatabase; +describe.each([ + () => ({ database: new InMemoryDatabase(), cleanup: undefined }), + () => { + const store = openTmpStore(true); + const database = new PersistedProvingJobDatabase(store); + const cleanup = () => store.close(); + return { database, cleanup }; + }, +])('ProvingBroker', createDb => { let broker: ProvingBroker; let jobTimeoutSec: number; let maxRetries: number; + let database: ProvingJobDatabase; + let cleanup: undefined | (() => Promise | void); const now = () => Math.floor(Date.now() / 1000); beforeEach(() => { jobTimeoutSec = 10; maxRetries = 2; - database = new InMemoryDatabase(); + ({ database, cleanup } = createDb()); + broker = new ProvingBroker(database, { jobTimeoutSec: jobTimeoutSec, timeoutIntervalSec: jobTimeoutSec / 4, @@ -42,6 +56,12 @@ describe('ProvingBroker', () => { }); }); + afterEach(async () => { + if (cleanup) { + await cleanup(); + } + }); + describe('Producer API', () => { beforeEach(async () => { await broker.start(); @@ -909,10 +929,6 @@ describe('ProvingBroker', () => { inputs: makePrivateBaseRollupInputs(), }); - expect(database.getProvingJob(id1)).not.toBeUndefined(); - expect(database.getProvingJobResult(id1)).not.toBeUndefined(); - expect(database.getProvingJob(id2)).not.toBeUndefined(); - await broker.start(); await expect(broker.getProvingJobStatus(id1)).resolves.toEqual({ @@ -922,27 +938,31 @@ describe('ProvingBroker', () => { await expect(broker.getProvingJobStatus(id2)).resolves.toEqual({ status: 'in-queue' }); + jest.spyOn(database, 'deleteProvingJobAndResult'); + await broker.removeAndCancelProvingJob(id1); await broker.removeAndCancelProvingJob(id2); + expect(database.deleteProvingJobAndResult).toHaveBeenCalledWith(id1); + expect(database.deleteProvingJobAndResult).toHaveBeenCalledWith(id2); + await expect(broker.getProvingJobStatus(id1)).resolves.toEqual({ status: 'not-found' }); await expect(broker.getProvingJobStatus(id2)).resolves.toEqual({ status: 'not-found' }); - - expect(database.getProvingJob(id1)).toBeUndefined(); - expect(database.getProvingJobResult(id1)).toBeUndefined(); - expect(database.getProvingJob(id2)).toBeUndefined(); }); it('saves job when enqueued', async () => { await broker.start(); - const id = makeProvingJobId(); - await broker.enqueueProvingJob({ - id, + const job: V2ProvingJob = { + id: makeProvingJobId(), type: ProvingRequestType.BASE_PARITY, blockNumber: 1, inputs: makeBaseParityInputs(), - }); - expect(database.getProvingJob(id)).not.toBeUndefined(); + }; + + jest.spyOn(database, 'addProvingJob'); + await broker.enqueueProvingJob(job); + + expect(database.addProvingJob).toHaveBeenCalledWith(job); }); it('does not retain job if database fails to save', async () => { @@ -963,23 +983,29 @@ describe('ProvingBroker', () => { it('saves job result', async () => { await broker.start(); - const id = makeProvingJobId(); - await broker.enqueueProvingJob({ - id, + + const job: V2ProvingJob = { + id: makeProvingJobId(), type: ProvingRequestType.BASE_PARITY, blockNumber: 1, inputs: makeBaseParityInputs(), - }); - await broker.reportProvingJobSuccess(id, { + }; + jest.spyOn(database, 'setProvingJobResult'); + + await broker.enqueueProvingJob(job); + + const result: V2ProofOutput = { type: ProvingRequestType.BASE_PARITY, value: makePublicInputsAndRecursiveProof( makeParityPublicInputs(RECURSIVE_PROOF_LENGTH), makeRecursiveProof(RECURSIVE_PROOF_LENGTH), VerificationKeyData.makeFake(), ), - }); - await assertJobStatus(id, 'resolved'); - expect(database.getProvingJobResult(id)).toEqual({ value: expect.any(Object) }); + }; + await broker.reportProvingJobSuccess(job.id, result); + + await assertJobStatus(job.id, 'resolved'); + expect(database.setProvingJobResult).toHaveBeenCalledWith(job.id, result); }); it('does not retain job result if database fails to save', async () => { @@ -1003,22 +1029,25 @@ describe('ProvingBroker', () => { }), ).rejects.toThrow(new Error('db error')); await assertJobStatus(id, 'in-queue'); - expect(database.getProvingJobResult(id)).toBeUndefined(); }); it('saves job error', async () => { await broker.start(); + const id = makeProvingJobId(); + jest.spyOn(database, 'setProvingJobError'); + await broker.enqueueProvingJob({ id, type: ProvingRequestType.BASE_PARITY, blockNumber: 1, inputs: makeBaseParityInputs(), }); + const error = new Error('test error'); await broker.reportProvingJobError(id, error); await assertJobStatus(id, 'rejected'); - expect(database.getProvingJobResult(id)).toEqual({ error: String(error) }); + expect(database.setProvingJobError).toHaveBeenCalledWith(id, error); }); it('does not retain job error if database fails to save', async () => { @@ -1033,15 +1062,14 @@ describe('ProvingBroker', () => { }); await expect(broker.reportProvingJobError(id, new Error())).rejects.toThrow(new Error('db error')); await assertJobStatus(id, 'in-queue'); - expect(database.getProvingJobResult(id)).toBeUndefined(); }); it('does not save job result if job is unknown', async () => { await broker.start(); const id = makeProvingJobId(); - expect(database.getProvingJob(id)).toBeUndefined(); - expect(database.getProvingJobResult(id)).toBeUndefined(); + jest.spyOn(database, 'setProvingJobResult'); + jest.spyOn(database, 'addProvingJob'); await broker.reportProvingJobSuccess(id, { type: ProvingRequestType.BASE_PARITY, @@ -1052,21 +1080,21 @@ describe('ProvingBroker', () => { ), }); - expect(database.getProvingJob(id)).toBeUndefined(); - expect(database.getProvingJobResult(id)).toBeUndefined(); + expect(database.setProvingJobResult).not.toHaveBeenCalled(); + expect(database.addProvingJob).not.toHaveBeenCalled(); }); it('does not save job error if job is unknown', async () => { await broker.start(); const id = makeProvingJobId(); - expect(database.getProvingJob(id)).toBeUndefined(); - expect(database.getProvingJobResult(id)).toBeUndefined(); + jest.spyOn(database, 'setProvingJobError'); + jest.spyOn(database, 'addProvingJob'); await broker.reportProvingJobError(id, new Error('test error')); - expect(database.getProvingJob(id)).toBeUndefined(); - expect(database.getProvingJobResult(id)).toBeUndefined(); + expect(database.setProvingJobError).not.toHaveBeenCalled(); + expect(database.addProvingJob).not.toHaveBeenCalled(); }); }); diff --git a/yarn-project/prover-client/src/proving_broker/proving_broker.ts b/yarn-project/prover-client/src/proving_broker/proving_broker.ts index 396bbfb48ec..2fe40eac234 100644 --- a/yarn-project/prover-client/src/proving_broker/proving_broker.ts +++ b/yarn-project/prover-client/src/proving_broker/proving_broker.ts @@ -12,8 +12,8 @@ import { PriorityMemoryQueue } from '@aztec/foundation/queue'; import assert from 'assert'; -import { type ProvingBrokerDatabase } from './proving_broker_database.js'; import type { ProvingJobConsumer, ProvingJobFilter, ProvingJobProducer } from './proving_broker_interface.js'; +import { type ProvingJobDatabase } from './proving_job_database.js'; type InProgressMetadata = { id: V2ProvingJobId; @@ -71,7 +71,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer { private maxRetries: number; public constructor( - private database: ProvingBrokerDatabase, + private database: ProvingJobDatabase, { jobTimeoutSec = 30, timeoutIntervalSec = 10, maxRetries = 3 }: ProofRequestBrokerConfig = {}, private logger = createDebugLogger('aztec:prover-client:proof-request-broker'), ) { diff --git a/yarn-project/prover-client/src/proving_broker/proving_job_controller.test.ts b/yarn-project/prover-client/src/proving_broker/proving_job_controller.test.ts new file mode 100644 index 00000000000..724d1d4606f --- /dev/null +++ b/yarn-project/prover-client/src/proving_broker/proving_job_controller.test.ts @@ -0,0 +1,91 @@ +import { ProvingRequestType, type V2ProvingJobId, makePublicInputsAndRecursiveProof } from '@aztec/circuit-types'; +import { RECURSIVE_PROOF_LENGTH, VerificationKeyData, makeRecursiveProof } from '@aztec/circuits.js'; +import { makeBaseParityInputs, makeParityPublicInputs } from '@aztec/circuits.js/testing'; +import { sleep } from '@aztec/foundation/sleep'; + +import { jest } from '@jest/globals'; + +import { MockProver } from '../test/mock_prover.js'; +import { ProvingJobController, ProvingJobStatus } from './proving_job_controller.js'; + +describe('ProvingJobController', () => { + let prover: MockProver; + let onComplete: jest.Mock; + let controller: ProvingJobController; + + beforeEach(() => { + prover = new MockProver(); + onComplete = jest.fn(); + controller = new ProvingJobController( + { + type: ProvingRequestType.BASE_PARITY, + blockNumber: 1, + id: '1' as V2ProvingJobId, + inputs: makeBaseParityInputs(), + }, + 0, + prover, + onComplete, + ); + }); + + it('reports IDLE status initially', () => { + expect(controller.getStatus()).toBe(ProvingJobStatus.IDLE); + }); + + it('reports PROVING status while busy', () => { + controller.start(); + expect(controller.getStatus()).toBe(ProvingJobStatus.PROVING); + }); + + it('reports DONE status after job is done', async () => { + controller.start(); + await sleep(1); // give promises a chance to complete + expect(controller.getStatus()).toBe(ProvingJobStatus.DONE); + }); + + it('calls onComplete with the proof', async () => { + const resp = makePublicInputsAndRecursiveProof( + makeParityPublicInputs(), + makeRecursiveProof(RECURSIVE_PROOF_LENGTH), + VerificationKeyData.makeFakeHonk(), + ); + jest.spyOn(prover, 'getBaseParityProof').mockResolvedValueOnce(resp); + + controller.start(); + await sleep(1); // give promises a chance to complete + expect(onComplete).toHaveBeenCalledWith(undefined, { + type: ProvingRequestType.BASE_PARITY, + value: resp, + }); + }); + + it('calls onComplete with the error', async () => { + const err = new Error('test error'); + jest.spyOn(prover, 'getBaseParityProof').mockRejectedValueOnce(err); + + controller.start(); + await sleep(1); + expect(onComplete).toHaveBeenCalledWith(err, undefined); + }); + + it('does not crash if onComplete throws', async () => { + const err = new Error('test error'); + onComplete.mockImplementationOnce(() => { + throw err; + }); + + controller.start(); + await sleep(1); + expect(onComplete).toHaveBeenCalled(); + }); + + it('does not crash if onComplete rejects', async () => { + const err = new Error('test error'); + onComplete.mockRejectedValueOnce(err); + + controller.start(); + await sleep(1); + expect(onComplete).toHaveBeenCalled(); + }); +}); diff --git a/yarn-project/prover-client/src/proving_broker/proving_job_controller.ts b/yarn-project/prover-client/src/proving_broker/proving_job_controller.ts new file mode 100644 index 00000000000..53d18b476a0 --- /dev/null +++ b/yarn-project/prover-client/src/proving_broker/proving_job_controller.ts @@ -0,0 +1,148 @@ +import { + ProvingRequestType, + type ServerCircuitProver, + type V2ProofOutput, + type V2ProvingJob, + type V2ProvingJobId, +} from '@aztec/circuit-types'; + +export enum ProvingJobStatus { + IDLE = 'idle', + PROVING = 'proving', + DONE = 'done', +} + +type ProvingJobCompletionCallback = ( + error: Error | undefined, + result: V2ProofOutput | undefined, +) => void | Promise; + +export class ProvingJobController { + private status: ProvingJobStatus = ProvingJobStatus.IDLE; + private promise?: Promise; + private abortController = new AbortController(); + + constructor( + private job: V2ProvingJob, + private startedAt: number, + private circuitProver: ServerCircuitProver, + private onComplete: ProvingJobCompletionCallback, + ) {} + + public start(): void { + if (this.status !== ProvingJobStatus.IDLE) { + return; + } + + this.status = ProvingJobStatus.PROVING; + this.promise = this.generateProof() + .then( + result => { + this.status = ProvingJobStatus.DONE; + return this.onComplete(undefined, result); + }, + error => { + this.status = ProvingJobStatus.DONE; + if (error.name === 'AbortError') { + // Ignore abort errors + return; + } + return this.onComplete(error, undefined); + }, + ) + .catch(_ => { + // ignore completion errors + }); + } + + public getStatus(): ProvingJobStatus { + return this.status; + } + + public abort(): void { + if (this.status !== ProvingJobStatus.PROVING) { + return; + } + + this.abortController.abort(); + } + + public getJobId(): V2ProvingJobId { + return this.job.id; + } + + public getStartedAt(): number { + return this.startedAt; + } + + private async generateProof(): Promise { + const { type, inputs } = this.job; + const signal = this.abortController.signal; + switch (type) { + case ProvingRequestType.PUBLIC_VM: { + const value = await this.circuitProver.getAvmProof(inputs, signal); + return { type, value }; + } + + case ProvingRequestType.PRIVATE_BASE_ROLLUP: { + const value = await this.circuitProver.getPrivateBaseRollupProof(inputs, signal); + return { type, value }; + } + + case ProvingRequestType.PUBLIC_BASE_ROLLUP: { + const value = await this.circuitProver.getPublicBaseRollupProof(inputs, signal); + return { type, value }; + } + + case ProvingRequestType.MERGE_ROLLUP: { + const value = await this.circuitProver.getMergeRollupProof(inputs, signal); + return { type, value }; + } + + case ProvingRequestType.EMPTY_BLOCK_ROOT_ROLLUP: { + const value = await this.circuitProver.getEmptyBlockRootRollupProof(inputs, signal); + return { type, value }; + } + + case ProvingRequestType.BLOCK_ROOT_ROLLUP: { + const value = await this.circuitProver.getBlockRootRollupProof(inputs, signal); + return { type, value }; + } + + case ProvingRequestType.BLOCK_MERGE_ROLLUP: { + const value = await this.circuitProver.getBlockMergeRollupProof(inputs, signal); + return { type, value }; + } + + case ProvingRequestType.ROOT_ROLLUP: { + const value = await this.circuitProver.getRootRollupProof(inputs, signal); + return { type, value }; + } + + case ProvingRequestType.BASE_PARITY: { + const value = await this.circuitProver.getBaseParityProof(inputs, signal); + return { type, value }; + } + + case ProvingRequestType.ROOT_PARITY: { + const value = await this.circuitProver.getRootParityProof(inputs, signal); + return { type, value }; + } + + case ProvingRequestType.PRIVATE_KERNEL_EMPTY: { + const value = await this.circuitProver.getEmptyPrivateKernelProof(inputs, signal); + return { type, value }; + } + + case ProvingRequestType.TUBE_PROOF: { + const value = await this.circuitProver.getTubeProof(inputs, signal); + return { type, value }; + } + + default: { + const _exhaustive: never = type; + return Promise.reject(new Error(`Invalid proof request type: ${type}`)); + } + } + } +} diff --git a/yarn-project/prover-client/src/proving_broker/proving_broker_database.ts b/yarn-project/prover-client/src/proving_broker/proving_job_database.ts similarity index 51% rename from yarn-project/prover-client/src/proving_broker/proving_broker_database.ts rename to yarn-project/prover-client/src/proving_broker/proving_job_database.ts index 0d7983f5252..99cae7147ac 100644 --- a/yarn-project/prover-client/src/proving_broker/proving_broker_database.ts +++ b/yarn-project/prover-client/src/proving_broker/proving_job_database.ts @@ -5,7 +5,10 @@ import { type V2ProvingJobResult, } from '@aztec/circuit-types'; -export interface ProvingBrokerDatabase { +/** + * A database for storing proof requests and their results + */ +export interface ProvingJobDatabase { /** * Saves a proof request so it can be retrieved later * @param request - The proof request to save @@ -39,43 +42,3 @@ export interface ProvingBrokerDatabase { */ setProvingJobError(id: V2ProvingJobId, err: Error): Promise; } - -export class InMemoryDatabase implements ProvingBrokerDatabase { - private jobs = new Map(); - private results = new Map(); - - getProvingJob(id: V2ProvingJobId): V2ProvingJob | undefined { - return this.jobs.get(id); - } - - getProvingJobResult(id: V2ProvingJobId): V2ProvingJobResult | undefined { - return this.results.get(id); - } - - addProvingJob(request: V2ProvingJob): Promise { - this.jobs.set(request.id, request); - return Promise.resolve(); - } - - setProvingJobResult(id: V2ProvingJobId, value: V2ProofOutput): Promise { - this.results.set(id, { value }); - return Promise.resolve(); - } - - setProvingJobError(id: V2ProvingJobId, error: Error): Promise { - this.results.set(id, { error: String(error) }); - return Promise.resolve(); - } - - deleteProvingJobAndResult(id: V2ProvingJobId): Promise { - this.jobs.delete(id); - this.results.delete(id); - return Promise.resolve(); - } - - *allProvingJobs(): Iterable<[V2ProvingJob, V2ProvingJobResult | undefined]> { - for (const item of this.jobs.values()) { - yield [item, this.results.get(item.id)] as const; - } - } -} diff --git a/yarn-project/prover-client/src/proving_broker/proving_job_database/memory.ts b/yarn-project/prover-client/src/proving_broker/proving_job_database/memory.ts new file mode 100644 index 00000000000..19acfaf88e7 --- /dev/null +++ b/yarn-project/prover-client/src/proving_broker/proving_job_database/memory.ts @@ -0,0 +1,43 @@ +import type { V2ProofOutput, V2ProvingJob, V2ProvingJobId, V2ProvingJobResult } from '@aztec/circuit-types'; + +import { type ProvingJobDatabase } from '../proving_job_database.js'; + +export class InMemoryDatabase implements ProvingJobDatabase { + private jobs = new Map(); + private results = new Map(); + + getProvingJob(id: V2ProvingJobId): V2ProvingJob | undefined { + return this.jobs.get(id); + } + + getProvingJobResult(id: V2ProvingJobId): V2ProvingJobResult | undefined { + return this.results.get(id); + } + + addProvingJob(request: V2ProvingJob): Promise { + this.jobs.set(request.id, request); + return Promise.resolve(); + } + + setProvingJobResult(id: V2ProvingJobId, value: V2ProofOutput): Promise { + this.results.set(id, { value }); + return Promise.resolve(); + } + + setProvingJobError(id: V2ProvingJobId, error: Error): Promise { + this.results.set(id, { error: String(error) }); + return Promise.resolve(); + } + + deleteProvingJobAndResult(id: V2ProvingJobId): Promise { + this.jobs.delete(id); + this.results.delete(id); + return Promise.resolve(); + } + + *allProvingJobs(): Iterable<[V2ProvingJob, V2ProvingJobResult | undefined]> { + for (const item of this.jobs.values()) { + yield [item, this.results.get(item.id)] as const; + } + } +} diff --git a/yarn-project/prover-client/src/proving_broker/proving_job_database/persisted.ts b/yarn-project/prover-client/src/proving_broker/proving_job_database/persisted.ts new file mode 100644 index 00000000000..748ad387124 --- /dev/null +++ b/yarn-project/prover-client/src/proving_broker/proving_job_database/persisted.ts @@ -0,0 +1,44 @@ +import { type V2ProofOutput, V2ProvingJob, type V2ProvingJobId, V2ProvingJobResult } from '@aztec/circuit-types'; +import { type AztecKVStore, type AztecMap } from '@aztec/kv-store'; + +import { type ProvingJobDatabase } from '../proving_job_database.js'; + +export class PersistedProvingJobDatabase implements ProvingJobDatabase { + private jobs: AztecMap; + private jobResults: AztecMap; + + constructor(private store: AztecKVStore) { + this.jobs = store.openMap('proving_jobs'); + this.jobResults = store.openMap('proving_job_results'); + } + + async addProvingJob(job: V2ProvingJob): Promise { + await this.jobs.set(job.id, JSON.stringify(job)); + } + + *allProvingJobs(): Iterable<[V2ProvingJob, V2ProvingJobResult | undefined]> { + for (const jobStr of this.jobs.values()) { + const job = V2ProvingJob.parse(JSON.parse(jobStr)); + const resultStr = this.jobResults.get(job.id); + const result = resultStr ? V2ProvingJobResult.parse(JSON.parse(resultStr)) : undefined; + yield [job, result]; + } + } + + deleteProvingJobAndResult(id: V2ProvingJobId): Promise { + return this.store.transaction(() => { + void this.jobs.delete(id); + void this.jobResults.delete(id); + }); + } + + async setProvingJobError(id: V2ProvingJobId, err: Error): Promise { + const res: V2ProvingJobResult = { error: err.message }; + await this.jobResults.set(id, JSON.stringify(res)); + } + + async setProvingJobResult(id: V2ProvingJobId, value: V2ProofOutput): Promise { + const res: V2ProvingJobResult = { value }; + await this.jobResults.set(id, JSON.stringify(res)); + } +} diff --git a/yarn-project/prover-client/src/test/bb_prover_base_rollup.test.ts b/yarn-project/prover-client/src/test/bb_prover_base_rollup.test.ts index de9828a3c70..e8b644a8a26 100644 --- a/yarn-project/prover-client/src/test/bb_prover_base_rollup.test.ts +++ b/yarn-project/prover-client/src/test/bb_prover_base_rollup.test.ts @@ -2,6 +2,7 @@ import { BBNativeRollupProver, type BBProverConfig } from '@aztec/bb-prover'; import { makeEmptyProcessedTx } from '@aztec/circuit-types'; import { PRIVATE_KERNEL_EMPTY_INDEX, + type PrivateBaseRollupHints, PrivateBaseRollupInputs, PrivateKernelEmptyInputData, PrivateTubeData, @@ -59,7 +60,7 @@ describe('prover/bb_prover/base-rollup', () => { const tubeData = new PrivateTubeData(tubeProof.inputs, tubeProof.proof, vkData); const baseRollupHints = await buildBaseRollupHints(tx, context.globalVariables, context.actualDb); - const baseRollupInputs = new PrivateBaseRollupInputs(tubeData, baseRollupHints); + const baseRollupInputs = new PrivateBaseRollupInputs(tubeData, baseRollupHints as PrivateBaseRollupHints); logger.verbose('Proving base rollups'); const proofOutputs = await context.prover.getPrivateBaseRollupProof(baseRollupInputs); diff --git a/yarn-project/prover-client/src/test/mock_prover.ts b/yarn-project/prover-client/src/test/mock_prover.ts index d4f253fa868..118ff214e14 100644 --- a/yarn-project/prover-client/src/test/mock_prover.ts +++ b/yarn-project/prover-client/src/test/mock_prover.ts @@ -8,11 +8,22 @@ import { import { AVM_PROOF_LENGTH_IN_FIELDS, AVM_VERIFICATION_KEY_LENGTH_IN_FIELDS, + type AvmCircuitInputs, type BaseOrMergeRollupPublicInputs, + type BaseParityInputs, + type BlockMergeRollupInputs, type BlockRootOrBlockMergePublicInputs, + type BlockRootRollupInputs, + type EmptyBlockRootRollupInputs, type KernelCircuitPublicInputs, + type MergeRollupInputs, NESTED_RECURSIVE_PROOF_LENGTH, + type PrivateBaseRollupInputs, + type PrivateKernelEmptyInputData, + type PublicBaseRollupInputs, RECURSIVE_PROOF_LENGTH, + type RootParityInputs, + type RootRollupInputs, type RootRollupPublicInputs, TUBE_PROOF_LENGTH, VerificationKeyData, @@ -30,7 +41,7 @@ import { export class MockProver implements ServerCircuitProver { constructor() {} - getAvmProof() { + getAvmProof(_inputs: AvmCircuitInputs, _signal?: AbortSignal, _epochNumber?: number) { return Promise.resolve( makeProofAndVerificationKey( makeEmptyRecursiveProof(AVM_PROOF_LENGTH_IN_FIELDS), @@ -39,7 +50,7 @@ export class MockProver implements ServerCircuitProver { ); } - getBaseParityProof() { + getBaseParityProof(_inputs: BaseParityInputs, _signal?: AbortSignal, _epochNumber?: number) { return Promise.resolve( makePublicInputsAndRecursiveProof( makeParityPublicInputs(), @@ -49,7 +60,7 @@ export class MockProver implements ServerCircuitProver { ); } - getRootParityProof() { + getRootParityProof(_inputs: RootParityInputs, _signal?: AbortSignal, _epochNumber?: number) { return Promise.resolve( makePublicInputsAndRecursiveProof( makeParityPublicInputs(), @@ -59,7 +70,11 @@ export class MockProver implements ServerCircuitProver { ); } - getPrivateBaseRollupProof(): Promise> { + getPrivateBaseRollupProof( + _baseRollupInput: PrivateBaseRollupInputs, + _signal?: AbortSignal, + _epochNumber?: number, + ): Promise> { return Promise.resolve( makePublicInputsAndRecursiveProof( makeBaseOrMergeRollupPublicInputs(), @@ -69,7 +84,11 @@ export class MockProver implements ServerCircuitProver { ); } - getPublicBaseRollupProof(): Promise> { + getPublicBaseRollupProof( + _inputs: PublicBaseRollupInputs, + _signal?: AbortSignal, + _epochNumber?: number, + ): Promise> { return Promise.resolve( makePublicInputsAndRecursiveProof( makeBaseOrMergeRollupPublicInputs(), @@ -79,7 +98,11 @@ export class MockProver implements ServerCircuitProver { ); } - getMergeRollupProof(): Promise> { + getMergeRollupProof( + _input: MergeRollupInputs, + _signal?: AbortSignal, + _epochNumber?: number, + ): Promise> { return Promise.resolve( makePublicInputsAndRecursiveProof( makeBaseOrMergeRollupPublicInputs(), @@ -89,7 +112,7 @@ export class MockProver implements ServerCircuitProver { ); } - getBlockMergeRollupProof() { + getBlockMergeRollupProof(_input: BlockMergeRollupInputs, _signal?: AbortSignal, _epochNumber?: number) { return Promise.resolve( makePublicInputsAndRecursiveProof( makeBlockRootOrBlockMergeRollupPublicInputs(), @@ -99,7 +122,11 @@ export class MockProver implements ServerCircuitProver { ); } - getEmptyBlockRootRollupProof(): Promise> { + getEmptyBlockRootRollupProof( + _input: EmptyBlockRootRollupInputs, + _signal?: AbortSignal, + _epochNumber?: number, + ): Promise> { return Promise.resolve( makePublicInputsAndRecursiveProof( makeBlockRootOrBlockMergeRollupPublicInputs(), @@ -109,7 +136,11 @@ export class MockProver implements ServerCircuitProver { ); } - getBlockRootRollupProof(): Promise> { + getBlockRootRollupProof( + _input: BlockRootRollupInputs, + _signal?: AbortSignal, + _epochNumber?: number, + ): Promise> { return Promise.resolve( makePublicInputsAndRecursiveProof( makeBlockRootOrBlockMergeRollupPublicInputs(), @@ -119,7 +150,11 @@ export class MockProver implements ServerCircuitProver { ); } - getEmptyPrivateKernelProof(): Promise> { + getEmptyPrivateKernelProof( + _inputs: PrivateKernelEmptyInputData, + _signal?: AbortSignal, + _epochNumber?: number, + ): Promise> { return Promise.resolve( makePublicInputsAndRecursiveProof( makeKernelCircuitPublicInputs(), @@ -129,7 +164,11 @@ export class MockProver implements ServerCircuitProver { ); } - getRootRollupProof(): Promise> { + getRootRollupProof( + _input: RootRollupInputs, + _signal?: AbortSignal, + _epochNumber?: number, + ): Promise> { return Promise.resolve( makePublicInputsAndRecursiveProof( makeRootRollupPublicInputs(), diff --git a/yarn-project/prover-node/src/factory.ts b/yarn-project/prover-node/src/factory.ts index 5632ba0428d..12ac2e0de92 100644 --- a/yarn-project/prover-node/src/factory.ts +++ b/yarn-project/prover-node/src/factory.ts @@ -7,7 +7,6 @@ import { type DataStoreConfig } from '@aztec/kv-store/config'; import { RollupAbi } from '@aztec/l1-artifacts'; import { createProverClient } from '@aztec/prover-client'; import { L1Publisher } from '@aztec/sequencer-client'; -import { createSimulationProvider } from '@aztec/simulator'; import { type TelemetryClient } from '@aztec/telemetry-client'; import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { createWorldStateSynchronizer } from '@aztec/world-state'; @@ -32,6 +31,7 @@ export async function createProverNode( log?: DebugLogger; aztecNodeTxProvider?: ProverCoordination; archiver?: Archiver; + publisher?: L1Publisher; } = {}, ) { const telemetry = deps.telemetry ?? new NoopTelemetryClient(); @@ -43,12 +43,10 @@ export async function createProverNode( const worldStateSynchronizer = await createWorldStateSynchronizer(worldStateConfig, archiver, telemetry); await worldStateSynchronizer.start(); - const simulationProvider = await createSimulationProvider(config, log); - const prover = await createProverClient(config, telemetry); // REFACTOR: Move publisher out of sequencer package and into an L1-related package - const publisher = new L1Publisher(config, telemetry); + const publisher = deps.publisher ?? new L1Publisher(config, telemetry); // If config.p2pEnabled is true, createProverCoordination will create a p2p client where quotes will be shared and tx's requested // If config.p2pEnabled is false, createProverCoordination request information from the AztecNode @@ -82,7 +80,6 @@ export async function createProverNode( archiver, worldStateSynchronizer, proverCoordination, - simulationProvider, quoteProvider, quoteSigner, claimsMonitor, diff --git a/yarn-project/prover-node/src/job/epoch-proving-job.ts b/yarn-project/prover-node/src/job/epoch-proving-job.ts index bed463abef8..c50e6682be4 100644 --- a/yarn-project/prover-node/src/job/epoch-proving-job.ts +++ b/yarn-project/prover-node/src/job/epoch-proving-job.ts @@ -12,14 +12,7 @@ import { type Tx, type TxHash, } from '@aztec/circuit-types'; -import { - KernelCircuitPublicInputs, - MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, - NULLIFIER_SUBTREE_HEIGHT, - PUBLIC_DATA_SUBTREE_HEIGHT, - PublicDataTreeLeaf, -} from '@aztec/circuits.js'; -import { padArrayEnd } from '@aztec/foundation/collection'; +import { KernelCircuitPublicInputs, NULLIFIER_SUBTREE_HEIGHT, PublicDataTreeLeaf } from '@aztec/circuits.js'; import { createDebugLogger } from '@aztec/foundation/log'; import { promiseWithResolvers } from '@aztec/foundation/promise'; import { Timer } from '@aztec/foundation/timer'; @@ -201,15 +194,14 @@ export class EpochProvingJob { emptyKernelOutput.end.nullifiers.map(n => n.toBuffer()), NULLIFIER_SUBTREE_HEIGHT, ); - const allPublicDataWrites = padArrayEnd( - emptyKernelOutput.end.publicDataWrites.map(({ leafSlot, value }) => new PublicDataTreeLeaf(leafSlot, value)), - PublicDataTreeLeaf.empty(), - MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, - ); + const allPublicDataWrites = emptyKernelOutput.end.publicDataWrites + .filter(write => !write.isEmpty()) + .map(({ leafSlot, value }) => new PublicDataTreeLeaf(leafSlot, value)); + await this.db.batchInsert( MerkleTreeId.PUBLIC_DATA_TREE, allPublicDataWrites.map(x => x.toBuffer()), - PUBLIC_DATA_SUBTREE_HEIGHT, + 0, ); } } diff --git a/yarn-project/prover-node/src/prover-node.test.ts b/yarn-project/prover-node/src/prover-node.test.ts index 3b1f88a1215..c023aff2c10 100644 --- a/yarn-project/prover-node/src/prover-node.test.ts +++ b/yarn-project/prover-node/src/prover-node.test.ts @@ -25,7 +25,7 @@ import { } from '@aztec/p2p'; import { createBootstrapNode, createTestLibP2PService } from '@aztec/p2p/mocks'; import { type L1Publisher } from '@aztec/sequencer-client'; -import { type PublicProcessorFactory, type SimulationProvider } from '@aztec/simulator'; +import { type PublicProcessorFactory } from '@aztec/simulator'; import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { jest } from '@jest/globals'; @@ -48,7 +48,6 @@ describe('prover-node', () => { let contractDataSource: MockProxy; let worldState: MockProxy; let coordination: MockProxy | ProverCoordination; - let simulator: MockProxy; let quoteProvider: MockProxy; let quoteSigner: MockProxy; let bondManager: MockProxy; @@ -97,7 +96,6 @@ describe('prover-node', () => { contractDataSource, worldState, coordination, - simulator, quoteProvider, quoteSigner, claimsMonitor, @@ -115,7 +113,6 @@ describe('prover-node', () => { contractDataSource = mock(); worldState = mock(); coordination = mock(); - simulator = mock(); quoteProvider = mock(); quoteSigner = mock(); bondManager = mock(); diff --git a/yarn-project/prover-node/src/prover-node.ts b/yarn-project/prover-node/src/prover-node.ts index 1b090315c8f..cff56201098 100644 --- a/yarn-project/prover-node/src/prover-node.ts +++ b/yarn-project/prover-node/src/prover-node.ts @@ -18,7 +18,7 @@ import { compact } from '@aztec/foundation/collection'; import { createDebugLogger } from '@aztec/foundation/log'; import { type Maybe } from '@aztec/foundation/types'; import { type L1Publisher } from '@aztec/sequencer-client'; -import { PublicProcessorFactory, type SimulationProvider } from '@aztec/simulator'; +import { PublicProcessorFactory } from '@aztec/simulator'; import { type TelemetryClient } from '@aztec/telemetry-client'; import { type BondManager } from './bond/bond-manager.js'; @@ -56,7 +56,6 @@ export class ProverNode implements ClaimsMonitorHandler, EpochMonitorHandler, Pr private readonly contractDataSource: ContractDataSource, private readonly worldState: WorldStateSynchronizer, private readonly coordination: ProverCoordination & Maybe, - private readonly simulator: SimulationProvider, private readonly quoteProvider: QuoteProvider, private readonly quoteSigner: QuoteSigner, private readonly claimsMonitor: ClaimsMonitor, @@ -143,7 +142,11 @@ export class ProverNode implements ClaimsMonitorHandler, EpochMonitorHandler, Pr const signed = await this.quoteSigner.sign(quote); // Send it to the coordinator - await this.sendEpochProofQuote(signed); + this.log.info( + `Sending quote for epoch ${epochNumber} with blocks ${blocks[0].number} to ${blocks.at(-1)!.number}`, + quote.toViemArgs(), + ); + await this.doSendEpochProofQuote(signed); } catch (err) { this.log.error(`Error handling epoch completed`, err); } @@ -178,11 +181,13 @@ export class ProverNode implements ClaimsMonitorHandler, EpochMonitorHandler, Pr this.log.info('Stopped ProverNode'); } - /** - * Sends an epoch proof quote to the coordinator. - */ + /** Sends an epoch proof quote to the coordinator. */ public sendEpochProofQuote(quote: EpochProofQuote): Promise { this.log.info(`Sending quote for epoch`, quote.toViemArgs().quote); + return this.doSendEpochProofQuote(quote); + } + + private doSendEpochProofQuote(quote: EpochProofQuote) { return this.coordination.addEpochProofQuote(quote); } @@ -243,11 +248,7 @@ export class ProverNode implements ClaimsMonitorHandler, EpochMonitorHandler, Pr const proverDb = await this.worldState.fork(fromBlock - 1); // Create a processor using the forked world state - const publicProcessorFactory = new PublicProcessorFactory( - this.contractDataSource, - this.simulator, - this.telemetryClient, - ); + const publicProcessorFactory = new PublicProcessorFactory(this.contractDataSource, this.telemetryClient); const cleanUp = async () => { await publicDb.close(); diff --git a/yarn-project/pxe/src/database/contracts/contract_artifact_db.ts b/yarn-project/pxe/src/database/contracts/contract_artifact_db.ts index 85f726d0973..3f6e6bb2311 100644 --- a/yarn-project/pxe/src/database/contracts/contract_artifact_db.ts +++ b/yarn-project/pxe/src/database/contracts/contract_artifact_db.ts @@ -9,6 +9,7 @@ export interface ContractArtifactDatabase { * Adds a new contract artifact to the database or updates an existing one. * @param id - Id of the corresponding contract class. * @param contract - Contract artifact to add. + * @throws - If there are duplicate private function selectors. */ addContractArtifact(id: Fr, contract: ContractArtifact): Promise; /** diff --git a/yarn-project/pxe/src/database/incoming_note_dao.test.ts b/yarn-project/pxe/src/database/incoming_note_dao.test.ts index a3cb0d5345c..1df9103e08f 100644 --- a/yarn-project/pxe/src/database/incoming_note_dao.test.ts +++ b/yarn-project/pxe/src/database/incoming_note_dao.test.ts @@ -1,38 +1,8 @@ -import { Note, randomTxHash } from '@aztec/circuit-types'; -import { AztecAddress, Fr, Point } from '@aztec/circuits.js'; -import { NoteSelector } from '@aztec/foundation/abi'; - import { IncomingNoteDao } from './incoming_note_dao.js'; -export const randomIncomingNoteDao = ({ - note = Note.random(), - contractAddress = AztecAddress.random(), - txHash = randomTxHash(), - storageSlot = Fr.random(), - noteTypeId = NoteSelector.random(), - nonce = Fr.random(), - noteHash = Fr.random(), - siloedNullifier = Fr.random(), - index = Fr.random().toBigInt(), - addressPoint = Point.random(), -}: Partial = {}) => { - return new IncomingNoteDao( - note, - contractAddress, - storageSlot, - noteTypeId, - txHash, - nonce, - noteHash, - siloedNullifier, - index, - addressPoint, - ); -}; - describe('Incoming Note DAO', () => { it('convert to and from buffer', () => { - const note = randomIncomingNoteDao(); + const note = IncomingNoteDao.random(); const buf = note.toBuffer(); expect(IncomingNoteDao.fromBuffer(buf)).toEqual(note); }); diff --git a/yarn-project/pxe/src/database/incoming_note_dao.ts b/yarn-project/pxe/src/database/incoming_note_dao.ts index cbd344b135c..d2dc2d38815 100644 --- a/yarn-project/pxe/src/database/incoming_note_dao.ts +++ b/yarn-project/pxe/src/database/incoming_note_dao.ts @@ -1,4 +1,4 @@ -import { type L1NotePayload, Note, TxHash } from '@aztec/circuit-types'; +import { type L1NotePayload, Note, TxHash, randomTxHash } from '@aztec/circuit-types'; import { AztecAddress, Fr, Point, type PublicKey } from '@aztec/circuits.js'; import { NoteSelector } from '@aztec/foundation/abi'; import { toBigIntBE } from '@aztec/foundation/bigint-buffer'; @@ -22,6 +22,10 @@ export class IncomingNoteDao implements NoteData { public noteTypeId: NoteSelector, /** The hash of the tx the note was created in. */ public txHash: TxHash, + /** The L2 block number in which the tx with this note was included. */ + public l2BlockNumber: number, + /** The L2 block hash in which the tx with this note was included. */ + public l2BlockHash: string, /** The nonce of the note. */ public nonce: Fr, /** @@ -44,6 +48,8 @@ export class IncomingNoteDao implements NoteData { note: Note, payload: L1NotePayload, noteInfo: NoteInfo, + l2BlockNumber: number, + l2BlockHash: string, dataStartIndexForTx: number, addressPoint: PublicKey, ) { @@ -54,6 +60,8 @@ export class IncomingNoteDao implements NoteData { payload.storageSlot, payload.noteTypeId, noteInfo.txHash, + l2BlockNumber, + l2BlockHash, noteInfo.nonce, noteInfo.noteHash, noteInfo.siloedNullifier, @@ -69,6 +77,8 @@ export class IncomingNoteDao implements NoteData { this.storageSlot, this.noteTypeId, this.txHash.buffer, + this.l2BlockNumber, + Fr.fromString(this.l2BlockHash), this.nonce, this.noteHash, this.siloedNullifier, @@ -76,6 +86,7 @@ export class IncomingNoteDao implements NoteData { this.addressPoint, ]); } + static fromBuffer(buffer: Buffer | BufferReader) { const reader = BufferReader.asReader(buffer); @@ -84,6 +95,8 @@ export class IncomingNoteDao implements NoteData { const storageSlot = Fr.fromBuffer(reader); const noteTypeId = reader.readObject(NoteSelector); const txHash = reader.readObject(TxHash); + const l2BlockNumber = reader.readNumber(); + const l2BlockHash = Fr.fromBuffer(reader).toString(); const nonce = Fr.fromBuffer(reader); const noteHash = Fr.fromBuffer(reader); const siloedNullifier = Fr.fromBuffer(reader); @@ -96,6 +109,8 @@ export class IncomingNoteDao implements NoteData { storageSlot, noteTypeId, txHash, + l2BlockNumber, + l2BlockHash, nonce, noteHash, siloedNullifier, @@ -122,4 +137,34 @@ export class IncomingNoteDao implements NoteData { const noteSize = 4 + this.note.items.length * Fr.SIZE_IN_BYTES; return noteSize + AztecAddress.SIZE_IN_BYTES + Fr.SIZE_IN_BYTES * 4 + TxHash.SIZE + Point.SIZE_IN_BYTES + indexSize; } + + static random({ + note = Note.random(), + contractAddress = AztecAddress.random(), + txHash = randomTxHash(), + storageSlot = Fr.random(), + noteTypeId = NoteSelector.random(), + nonce = Fr.random(), + l2BlockNumber = Math.floor(Math.random() * 1000), + l2BlockHash = Fr.random().toString(), + noteHash = Fr.random(), + siloedNullifier = Fr.random(), + index = Fr.random().toBigInt(), + addressPoint = Point.random(), + }: Partial = {}) { + return new IncomingNoteDao( + note, + contractAddress, + storageSlot, + noteTypeId, + txHash, + l2BlockNumber, + l2BlockHash, + nonce, + noteHash, + siloedNullifier, + index, + addressPoint, + ); + } } diff --git a/yarn-project/pxe/src/database/kv_pxe_database.ts b/yarn-project/pxe/src/database/kv_pxe_database.ts index 2fea8a0452b..287af7b6bbd 100644 --- a/yarn-project/pxe/src/database/kv_pxe_database.ts +++ b/yarn-project/pxe/src/database/kv_pxe_database.ts @@ -1,4 +1,10 @@ -import { type IncomingNotesFilter, MerkleTreeId, NoteStatus, type OutgoingNotesFilter } from '@aztec/circuit-types'; +import { + type InBlock, + type IncomingNotesFilter, + MerkleTreeId, + NoteStatus, + type OutgoingNotesFilter, +} from '@aztec/circuit-types'; import { AztecAddress, CompleteAddress, @@ -7,9 +13,8 @@ import { type IndexedTaggingSecret, type PublicKey, SerializableContractInstance, - computePoint, } from '@aztec/circuits.js'; -import { type ContractArtifact } from '@aztec/foundation/abi'; +import { type ContractArtifact, FunctionSelector, FunctionType } from '@aztec/foundation/abi'; import { toBufferBE } from '@aztec/foundation/bigint-buffer'; import { Fr } from '@aztec/foundation/fields'; import { @@ -39,11 +44,14 @@ export class KVPxeDatabase implements PxeDatabase { #notes: AztecMap; #nullifiedNotes: AztecMap; #nullifierToNoteId: AztecMap; + #nullifiersByBlockNumber: AztecMultiMap; + #nullifiedNotesToScope: AztecMultiMap; #nullifiedNotesByContract: AztecMultiMap; #nullifiedNotesByStorageSlot: AztecMultiMap; #nullifiedNotesByTxHash: AztecMultiMap; #nullifiedNotesByAddressPoint: AztecMultiMap; + #nullifiedNotesByNullifier: AztecMap; #syncedBlockPerPublicKey: AztecMap; #contractArtifacts: AztecMap; #contractInstances: AztecMap; @@ -56,6 +64,7 @@ export class KVPxeDatabase implements PxeDatabase { #outgoingNotesByOvpkM: AztecMultiMap; #scopes: AztecSet; + #notesToScope: AztecMultiMap; #notesByContractAndScope: Map>; #notesByStorageSlotAndScope: Map>; #notesByTxHashAndScope: Map>; @@ -87,11 +96,14 @@ export class KVPxeDatabase implements PxeDatabase { this.#notes = db.openMap('notes'); this.#nullifiedNotes = db.openMap('nullified_notes'); this.#nullifierToNoteId = db.openMap('nullifier_to_note'); + this.#nullifiersByBlockNumber = db.openMultiMap('nullifier_to_block_number'); + this.#nullifiedNotesToScope = db.openMultiMap('nullified_notes_to_scope'); this.#nullifiedNotesByContract = db.openMultiMap('nullified_notes_by_contract'); this.#nullifiedNotesByStorageSlot = db.openMultiMap('nullified_notes_by_storage_slot'); this.#nullifiedNotesByTxHash = db.openMultiMap('nullified_notes_by_tx_hash'); this.#nullifiedNotesByAddressPoint = db.openMultiMap('nullified_notes_by_address_point'); + this.#nullifiedNotesByNullifier = db.openMap('nullified_notes_by_nullifier'); this.#outgoingNotes = db.openMap('outgoing_notes'); this.#outgoingNotesByContract = db.openMultiMap('outgoing_notes_by_contract'); @@ -100,6 +112,7 @@ export class KVPxeDatabase implements PxeDatabase { this.#outgoingNotesByOvpkM = db.openMultiMap('outgoing_notes_by_ovpk_m'); this.#scopes = db.openSet('scopes'); + this.#notesToScope = db.openMultiMap('notes_to_scope'); this.#notesByContractAndScope = new Map>(); this.#notesByStorageSlotAndScope = new Map>(); this.#notesByTxHashAndScope = new Map>(); @@ -128,6 +141,19 @@ export class KVPxeDatabase implements PxeDatabase { } public async addContractArtifact(id: Fr, contract: ContractArtifact): Promise { + const privateSelectors = contract.functions + .filter(functionArtifact => functionArtifact.functionType === FunctionType.PRIVATE) + .map(privateFunctionArtifact => + FunctionSelector.fromNameAndParameters( + privateFunctionArtifact.name, + privateFunctionArtifact.parameters, + ).toString(), + ); + + if (privateSelectors.length !== new Set(privateSelectors).size) { + throw new Error('Repeated function selectors of private functions'); + } + await this.#contractArtifacts.set(id.toString(), contractArtifactToBuffer(contract)); } @@ -195,6 +221,7 @@ export class KVPxeDatabase implements PxeDatabase { // Had we stored them by their nullifier, they would be returned in random order const noteIndex = toBufferBE(dao.index, 32).toString('hex'); void this.#notes.set(noteIndex, dao.toBuffer()); + void this.#notesToScope.set(noteIndex, scope.toString()); void this.#nullifierToNoteId.set(dao.siloedNullifier.toString(), noteIndex); void this.#notesByContractAndScope.get(scope.toString())!.set(dao.contractAddress.toString(), noteIndex); @@ -214,8 +241,92 @@ export class KVPxeDatabase implements PxeDatabase { }); } + public removeNotesAfter(blockNumber: number): Promise { + return this.db.transaction(() => { + for (const note of this.#notes.values()) { + const noteDao = IncomingNoteDao.fromBuffer(note); + if (noteDao.l2BlockNumber > blockNumber) { + const noteIndex = toBufferBE(noteDao.index, 32).toString('hex'); + void this.#notes.delete(noteIndex); + void this.#notesToScope.delete(noteIndex); + void this.#nullifierToNoteId.delete(noteDao.siloedNullifier.toString()); + for (const scope of this.#scopes.entries()) { + void this.#notesByAddressPointAndScope.get(scope)!.deleteValue(noteDao.addressPoint.toString(), noteIndex); + void this.#notesByTxHashAndScope.get(scope)!.deleteValue(noteDao.txHash.toString(), noteIndex); + void this.#notesByContractAndScope.get(scope)!.deleteValue(noteDao.contractAddress.toString(), noteIndex); + void this.#notesByStorageSlotAndScope.get(scope)!.deleteValue(noteDao.storageSlot.toString(), noteIndex); + } + } + } + + for (const note of this.#outgoingNotes.values()) { + const noteDao = OutgoingNoteDao.fromBuffer(note); + if (noteDao.l2BlockNumber > blockNumber) { + const noteIndex = toBufferBE(noteDao.index, 32).toString('hex'); + void this.#outgoingNotes.delete(noteIndex); + void this.#outgoingNotesByContract.deleteValue(noteDao.contractAddress.toString(), noteIndex); + void this.#outgoingNotesByStorageSlot.deleteValue(noteDao.storageSlot.toString(), noteIndex); + void this.#outgoingNotesByTxHash.deleteValue(noteDao.txHash.toString(), noteIndex); + void this.#outgoingNotesByOvpkM.deleteValue(noteDao.ovpkM.toString(), noteIndex); + } + } + }); + } + + public async unnullifyNotesAfter(blockNumber: number): Promise { + const nullifiersToUndo: string[] = []; + const currentBlockNumber = blockNumber + 1; + const maxBlockNumber = this.getBlockNumber() ?? currentBlockNumber; + for (let i = currentBlockNumber; i <= maxBlockNumber; i++) { + nullifiersToUndo.push(...this.#nullifiersByBlockNumber.getValues(i)); + } + + const notesIndexesToReinsert = await this.db.transaction(() => + nullifiersToUndo.map(nullifier => this.#nullifiedNotesByNullifier.get(nullifier)), + ); + const nullifiedNoteBuffers = await this.db.transaction(() => { + return notesIndexesToReinsert + .filter(noteIndex => noteIndex != undefined) + .map(noteIndex => this.#nullifiedNotes.get(noteIndex!)); + }); + const noteDaos = nullifiedNoteBuffers + .filter(buffer => buffer != undefined) + .map(buffer => IncomingNoteDao.fromBuffer(buffer!)); + + await this.db.transaction(() => { + for (const dao of noteDaos) { + const noteIndex = toBufferBE(dao.index, 32).toString('hex'); + void this.#notes.set(noteIndex, dao.toBuffer()); + void this.#nullifierToNoteId.set(dao.siloedNullifier.toString(), noteIndex); + + let scopes = Array.from(this.#nullifiedNotesToScope.getValues(noteIndex) ?? []); + + if (scopes.length === 0) { + scopes = [new AztecAddress(dao.addressPoint.x).toString()]; + } + + for (const scope of scopes) { + void this.#notesByContractAndScope.get(scope)!.set(dao.contractAddress.toString(), noteIndex); + void this.#notesByStorageSlotAndScope.get(scope)!.set(dao.storageSlot.toString(), noteIndex); + void this.#notesByTxHashAndScope.get(scope)!.set(dao.txHash.toString(), noteIndex); + void this.#notesByAddressPointAndScope.get(scope)!.set(dao.addressPoint.toString(), noteIndex); + void this.#notesToScope.set(noteIndex, scope); + } + + void this.#nullifiedNotes.delete(noteIndex); + void this.#nullifiedNotesToScope.delete(noteIndex); + void this.#nullifiersByBlockNumber.deleteValue(dao.l2BlockNumber, dao.siloedNullifier.toString()); + void this.#nullifiedNotesByContract.deleteValue(dao.contractAddress.toString(), noteIndex); + void this.#nullifiedNotesByStorageSlot.deleteValue(dao.storageSlot.toString(), noteIndex); + void this.#nullifiedNotesByTxHash.deleteValue(dao.txHash.toString(), noteIndex); + void this.#nullifiedNotesByAddressPoint.deleteValue(dao.addressPoint.toString(), noteIndex); + void this.#nullifiedNotesByNullifier.delete(dao.siloedNullifier.toString()); + } + }); + } + getIncomingNotes(filter: IncomingNotesFilter): Promise { - const publicKey: PublicKey | undefined = filter.owner ? computePoint(filter.owner) : undefined; + const publicKey: PublicKey | undefined = filter.owner ? filter.owner.toAddressPoint() : undefined; filter.status = filter.status ?? NoteStatus.ACTIVE; @@ -350,7 +461,7 @@ export class KVPxeDatabase implements PxeDatabase { return Promise.resolve(notes); } - removeNullifiedNotes(nullifiers: Fr[], accountAddressPoint: PublicKey): Promise { + removeNullifiedNotes(nullifiers: InBlock[], accountAddressPoint: PublicKey): Promise { if (nullifiers.length === 0) { return Promise.resolve([]); } @@ -358,7 +469,8 @@ export class KVPxeDatabase implements PxeDatabase { return this.#db.transaction(() => { const nullifiedNotes: IncomingNoteDao[] = []; - for (const nullifier of nullifiers) { + for (const blockScopedNullifier of nullifiers) { + const { data: nullifier, l2BlockNumber: blockNumber } = blockScopedNullifier; const noteIndex = this.#nullifierToNoteId.get(nullifier.toString()); if (!noteIndex) { continue; @@ -370,7 +482,7 @@ export class KVPxeDatabase implements PxeDatabase { // note doesn't exist. Maybe it got nullified already continue; } - + const noteScopes = this.#notesToScope.getValues(noteIndex) ?? []; const note = IncomingNoteDao.fromBuffer(noteBuffer); if (!note.addressPoint.equals(accountAddressPoint)) { // tried to nullify someone else's note @@ -380,19 +492,27 @@ export class KVPxeDatabase implements PxeDatabase { nullifiedNotes.push(note); void this.#notes.delete(noteIndex); + void this.#notesToScope.delete(noteIndex); - for (const scope in this.#scopes.entries()) { + for (const scope of this.#scopes.entries()) { void this.#notesByAddressPointAndScope.get(scope)!.deleteValue(accountAddressPoint.toString(), noteIndex); void this.#notesByTxHashAndScope.get(scope)!.deleteValue(note.txHash.toString(), noteIndex); void this.#notesByContractAndScope.get(scope)!.deleteValue(note.contractAddress.toString(), noteIndex); void this.#notesByStorageSlotAndScope.get(scope)!.deleteValue(note.storageSlot.toString(), noteIndex); } + if (noteScopes !== undefined) { + for (const scope of noteScopes) { + void this.#nullifiedNotesToScope.set(noteIndex, scope); + } + } void this.#nullifiedNotes.set(noteIndex, note.toBuffer()); + void this.#nullifiersByBlockNumber.set(blockNumber, nullifier.toString()); void this.#nullifiedNotesByContract.set(note.contractAddress.toString(), noteIndex); void this.#nullifiedNotesByStorageSlot.set(note.storageSlot.toString(), noteIndex); void this.#nullifiedNotesByTxHash.set(note.txHash.toString(), noteIndex); void this.#nullifiedNotesByAddressPoint.set(note.addressPoint.toString(), noteIndex); + void this.#nullifiedNotesByNullifier.set(nullifier.toString(), noteIndex); void this.#nullifierToNoteId.delete(nullifier.toString()); } @@ -548,25 +668,19 @@ export class KVPxeDatabase implements PxeDatabase { return incomingNotesSize + outgoingNotesSize + treeRootsSize + authWitsSize + addressesSize; } - async incrementTaggingSecretsIndexesAsSender(appTaggingSecrets: Fr[]): Promise { - await this.#incrementTaggingSecretsIndexes(appTaggingSecrets, this.#taggingSecretIndexesForSenders); + async setTaggingSecretsIndexesAsSender(indexedSecrets: IndexedTaggingSecret[]): Promise { + await this.#setTaggingSecretsIndexes(indexedSecrets, this.#taggingSecretIndexesForSenders); } - async #incrementTaggingSecretsIndexes(appTaggingSecrets: Fr[], storageMap: AztecMap): Promise { - const indexes = await this.#getTaggingSecretsIndexes(appTaggingSecrets, storageMap); - await this.db.transaction(() => { - indexes.forEach((taggingSecretIndex, listIndex) => { - const nextIndex = taggingSecretIndex + 1; - void storageMap.set(appTaggingSecrets[listIndex].toString(), nextIndex); - }); - }); + async setTaggingSecretsIndexesAsRecipient(indexedSecrets: IndexedTaggingSecret[]): Promise { + await this.#setTaggingSecretsIndexes(indexedSecrets, this.#taggingSecretIndexesForRecipients); } - async setTaggingSecretsIndexesAsRecipient(indexedSecrets: IndexedTaggingSecret[]): Promise { - await this.db.transaction(() => { - indexedSecrets.forEach(indexedSecret => { - void this.#taggingSecretIndexesForRecipients.set(indexedSecret.secret.toString(), indexedSecret.index); - }); + #setTaggingSecretsIndexes(indexedSecrets: IndexedTaggingSecret[], storageMap: AztecMap) { + return this.db.transaction(() => { + indexedSecrets.forEach( + indexedSecret => void storageMap.set(indexedSecret.secret.toString(), indexedSecret.index), + ); }); } @@ -581,4 +695,15 @@ export class KVPxeDatabase implements PxeDatabase { #getTaggingSecretsIndexes(appTaggingSecrets: Fr[], storageMap: AztecMap): Promise { return this.db.transaction(() => appTaggingSecrets.map(secret => storageMap.get(`${secret.toString()}`) ?? 0)); } + + async resetNoteSyncData(): Promise { + await this.db.transaction(() => { + for (const recipient of this.#taggingSecretIndexesForRecipients.keys()) { + void this.#taggingSecretIndexesForRecipients.delete(recipient); + } + for (const sender of this.#taggingSecretIndexesForSenders.keys()) { + void this.#taggingSecretIndexesForSenders.delete(sender); + } + }); + } } diff --git a/yarn-project/pxe/src/database/outgoing_note_dao.test.ts b/yarn-project/pxe/src/database/outgoing_note_dao.test.ts index 04b2b2201f7..0c293ba13eb 100644 --- a/yarn-project/pxe/src/database/outgoing_note_dao.test.ts +++ b/yarn-project/pxe/src/database/outgoing_note_dao.test.ts @@ -1,26 +1,8 @@ -import { Note, randomTxHash } from '@aztec/circuit-types'; -import { AztecAddress, Fr, Point } from '@aztec/circuits.js'; -import { NoteSelector } from '@aztec/foundation/abi'; - import { OutgoingNoteDao } from './outgoing_note_dao.js'; -export const randomOutgoingNoteDao = ({ - note = Note.random(), - contractAddress = AztecAddress.random(), - txHash = randomTxHash(), - storageSlot = Fr.random(), - noteTypeId = NoteSelector.random(), - nonce = Fr.random(), - noteHash = Fr.random(), - index = Fr.random().toBigInt(), - ovpkM = Point.random(), -}: Partial = {}) => { - return new OutgoingNoteDao(note, contractAddress, storageSlot, noteTypeId, txHash, nonce, noteHash, index, ovpkM); -}; - describe('Outgoing Note DAO', () => { it('convert to and from buffer', () => { - const note = randomOutgoingNoteDao(); + const note = OutgoingNoteDao.random(); const buf = note.toBuffer(); expect(OutgoingNoteDao.fromBuffer(buf)).toEqual(note); }); diff --git a/yarn-project/pxe/src/database/outgoing_note_dao.ts b/yarn-project/pxe/src/database/outgoing_note_dao.ts index 04bb7d4835c..386b23ecd57 100644 --- a/yarn-project/pxe/src/database/outgoing_note_dao.ts +++ b/yarn-project/pxe/src/database/outgoing_note_dao.ts @@ -1,4 +1,4 @@ -import { type L1NotePayload, Note, TxHash } from '@aztec/circuit-types'; +import { type L1NotePayload, Note, TxHash, randomTxHash } from '@aztec/circuit-types'; import { AztecAddress, Fr, Point, type PublicKey } from '@aztec/circuits.js'; import { NoteSelector } from '@aztec/foundation/abi'; import { toBigIntBE } from '@aztec/foundation/bigint-buffer'; @@ -21,6 +21,10 @@ export class OutgoingNoteDao { public noteTypeId: NoteSelector, /** The hash of the tx the note was created in. */ public txHash: TxHash, + /** The L2 block number in which the tx with this note was included. */ + public l2BlockNumber: number, + /** The L2 block hash in which the tx with this note was included. */ + public l2BlockHash: string, /** The nonce of the note. */ public nonce: Fr, /** @@ -38,6 +42,8 @@ export class OutgoingNoteDao { note: Note, payload: L1NotePayload, noteInfo: NoteInfo, + l2BlockNumber: number, + l2BlockHash: string, dataStartIndexForTx: number, ovpkM: PublicKey, ) { @@ -48,6 +54,8 @@ export class OutgoingNoteDao { payload.storageSlot, payload.noteTypeId, noteInfo.txHash, + l2BlockNumber, + l2BlockHash, noteInfo.nonce, noteInfo.noteHash, noteHashIndexInTheWholeTree, @@ -62,6 +70,8 @@ export class OutgoingNoteDao { this.storageSlot, this.noteTypeId, this.txHash.buffer, + this.l2BlockNumber, + Fr.fromString(this.l2BlockHash), this.nonce, this.noteHash, this.index, @@ -76,6 +86,8 @@ export class OutgoingNoteDao { const storageSlot = Fr.fromBuffer(reader); const noteTypeId = reader.readObject(NoteSelector); const txHash = new TxHash(reader.readBytes(TxHash.SIZE)); + const l2BlockNumber = reader.readNumber(); + const l2BlockHash = Fr.fromBuffer(reader).toString(); const nonce = Fr.fromBuffer(reader); const noteHash = Fr.fromBuffer(reader); const index = toBigIntBE(reader.readBytes(32)); @@ -87,6 +99,8 @@ export class OutgoingNoteDao { storageSlot, noteTypeId, txHash, + l2BlockNumber, + l2BlockHash, nonce, noteHash, index, @@ -111,4 +125,32 @@ export class OutgoingNoteDao { const noteSize = 4 + this.note.items.length * Fr.SIZE_IN_BYTES; return noteSize + AztecAddress.SIZE_IN_BYTES + Fr.SIZE_IN_BYTES * 2 + TxHash.SIZE + Point.SIZE_IN_BYTES; } + + static random({ + note = Note.random(), + contractAddress = AztecAddress.random(), + txHash = randomTxHash(), + storageSlot = Fr.random(), + noteTypeId = NoteSelector.random(), + nonce = Fr.random(), + l2BlockNumber = Math.floor(Math.random() * 1000), + l2BlockHash = Fr.random().toString(), + noteHash = Fr.random(), + index = Fr.random().toBigInt(), + ovpkM = Point.random(), + }: Partial = {}) { + return new OutgoingNoteDao( + note, + contractAddress, + storageSlot, + noteTypeId, + txHash, + l2BlockNumber, + l2BlockHash, + nonce, + noteHash, + index, + ovpkM, + ); + } } diff --git a/yarn-project/pxe/src/database/pxe_database.ts b/yarn-project/pxe/src/database/pxe_database.ts index d4cdb12bcec..8b884041bb9 100644 --- a/yarn-project/pxe/src/database/pxe_database.ts +++ b/yarn-project/pxe/src/database/pxe_database.ts @@ -1,4 +1,4 @@ -import { type IncomingNotesFilter, type OutgoingNotesFilter } from '@aztec/circuit-types'; +import { type InBlock, type IncomingNotesFilter, type OutgoingNotesFilter } from '@aztec/circuit-types'; import { type CompleteAddress, type ContractInstanceWithAddress, @@ -96,7 +96,7 @@ export interface PxeDatabase extends ContractArtifactDatabase, ContractInstanceD * @param account - A PublicKey instance representing the account for which the records are being removed. * @returns Removed notes. */ - removeNullifiedNotes(nullifiers: Fr[], account: PublicKey): Promise; + removeNullifiedNotes(nullifiers: InBlock[], account: PublicKey): Promise; /** * Gets the most recently processed block number. @@ -202,11 +202,11 @@ export interface PxeDatabase extends ContractArtifactDatabase, ContractInstanceD getTaggingSecretsIndexesAsSender(appTaggingSecrets: Fr[]): Promise; /** - * Increments the index for the provided app siloed tagging secrets in the senders database - * To be used when the generated tags have been used as sender + * Sets the index for the provided app siloed tagging secrets + * To be used when the generated tags have been "seen" as a sender * @param appTaggingSecrets - The app siloed tagging secrets. */ - incrementTaggingSecretsIndexesAsSender(appTaggingSecrets: Fr[]): Promise; + setTaggingSecretsIndexesAsSender(indexedTaggingSecrets: IndexedTaggingSecret[]): Promise; /** * Sets the index for the provided app siloed tagging secrets @@ -214,4 +214,24 @@ export interface PxeDatabase extends ContractArtifactDatabase, ContractInstanceD * @param appTaggingSecrets - The app siloed tagging secrets. */ setTaggingSecretsIndexesAsRecipient(indexedTaggingSecrets: IndexedTaggingSecret[]): Promise; + + /** + * Deletes all notes synched after this block number. + * @param blockNumber - All notes strictly after this block number are removed. + */ + removeNotesAfter(blockNumber: number): Promise; + + /** + * Restores notes nullified after the given block. + * @param blockNumber - All nullifiers strictly after this block are removed. + */ + unnullifyNotesAfter(blockNumber: number): Promise; + + /** + * Resets the indexes used to sync notes to 0 for every sender and recipient, causing the next sync process to + * start from scratch, taking longer than usual. + * This can help fix desynchronization issues, including finding logs that had previously been overlooked, and + * is also required to deal with chain reorgs. + */ + resetNoteSyncData(): Promise; } diff --git a/yarn-project/pxe/src/database/pxe_database_test_suite.ts b/yarn-project/pxe/src/database/pxe_database_test_suite.ts index 1fefa614bad..4fcf59993b1 100644 --- a/yarn-project/pxe/src/database/pxe_database_test_suite.ts +++ b/yarn-project/pxe/src/database/pxe_database_test_suite.ts @@ -5,17 +5,16 @@ import { INITIAL_L2_BLOCK_NUM, PublicKeys, SerializableContractInstance, - computePoint, } from '@aztec/circuits.js'; import { makeHeader } from '@aztec/circuits.js/testing'; +import { FunctionType } from '@aztec/foundation/abi'; import { randomInt } from '@aztec/foundation/crypto'; import { Fr, Point } from '@aztec/foundation/fields'; import { BenchmarkingContractArtifact } from '@aztec/noir-contracts.js/Benchmarking'; +import { TestContractArtifact } from '@aztec/noir-contracts.js/Test'; -import { type IncomingNoteDao } from './incoming_note_dao.js'; -import { randomIncomingNoteDao } from './incoming_note_dao.test.js'; -import { type OutgoingNoteDao } from './outgoing_note_dao.js'; -import { randomOutgoingNoteDao } from './outgoing_note_dao.test.js'; +import { IncomingNoteDao } from './incoming_note_dao.js'; +import { OutgoingNoteDao } from './outgoing_note_dao.js'; import { type PxeDatabase } from './pxe_database.js'; /** @@ -102,7 +101,7 @@ export function describePxeDatabase(getDatabase: () => PxeDatabase) { [ () => ({ owner: owners[0].address }), - () => notes.filter(note => note.addressPoint.equals(computePoint(owners[0].address))), + () => notes.filter(note => note.addressPoint.equals(owners[0].address.toAddressPoint())), ], [ @@ -121,11 +120,12 @@ export function describePxeDatabase(getDatabase: () => PxeDatabase) { storageSlots = Array.from({ length: 2 }).map(() => Fr.random()); notes = Array.from({ length: 10 }).map((_, i) => - randomIncomingNoteDao({ + IncomingNoteDao.random({ contractAddress: contractAddresses[i % contractAddresses.length], storageSlot: storageSlots[i % storageSlots.length], - addressPoint: computePoint(owners[i % owners.length].address), + addressPoint: owners[i % owners.length].address.toAddressPoint(), index: BigInt(i), + l2BlockNumber: i, }), ); @@ -156,9 +156,13 @@ export function describePxeDatabase(getDatabase: () => PxeDatabase) { // Nullify all notes and use the same filter as other test cases for (const owner of owners) { - const notesToNullify = notes.filter(note => note.addressPoint.equals(computePoint(owner.address))); - const nullifiers = notesToNullify.map(note => note.siloedNullifier); - await expect(database.removeNullifiedNotes(nullifiers, computePoint(owner.address))).resolves.toEqual( + const notesToNullify = notes.filter(note => note.addressPoint.equals(owner.address.toAddressPoint())); + const nullifiers = notesToNullify.map(note => ({ + data: note.siloedNullifier, + l2BlockNumber: note.l2BlockNumber, + l2BlockHash: note.l2BlockHash, + })); + await expect(database.removeNullifiedNotes(nullifiers, owner.address.toAddressPoint())).resolves.toEqual( notesToNullify, ); } @@ -171,8 +175,12 @@ export function describePxeDatabase(getDatabase: () => PxeDatabase) { it('skips nullified notes by default or when requesting active', async () => { await database.addNotes(notes, []); - const notesToNullify = notes.filter(note => note.addressPoint.equals(computePoint(owners[0].address))); - const nullifiers = notesToNullify.map(note => note.siloedNullifier); + const notesToNullify = notes.filter(note => note.addressPoint.equals(owners[0].address.toAddressPoint())); + const nullifiers = notesToNullify.map(note => ({ + data: note.siloedNullifier, + l2BlockNumber: note.l2BlockNumber, + l2BlockHash: note.l2BlockHash, + })); await expect(database.removeNullifiedNotes(nullifiers, notesToNullify[0].addressPoint)).resolves.toEqual( notesToNullify, ); @@ -184,11 +192,35 @@ export function describePxeDatabase(getDatabase: () => PxeDatabase) { expect(actualNotesWithActive).toEqual(notes.filter(note => !notesToNullify.includes(note))); }); + it('handles note unnullification', async () => { + await database.setHeader(makeHeader(randomInt(1000), 100, 0 /** slot number */)); + await database.addNotes(notes, []); + + const notesToNullify = notes.filter(note => note.addressPoint.equals(owners[0].address.toAddressPoint())); + const nullifiers = notesToNullify.map(note => ({ + data: note.siloedNullifier, + l2BlockNumber: 99, + l2BlockHash: Fr.random().toString(), + })); + await expect(database.removeNullifiedNotes(nullifiers, notesToNullify[0].addressPoint)).resolves.toEqual( + notesToNullify, + ); + await expect(database.unnullifyNotesAfter(98)).resolves.toEqual(undefined); + + const result = await database.getIncomingNotes({ status: NoteStatus.ACTIVE, owner: owners[0].address }); + + expect(result.sort()).toEqual([...notesToNullify].sort()); + }); + it('returns active and nullified notes when requesting either', async () => { await database.addNotes(notes, []); - const notesToNullify = notes.filter(note => note.addressPoint.equals(computePoint(owners[0].address))); - const nullifiers = notesToNullify.map(note => note.siloedNullifier); + const notesToNullify = notes.filter(note => note.addressPoint.equals(owners[0].address.toAddressPoint())); + const nullifiers = notesToNullify.map(note => ({ + data: note.siloedNullifier, + l2BlockNumber: note.l2BlockNumber, + l2BlockHash: note.l2BlockHash, + })); await expect(database.removeNullifiedNotes(nullifiers, notesToNullify[0].addressPoint)).resolves.toEqual( notesToNullify, ); @@ -246,7 +278,16 @@ export function describePxeDatabase(getDatabase: () => PxeDatabase) { ).resolves.toEqual([notes[0]]); await expect( - database.removeNullifiedNotes([notes[0].siloedNullifier], computePoint(owners[0].address)), + database.removeNullifiedNotes( + [ + { + data: notes[0].siloedNullifier, + l2BlockHash: notes[0].l2BlockHash, + l2BlockNumber: notes[0].l2BlockNumber, + }, + ], + owners[0].address.toAddressPoint(), + ), ).resolves.toEqual([notes[0]]); await expect( @@ -260,6 +301,14 @@ export function describePxeDatabase(getDatabase: () => PxeDatabase) { }), ).resolves.toEqual([]); }); + + it('removes notes after a given block', async () => { + await database.addNotes(notes, [], owners[0].address); + + await database.removeNotesAfter(5); + const result = await database.getIncomingNotes({ scopes: [owners[0].address] }); + expect(new Set(result)).toEqual(new Set(notes.slice(0, 6))); + }); }); describe('outgoing notes', () => { @@ -307,7 +356,7 @@ export function describePxeDatabase(getDatabase: () => PxeDatabase) { storageSlots = Array.from({ length: 2 }).map(() => Fr.random()); notes = Array.from({ length: 10 }).map((_, i) => - randomOutgoingNoteDao({ + OutgoingNoteDao.random({ contractAddress: contractAddresses[i % contractAddresses.length], storageSlot: storageSlots[i % storageSlots.length], ovpkM: owners[i % owners.length].publicKeys.masterOutgoingViewingPublicKey, @@ -391,6 +440,19 @@ export function describePxeDatabase(getDatabase: () => PxeDatabase) { await expect(database.getContractArtifact(id)).resolves.toEqual(artifact); }); + it('does not store a contract artifact with a duplicate private function selector', async () => { + const artifact = TestContractArtifact; + const index = artifact.functions.findIndex(fn => fn.functionType === FunctionType.PRIVATE); + + const copiedFn = structuredClone(artifact.functions[index]); + artifact.functions.push(copiedFn); + + const id = Fr.random(); + await expect(database.addContractArtifact(id, artifact)).rejects.toThrow( + 'Repeated function selectors of private functions', + ); + }); + it('stores a contract instance', async () => { const address = AztecAddress.random(); const instance = SerializableContractInstance.random().withAddress(address); diff --git a/yarn-project/pxe/src/kernel_oracle/index.ts b/yarn-project/pxe/src/kernel_oracle/index.ts index d2115d2cc40..a66ec8db465 100644 --- a/yarn-project/pxe/src/kernel_oracle/index.ts +++ b/yarn-project/pxe/src/kernel_oracle/index.ts @@ -70,7 +70,7 @@ export class KernelOracle implements ProvingDataOracle { } async getNoteHashTreeRoot(): Promise { - const header = await this.node.getHeader(this.blockNumber); + const header = await this.node.getBlockHeader(this.blockNumber); return header.state.partial.noteHashTree.root; } diff --git a/yarn-project/pxe/src/note_decryption_utils/produce_note_daos.ts b/yarn-project/pxe/src/note_decryption_utils/produce_note_daos.ts index 8e857d51330..fc3e1918ce1 100644 --- a/yarn-project/pxe/src/note_decryption_utils/produce_note_daos.ts +++ b/yarn-project/pxe/src/note_decryption_utils/produce_note_daos.ts @@ -34,6 +34,8 @@ export async function produceNoteDaos( ovpkM: PublicKey | undefined, payload: L1NotePayload, txHash: TxHash, + l2BlockNumber: number, + l2BlockHash: string, noteHashes: Fr[], dataStartIndexForTx: number, excludedIndices: Set, @@ -56,6 +58,8 @@ export async function produceNoteDaos( addressPoint, payload, txHash, + l2BlockNumber, + l2BlockHash, noteHashes, dataStartIndexForTx, excludedIndices, @@ -74,6 +78,8 @@ export async function produceNoteDaos( incomingNote.storageSlot, incomingNote.noteTypeId, incomingNote.txHash, + incomingNote.l2BlockNumber, + incomingNote.l2BlockHash, incomingNote.nonce, incomingNote.noteHash, incomingNote.index, @@ -86,6 +92,8 @@ export async function produceNoteDaos( ovpkM, payload, txHash, + l2BlockNumber, + l2BlockHash, noteHashes, dataStartIndexForTx, excludedIndices, diff --git a/yarn-project/pxe/src/note_decryption_utils/produce_note_daos_for_key.ts b/yarn-project/pxe/src/note_decryption_utils/produce_note_daos_for_key.ts index 9e530b387d1..eeeb6c9ee9e 100644 --- a/yarn-project/pxe/src/note_decryption_utils/produce_note_daos_for_key.ts +++ b/yarn-project/pxe/src/note_decryption_utils/produce_note_daos_for_key.ts @@ -13,6 +13,8 @@ export async function produceNoteDaosForKey( pkM: PublicKey, payload: L1NotePayload, txHash: TxHash, + l2BlockNumber: number, + l2BlockHash: string, noteHashes: Fr[], dataStartIndexForTx: number, excludedIndices: Set, @@ -21,6 +23,8 @@ export async function produceNoteDaosForKey( note: Note, payload: L1NotePayload, noteInfo: NoteInfo, + l2BlockNumber: number, + l2BlockHash: string, dataStartIndexForTx: number, pkM: PublicKey, ) => T, @@ -44,7 +48,7 @@ export async function produceNoteDaosForKey( ); excludedIndices?.add(noteInfo.noteHashIndex); - noteDao = daoConstructor(note, payload, noteInfo, dataStartIndexForTx, pkM); + noteDao = daoConstructor(note, payload, noteInfo, l2BlockNumber, l2BlockHash, dataStartIndexForTx, pkM); } catch (e) { logger.error(`Could not process note because of "${e}". Discarding note...`); } diff --git a/yarn-project/pxe/src/pxe_service/create_pxe_service.ts b/yarn-project/pxe/src/pxe_service/create_pxe_service.ts index 3fa9abe5b1e..a3bdd43b105 100644 --- a/yarn-project/pxe/src/pxe_service/create_pxe_service.ts +++ b/yarn-project/pxe/src/pxe_service/create_pxe_service.ts @@ -3,6 +3,7 @@ import { type AztecNode, type PrivateKernelProver } from '@aztec/circuit-types'; import { randomBytes } from '@aztec/foundation/crypto'; import { createDebugLogger } from '@aztec/foundation/log'; import { KeyStore } from '@aztec/key-store'; +import { L2TipsStore } from '@aztec/kv-store/stores'; import { createStore } from '@aztec/kv-store/utils'; import { type PXEServiceConfig } from '../config/index.js'; @@ -39,12 +40,14 @@ export async function createPXEService( const keyStore = new KeyStore( await createStore('pxe_key_store', configWithContracts, createDebugLogger('aztec:pxe:keystore:lmdb')), ); - const db = new KVPxeDatabase( - await createStore('pxe_data', configWithContracts, createDebugLogger('aztec:pxe:data:lmdb')), - ); + + const store = await createStore('pxe_data', configWithContracts, createDebugLogger('aztec:pxe:data:lmdb')); + + const db = new KVPxeDatabase(store); + const tips = new L2TipsStore(store, 'pxe'); const prover = proofCreator ?? (await createProver(config, logSuffix)); - const server = new PXEService(keyStore, aztecNode, db, prover, config, logSuffix); + const server = new PXEService(keyStore, aztecNode, db, tips, prover, config, logSuffix); await server.start(); return server; } diff --git a/yarn-project/pxe/src/pxe_service/pxe_service.ts b/yarn-project/pxe/src/pxe_service/pxe_service.ts index 62e96081780..b7ed140aa73 100644 --- a/yarn-project/pxe/src/pxe_service/pxe_service.ts +++ b/yarn-project/pxe/src/pxe_service/pxe_service.ts @@ -6,6 +6,7 @@ import { type ExtendedNote, type FunctionCall, type GetUnencryptedLogsResponse, + type InBlock, type IncomingNotesFilter, L1EventPayload, type L2Block, @@ -22,7 +23,6 @@ import { type SiblingPath, SimulationError, type Tx, - type TxEffect, type TxExecutionRequest, type TxHash, TxProvingResult, @@ -43,7 +43,6 @@ import { computeAddressSecret, computeContractAddressFromInstance, computeContractClassId, - computePoint, getContractClassFromArtifact, } from '@aztec/circuits.js'; import { computeNoteHashNonce, siloNullifier } from '@aztec/circuits.js/hash'; @@ -58,6 +57,7 @@ import { Fr, type Point } from '@aztec/foundation/fields'; import { type DebugLogger, createDebugLogger } from '@aztec/foundation/log'; import { SerialQueue } from '@aztec/foundation/queue'; import { type KeyStore } from '@aztec/key-store'; +import { type L2TipsStore } from '@aztec/kv-store/stores'; import { ProtocolContractAddress, getCanonicalProtocolContract, @@ -93,12 +93,13 @@ export class PXEService implements PXE { private keyStore: KeyStore, private node: AztecNode, private db: PxeDatabase, + tipsStore: L2TipsStore, private proofCreator: PrivateKernelProver, - private config: PXEServiceConfig, + config: PXEServiceConfig, logSuffix?: string, ) { this.log = createDebugLogger(logSuffix ? `aztec:pxe_service_${logSuffix}` : `aztec:pxe_service`); - this.synchronizer = new Synchronizer(node, db, this.jobQueue, logSuffix); + this.synchronizer = new Synchronizer(node, db, tipsStore, config, logSuffix); this.contractDataOracle = new ContractDataOracle(db); this.simulator = getAcirSimulator(db, node, keyStore, this.contractDataOracle); this.packageVersion = getPackageInfo().version; @@ -112,8 +113,7 @@ export class PXEService implements PXE { * @returns A promise that resolves when the server has started successfully. */ public async start() { - const { l2BlockPollingIntervalMS } = this.config; - await this.synchronizer.start(1, l2BlockPollingIntervalMS); + await this.synchronizer.start(); await this.#registerProtocolContracts(); const info = await this.getNodeInfo(); this.log.info(`Started PXE connected to chain ${info.l1ChainId} version ${info.protocolVersion}`); @@ -242,21 +242,24 @@ export class PXEService implements PXE { if (artifact) { // If the user provides an artifact, validate it against the expected class id and register it - const contractClassId = computeContractClassId(getContractClassFromArtifact(artifact)); + const contractClass = getContractClassFromArtifact(artifact); + const contractClassId = computeContractClassId(contractClass); if (!contractClassId.equals(instance.contractClassId)) { throw new Error( `Artifact does not match expected class id (computed ${contractClassId} but instance refers to ${instance.contractClassId})`, ); } - if ( - // Computed address from the instance does not match address inside instance - !computeContractAddressFromInstance(instance).equals(instance.address) - ) { + if (!computeContractAddressFromInstance(instance).equals(instance.address)) { throw new Error('Added a contract in which the address does not match the contract instance.'); } await this.db.addContractArtifact(contractClassId, artifact); + + // TODO: PXE may not want to broadcast the artifact to the network await this.node.addContractArtifact(instance.address, artifact); + + // TODO(#10007): Node should get public contract class from the registration event, not from PXE registration + await this.node.addContractClass({ ...contractClass, privateFunctions: [], unconstrainedFunctions: [] }); } else { // Otherwise, make sure there is an artifact already registered for that class id artifact = await this.db.getContractArtifact(instance.contractClassId); @@ -289,7 +292,7 @@ export class PXEService implements PXE { let owner = filter.owner; if (owner === undefined) { const completeAddresses = (await this.db.getCompleteAddresses()).find(completeAddress => - computePoint(completeAddress.address).equals(dao.addressPoint), + completeAddress.address.toAddressPoint().equals(dao.addressPoint), ); if (completeAddresses === undefined) { throw new Error(`Cannot find complete address for addressPoint ${dao.addressPoint.toString()}`); @@ -350,7 +353,7 @@ export class PXEService implements PXE { throw new Error(`Unknown account: ${note.owner.toString()}`); } - const nonces = await this.#getNoteNonces(note); + const { data: nonces, l2BlockNumber, l2BlockHash } = await this.#getNoteNonces(note); if (nonces.length === 0) { throw new Error(`Cannot find the note in tx: ${note.txHash}.`); } @@ -385,11 +388,13 @@ export class PXEService implements PXE { note.storageSlot, note.noteTypeId, note.txHash, + l2BlockNumber, + l2BlockHash, nonce, noteHash, siloedNullifier, index, - computePoint(owner.address), + owner.address.toAddressPoint(), ), scope, ); @@ -397,7 +402,7 @@ export class PXEService implements PXE { } public async addNullifiedNote(note: ExtendedNote) { - const nonces = await this.#getNoteNonces(note); + const { data: nonces, l2BlockHash, l2BlockNumber } = await this.#getNoteNonces(note); if (nonces.length === 0) { throw new Error(`Cannot find the note in tx: ${note.txHash}.`); } @@ -428,11 +433,13 @@ export class PXEService implements PXE { note.storageSlot, note.noteTypeId, note.txHash, + l2BlockNumber, + l2BlockHash, nonce, noteHash, Fr.ZERO, // We are not able to derive index, - computePoint(note.owner), + note.owner.toAddressPoint(), ), ); } @@ -444,15 +451,15 @@ export class PXEService implements PXE { * @returns The nonces of the note. * @remarks More than a single nonce may be returned since there might be more than one nonce for a given note. */ - async #getNoteNonces(note: ExtendedNote): Promise { + async #getNoteNonces(note: ExtendedNote): Promise> { const tx = await this.node.getTxEffect(note.txHash); if (!tx) { throw new Error(`Unknown tx: ${note.txHash}`); } const nonces: Fr[] = []; - const firstNullifier = tx.nullifiers[0]; - const hashes = tx.noteHashes; + const firstNullifier = tx.data.nullifiers[0]; + const hashes = tx.data.noteHashes; for (let i = 0; i < hashes.length; ++i) { const hash = hashes[i]; if (hash.equals(Fr.ZERO)) { @@ -473,7 +480,7 @@ export class PXEService implements PXE { } } - return nonces; + return { l2BlockHash: tx.l2BlockHash, l2BlockNumber: tx.l2BlockNumber, data: nonces }; } public async getBlock(blockNumber: number): Promise { @@ -496,10 +503,19 @@ export class PXEService implements PXE { txRequest: TxExecutionRequest, privateExecutionResult: PrivateExecutionResult, ): Promise { - return this.jobQueue.put(async () => { - const { publicInputs, clientIvcProof } = await this.#prove(txRequest, this.proofCreator, privateExecutionResult); - return new TxProvingResult(privateExecutionResult, publicInputs, clientIvcProof!); - }); + return this.jobQueue + .put(async () => { + const { publicInputs, clientIvcProof } = await this.#prove( + txRequest, + this.proofCreator, + privateExecutionResult, + ); + return new TxProvingResult(privateExecutionResult, publicInputs, clientIvcProof!); + }) + .catch(err => { + this.log.error(err); + throw err; + }); } // TODO(#7456) Prevent msgSender being defined here for the first call @@ -511,47 +527,52 @@ export class PXEService implements PXE { profile: boolean = false, scopes?: AztecAddress[], ): Promise { - return await this.jobQueue.put(async () => { - const privateExecutionResult = await this.#executePrivate(txRequest, msgSender, scopes); - - let publicInputs: PrivateKernelTailCircuitPublicInputs; - let profileResult; - if (profile) { - ({ publicInputs, profileResult } = await this.#profileKernelProver( - txRequest, - this.proofCreator, - privateExecutionResult, - )); - } else { - publicInputs = await this.#simulateKernels(txRequest, privateExecutionResult); - } + return await this.jobQueue + .put(async () => { + const privateExecutionResult = await this.#executePrivate(txRequest, msgSender, scopes); + + let publicInputs: PrivateKernelTailCircuitPublicInputs; + let profileResult; + if (profile) { + ({ publicInputs, profileResult } = await this.#profileKernelProver( + txRequest, + this.proofCreator, + privateExecutionResult, + )); + } else { + publicInputs = await this.#simulateKernels(txRequest, privateExecutionResult); + } - const privateSimulationResult = new PrivateSimulationResult(privateExecutionResult, publicInputs); - const simulatedTx = privateSimulationResult.toSimulatedTx(); - let publicOutput: PublicSimulationOutput | undefined; - if (simulatePublic) { - publicOutput = await this.#simulatePublicCalls(simulatedTx); - } + const privateSimulationResult = new PrivateSimulationResult(privateExecutionResult, publicInputs); + const simulatedTx = privateSimulationResult.toSimulatedTx(); + let publicOutput: PublicSimulationOutput | undefined; + if (simulatePublic) { + publicOutput = await this.#simulatePublicCalls(simulatedTx); + } - if (!skipTxValidation) { - if (!(await this.node.isValidTx(simulatedTx, true))) { - throw new Error('The simulated transaction is unable to be added to state and is invalid.'); + if (!skipTxValidation) { + if (!(await this.node.isValidTx(simulatedTx, true))) { + throw new Error('The simulated transaction is unable to be added to state and is invalid.'); + } } - } - // We log only if the msgSender is undefined, as simulating with a different msgSender - // is unlikely to be a real transaction, and likely to be only used to read data. - // Meaning that it will not necessarily have produced a nullifier (and thus have no TxHash) - // If we log, the `getTxHash` function will throw. - if (!msgSender) { - this.log.info(`Executed local simulation for ${simulatedTx.getTxHash()}`); - } - return TxSimulationResult.fromPrivateSimulationResultAndPublicOutput( - privateSimulationResult, - publicOutput, - profileResult, - ); - }); + // We log only if the msgSender is undefined, as simulating with a different msgSender + // is unlikely to be a real transaction, and likely to be only used to read data. + // Meaning that it will not necessarily have produced a nullifier (and thus have no TxHash) + // If we log, the `getTxHash` function will throw. + if (!msgSender) { + this.log.info(`Executed local simulation for ${simulatedTx.getTxHash()}`); + } + return TxSimulationResult.fromPrivateSimulationResultAndPublicOutput( + privateSimulationResult, + publicOutput, + profileResult, + ); + }) + .catch(err => { + this.log.error(err); + throw err; + }); } public async sendTx(tx: Tx): Promise { @@ -560,7 +581,10 @@ export class PXEService implements PXE { throw new Error(`A settled tx with equal hash ${txHash.toString()} exists.`); } this.log.info(`Sending transaction ${txHash}`); - await this.node.sendTx(tx); + await this.node.sendTx(tx).catch(err => { + this.log.error(err); + throw err; + }); this.log.info(`Sent transaction ${txHash}`); return txHash; } @@ -573,21 +597,26 @@ export class PXEService implements PXE { scopes?: AztecAddress[], ): Promise { // all simulations must be serialized w.r.t. the synchronizer - return await this.jobQueue.put(async () => { - // TODO - Should check if `from` has the permission to call the view function. - const functionCall = await this.#getFunctionCall(functionName, args, to); - const executionResult = await this.#simulateUnconstrained(functionCall, scopes); - - // TODO - Return typed result based on the function artifact. - return executionResult; - }); + return await this.jobQueue + .put(async () => { + // TODO - Should check if `from` has the permission to call the view function. + const functionCall = await this.#getFunctionCall(functionName, args, to); + const executionResult = await this.#simulateUnconstrained(functionCall, scopes); + + // TODO - Return typed result based on the function artifact. + return executionResult; + }) + .catch(err => { + this.log.error(err); + throw err; + }); } public getTxReceipt(txHash: TxHash): Promise { return this.node.getTxReceipt(txHash); } - public getTxEffect(txHash: TxHash): Promise { + public getTxEffect(txHash: TxHash) { return this.node.getTxEffect(txHash); } @@ -773,7 +802,11 @@ export class PXEService implements PXE { return result; } catch (err) { if (err instanceof SimulationError) { - await enrichPublicSimulationError(err, this.contractDataOracle, this.db, this.log); + try { + await enrichPublicSimulationError(err, this.contractDataOracle, this.db, this.log); + } catch (enrichErr) { + this.log.error(`Failed to enrich public simulation error: ${enrichErr}`); + } } throw err; } @@ -944,4 +977,8 @@ export class PXEService implements PXE { return decodedEvents; } + + async resetNoteSyncData() { + return await this.db.resetNoteSyncData(); + } } diff --git a/yarn-project/pxe/src/pxe_service/test/pxe_service.test.ts b/yarn-project/pxe/src/pxe_service/test/pxe_service.test.ts index c4fd6d2a627..4acbc87e2a4 100644 --- a/yarn-project/pxe/src/pxe_service/test/pxe_service.test.ts +++ b/yarn-project/pxe/src/pxe_service/test/pxe_service.test.ts @@ -1,8 +1,9 @@ -import { type AztecNode, type PXE, TxEffect, mockTx } from '@aztec/circuit-types'; +import { type AztecNode, type PXE, TxEffect, mockTx, randomInBlock } from '@aztec/circuit-types'; import { INITIAL_L2_BLOCK_NUM } from '@aztec/circuits.js/constants'; import { type L1ContractAddresses } from '@aztec/ethereum'; import { EthAddress } from '@aztec/foundation/eth-address'; import { KeyStore } from '@aztec/key-store'; +import { L2TipsStore } from '@aztec/kv-store/stores'; import { openTmpStore } from '@aztec/kv-store/utils'; import { type MockProxy, mock } from 'jest-mock-extended'; @@ -19,6 +20,7 @@ function createPXEService(): Promise { const keyStore = new KeyStore(kvStore); const node = mock(); const db = new KVPxeDatabase(kvStore); + const tips = new L2TipsStore(kvStore, 'pxe'); const config: PXEServiceConfig = { l2BlockPollingIntervalMS: 100, l2StartingBlock: INITIAL_L2_BLOCK_NUM, @@ -45,7 +47,7 @@ function createPXEService(): Promise { }; node.getL1ContractAddresses.mockResolvedValue(mockedContracts); - return Promise.resolve(new PXEService(keyStore, node, db, new TestPrivateKernelProver(), config)); + return Promise.resolve(new PXEService(keyStore, node, db, tips, new TestPrivateKernelProver(), config)); } pxeTestSuite('PXEService', createPXEService); @@ -55,11 +57,13 @@ describe('PXEService', () => { let node: MockProxy; let db: PxeDatabase; let config: PXEServiceConfig; + let tips: L2TipsStore; beforeEach(() => { const kvStore = openTmpStore(); keyStore = new KeyStore(kvStore); node = mock(); + tips = new L2TipsStore(kvStore, 'pxe'); db = new KVPxeDatabase(kvStore); config = { l2BlockPollingIntervalMS: 100, @@ -75,9 +79,9 @@ describe('PXEService', () => { const settledTx = TxEffect.random(); const duplicateTx = mockTx(); - node.getTxEffect.mockResolvedValue(settledTx); + node.getTxEffect.mockResolvedValue(randomInBlock(settledTx)); - const pxe = new PXEService(keyStore, node, db, new TestPrivateKernelProver(), config); + const pxe = new PXEService(keyStore, node, db, tips, new TestPrivateKernelProver(), config); await expect(pxe.sendTx(duplicateTx)).rejects.toThrow(/A settled tx with equal hash/); }); }); diff --git a/yarn-project/pxe/src/simulator_oracle/index.ts b/yarn-project/pxe/src/simulator_oracle/index.ts index 73cf685b016..a7ca08660f2 100644 --- a/yarn-project/pxe/src/simulator_oracle/index.ts +++ b/yarn-project/pxe/src/simulator_oracle/index.ts @@ -1,5 +1,6 @@ import { type AztecNode, + type InBlock, L1NotePayload, type L2Block, type L2BlockNumber, @@ -22,7 +23,6 @@ import { type KeyValidationRequest, type L1_TO_L2_MSG_TREE_HEIGHT, computeAddressSecret, - computePoint, computeTaggingSecret, } from '@aztec/circuits.js'; import { type FunctionArtifact, getFunctionArtifact } from '@aztec/foundation/abi'; @@ -268,8 +268,11 @@ export class SimulatorOracle implements DBOracle { sender: AztecAddress, recipient: AztecAddress, ): Promise { + await this.syncTaggedLogsAsSender(contractAddress, sender, recipient); + const secret = await this.#calculateTaggingSecret(contractAddress, sender, recipient); const [index] = await this.db.getTaggingSecretsIndexesAsSender([secret]); + return new IndexedTaggingSecret(secret, index); } @@ -289,7 +292,9 @@ export class SimulatorOracle implements DBOracle { this.log.verbose( `Incrementing secret ${secret} as sender ${sender} for recipient: ${recipient} at contract: ${contractName}(${contractAddress})`, ); - await this.db.incrementTaggingSecretsIndexesAsSender([secret]); + + const [index] = await this.db.getTaggingSecretsIndexesAsSender([secret]); + await this.db.setTaggingSecretsIndexesAsSender([new IndexedTaggingSecret(secret, index + 1)]); } async #calculateTaggingSecret(contractAddress: AztecAddress, sender: AztecAddress, recipient: AztecAddress) { @@ -329,6 +334,70 @@ export class SimulatorOracle implements DBOracle { return appTaggingSecrets.map((secret, i) => new IndexedTaggingSecret(secret, indexes[i])); } + /** + * Updates the local index of the shared tagging secret of a sender / recipient pair + * if a log with a larger index is found from the node. + * @param contractAddress - The address of the contract that the logs are tagged for + * @param sender - The address of the sender, we must know the sender's ivsk_m. + * @param recipient - The address of the recipient. + */ + public async syncTaggedLogsAsSender( + contractAddress: AztecAddress, + sender: AztecAddress, + recipient: AztecAddress, + ): Promise { + const appTaggingSecret = await this.#calculateTaggingSecret(contractAddress, sender, recipient); + let [currentIndex] = await this.db.getTaggingSecretsIndexesAsSender([appTaggingSecret]); + + const INDEX_OFFSET = 10; + + let previousEmptyBack = 0; + let currentEmptyBack = 0; + let currentEmptyFront: number; + + // The below code is trying to find the index of the start of the first window in which for all elements of window, we do not see logs. + // We take our window size, and fetch the node for these logs. We store both the amount of empty consecutive slots from the front and the back. + // We use our current empty consecutive slots from the front, as well as the previous consecutive empty slots from the back to see if we ever hit a time where there + // is a window in which we see the combination of them to be greater than the window's size. If true, we rewind current index to the start of said window and use it. + // Assuming two windows of 5: + // [0, 1, 0, 1, 0], [0, 0, 0, 0, 0] + // We can see that when processing the second window, the previous amount of empty slots from the back of the window (1), added with the empty elements from the front of the window (5) + // is greater than 5 (6) and therefore we have found a window to use. + // We simply need to take the number of elements (10) - the size of the window (5) - the number of consecutive empty elements from the back of the last window (1) = 4; + // This is the first index of our desired window. + // Note that if we ever see a situation like so: + // [0, 1, 0, 1, 0], [0, 0, 0, 0, 1] + // This also returns the correct index (4), but this is indicative of a problem / desync. i.e. we should never have a window that has a log that exists after the window. + + do { + const currentTags = [...new Array(INDEX_OFFSET)].map((_, i) => { + const indexedAppTaggingSecret = new IndexedTaggingSecret(appTaggingSecret, currentIndex + i); + return indexedAppTaggingSecret.computeTag(recipient); + }); + previousEmptyBack = currentEmptyBack; + + const possibleLogs = await this.aztecNode.getLogsByTags(currentTags); + + const indexOfFirstLog = possibleLogs.findIndex(possibleLog => possibleLog.length !== 0); + currentEmptyFront = indexOfFirstLog === -1 ? INDEX_OFFSET : indexOfFirstLog; + + const indexOfLastLog = possibleLogs.findLastIndex(possibleLog => possibleLog.length !== 0); + currentEmptyBack = indexOfLastLog === -1 ? INDEX_OFFSET : INDEX_OFFSET - 1 - indexOfLastLog; + + currentIndex += INDEX_OFFSET; + } while (currentEmptyFront + previousEmptyBack < INDEX_OFFSET); + + // We unwind the entire current window and the amount of consecutive empty slots from the previous window + const newIndex = currentIndex - (INDEX_OFFSET + previousEmptyBack); + + await this.db.setTaggingSecretsIndexesAsSender([new IndexedTaggingSecret(appTaggingSecret, newIndex)]); + + const contractName = await this.contractDataOracle.getDebugContractName(contractAddress); + this.log.debug( + `Syncing logs for sender ${sender}, secret ${appTaggingSecret}:${currentIndex} at contract: ${contractName}(${contractAddress})`, + ); + } + /** * Synchronizes the logs tagged with scoped addresses and all the senders in the addressbook. * Returns the unsynched logs and updates the indexes of the secrets used to tag them until there are no more logs to sync. @@ -441,7 +510,12 @@ export class SimulatorOracle implements DBOracle { result.set( recipient.toString(), - logs.filter(log => log.blockNumber <= maxBlockNumber), + // Remove logs with a block number higher than the max block number + // Duplicates are likely to happen due to the sliding window, so we also filter them out + logs.filter( + (log, index, self) => + log.blockNumber <= maxBlockNumber && index === self.findIndex(otherLog => otherLog.equals(log)), + ), ); } return result; @@ -469,7 +543,7 @@ export class SimulatorOracle implements DBOracle { const incomingNotes: IncomingNoteDao[] = []; const outgoingNotes: OutgoingNoteDao[] = []; - const txEffectsCache = new Map(); + const txEffectsCache = new Map | undefined>(); for (const scopedLog of scopedLogs) { const incomingNotePayload = L1NotePayload.decryptAsIncoming( @@ -510,11 +584,13 @@ export class SimulatorOracle implements DBOracle { // mocking ESM exports, we have to pollute the method even more by providing a simulator parameter so tests can inject a fake one. simulator ?? getAcirSimulator(this.db, this.aztecNode, this.keyStore, this.contractDataOracle), this.db, - incomingNotePayload ? computePoint(recipient) : undefined, + incomingNotePayload ? recipient.toAddressPoint() : undefined, outgoingNotePayload ? recipientCompleteAddress.publicKeys.masterOutgoingViewingPublicKey : undefined, payload!, - txEffect.txHash, - txEffect.noteHashes, + txEffect.data.txHash, + txEffect.l2BlockNumber, + txEffect.l2BlockHash, + txEffect.data.noteHashes, scopedLog.dataStartIndexForTx, excludedIndices.get(scopedLog.txHash.toString())!, this.log, @@ -557,17 +633,19 @@ export class SimulatorOracle implements DBOracle { } const nullifiedNotes: IncomingNoteDao[] = []; const currentNotesForRecipient = await this.db.getIncomingNotes({ owner: recipient }); - const nullifierIndexes = await this.aztecNode.findLeavesIndexes( - 'latest', - MerkleTreeId.NULLIFIER_TREE, - currentNotesForRecipient.map(note => note.siloedNullifier), - ); - - const foundNullifiers = currentNotesForRecipient - .filter((_, i) => nullifierIndexes[i] !== undefined) - .map(note => note.siloedNullifier); + const nullifiersToCheck = currentNotesForRecipient.map(note => note.siloedNullifier); + const currentBlockNumber = await this.getBlockNumber(); + const nullifierIndexes = await this.aztecNode.findNullifiersIndexesWithBlock(currentBlockNumber, nullifiersToCheck); + + const foundNullifiers = nullifiersToCheck + .map((nullifier, i) => { + if (nullifierIndexes[i] !== undefined) { + return { ...nullifierIndexes[i], ...{ data: nullifier } } as InBlock; + } + }) + .filter(nullifier => nullifier !== undefined) as InBlock[]; - await this.db.removeNullifiedNotes(foundNullifiers, computePoint(recipient)); + await this.db.removeNullifiedNotes(foundNullifiers, recipient.toAddressPoint()); nullifiedNotes.forEach(noteDao => { this.log.verbose( `Removed note for contract ${noteDao.contractAddress} at slot ${ diff --git a/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts b/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts index f4630610b6e..388253c9005 100644 --- a/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts +++ b/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts @@ -7,6 +7,7 @@ import { type TxEffect, TxHash, TxScopedL2Log, + randomInBlock, } from '@aztec/circuit-types'; import { AztecAddress, @@ -37,7 +38,7 @@ import { type PxeDatabase } from '../database/index.js'; import { KVPxeDatabase } from '../database/kv_pxe_database.js'; import { type OutgoingNoteDao } from '../database/outgoing_note_dao.js'; import { ContractDataOracle } from '../index.js'; -import { type SimulatorOracle } from './index.js'; +import { SimulatorOracle } from './index.js'; const TXS_PER_BLOCK = 4; const NUM_NOTE_HASHES_PER_BLOCK = TXS_PER_BLOCK * MAX_NOTE_HASHES_PER_TX; @@ -129,8 +130,7 @@ describe('Simulator oracle', () => { contractDataOracle = new ContractDataOracle(database); jest.spyOn(contractDataOracle, 'getDebugContractName').mockImplementation(() => Promise.resolve('TestContract')); keyStore = new KeyStore(db); - const simulatorOracleModule = await import('../simulator_oracle/index.js'); - simulatorOracle = new simulatorOracleModule.SimulatorOracle(contractDataOracle, database, keyStore, aztecNode); + simulatorOracle = new SimulatorOracle(contractDataOracle, database, keyStore, aztecNode); // Set up contract address contractAddress = AztecAddress.random(); // Set up recipient account @@ -143,7 +143,7 @@ describe('Simulator oracle', () => { describe('sync tagged logs', () => { const NUM_SENDERS = 10; const SENDER_OFFSET_WINDOW_SIZE = 10; - let senders: { completeAddress: CompleteAddress; ivsk: Fq }[]; + let senders: { completeAddress: CompleteAddress; ivsk: Fq; secretKey: Fr }[]; function generateMockLogs(senderOffset: number) { const logs: { [k: string]: TxScopedL2Log[] } = {}; @@ -231,7 +231,7 @@ describe('Simulator oracle', () => { const partialAddress = Fr.random(); const address = computeAddress(keys.publicKeys, partialAddress); const completeAddress = new CompleteAddress(address, keys.publicKeys, partialAddress); - return { completeAddress, ivsk: keys.masterIncomingViewingSecretKey }; + return { completeAddress, ivsk: keys.masterIncomingViewingSecretKey, secretKey: new Fr(index) }; }); for (const sender of senders) { await database.addContactAddress(sender.completeAddress.address); @@ -267,6 +267,59 @@ describe('Simulator oracle', () => { expect(aztecNode.getLogsByTags.mock.calls.length).toBe(2 + SENDER_OFFSET_WINDOW_SIZE); }); + it('should sync tagged logs as senders', async () => { + for (const sender of senders) { + await database.addCompleteAddress(sender.completeAddress); + await keyStore.addAccount(sender.secretKey, sender.completeAddress.partialAddress); + } + + let senderOffset = 0; + generateMockLogs(senderOffset); + + // Recompute the secrets (as recipient) to ensure indexes are updated + const ivsk = await keyStore.getMasterIncomingViewingSecretKey(recipient.address); + const secrets = senders.map(sender => { + const firstSenderSharedSecret = computeTaggingSecret(recipient, ivsk, sender.completeAddress.address); + return poseidon2Hash([firstSenderSharedSecret.x, firstSenderSharedSecret.y, contractAddress]); + }); + + const indexesAsSender = await database.getTaggingSecretsIndexesAsSender(secrets); + expect(indexesAsSender).toStrictEqual([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + + expect(aztecNode.getLogsByTags.mock.calls.length).toBe(0); + + for (let i = 0; i < senders.length; i++) { + await simulatorOracle.syncTaggedLogsAsSender( + contractAddress, + senders[i].completeAddress.address, + recipient.address, + ); + } + + let indexesAsSenderAfterSync = await database.getTaggingSecretsIndexesAsSender(secrets); + expect(indexesAsSenderAfterSync).toStrictEqual([1, 1, 1, 1, 1, 2, 2, 2, 2, 2]); + + // Two windows are fetch for each sender + expect(aztecNode.getLogsByTags.mock.calls.length).toBe(NUM_SENDERS * 2); + aztecNode.getLogsByTags.mockReset(); + + // We add more logs at the end of the window to make sure we only detect them and bump the indexes if it lies within our window + senderOffset = 10; + generateMockLogs(senderOffset); + for (let i = 0; i < senders.length; i++) { + await simulatorOracle.syncTaggedLogsAsSender( + contractAddress, + senders[i].completeAddress.address, + recipient.address, + ); + } + + indexesAsSenderAfterSync = await database.getTaggingSecretsIndexesAsSender(secrets); + expect(indexesAsSenderAfterSync).toStrictEqual([11, 11, 11, 11, 11, 12, 12, 12, 12, 12]); + + expect(aztecNode.getLogsByTags.mock.calls.length).toBe(NUM_SENDERS * 2); + }); + it('should sync tagged logs with a sender index offset', async () => { const senderOffset = 5; generateMockLogs(senderOffset); @@ -290,7 +343,7 @@ describe('Simulator oracle', () => { expect(indexes).toEqual([6, 6, 6, 6, 6, 7, 7, 7, 7, 7]); // We should have called the node 17 times: - // 5 times with no results (sender offset) + 2 times with logs (slide the window) + 10 times with no results (window size) + // 5 times with no results (sender offset) + 2 times with logs (sliding the window) + 10 times with no results (window size) expect(aztecNode.getLogsByTags.mock.calls.length).toBe(5 + 2 + SENDER_OFFSET_WINDOW_SIZE); }); @@ -305,19 +358,29 @@ describe('Simulator oracle', () => { return poseidon2Hash([firstSenderSharedSecret.x, firstSenderSharedSecret.y, contractAddress]); }); + // Increase our indexes to 2 await database.setTaggingSecretsIndexesAsRecipient(secrets.map(secret => new IndexedTaggingSecret(secret, 2))); const syncedLogs = await simulatorOracle.syncTaggedLogs(contractAddress, 3); - // Even if our index as recipient is higher than what the recipient sent, we should be able to find the logs + // Even if our index as recipient is higher than what the sender sent, we should be able to find the logs + // since the window starts at Math.max(0, 2 - window_size) = 0 expect(syncedLogs.get(recipient.address.toString())).toHaveLength(NUM_SENDERS + 1 + NUM_SENDERS / 2); + // First sender should have 2 logs, but keep index 2 since they were built using the same tag + // Next 4 senders hould also have index 2 = offset + 1 + // Last 5 senders should have index 3 = offset + 2 + const indexes = await database.getTaggingSecretsIndexesAsRecipient(secrets); + + expect(indexes).toHaveLength(NUM_SENDERS); + expect(indexes).toEqual([2, 2, 2, 2, 2, 3, 3, 3, 3, 3]); + // We should have called the node 13 times: // 1 time without logs + 2 times with logs (sliding the window) + 10 times with no results (window size) expect(aztecNode.getLogsByTags.mock.calls.length).toBe(3 + SENDER_OFFSET_WINDOW_SIZE); }); - it("should sync not tagged logs for which indexes are not updated if they're outside the window", async () => { + it("should not sync tagged logs for which indexes are not updated if they're outside the window", async () => { const senderOffset = 0; generateMockLogs(senderOffset); @@ -334,14 +397,62 @@ describe('Simulator oracle', () => { const syncedLogs = await simulatorOracle.syncTaggedLogs(contractAddress, 3); - // Only half of the logs should be synced since we start from index 1 = offset + 1, the other half should be skipped + // Only half of the logs should be synced since we start from index 1 = (11 - window_size), the other half should be skipped expect(syncedLogs.get(recipient.address.toString())).toHaveLength(NUM_SENDERS / 2); + // Indexes should remain where we set them (window_size + 1) + const indexes = await database.getTaggingSecretsIndexesAsRecipient(secrets); + + expect(indexes).toHaveLength(NUM_SENDERS); + expect(indexes).toEqual([11, 11, 11, 11, 11, 11, 11, 11, 11, 11]); + // We should have called the node SENDER_OFFSET_WINDOW_SIZE + 1 (with logs) + SENDER_OFFSET_WINDOW_SIZE: // Once for index 1 (NUM_SENDERS/2 logs) + 2 times the sliding window (no logs each time) expect(aztecNode.getLogsByTags.mock.calls.length).toBe(1 + 2 * SENDER_OFFSET_WINDOW_SIZE); }); + it('should sync tagged logs from scratch after a DB wipe', async () => { + const senderOffset = 0; + generateMockLogs(senderOffset); + + // Recompute the secrets (as recipient) to update indexes + const ivsk = await keyStore.getMasterIncomingViewingSecretKey(recipient.address); + const secrets = senders.map(sender => { + const firstSenderSharedSecret = computeTaggingSecret(recipient, ivsk, sender.completeAddress.address); + return poseidon2Hash([firstSenderSharedSecret.x, firstSenderSharedSecret.y, contractAddress]); + }); + + await database.setTaggingSecretsIndexesAsRecipient( + secrets.map(secret => new IndexedTaggingSecret(secret, SENDER_OFFSET_WINDOW_SIZE + 2)), + ); + + let syncedLogs = await simulatorOracle.syncTaggedLogs(contractAddress, 3); + + // No logs should be synced since we start from index 2 = 12 - window_size + expect(syncedLogs.get(recipient.address.toString())).toHaveLength(0); + // We should have called the node 21 times (window size + current_index + window size) + expect(aztecNode.getLogsByTags.mock.calls.length).toBe(2 * SENDER_OFFSET_WINDOW_SIZE + 1); + + aztecNode.getLogsByTags.mockClear(); + + // Wipe the database + await database.resetNoteSyncData(); + + syncedLogs = await simulatorOracle.syncTaggedLogs(contractAddress, 3); + + // First sender should have 2 logs, but keep index 1 since they were built using the same tag + // Next 4 senders hould also have index 1 = offset + 1 + // Last 5 senders should have index 2 = offset + 2 + const indexes = await database.getTaggingSecretsIndexesAsRecipient(secrets); + + expect(indexes).toHaveLength(NUM_SENDERS); + expect(indexes).toEqual([1, 1, 1, 1, 1, 2, 2, 2, 2, 2]); + + // We should have called the node 12 times: + // 2 times with logs (sliding the window) + 10 times with no results (window size) + expect(aztecNode.getLogsByTags.mock.calls.length).toBe(2 + SENDER_OFFSET_WINDOW_SIZE); + }); + it('should not sync tagged logs with a blockNumber > maxBlockNumber', async () => { const senderOffset = 0; generateMockLogs(senderOffset); @@ -423,13 +534,13 @@ describe('Simulator oracle', () => { }); aztecNode.getTxEffect.mockImplementation(txHash => { - return Promise.resolve(txEffectsMap[txHash.toString()] as TxEffect); + return Promise.resolve(randomInBlock(txEffectsMap[txHash.toString()] as TxEffect)); }); - aztecNode.findLeavesIndexes.mockImplementation((_blockNumber, _treeId, leafValues) => + aztecNode.findNullifiersIndexesWithBlock.mockImplementation((_blockNumber, requestedNullifiers) => Promise.resolve( - Array(leafValues.length - nullifiers) + Array(requestedNullifiers.length - nullifiers) .fill(undefined) - .concat(Array(nullifiers).fill(1n)), + .concat(Array(nullifiers).fill({ data: 1n, l2BlockNumber: 1n, l2BlockHash: '0x' })), ), ); return taggedLogs; diff --git a/yarn-project/pxe/src/synchronizer/synchronizer.test.ts b/yarn-project/pxe/src/synchronizer/synchronizer.test.ts index 8b0c6810eda..e78c0dbb4ab 100644 --- a/yarn-project/pxe/src/synchronizer/synchronizer.test.ts +++ b/yarn-project/pxe/src/synchronizer/synchronizer.test.ts @@ -1,94 +1,59 @@ -import { type AztecNode, L2Block } from '@aztec/circuit-types'; -import { type Header } from '@aztec/circuits.js'; -import { makeHeader } from '@aztec/circuits.js/testing'; -import { randomInt } from '@aztec/foundation/crypto'; -import { SerialQueue } from '@aztec/foundation/queue'; +import { type AztecNode, L2Block, type L2BlockStream } from '@aztec/circuit-types'; +import { L2TipsStore } from '@aztec/kv-store/stores'; import { openTmpStore } from '@aztec/kv-store/utils'; +import { jest } from '@jest/globals'; import { type MockProxy, mock } from 'jest-mock-extended'; +import times from 'lodash.times'; import { type PxeDatabase } from '../database/index.js'; import { KVPxeDatabase } from '../database/kv_pxe_database.js'; import { Synchronizer } from './synchronizer.js'; describe('Synchronizer', () => { - let aztecNode: MockProxy; let database: PxeDatabase; - let synchronizer: TestSynchronizer; - let jobQueue: SerialQueue; - const initialSyncBlockNumber = 3; - let headerBlock3: Header; - - beforeEach(() => { - headerBlock3 = makeHeader(randomInt(1000), initialSyncBlockNumber, initialSyncBlockNumber); + let synchronizer: Synchronizer; + let tipsStore: L2TipsStore; // eslint-disable-line @typescript-eslint/no-unused-vars - aztecNode = mock(); - database = new KVPxeDatabase(openTmpStore()); - jobQueue = new SerialQueue(); - synchronizer = new TestSynchronizer(aztecNode, database, jobQueue); - }); - - it('sets header from aztec node on initial sync', async () => { - aztecNode.getBlockNumber.mockResolvedValue(initialSyncBlockNumber); - aztecNode.getHeader.mockResolvedValue(headerBlock3); + let aztecNode: MockProxy; + let blockStream: MockProxy; - await synchronizer.initialSync(); + const TestSynchronizer = class extends Synchronizer { + protected override createBlockStream(): L2BlockStream { + return blockStream; + } + }; - expect(database.getHeader()).toEqual(headerBlock3); + beforeEach(() => { + const store = openTmpStore(); + blockStream = mock(); + aztecNode = mock(); + database = new KVPxeDatabase(store); + tipsStore = new L2TipsStore(store, 'pxe'); + synchronizer = new TestSynchronizer(aztecNode, database, tipsStore); }); it('sets header from latest block', async () => { const block = L2Block.random(1, 4); - aztecNode.getLogs.mockResolvedValueOnce([block.body.encryptedLogs]).mockResolvedValue([block.body.unencryptedLogs]); - aztecNode.getBlocks.mockResolvedValue([block]); - - await synchronizer.work(); + await synchronizer.handleBlockStreamEvent({ type: 'blocks-added', blocks: [block] }); const obtainedHeader = database.getHeader(); expect(obtainedHeader).toEqual(block.header); }); - it('overrides header from initial sync once current block number is larger', async () => { - // Initial sync is done on block with height 3 - aztecNode.getBlockNumber.mockResolvedValue(initialSyncBlockNumber); - aztecNode.getHeader.mockResolvedValue(headerBlock3); - - await synchronizer.initialSync(); - const header0 = database.getHeader(); - expect(header0).toEqual(headerBlock3); - - // We then process block with height 1, this should not change the header - const block1 = L2Block.random(1, 4); - - aztecNode.getLogs - .mockResolvedValueOnce([block1.body.encryptedLogs]) - .mockResolvedValue([block1.body.unencryptedLogs]); - - aztecNode.getBlocks.mockResolvedValue([block1]); + it('removes notes from db on a reorg', async () => { + const removeNotesAfter = jest.spyOn(database, 'removeNotesAfter').mockImplementation(() => Promise.resolve()); + const unnullifyNotesAfter = jest.spyOn(database, 'unnullifyNotesAfter').mockImplementation(() => Promise.resolve()); + const resetNoteSyncData = jest.spyOn(database, 'resetNoteSyncData').mockImplementation(() => Promise.resolve()); + aztecNode.getBlockHeader.mockImplementation(blockNumber => + Promise.resolve(L2Block.random(blockNumber as number).header), + ); - await synchronizer.work(); - const header1 = database.getHeader(); - expect(header1).toEqual(headerBlock3); - expect(header1).not.toEqual(block1.header); + await synchronizer.handleBlockStreamEvent({ type: 'blocks-added', blocks: times(5, L2Block.random) }); + await synchronizer.handleBlockStreamEvent({ type: 'chain-pruned', blockNumber: 3 }); - // But they should change when we process block with height 5 - const block5 = L2Block.random(5, 4); - - aztecNode.getBlocks.mockResolvedValue([block5]); - - await synchronizer.work(); - const header5 = database.getHeader(); - expect(header5).not.toEqual(headerBlock3); - expect(header5).toEqual(block5.header); + expect(removeNotesAfter).toHaveBeenCalledWith(3); + expect(unnullifyNotesAfter).toHaveBeenCalledWith(3); + expect(resetNoteSyncData).toHaveBeenCalled(); }); }); - -class TestSynchronizer extends Synchronizer { - public override work(limit = 1) { - return super.work(limit); - } - - public override initialSync(): Promise { - return super.initialSync(); - } -} diff --git a/yarn-project/pxe/src/synchronizer/synchronizer.ts b/yarn-project/pxe/src/synchronizer/synchronizer.ts index 3ed458b2db1..d527a38b535 100644 --- a/yarn-project/pxe/src/synchronizer/synchronizer.ts +++ b/yarn-project/pxe/src/synchronizer/synchronizer.ts @@ -1,9 +1,14 @@ -import { type AztecNode, type L2Block } from '@aztec/circuit-types'; +import { + type AztecNode, + L2BlockStream, + type L2BlockStreamEvent, + type L2BlockStreamEventHandler, +} from '@aztec/circuit-types'; import { INITIAL_L2_BLOCK_NUM } from '@aztec/circuits.js'; import { type DebugLogger, createDebugLogger } from '@aztec/foundation/log'; -import { type SerialQueue } from '@aztec/foundation/queue'; -import { RunningPromise } from '@aztec/foundation/running-promise'; +import { type L2TipsStore } from '@aztec/kv-store/stores'; +import { type PXEConfig } from '../config/index.js'; import { type PxeDatabase } from '../database/index.js'; /** @@ -13,14 +18,51 @@ import { type PxeDatabase } from '../database/index.js'; * details, and fetch transactions by hash. The Synchronizer ensures that it maintains the note processors * in sync with the blockchain while handling retries and errors gracefully. */ -export class Synchronizer { - private runningPromise?: RunningPromise; +export class Synchronizer implements L2BlockStreamEventHandler { private running = false; private initialSyncBlockNumber = INITIAL_L2_BLOCK_NUM - 1; private log: DebugLogger; + protected readonly blockStream: L2BlockStream; - constructor(private node: AztecNode, private db: PxeDatabase, private jobQueue: SerialQueue, logSuffix = '') { + constructor( + private node: AztecNode, + private db: PxeDatabase, + private l2TipsStore: L2TipsStore, + config: Partial> = {}, + logSuffix?: string, + ) { this.log = createDebugLogger(logSuffix ? `aztec:pxe_synchronizer_${logSuffix}` : 'aztec:pxe_synchronizer'); + this.blockStream = this.createBlockStream(config); + } + + protected createBlockStream(config: Partial>) { + return new L2BlockStream(this.node, this.l2TipsStore, this, { + pollIntervalMS: config.l2BlockPollingIntervalMS, + startingBlock: config.l2StartingBlock, + }); + } + + /** Handle events emitted by the block stream. */ + public async handleBlockStreamEvent(event: L2BlockStreamEvent): Promise { + await this.l2TipsStore.handleBlockStreamEvent(event); + + switch (event.type) { + case 'blocks-added': + this.log.verbose(`Processing blocks ${event.blocks[0].number} to ${event.blocks.at(-1)!.number}`); + await this.db.setHeader(event.blocks.at(-1)!.header); + break; + case 'chain-pruned': + this.log.info(`Pruning data after block ${event.blockNumber} due to reorg`); + // We first unnullify and then remove so that unnullified notes that were created after the block number end up deleted. + await this.db.unnullifyNotesAfter(event.blockNumber); + await this.db.removeNotesAfter(event.blockNumber); + // Remove all note tagging indexes to force a full resync. This is suboptimal, but unless we track the + // block number in which each index is used it's all we can do. + await this.db.resetNoteSyncData(); + // Update the header to the last block. + await this.db.setHeader(await this.node.getBlockHeader(event.blockNumber)); + break; + } } /** @@ -31,79 +73,21 @@ export class Synchronizer { * @param limit - The maximum number of encrypted, unencrypted logs and blocks to fetch in each iteration. * @param retryInterval - The time interval (in ms) to wait before retrying if no data is available. */ - public async start(limit = 1, retryInterval = 1000) { + public async start() { if (this.running) { return; } this.running = true; - await this.jobQueue.put(() => this.initialSync()); + // REFACTOR: We should know the header of the genesis block without having to request it from the node. + await this.db.setHeader(await this.node.getBlockHeader(0)); + + await this.trigger(); this.log.info('Initial sync complete'); - this.runningPromise = new RunningPromise(() => this.sync(limit), retryInterval); - this.runningPromise.start(); + this.blockStream.start(); this.log.debug('Started loop'); } - protected async initialSync() { - // fast forward to the latest block - const latestHeader = await this.node.getHeader(); - this.initialSyncBlockNumber = Number(latestHeader.globalVariables.blockNumber.toBigInt()); - await this.db.setHeader(latestHeader); - } - - /** - * Fetches encrypted logs and blocks from the Aztec node and processes them for all note processors. - * If needed, catches up note processors that are lagging behind the main sync, e.g. because we just added a new account. - * - * Uses the job queue to ensure that - * - sync does not overlap with pxe simulations. - * - one sync is running at a time. - * - * @param limit - The maximum number of encrypted, unencrypted logs and blocks to fetch in each iteration. - * @returns a promise that resolves when the sync is complete - */ - protected sync(limit: number) { - return this.jobQueue.put(async () => { - let moreWork = true; - // keep external this.running flag to interrupt greedy sync - while (moreWork && this.running) { - moreWork = await this.work(limit); - } - }); - } - - /** - * Fetches encrypted logs and blocks from the Aztec node and processes them for all note processors. - * - * @param limit - The maximum number of encrypted, unencrypted logs and blocks to fetch in each iteration. - * @returns true if there could be more work, false if we're caught up or there was an error. - */ - protected async work(limit = 1): Promise { - const from = this.getSynchedBlockNumber() + 1; - try { - const blocks = await this.node.getBlocks(from, limit); - if (blocks.length === 0) { - return false; - } - - // Update latest tree roots from the most recent block - const latestBlock = blocks[blocks.length - 1]; - await this.setHeaderFromBlock(latestBlock); - return true; - } catch (err) { - this.log.error(`Error in synchronizer work`, err); - return false; - } - } - - private async setHeaderFromBlock(latestBlock: L2Block) { - if (latestBlock.number < this.initialSyncBlockNumber) { - return; - } - - await this.db.setHeader(latestBlock.header); - } - /** * Stops the synchronizer gracefully, interrupting any ongoing sleep and waiting for the current * iteration to complete before setting the running state to false. Once stopped, the synchronizer @@ -113,10 +97,15 @@ export class Synchronizer { */ public async stop() { this.running = false; - await this.runningPromise?.stop(); + await this.blockStream.stop(); this.log.info('Stopped'); } + /** Triggers a single run. */ + public async trigger() { + await this.blockStream.sync(); + } + private getSynchedBlockNumber() { return this.db.getBlockNumber() ?? this.initialSyncBlockNumber; } diff --git a/yarn-project/scripts/src/benchmarks/aggregate.ts b/yarn-project/scripts/src/benchmarks/aggregate.ts index 5b82bd208ce..6ad9da18c81 100644 --- a/yarn-project/scripts/src/benchmarks/aggregate.ts +++ b/yarn-project/scripts/src/benchmarks/aggregate.ts @@ -160,7 +160,6 @@ function processCircuitProving(entry: CircuitProvingStats, results: BenchmarkCol function processAvmSimulation(entry: AvmSimulationStats, results: BenchmarkCollectedResults) { append(results, 'avm_simulation_time_ms', entry.appCircuitName, entry.duration); - append(results, 'avm_simulation_bytecode_size_in_bytes', entry.appCircuitName, entry.bytecodeSize); } function processDbAccess(entry: PublicDBAccessStats, results: BenchmarkCollectedResults) { diff --git a/yarn-project/scripts/src/benchmarks/markdown.ts b/yarn-project/scripts/src/benchmarks/markdown.ts index de306a4434a..5c7d55390d2 100644 --- a/yarn-project/scripts/src/benchmarks/markdown.ts +++ b/yarn-project/scripts/src/benchmarks/markdown.ts @@ -192,7 +192,6 @@ export function getMarkdown(prNumber: number) { const kernelCircuitMetrics = Metrics.filter(m => m.groupBy === 'protocol-circuit-name').map(m => m.name); const appCircuitMetrics = Metrics.filter(m => m.groupBy === 'app-circuit-name') .filter(m => m.name !== 'avm_simulation_time_ms') - .filter(m => m.name !== 'avm_simulation_bytecode_size_in_bytes') .map(m => m.name); const metricsByClassesRegistered = Metrics.filter(m => m.groupBy === 'classes-registered').map(m => m.name); const metricsByFeePaymentMethod = Metrics.filter(m => m.groupBy === 'fee-payment-method').map(m => m.name); @@ -261,7 +260,7 @@ ${getTableContent( Time to simulate various public functions in the AVM. ${getTableContent( - transpose(pick(benchmark, ['avm_simulation_time_ms', 'avm_simulation_bytecode_size_in_bytes'])), + transpose(pick(benchmark, ['avm_simulation_time_ms'])), transpose(baseBenchmark), '', 'Function', diff --git a/yarn-project/sequencer-client/src/block_builder/light.test.ts b/yarn-project/sequencer-client/src/block_builder/light.test.ts index ebb0dbcf9ad..43cbd91a83b 100644 --- a/yarn-project/sequencer-client/src/block_builder/light.test.ts +++ b/yarn-project/sequencer-client/src/block_builder/light.test.ts @@ -23,6 +23,7 @@ import { NUM_BASE_PARITY_PER_ROOT_PARITY, type ParityPublicInputs, PreviousRollupData, + type PrivateBaseRollupHints, PrivateBaseRollupInputs, PrivateTubeData, type RecursiveProof, @@ -268,7 +269,7 @@ describe('LightBlockBuilder', () => { const vkData = new VkWitnessData(TubeVk, vkIndex, vkPath); const tubeData = new PrivateTubeData(tx.data.toKernelCircuitPublicInputs(), emptyProof, vkData); const hints = await buildBaseRollupHints(tx, globalVariables, expectsFork); - const inputs = new PrivateBaseRollupInputs(tubeData, hints); + const inputs = new PrivateBaseRollupInputs(tubeData, hints as PrivateBaseRollupHints); const result = await simulator.getPrivateBaseRollupProof(inputs); rollupOutputs.push(result.inputs); } diff --git a/yarn-project/sequencer-client/src/block_builder/light.ts b/yarn-project/sequencer-client/src/block_builder/light.ts index 90075c3f020..44d92ca2a23 100644 --- a/yarn-project/sequencer-client/src/block_builder/light.ts +++ b/yarn-project/sequencer-client/src/block_builder/light.ts @@ -32,7 +32,7 @@ export class LightweightBlockBuilder implements BlockBuilder { constructor(private db: MerkleTreeWriteOperations, private telemetry: TelemetryClient) {} async startNewBlock(numTxs: number, globalVariables: GlobalVariables, l1ToL2Messages: Fr[]): Promise { - this.logger.verbose('Starting new block', { numTxs, globalVariables, l1ToL2Messages }); + this.logger.verbose('Starting new block', { numTxs, globalVariables: globalVariables.toJSON(), l1ToL2Messages }); this.numTxs = numTxs; this.globalVariables = globalVariables; this.l1ToL2Messages = padArrayEnd(l1ToL2Messages, Fr.ZERO, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP); diff --git a/yarn-project/sequencer-client/src/client/sequencer-client.ts b/yarn-project/sequencer-client/src/client/sequencer-client.ts index a9618e8b980..404b062696a 100644 --- a/yarn-project/sequencer-client/src/client/sequencer-client.ts +++ b/yarn-project/sequencer-client/src/client/sequencer-client.ts @@ -2,7 +2,7 @@ import { type L1ToL2MessageSource, type L2BlockSource, type WorldStateSynchroniz import { type ContractDataSource } from '@aztec/circuits.js'; import { type EthAddress } from '@aztec/foundation/eth-address'; import { type P2P } from '@aztec/p2p'; -import { PublicProcessorFactory, type SimulationProvider } from '@aztec/simulator'; +import { PublicProcessorFactory } from '@aztec/simulator'; import { type TelemetryClient } from '@aztec/telemetry-client'; import { type ValidatorClient } from '@aztec/validator-client'; @@ -34,19 +34,30 @@ export class SequencerClient { */ public static async new( config: SequencerClientConfig, - validatorClient: ValidatorClient | undefined, // allowed to be undefined while we migrate - p2pClient: P2P, - worldStateSynchronizer: WorldStateSynchronizer, - contractDataSource: ContractDataSource, - l2BlockSource: L2BlockSource, - l1ToL2MessageSource: L1ToL2MessageSource, - simulationProvider: SimulationProvider, - telemetryClient: TelemetryClient, + deps: { + validatorClient: ValidatorClient | undefined; // allowed to be undefined while we migrate + p2pClient: P2P; + worldStateSynchronizer: WorldStateSynchronizer; + contractDataSource: ContractDataSource; + l2BlockSource: L2BlockSource; + l1ToL2MessageSource: L1ToL2MessageSource; + telemetry: TelemetryClient; + publisher?: L1Publisher; + }, ) { - const publisher = new L1Publisher(config, telemetryClient); + const { + validatorClient, + p2pClient, + worldStateSynchronizer, + contractDataSource, + l2BlockSource, + l1ToL2MessageSource, + telemetry: telemetryClient, + } = deps; + const publisher = deps.publisher ?? new L1Publisher(config, telemetryClient); const globalsBuilder = new GlobalVariableBuilder(config); - const publicProcessorFactory = new PublicProcessorFactory(contractDataSource, simulationProvider, telemetryClient); + const publicProcessorFactory = new PublicProcessorFactory(contractDataSource, telemetryClient); const rollup = publisher.getRollupContract(); const [l1GenesisTime, slotDuration] = await Promise.all([ diff --git a/yarn-project/sequencer-client/src/index.ts b/yarn-project/sequencer-client/src/index.ts index 66c24396853..1718ed0a3a6 100644 --- a/yarn-project/sequencer-client/src/index.ts +++ b/yarn-project/sequencer-client/src/index.ts @@ -4,4 +4,5 @@ export * from './publisher/index.js'; export * from './sequencer/index.js'; // Used by the node to simulate public parts of transactions. Should these be moved to a shared library? +// ISSUE(#9832) export * from './global_variable_builder/index.js'; diff --git a/yarn-project/sequencer-client/src/publisher/index.ts b/yarn-project/sequencer-client/src/publisher/index.ts index 97e14e96262..5590025020a 100644 --- a/yarn-project/sequencer-client/src/publisher/index.ts +++ b/yarn-project/sequencer-client/src/publisher/index.ts @@ -1,2 +1,3 @@ export { L1Publisher, L1SubmitEpochProofArgs } from './l1-publisher.js'; +export * from './test-l1-publisher.js'; export * from './config.js'; diff --git a/yarn-project/sequencer-client/src/publisher/l1-publisher.test.ts b/yarn-project/sequencer-client/src/publisher/l1-publisher.test.ts index 3fd8ced0837..6c817d70a2c 100644 --- a/yarn-project/sequencer-client/src/publisher/l1-publisher.test.ts +++ b/yarn-project/sequencer-client/src/publisher/l1-publisher.test.ts @@ -120,10 +120,12 @@ describe('L1Publisher', () => { expect(result).toEqual(true); const args = [ - `0x${header.toString('hex')}`, - `0x${archive.toString('hex')}`, - `0x${blockHash.toString('hex')}`, - [], + { + header: `0x${header.toString('hex')}`, + archive: `0x${archive.toString('hex')}`, + blockHash: `0x${blockHash.toString('hex')}`, + txHashes: [], + }, [], `0x${body.toString('hex')}`, ] as const; diff --git a/yarn-project/sequencer-client/src/publisher/l1-publisher.ts b/yarn-project/sequencer-client/src/publisher/l1-publisher.ts index 229ba4f7d75..9226059ab6c 100644 --- a/yarn-project/sequencer-client/src/publisher/l1-publisher.ts +++ b/yarn-project/sequencer-client/src/publisher/l1-publisher.ts @@ -17,7 +17,7 @@ import { type Proof, type RootRollupPublicInputs, } from '@aztec/circuits.js'; -import { type L1ContractsConfig, createEthereumChain } from '@aztec/ethereum'; +import { type EthereumChain, type L1ContractsConfig, createEthereumChain } from '@aztec/ethereum'; import { makeTuple } from '@aztec/foundation/array'; import { areArraysEqual, compactArray, times } from '@aztec/foundation/collection'; import { type Signature } from '@aztec/foundation/eth-signature'; @@ -58,7 +58,6 @@ import { publicActions, } from 'viem'; import { privateKeyToAccount } from 'viem/accounts'; -import type * as chains from 'viem/chains'; import { type PublisherConfig, type TxSenderConfig } from './config.js'; import { L1PublisherMetrics } from './l1-publisher-metrics.js'; @@ -145,19 +144,19 @@ export class L1Publisher { protected log = createDebugLogger('aztec:sequencer:publisher'); - private rollupContract: GetContractReturnType< + protected rollupContract: GetContractReturnType< typeof RollupAbi, - WalletClient + WalletClient >; - private governanceProposerContract?: GetContractReturnType< + protected governanceProposerContract?: GetContractReturnType< typeof GovernanceProposerAbi, - WalletClient + WalletClient > = undefined; - private publicClient: PublicClient; - private walletClient: WalletClient; - private account: PrivateKeyAccount; - private ethereumSlotDuration: bigint; + protected publicClient: PublicClient; + protected walletClient: WalletClient; + protected account: PrivateKeyAccount; + protected ethereumSlotDuration: bigint; public static PROPOSE_GAS_GUESS: bigint = 12_000_000n; public static PROPOSE_AND_CLAIM_GAS_GUESS: bigint = this.PROPOSE_GAS_GUESS + 100_000n; @@ -175,11 +174,7 @@ export class L1Publisher { this.account = privateKeyToAccount(publisherPrivateKey); this.log.debug(`Publishing from address ${this.account.address}`); - this.walletClient = createWalletClient({ - account: this.account, - chain: chain.chainInfo, - transport: http(chain.rpcUrl), - }); + this.walletClient = this.createWalletClient(this.account, chain); this.publicClient = createPublicClient({ chain: chain.chainInfo, @@ -202,6 +197,17 @@ export class L1Publisher { } } + protected createWalletClient( + account: PrivateKeyAccount, + chain: EthereumChain, + ): WalletClient { + return createWalletClient({ + account, + chain: chain.chainInfo, + transport: http(chain.rpcUrl), + }); + } + public getPayLoad() { return this.payload; } @@ -226,7 +232,7 @@ export class L1Publisher { public getRollupContract(): GetContractReturnType< typeof RollupAbi, - WalletClient + WalletClient > { return this.rollupContract; } @@ -732,10 +738,12 @@ export class L1Publisher { : []; const txHashes = encodedData.txHashes ? encodedData.txHashes.map(txHash => txHash.to0xString()) : []; const args = [ - `0x${encodedData.header.toString('hex')}`, - `0x${encodedData.archive.toString('hex')}`, - `0x${encodedData.blockHash.toString('hex')}`, - txHashes, + { + header: `0x${encodedData.header.toString('hex')}`, + archive: `0x${encodedData.archive.toString('hex')}`, + blockHash: `0x${encodedData.blockHash.toString('hex')}`, + txHashes, + }, attestations, `0x${encodedData.body.toString('hex')}`, ] as const; diff --git a/yarn-project/sequencer-client/src/publisher/test-l1-publisher.ts b/yarn-project/sequencer-client/src/publisher/test-l1-publisher.ts new file mode 100644 index 00000000000..f60dd608138 --- /dev/null +++ b/yarn-project/sequencer-client/src/publisher/test-l1-publisher.ts @@ -0,0 +1,20 @@ +import { type EthereumChain } from '@aztec/ethereum'; +import { type Delayer, withDelayer } from '@aztec/ethereum/test'; + +import { type Chain, type HttpTransport, type PrivateKeyAccount, type WalletClient } from 'viem'; + +import { L1Publisher } from './l1-publisher.js'; + +export class TestL1Publisher extends L1Publisher { + public delayer: Delayer | undefined; + + protected override createWalletClient( + account: PrivateKeyAccount, + chain: EthereumChain, + ): WalletClient { + const baseClient = super.createWalletClient(account, chain); + const { client, delayer } = withDelayer(baseClient, { ethereumSlotDuration: this.ethereumSlotDuration }); + this.delayer = delayer; + return client; + } +} diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts index 59b65a3c875..e1b3f8bb71a 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts @@ -232,8 +232,10 @@ describe('sequencer', () => { }, // It would be nice to add the other states, but we would need to inject delays within the `work` loop ])('does not build a block if it does not have enough time left in the slot', async ({ delayedState }) => { - // trick the sequencer into thinking that we are just too far into the slot - sequencer.setL1GenesisTime(Math.floor(Date.now() / 1000) - (sequencer.getTimeTable()[delayedState] + 1)); + // trick the sequencer into thinking that we are just too far into slot 1 + sequencer.setL1GenesisTime( + Math.floor(Date.now() / 1000) - slotDuration * 1 - (sequencer.getTimeTable()[delayedState] + 1), + ); const tx = mockTxForRollup(); tx.data.constants.txContext.chainId = chainId; @@ -841,7 +843,7 @@ class TestSubject extends Sequencer { } public override doRealWork() { - this.setState(SequencerState.IDLE, true /** force */); + this.setState(SequencerState.IDLE, 0, true /** force */); return super.doRealWork(); } } diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.ts index 35a4bd945d1..17e371c0738 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.ts @@ -15,6 +15,7 @@ import { AppendOnlyTreeSnapshot, ContentCommitment, GENESIS_ARCHIVE_ROOT, + type GlobalVariables, Header, StateReference, } from '@aztec/circuits.js'; @@ -112,6 +113,9 @@ export class Sequencer { this.updateConfig(config); this.metrics = new SequencerMetrics(telemetry, () => this.state, 'Sequencer'); this.log.verbose(`Initialized sequencer with ${this.minTxsPerBLock}-${this.maxTxsPerBlock} txs per block.`); + + // Register the block builder with the validator client for re-execution + this.validatorClient?.registerBlockBuilder(this.buildBlock.bind(this)); } get tracer(): Tracer { @@ -167,11 +171,11 @@ export class Sequencer { [SequencerState.IDLE]: this.aztecSlotDuration, [SequencerState.SYNCHRONIZING]: this.aztecSlotDuration, [SequencerState.PROPOSER_CHECK]: this.aztecSlotDuration, // We always want to allow the full slot to check if we are the proposer - [SequencerState.WAITING_FOR_TXS]: 3, - [SequencerState.CREATING_BLOCK]: 5, - [SequencerState.PUBLISHING_BLOCK_TO_PEERS]: 5 + this.maxTxsPerBlock * 2, // if we take 5 seconds to create block, then 4 transactions at 2 seconds each - [SequencerState.WAITING_FOR_ATTESTATIONS]: 5 + this.maxTxsPerBlock * 2 + 3, // it shouldn't take 3 seconds to publish to peers - [SequencerState.PUBLISHING_BLOCK]: 5 + this.maxTxsPerBlock * 2 + 3 + 5, // wait 5 seconds for attestations + [SequencerState.WAITING_FOR_TXS]: 5, + [SequencerState.CREATING_BLOCK]: 7, + [SequencerState.PUBLISHING_BLOCK_TO_PEERS]: 7 + this.maxTxsPerBlock * 2, // if we take 5 seconds to create block, then 4 transactions at 2 seconds each + [SequencerState.WAITING_FOR_ATTESTATIONS]: 7 + this.maxTxsPerBlock * 2 + 3, // it shouldn't take 3 seconds to publish to peers + [SequencerState.PUBLISHING_BLOCK]: 7 + this.maxTxsPerBlock * 2 + 3 + 5, // wait 5 seconds for attestations }; if (this.enforceTimeTable && newTimeTable[SequencerState.PUBLISHING_BLOCK] > this.aztecSlotDuration) { throw new Error('Sequencer cannot publish block in less than a slot'); @@ -185,7 +189,7 @@ export class Sequencer { public start() { this.runningPromise = new RunningPromise(this.work.bind(this), this.pollingIntervalMs); this.runningPromise.start(); - this.setState(SequencerState.IDLE, true /** force */); + this.setState(SequencerState.IDLE, 0, true /** force */); this.log.info('Sequencer started'); return Promise.resolve(); } @@ -197,7 +201,7 @@ export class Sequencer { this.log.debug(`Stopping sequencer`); await this.runningPromise?.stop(); this.publisher.interrupt(); - this.setState(SequencerState.STOPPED, true /** force */); + this.setState(SequencerState.STOPPED, 0, true /** force */); this.log.info('Stopped sequencer'); } @@ -208,7 +212,7 @@ export class Sequencer { this.log.info('Restarting sequencer'); this.publisher.restart(); this.runningPromise!.start(); - this.setState(SequencerState.IDLE, true /** force */); + this.setState(SequencerState.IDLE, 0, true /** force */); } /** @@ -228,7 +232,7 @@ export class Sequencer { * - If our block for some reason is not included, revert the state */ protected async doRealWork() { - this.setState(SequencerState.SYNCHRONIZING); + this.setState(SequencerState.SYNCHRONIZING, 0); // Update state when the previous block has been synced const prevBlockSynced = await this.isBlockSynced(); // Do not go forward with new block if the previous one has not been mined and processed @@ -239,7 +243,7 @@ export class Sequencer { this.log.debug('Previous block has been mined and processed'); - this.setState(SequencerState.PROPOSER_CHECK); + this.setState(SequencerState.PROPOSER_CHECK, 0); const chainTip = await this.l2BlockSource.getBlock(-1); const historicalHeader = chainTip?.header; @@ -273,8 +277,9 @@ export class Sequencer { if (!this.shouldProposeBlock(historicalHeader, {})) { return; } + const secondsIntoSlot = getSecondsIntoSlot(this.l1GenesisTime, this.aztecSlotDuration, Number(slot)); - this.setState(SequencerState.WAITING_FOR_TXS); + this.setState(SequencerState.WAITING_FOR_TXS, secondsIntoSlot); // Get txs to build the new block. const pendingTxs = this.p2pClient.getTxs('pending'); @@ -319,7 +324,7 @@ export class Sequencer { } catch (err) { this.log.error(`Error assembling block`, (err as any).stack); } - this.setState(SequencerState.IDLE); + this.setState(SequencerState.IDLE, 0); } protected async work() { @@ -333,7 +338,7 @@ export class Sequencer { throw err; } } finally { - this.setState(SequencerState.IDLE); + this.setState(SequencerState.IDLE, 0); } } @@ -392,14 +397,13 @@ export class Sequencer { return true; } - setState(proposedState: SequencerState, force: boolean = false) { + setState(proposedState: SequencerState, secondsIntoSlot: number, force: boolean = false) { if (this.state === SequencerState.STOPPED && force !== true) { this.log.warn( `Cannot set sequencer from ${this.state} to ${proposedState} as it is stopped. Set force=true to override.`, ); return; } - const secondsIntoSlot = getSecondsIntoSlot(this.l1GenesisTime, this.aztecSlotDuration); if (!this.doIHaveEnoughTimeLeft(proposedState, secondsIntoSlot)) { throw new SequencerTooSlowError(this.state, proposedState, this.timeTable[proposedState], secondsIntoSlot); } @@ -473,37 +477,21 @@ export class Sequencer { } /** - * @notice Build and propose a block to the chain + * Build a block * - * @dev MUST throw instead of exiting early to ensure that world-state - * is being rolled back if the block is dropped. + * Shared between the sequencer and the validator for re-execution * * @param validTxs - The valid transactions to construct the block from - * @param proposalHeader - The partial header constructed for the proposal + * @param newGlobalVariables - The global variables for the new block * @param historicalHeader - The historical header of the parent + * @param interrupt - The interrupt callback, used to validate the block for submission and check if we should propose the block */ - @trackSpan('Sequencer.buildBlockAndAttemptToPublish', (_validTxs, proposalHeader, _historicalHeader) => ({ - [Attributes.BLOCK_NUMBER]: proposalHeader.globalVariables.blockNumber.toNumber(), - })) - private async buildBlockAndAttemptToPublish( + private async buildBlock( validTxs: Tx[], - proposalHeader: Header, - historicalHeader: Header | undefined, - ): Promise { - await this.publisher.validateBlockForSubmission(proposalHeader); - - const newGlobalVariables = proposalHeader.globalVariables; - - this.metrics.recordNewBlock(newGlobalVariables.blockNumber.toNumber(), validTxs.length); - const workTimer = new Timer(); - this.setState(SequencerState.CREATING_BLOCK); - this.log.info( - `Building blockNumber=${newGlobalVariables.blockNumber.toNumber()} txCount=${ - validTxs.length - } slotNumber=${newGlobalVariables.slotNumber.toNumber()}`, - ); - - // Get l1 to l2 messages from the contract + newGlobalVariables: GlobalVariables, + historicalHeader?: Header, + interrupt?: (processedTxs: ProcessedTx[]) => Promise, + ) { this.log.debug('Requesting L1 to L2 messages from contract'); const l1ToL2Messages = await this.l1ToL2MessageSource.getL1ToL2Messages(newGlobalVariables.blockNumber.toBigInt()); this.log.verbose( @@ -513,11 +501,15 @@ export class Sequencer { const numRealTxs = validTxs.length; const blockSize = Math.max(2, numRealTxs); + // Sync to the previous block at least + await this.worldState.syncImmediate(newGlobalVariables.blockNumber.toNumber() - 1); + this.log.verbose(`Synced to previous block ${newGlobalVariables.blockNumber.toNumber() - 1}`); + // NB: separating the dbs because both should update the state const publicProcessorFork = await this.worldState.fork(); const orchestratorFork = await this.worldState.fork(); + try { - // We create a fresh processor each time to reset any cached state (eg storage writes) const processor = this.publicProcessorFactory.create(publicProcessorFork, historicalHeader, newGlobalVariables); const blockBuildingTimer = new Timer(); const blockBuilder = this.blockBuilderFactory.create(orchestratorFork); @@ -537,6 +529,62 @@ export class Sequencer { await this.p2pClient.deleteTxs(Tx.getHashes(failedTxData)); } + await interrupt?.(processedTxs); + + // All real transactions have been added, set the block as full and complete the proving. + const block = await blockBuilder.setBlockCompleted(); + + return { block, publicProcessorDuration, numProcessedTxs: processedTxs.length, blockBuildingTimer }; + } finally { + // We create a fresh processor each time to reset any cached state (eg storage writes) + await publicProcessorFork.close(); + await orchestratorFork.close(); + } + } + + /** + * @notice Build and propose a block to the chain + * + * @dev MUST throw instead of exiting early to ensure that world-state + * is being rolled back if the block is dropped. + * + * @param validTxs - The valid transactions to construct the block from + * @param proposalHeader - The partial header constructed for the proposal + * @param historicalHeader - The historical header of the parent + */ + @trackSpan('Sequencer.buildBlockAndAttemptToPublish', (_validTxs, proposalHeader, _historicalHeader) => ({ + [Attributes.BLOCK_NUMBER]: proposalHeader.globalVariables.blockNumber.toNumber(), + })) + private async buildBlockAndAttemptToPublish( + validTxs: Tx[], + proposalHeader: Header, + historicalHeader: Header | undefined, + ): Promise { + await this.publisher.validateBlockForSubmission(proposalHeader); + + const newGlobalVariables = proposalHeader.globalVariables; + + this.metrics.recordNewBlock(newGlobalVariables.blockNumber.toNumber(), validTxs.length); + const workTimer = new Timer(); + const secondsIntoSlot = getSecondsIntoSlot( + this.l1GenesisTime, + this.aztecSlotDuration, + newGlobalVariables.slotNumber.toNumber(), + ); + this.setState(SequencerState.CREATING_BLOCK, secondsIntoSlot); + this.log.info( + `Building blockNumber=${newGlobalVariables.blockNumber.toNumber()} txCount=${ + validTxs.length + } slotNumber=${newGlobalVariables.slotNumber.toNumber()}`, + ); + + /** + * BuildBlock is shared between the sequencer and the validator for re-execution + * We use the interrupt callback to validate the block for submission and check if we should propose the block + * + * If we fail, we throw an error in order to roll back + */ + const interrupt = async (processedTxs: ProcessedTx[]) => { await this.publisher.validateBlockForSubmission(proposalHeader); if ( @@ -548,9 +596,15 @@ export class Sequencer { // TODO: Roll back changes to world state throw new Error('Should not propose the block'); } + }; - // All real transactions have been added, set the block as full and complete the proving. - const block = await blockBuilder.setBlockCompleted(); + try { + const { block, publicProcessorDuration, numProcessedTxs, blockBuildingTimer } = await this.buildBlock( + validTxs, + newGlobalVariables, + historicalHeader, + interrupt, + ); // TODO(@PhilWindle) We should probably periodically check for things like another // block being published before ours instead of just waiting on our block @@ -558,7 +612,7 @@ export class Sequencer { await this.publisher.validateBlockForSubmission(block.header); const workDuration = workTimer.ms(); - this.log.verbose( + this.log.info( `Assembled block ${block.number} (txEffectsHash: ${block.header.contentCommitment.txsEffectsHash.toString( 'hex', )})`, @@ -573,35 +627,30 @@ export class Sequencer { ); if (this.isFlushing) { - this.log.verbose(`Flushing completed`); + this.log.info(`Flushing completed`); } const txHashes = validTxs.map(tx => tx.getTxHash()); this.isFlushing = false; - this.log.verbose('Collecting attestations'); + this.log.info('Collecting attestations'); const attestations = await this.collectAttestations(block, txHashes); - this.log.verbose('Attestations collected'); + this.log.info('Attestations collected'); - this.log.verbose('Collecting proof quotes'); + this.log.info('Collecting proof quotes'); const proofQuote = await this.createProofClaimForPreviousEpoch(newGlobalVariables.slotNumber.toBigInt()); - this.log.verbose(proofQuote ? `Using proof quote ${inspect(proofQuote.payload)}` : 'No proof quote available'); - - try { - await this.publishL2Block(block, attestations, txHashes, proofQuote); - this.metrics.recordPublishedBlock(workDuration); - this.log.info( - `Submitted rollup block ${block.number} with ${processedTxs.length} transactions duration=${Math.ceil( - workDuration, - )}ms (Submitter: ${this.publisher.getSenderAddress()})`, - ); - } catch (err) { - this.metrics.recordFailedBlock(); - throw err; - } - } finally { - await publicProcessorFork.close(); - await orchestratorFork.close(); + this.log.info(proofQuote ? `Using proof quote ${inspect(proofQuote.payload)}` : 'No proof quote available'); + + await this.publishL2Block(block, attestations, txHashes, proofQuote); + this.metrics.recordPublishedBlock(workDuration); + this.log.info( + `Submitted rollup block ${block.number} with ${numProcessedTxs} transactions duration=${Math.ceil( + workDuration, + )}ms (Submitter: ${this.publisher.getSenderAddress()})`, + ); + } catch (err) { + this.metrics.recordFailedBlock(); + throw err; } } @@ -621,7 +670,7 @@ export class Sequencer { this.log.debug(`Attesting committee length ${committee.length}`); if (committee.length === 0) { - this.log.debug(`Attesting committee length is 0, skipping`); + this.log.verbose(`Attesting committee length is 0, skipping`); return undefined; } @@ -633,16 +682,28 @@ export class Sequencer { const numberOfRequiredAttestations = Math.floor((committee.length * 2) / 3) + 1; - this.log.verbose('Creating block proposal'); + this.log.info('Creating block proposal'); const proposal = await this.validatorClient.createBlockProposal(block.header, block.archive.root, txHashes); - this.setState(SequencerState.PUBLISHING_BLOCK_TO_PEERS); - this.log.verbose('Broadcasting block proposal to validators'); + let secondsIntoSlot = getSecondsIntoSlot( + this.l1GenesisTime, + this.aztecSlotDuration, + block.header.globalVariables.slotNumber.toNumber(), + ); + + this.setState(SequencerState.PUBLISHING_BLOCK_TO_PEERS, secondsIntoSlot); + this.log.info('Broadcasting block proposal to validators'); this.validatorClient.broadcastBlockProposal(proposal); - this.setState(SequencerState.WAITING_FOR_ATTESTATIONS); + secondsIntoSlot = getSecondsIntoSlot( + this.l1GenesisTime, + this.aztecSlotDuration, + block.header.globalVariables.slotNumber.toNumber(), + ); + + this.setState(SequencerState.WAITING_FOR_ATTESTATIONS, secondsIntoSlot); const attestations = await this.validatorClient.collectAttestations(proposal, numberOfRequiredAttestations); - this.log.verbose(`Collected attestations from validators, number of attestations: ${attestations.length}`); + this.log.info(`Collected attestations from validators, number of attestations: ${attestations.length}`); // note: the smart contract requires that the signatures are provided in the order of the committee return orderAttestations(attestations, committee); @@ -659,7 +720,7 @@ export class Sequencer { // Get quotes for the epoch to be proven const quotes = await this.p2pClient.getEpochProofQuotes(epochToProve); - this.log.verbose(`Retrieved ${quotes.length} quotes, slot: ${slotNumber}, epoch to prove: ${epochToProve}`); + this.log.info(`Retrieved ${quotes.length} quotes, slot: ${slotNumber}, epoch to prove: ${epochToProve}`); for (const quote of quotes) { this.log.verbose(inspect(quote.payload)); } @@ -670,7 +731,7 @@ export class Sequencer { const validQuotes = (await validQuotesPromise).filter((q): q is EpochProofQuote => !!q); if (!validQuotes.length) { - this.log.verbose(`Failed to find any valid proof quotes`); + this.log.warn(`Failed to find any valid proof quotes`); return undefined; } // pick the quote with the lowest fee @@ -697,8 +758,13 @@ export class Sequencer { txHashes?: TxHash[], proofQuote?: EpochProofQuote, ) { + const secondsIntoSlot = getSecondsIntoSlot( + this.l1GenesisTime, + this.aztecSlotDuration, + block.header.globalVariables.slotNumber.toNumber(), + ); // Publishes new block to the network and awaits the tx to be mined - this.setState(SequencerState.PUBLISHING_BLOCK); + this.setState(SequencerState.PUBLISHING_BLOCK, secondsIntoSlot); const publishedL2Block = await this.publisher.proposeL2Block(block, attestations, txHashes, proofQuote); if (!publishedL2Block) { diff --git a/yarn-project/sequencer-client/src/sequencer/utils.ts b/yarn-project/sequencer-client/src/sequencer/utils.ts index c423c29ace4..4c16e8c8a9b 100644 --- a/yarn-project/sequencer-client/src/sequencer/utils.ts +++ b/yarn-project/sequencer-client/src/sequencer/utils.ts @@ -73,6 +73,7 @@ export function orderAttestations(attestations: BlockAttestation[], orderAddress return orderedAttestations; } -export function getSecondsIntoSlot(l1GenesisTime: number, aztecSlotDuration: number): number { - return (Date.now() / 1000 - l1GenesisTime) % aztecSlotDuration; +export function getSecondsIntoSlot(l1GenesisTime: number, aztecSlotDuration: number, slotNumber: number): number { + const slotStartTimestamp = l1GenesisTime + slotNumber * aztecSlotDuration; + return Date.now() / 1000 - slotStartTimestamp; } diff --git a/yarn-project/simulator/package.json b/yarn-project/simulator/package.json index 7489b5f8596..2832153c30a 100644 --- a/yarn-project/simulator/package.json +++ b/yarn-project/simulator/package.json @@ -4,7 +4,7 @@ "type": "module", "exports": { ".": "./dest/index.js", - "./avm/fixtures": "./dest/avm/fixtures/index.js" + "./public/fixtures": "./dest/public/fixtures/index.js" }, "typedocOptions": { "entryPoints": [ @@ -71,6 +71,7 @@ "@noir-lang/noirc_abi": "portal:../../noir/packages/noirc_abi", "@noir-lang/types": "portal:../../noir/packages/types", "levelup": "^5.1.1", + "lodash.clonedeep": "^4.5.0", "memdown": "^6.1.1", "tslib": "^2.4.0" }, @@ -81,6 +82,7 @@ "@jest/globals": "^29.5.0", "@types/jest": "^29.5.0", "@types/levelup": "^5.1.3", + "@types/lodash.clonedeep": "^4.5.7", "@types/lodash.merge": "^4.6.9", "@types/memdown": "^3.0.2", "@types/node": "^18.7.23", diff --git a/yarn-project/simulator/src/avm/avm_contract_call_result.ts b/yarn-project/simulator/src/avm/avm_contract_call_result.ts index 7db56bc44de..bb1df3ff702 100644 --- a/yarn-project/simulator/src/avm/avm_contract_call_result.ts +++ b/yarn-project/simulator/src/avm/avm_contract_call_result.ts @@ -1,15 +1,52 @@ +import { type SimulationError } from '@aztec/circuit-types'; +import { Gas } from '@aztec/circuits.js'; import { type Fr } from '@aztec/foundation/fields'; +import { inspect } from 'util'; + +import { createSimulationError } from '../common/errors.js'; +import { type Gas as AvmGas } from './avm_gas.js'; import { type AvmRevertReason } from './errors.js'; /** * Results of an contract call's execution in the AVM. */ export class AvmContractCallResult { - constructor(public reverted: boolean, public output: Fr[], public revertReason?: AvmRevertReason) {} + constructor( + public reverted: boolean, + public output: Fr[], + public gasLeft: AvmGas, + public revertReason?: AvmRevertReason, + ) {} + + toString(): string { + let resultsStr = `reverted: ${this.reverted}, output: ${this.output}, gasLeft: ${inspect(this.gasLeft)}`; + if (this.revertReason) { + resultsStr += `, revertReason: ${this.revertReason}`; + } + return resultsStr; + } + + finalize(): AvmFinalizedCallResult { + const revertReason = this.revertReason ? createSimulationError(this.revertReason, this.output) : undefined; + return new AvmFinalizedCallResult(this.reverted, this.output, Gas.from(this.gasLeft), revertReason); + } +} + +/** + * Similar to AvmContractCallResult, but with a SimulationError and standard Gas + * which are useful for consumption external to core AVM simulation. + */ +export class AvmFinalizedCallResult { + constructor( + public reverted: boolean, + public output: Fr[], + public gasLeft: Gas, + public revertReason?: SimulationError, + ) {} toString(): string { - let resultsStr = `reverted: ${this.reverted}, output: ${this.output}`; + let resultsStr = `reverted: ${this.reverted}, output: ${this.output}, gasLeft: ${inspect(this.gasLeft)}`; if (this.revertReason) { resultsStr += `, revertReason: ${this.revertReason}`; } diff --git a/yarn-project/simulator/src/avm/avm_memory_types.ts b/yarn-project/simulator/src/avm/avm_memory_types.ts index d22e86bc9b0..96420655547 100644 --- a/yarn-project/simulator/src/avm/avm_memory_types.ts +++ b/yarn-project/simulator/src/avm/avm_memory_types.ts @@ -232,8 +232,8 @@ export class TaggedMemory implements TaggedMemoryInterface { // Whether to track and validate memory accesses for each instruction. static readonly TRACK_MEMORY_ACCESSES = process.env.NODE_ENV === 'test'; - // FIXME: memory should be 2^32, but TS doesn't allow for arrays that big. - static readonly MAX_MEMORY_SIZE = Number((1n << 32n) - 2n); + // FIXME: memory should be 2^32, but TS max array size is: 2^32 - 1 + static readonly MAX_MEMORY_SIZE = Number((1n << 32n) - 1n); private _mem: MemoryValue[]; constructor() { @@ -264,8 +264,7 @@ export class TaggedMemory implements TaggedMemoryInterface { } public getSlice(offset: number, size: number): MemoryValue[] { - assert(offset < TaggedMemory.MAX_MEMORY_SIZE); - assert(offset + size < TaggedMemory.MAX_MEMORY_SIZE); + assert(offset + size <= TaggedMemory.MAX_MEMORY_SIZE); const value = this._mem.slice(offset, offset + size); TaggedMemory.log.debug(`getSlice(${offset}, ${size}) = ${value}`); for (let i = 0; i < value.length; i++) { @@ -278,14 +277,12 @@ export class TaggedMemory implements TaggedMemoryInterface { } public getSliceAs(offset: number, size: number): T[] { - assert(offset < TaggedMemory.MAX_MEMORY_SIZE); - assert(offset + size < TaggedMemory.MAX_MEMORY_SIZE); + assert(offset + size <= TaggedMemory.MAX_MEMORY_SIZE); return this.getSlice(offset, size) as T[]; } public getSliceTags(offset: number, size: number): TypeTag[] { - assert(offset < TaggedMemory.MAX_MEMORY_SIZE); - assert(offset + size < TaggedMemory.MAX_MEMORY_SIZE); + assert(offset + size <= TaggedMemory.MAX_MEMORY_SIZE); return this._mem.slice(offset, offset + size).map(TaggedMemory.getTag); } @@ -296,8 +293,7 @@ export class TaggedMemory implements TaggedMemoryInterface { } public setSlice(offset: number, vs: MemoryValue[]) { - assert(offset < TaggedMemory.MAX_MEMORY_SIZE); - assert(offset + vs.length < TaggedMemory.MAX_MEMORY_SIZE); + assert(offset + vs.length <= TaggedMemory.MAX_MEMORY_SIZE); // We may need to extend the memory size, otherwise splice doesn't insert. if (offset + vs.length > this._mem.length) { this._mem.length = offset + vs.length; diff --git a/yarn-project/simulator/src/avm/avm_simulator.test.ts b/yarn-project/simulator/src/avm/avm_simulator.test.ts index f05c18b3e01..1c170d7f1c2 100644 --- a/yarn-project/simulator/src/avm/avm_simulator.test.ts +++ b/yarn-project/simulator/src/avm/avm_simulator.test.ts @@ -1,9 +1,7 @@ import { MerkleTreeId, type MerkleTreeWriteOperations } from '@aztec/circuit-types'; import { - type ContractDataSource, GasFees, GlobalVariables, - PublicDataTreeLeaf, PublicDataTreeLeafPreimage, type PublicFunction, PublicKeys, @@ -25,12 +23,13 @@ import { randomInt } from 'crypto'; import { mock } from 'jest-mock-extended'; import { PublicEnqueuedCallSideEffectTrace } from '../public/enqueued_call_side_effect_trace.js'; -import { WorldStateDB } from '../public/public_db_sources.js'; +import { type WorldStateDB } from '../public/public_db_sources.js'; import { type PublicSideEffectTraceInterface } from '../public/side_effect_trace_interface.js'; import { type AvmContext } from './avm_context.js'; import { type AvmExecutionEnvironment } from './avm_execution_environment.js'; import { type MemoryValue, TypeTag, type Uint8, type Uint64 } from './avm_memory_types.js'; import { AvmSimulator } from './avm_simulator.js'; +import { AvmEphemeralForest } from './avm_tree.js'; import { isAvmBytecode, markBytecodeAsAvm } from './bytecode_utils.js'; import { getAvmTestContractArtifact, @@ -46,7 +45,7 @@ import { randomMemoryUint64s, resolveAvmTestContractAssertionMessage, } from './fixtures/index.js'; -import { type AvmPersistableStateManager, getLeafOrLowLeaf } from './journal/journal.js'; +import { type AvmPersistableStateManager } from './journal/journal.js'; import { Add, CalldataCopy, @@ -138,7 +137,7 @@ describe('AVM simulator: transpiled Noir contracts', () => { const contractClass = makeContractClassPublic(0, publicFn); const contractInstance = makeContractInstanceFromClassId(contractClass.id); - // The values here should match those in `avm_simulator.test.ts` + // The values here should match those in getContractInstance test case const instanceGet = new SerializableContractInstance({ version: 1, salt: new Fr(0x123), @@ -153,6 +152,11 @@ describe('AVM simulator: transpiled Noir contracts', () => { ), }).withAddress(contractInstance.address); const worldStateDB = mock(); + const tmp = openTmpStore(); + const telemetryClient = new NoopTelemetryClient(); + const merkleTree = await (await MerkleTrees.new(tmp, telemetryClient)).fork(); + worldStateDB.getMerkleInterface.mockReturnValue(merkleTree); + worldStateDB.getContractInstance .mockResolvedValueOnce(contractInstance) .mockResolvedValueOnce(instanceGet) // test gets deployer @@ -165,9 +169,7 @@ describe('AVM simulator: transpiled Noir contracts', () => { mockStorageRead(worldStateDB, storageValue); const trace = mock(); - const telemetry = new NoopTelemetryClient(); - const merkleTrees = await (await MerkleTrees.new(openTmpStore(), telemetry)).fork(); - worldStateDB.getMerkleInterface.mockReturnValue(merkleTrees); + const merkleTrees = await AvmEphemeralForest.create(worldStateDB.getMerkleInterface()); const persistableState = initPersistableStateManager({ worldStateDB, trace, merkleTrees }); const environment = initExecutionEnvironment({ functionSelector, @@ -188,6 +190,16 @@ describe('AVM simulator: transpiled Noir contracts', () => { expect(results.reverted).toBe(false); }); + + it('execution of a non-existent contract immediately reverts', async () => { + const context = initContext(); + const results = await new AvmSimulator(context).execute(); + + expect(results.reverted).toBe(true); + expect(results.output).toEqual([]); + expect(results.gasLeft).toEqual({ l2Gas: 0, daGas: 0 }); + }); + it('addition', async () => { const calldata: Fr[] = [new Fr(1), new Fr(2)]; const context = initContext({ env: initExecutionEnvironment({ calldata }) }); @@ -891,6 +903,7 @@ describe('AVM simulator: transpiled Noir contracts', () => { environment: AvmExecutionEnvironment, nestedTrace: PublicSideEffectTraceInterface, isStaticCall: boolean = false, + exists: boolean = true, ) => { expect(trace.traceNestedCall).toHaveBeenCalledTimes(1); expect(trace.traceNestedCall).toHaveBeenCalledWith( @@ -900,18 +913,34 @@ describe('AVM simulator: transpiled Noir contracts', () => { contractCallDepth: new Fr(1), // top call is depth 0, nested is depth 1 globals: environment.globals, // just confirming that nested env looks roughly right isStaticCall: isStaticCall, - // TODO(7121): can't check calldata like this since it is modified on environment construction - // with AvmContextInputs. These should eventually go away. - //calldata: expect.arrayContaining(environment.calldata), // top-level call forwards args + // top-level calls forward args in these tests, + // but nested calls in these tests use public_dispatch, so selector is inserted as first arg + calldata: expect.arrayContaining([/*selector=*/ expect.anything(), ...environment.calldata]), }), /*startGasLeft=*/ expect.anything(), - /*endGasLeft=*/ expect.anything(), - /*bytecode=*/ expect.anything(), + /*bytecode=*/ exists ? expect.anything() : undefined, /*avmCallResults=*/ expect.anything(), // we don't have the NESTED call's results to check /*functionName=*/ expect.anything(), ); }; + it(`Nested call to non-existent contract`, async () => { + const calldata = [value0, value1]; + const context = createContext(calldata); + const callBytecode = getAvmTestContractBytecode('nested_call_to_add'); + // We don't mock getBytecode for the nested contract, so it will not exist + // which should cause the nested call to immediately revert + + const nestedTrace = mock(); + mockTraceFork(trace, nestedTrace); + + const results = await new AvmSimulator(context).executeBytecode(callBytecode); + expect(results.reverted).toBe(true); + expect(results.output).toEqual([]); + + expectTracedNestedCall(context.environment, nestedTrace, /*isStaticCall=*/ false, /*exists=*/ false); + }); + it(`Nested call`, async () => { const calldata = [value0, value1]; const context = createContext(calldata); @@ -1129,39 +1158,33 @@ describe('AVM simulator: transpiled Noir contracts', () => { const sender = AztecAddress.fromNumber(42); const value0 = new Fr(420); - const value1 = new Fr(69); const slotNumber0 = 1; // must update Noir contract if changing this - const slotNumber1 = 2; // must update Noir contract if changing this const slot0 = new Fr(slotNumber0); - const slot1 = new Fr(slotNumber1); const leafSlot0 = computePublicDataTreeLeafSlot(address, slot0); - const leafSlot1 = computePublicDataTreeLeafSlot(address, slot1); - const publicDataTreeLeaf0 = new PublicDataTreeLeaf(leafSlot0, value0); - const _publicDataTreeLeaf1 = new PublicDataTreeLeaf(leafSlot1, value1); - - const INTERNAL_PUBLIC_DATA_SUBTREE_HEIGHT = 0; - - const listSlotNumber0 = 2; // must update Noir contract if changing this - const listSlotNumber1 = listSlotNumber0 + 1; - const listSlot0 = new Fr(listSlotNumber0); - const listSlot1 = new Fr(listSlotNumber1); - const _leafListSlot0 = computePublicDataTreeLeafSlot(address, listSlot0); - const _leafListSlot1 = computePublicDataTreeLeafSlot(address, listSlot1); let worldStateDB: WorldStateDB; let merkleTrees: MerkleTreeWriteOperations; let trace: PublicSideEffectTraceInterface; let persistableState: AvmPersistableStateManager; + let ephemeralForest: AvmEphemeralForest; beforeEach(async () => { trace = mock(); - const telemetry = new NoopTelemetryClient(); - merkleTrees = await (await MerkleTrees.new(openTmpStore(), telemetry)).fork(); - worldStateDB = new WorldStateDB(merkleTrees, mock() as ContractDataSource); - - persistableState = initPersistableStateManager({ worldStateDB, trace, doMerkleOperations: true, merkleTrees }); + worldStateDB = mock(); + const tmp = openTmpStore(); + const telemetryClient = new NoopTelemetryClient(); + merkleTrees = await (await MerkleTrees.new(tmp, telemetryClient)).fork(); + (worldStateDB as jest.Mocked).getMerkleInterface.mockReturnValue(merkleTrees); + ephemeralForest = await AvmEphemeralForest.create(worldStateDB.getMerkleInterface()); + + persistableState = initPersistableStateManager({ + worldStateDB, + trace, + doMerkleOperations: true, + merkleTrees: ephemeralForest, + }); }); const createContext = (calldata: Fr[] = []) => { @@ -1174,13 +1197,17 @@ describe('AVM simulator: transpiled Noir contracts', () => { describe('Public storage accesses', () => { it('Should set value in storage (single)', async () => { const calldata = [value0]; + const { + preimage: lowLeafPreimage, + index: lowLeafIndex, + update: leafAlreadyPresent, + } = await ephemeralForest.getLeafOrLowLeafInfo( + MerkleTreeId.PUBLIC_DATA_TREE, + leafSlot0, + ); + + const lowLeafPath = await ephemeralForest.getSiblingPath(MerkleTreeId.PUBLIC_DATA_TREE, lowLeafIndex); - const [lowLeafIndex, lowLeafPreimage, lowLeafPath, leafAlreadyPresent] = - await getLeafOrLowLeaf( - MerkleTreeId.PUBLIC_DATA_TREE, - leafSlot0.toBigInt(), - merkleTrees, - ); // leafSlot0 should NOT be present in the tree! expect(leafAlreadyPresent).toEqual(false); expect(lowLeafPreimage.slot).not.toEqual(leafSlot0); @@ -1215,12 +1242,17 @@ describe('AVM simulator: transpiled Noir contracts', () => { it('Should read value in storage (single) - never written', async () => { const context = createContext(); - const [lowLeafIndex, lowLeafPreimage, lowLeafPath, leafAlreadyPresent] = - await getLeafOrLowLeaf( - MerkleTreeId.PUBLIC_DATA_TREE, - leafSlot0.toBigInt(), - merkleTrees, - ); + const { + preimage: lowLeafPreimage, + index: lowLeafIndex, + update: leafAlreadyPresent, + } = await ephemeralForest.getLeafOrLowLeafInfo( + MerkleTreeId.PUBLIC_DATA_TREE, + leafSlot0, + ); + + const lowLeafPath = await ephemeralForest.getSiblingPath(MerkleTreeId.PUBLIC_DATA_TREE, lowLeafIndex); + // leafSlot0 should NOT be present in the tree! expect(leafAlreadyPresent).toEqual(false); expect(lowLeafPreimage.slot).not.toEqual(leafSlot0); @@ -1244,17 +1276,19 @@ describe('AVM simulator: transpiled Noir contracts', () => { it('Should read value in storage (single) - written before, leaf exists', async () => { const context = createContext(); - - await merkleTrees.batchInsert( - MerkleTreeId.PUBLIC_DATA_TREE, - [publicDataTreeLeaf0.toBuffer()], - INTERNAL_PUBLIC_DATA_SUBTREE_HEIGHT, + (worldStateDB as jest.Mocked).storageRead.mockImplementationOnce( + (_contractAddress: AztecAddress, _slot: Fr) => Promise.resolve(value0), ); - const [leafIndex, leafPreimage, leafPath] = await getLeafOrLowLeaf( + + await ephemeralForest.writePublicStorage(leafSlot0, value0); + + const { preimage: leafPreimage, index: leafIndex } = await ephemeralForest.getLeafOrLowLeafInfo< MerkleTreeId.PUBLIC_DATA_TREE, - leafSlot0.toBigInt(), - merkleTrees, - ); + PublicDataTreeLeafPreimage + >(MerkleTreeId.PUBLIC_DATA_TREE, leafSlot0); + + const leafPath = await ephemeralForest.getSiblingPath(MerkleTreeId.PUBLIC_DATA_TREE, leafIndex); + // leafSlot0 should be present in the tree! expect(leafPreimage.slot).toEqual(leafSlot0); expect(leafPreimage.value).toEqual(value0); @@ -1279,12 +1313,16 @@ describe('AVM simulator: transpiled Noir contracts', () => { it('Should set and read value in storage (single)', async () => { const calldata = [value0]; - const [lowLeafIndex, lowLeafPreimage, lowLeafPath, leafAlreadyPresent] = - await getLeafOrLowLeaf( - MerkleTreeId.PUBLIC_DATA_TREE, - leafSlot0.toBigInt(), - merkleTrees, - ); + const { + preimage: lowLeafPreimage, + index: lowLeafIndex, + update: leafAlreadyPresent, + } = await ephemeralForest.getLeafOrLowLeafInfo( + MerkleTreeId.PUBLIC_DATA_TREE, + leafSlot0, + ); + const lowLeafPath = await ephemeralForest.getSiblingPath(MerkleTreeId.PUBLIC_DATA_TREE, lowLeafIndex); + // leafSlot0 should NOT be present in the tree! expect(leafAlreadyPresent).toEqual(false); expect(lowLeafPreimage.slot).not.toEqual(leafSlot0); @@ -1302,11 +1340,13 @@ describe('AVM simulator: transpiled Noir contracts', () => { expect(results.reverted).toBe(false); expect(results.output).toEqual([value0]); - const [leafIndex, leafPreimage, leafPath] = await getLeafOrLowLeaf( + const { preimage: leafPreimage, index: leafIndex } = await ephemeralForest.getLeafOrLowLeafInfo< MerkleTreeId.PUBLIC_DATA_TREE, - leafSlot0.toBigInt(), - merkleTrees, - ); + PublicDataTreeLeafPreimage + >(MerkleTreeId.PUBLIC_DATA_TREE, leafSlot0); + + const leafPath = await ephemeralForest.getSiblingPath(MerkleTreeId.PUBLIC_DATA_TREE, leafIndex); + // leafSlot0 should now be present in the tree! expect(leafPreimage.slot).toEqual(leafSlot0); expect(leafPreimage.value).toEqual(value0); diff --git a/yarn-project/simulator/src/avm/avm_simulator.ts b/yarn-project/simulator/src/avm/avm_simulator.ts index dd97bf194dd..24eb0dafe2a 100644 --- a/yarn-project/simulator/src/avm/avm_simulator.ts +++ b/yarn-project/simulator/src/avm/avm_simulator.ts @@ -1,20 +1,29 @@ -import { MAX_L2_GAS_PER_ENQUEUED_CALL } from '@aztec/circuits.js'; +import { + type AztecAddress, + Fr, + type FunctionSelector, + type GlobalVariables, + MAX_L2_GAS_PER_ENQUEUED_CALL, +} from '@aztec/circuits.js'; import { type DebugLogger, createDebugLogger } from '@aztec/foundation/log'; import { strict as assert } from 'assert'; import { SideEffectLimitReachedError } from '../public/side_effect_errors.js'; -import type { AvmContext } from './avm_context.js'; +import { AvmContext } from './avm_context.js'; import { AvmContractCallResult } from './avm_contract_call_result.js'; +import { AvmExecutionEnvironment } from './avm_execution_environment.js'; import { type Gas } from './avm_gas.js'; +import { AvmMachineState } from './avm_machine_state.js'; import { isAvmBytecode } from './bytecode_utils.js'; import { AvmExecutionError, + AvmRevertReason, InvalidProgramCounterError, - NoBytecodeForContractError, revertReasonFromExceptionalHalt, revertReasonFromExplicitRevert, } from './errors.js'; +import { type AvmPersistableStateManager } from './journal/journal.js'; import { decodeInstructionFromBytecode } from './serialization/bytecode_serialization.js'; type OpcodeTally = { @@ -41,16 +50,57 @@ export class AvmSimulator { this.log = createDebugLogger(`aztec:avm_simulator:core(f:${context.environment.functionSelector.toString()})`); } + public static create( + stateManager: AvmPersistableStateManager, + address: AztecAddress, + sender: AztecAddress, + functionSelector: FunctionSelector, // may be temporary (#7224) + transactionFee: Fr, + globals: GlobalVariables, + isStaticCall: boolean, + calldata: Fr[], + allocatedGas: Gas, + ) { + const avmExecutionEnv = new AvmExecutionEnvironment( + address, + sender, + functionSelector, + /*contractCallDepth=*/ Fr.zero(), + transactionFee, + globals, + isStaticCall, + calldata, + ); + + const avmMachineState = new AvmMachineState(allocatedGas); + const avmContext = new AvmContext(stateManager, avmExecutionEnv, avmMachineState); + return new AvmSimulator(avmContext); + } + /** * Fetch the bytecode and execute it in the current context. */ public async execute(): Promise { const bytecode = await this.context.persistableState.getBytecode(this.context.environment.address); - // This assumes that we will not be able to send messages to accounts without code - // Pending classes and instances impl details if (!bytecode) { - throw new NoBytecodeForContractError(this.context.environment.address); + // revert, consuming all gas + const message = `No bytecode found at: ${this.context.environment.address}. Reverting...`; + const revertReason = new AvmRevertReason( + message, + /*failingFunction=*/ { + contractAddress: this.context.environment.address, + functionSelector: this.context.environment.functionSelector, + }, + /*noirCallStack=*/ [], + ); + this.log.warn(message); + return new AvmContractCallResult( + /*reverted=*/ true, + /*output=*/ [], + /*gasLeft=*/ { l2Gas: 0, daGas: 0 }, + revertReason, + ); } return await this.executeBytecode(bytecode); @@ -119,7 +169,7 @@ export class AvmSimulator { const output = machineState.getOutput(); const reverted = machineState.getReverted(); const revertReason = reverted ? revertReasonFromExplicitRevert(output, this.context) : undefined; - const results = new AvmContractCallResult(reverted, output, revertReason); + const results = new AvmContractCallResult(reverted, output, machineState.gasLeft, revertReason); this.log.debug(`Context execution results: ${results.toString()}`); this.printOpcodeTallies(); @@ -134,7 +184,7 @@ export class AvmSimulator { const revertReason = revertReasonFromExceptionalHalt(err, this.context); // Note: "exceptional halts" cannot return data, hence []. - const results = new AvmContractCallResult(/*reverted=*/ true, /*output=*/ [], revertReason); + const results = new AvmContractCallResult(/*reverted=*/ true, /*output=*/ [], machineState.gasLeft, revertReason); this.log.debug(`Context execution results: ${results.toString()}`); this.printOpcodeTallies(); diff --git a/yarn-project/simulator/src/avm/avm_tree.test.ts b/yarn-project/simulator/src/avm/avm_tree.test.ts new file mode 100644 index 00000000000..45c328bfa0f --- /dev/null +++ b/yarn-project/simulator/src/avm/avm_tree.test.ts @@ -0,0 +1,511 @@ +import { + type BatchInsertionResult, + type IndexedTreeId, + MerkleTreeId, + type MerkleTreeWriteOperations, +} from '@aztec/circuit-types'; +import { + NOTE_HASH_TREE_HEIGHT, + type NULLIFIER_TREE_HEIGHT, + type NullifierLeafPreimage, + type PUBLIC_DATA_TREE_HEIGHT, + PublicDataTreeLeaf, + type PublicDataTreeLeafPreimage, +} from '@aztec/circuits.js'; +import { poseidon2Hash } from '@aztec/foundation/crypto'; +import { Fr } from '@aztec/foundation/fields'; +import { type IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; +import { openTmpStore } from '@aztec/kv-store/utils'; +import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; +import { MerkleTrees } from '@aztec/world-state'; + +import { AvmEphemeralForest, EphemeralAvmTree, type IndexedInsertionResult } from './avm_tree.js'; + +let worldStateTrees: MerkleTrees; +let copyState: MerkleTreeWriteOperations; +// Up to 64 dummy note hashes +let noteHashes: Fr[]; +let indexedHashes: Fr[]; +let slots: Fr[]; +let values: Fr[]; +let getSiblingIndex = 21n; + +/****************************************************/ +/*************** Test Helper Functions **************/ +/****************************************************/ + +// Helper to check the equality of the insertion results (low witness, insertion path) +const checkEqualityOfInsertionResults = ( + containerResults: IndexedInsertionResult[], + wsResults: BatchInsertionResult[], +) => { + if (containerResults.length !== wsResults.length) { + throw new Error('Results length mismatch'); + } + for (let i = 0; i < containerResults.length; i++) { + const containerResult = containerResults[i]; + const wsResult = wsResults[i]; + expect(containerResult.lowWitness.siblingPath).toEqual(wsResult.lowLeavesWitnessData![0].siblingPath.toFields()); + expect(containerResult.lowWitness.index).toEqual(wsResult.lowLeavesWitnessData![0].index); + expect(containerResult.lowWitness.preimage).toEqual(wsResult.lowLeavesWitnessData![0].leafPreimage); + expect(containerResult.insertionPath).toEqual(wsResult.newSubtreeSiblingPath.toFields()); + } +}; + +const getWorldStateRoot = async (treeId: MerkleTreeId) => { + return (await worldStateTrees.getTreeInfo(treeId, /*includeUncommitted=*/ true)).root; +}; + +const getWorldStateSiblingPath = (treeId: MerkleTreeId, index: bigint) => { + return worldStateTrees.getSiblingPath(treeId, index, /*includeUncommitted=*/ true); +}; + +const publicDataInsertWorldState = ( + slot: Fr, + value: Fr, +): Promise> => { + return worldStateTrees.batchInsert( + MerkleTreeId.PUBLIC_DATA_TREE, + [new PublicDataTreeLeaf(slot, value).toBuffer()], + 0, + ); +}; + +const nullifierInsertWorldState = ( + nullifier: Fr, +): Promise> => { + return worldStateTrees.batchInsert(MerkleTreeId.NULLIFIER_TREE, [nullifier.toBuffer()], 0); +}; + +// Set up some recurring state for the tests +beforeEach(async () => { + const store = openTmpStore(true); + worldStateTrees = await MerkleTrees.new(store, new NoopTelemetryClient()); + copyState = await worldStateTrees.fork(); + + noteHashes = Array.from({ length: 64 }, (_, i) => new Fr(i)); + // We do + 128 since the first 128 leaves are already filled in the indexed trees (nullifier, public data) + indexedHashes = Array.from({ length: 64 }, (_, i) => new Fr(i + 128)); + + slots = Array.from({ length: 64 }, (_, i) => new Fr(i + 128)); + values = Array.from({ length: 64 }, (_, i) => new Fr(i + 256)); +}); + +/****************************************************/ +/*************** Test Cases *************************/ +/****************************************************/ +/** + * Each test follows a similar structure with an extra test specific to the public data tree + * For each test, we perform a consistency check by comparing the roots and sibling paths (from a random index) or from the insertion results + * between the ephemeral container and the world state trees. + * NOTE HASH TREE TESTS + * 1. Given an empty state, we can append some leaves and remain consistent + * 2. Given a prefilled worldStateDB, we can fork the worldStateDB into the ephemeral container and append some leaves + * + * PUBLIC DATA TREE TESTS + * 1. Given an empty state, we can append some leaves and remain consistent + * 2. Given a prefilled worldStateDB, we can fork the worldStateDB into the ephemeral container and append some leaves + * 3. Given an state with a single leaf, we can update the leaf and append new leaves and remain consistent. + * + * NULLIFIER TREE TEST + * 1. Given an empty state, we can append some leaves and remain consistent + * 2. Given a prefilled worldStateDB, we can fork the worldStateDB into the ephemeral container and append some leaves + * 3. Given an state with a single leaf, we can update the leaf and append new leaves and remain consistent. + * + * FORKING AND MERGING TEST + * 1. Appending to a forked ephemeral container should continue to remain consistent with the world state trees + * 2. Forking, rolling back and forking again should remain consistent with the world state trees + */ + +describe('Simple Note Hash Consistency', () => { + const treeId = MerkleTreeId.NOTE_HASH_TREE; + it('Should do a simple append and check the root starting with empty', async () => { + const treeContainer = await AvmEphemeralForest.create(copyState); + for (const noteHash of noteHashes) { + treeContainer.appendNoteHash(noteHash); + } + await worldStateTrees.appendLeaves(treeId, noteHashes); + + // Check that the roots are consistent + const wsRoot = await getWorldStateRoot(treeId); + const computedRoot = treeContainer.treeMap.get(treeId)!.getRoot(); + expect(computedRoot.toBuffer()).toEqual(wsRoot); + + // Check a sibling path from a random index is consistent + const wsSiblingPath = await getWorldStateSiblingPath(treeId, getSiblingIndex); + const siblingPath = await treeContainer.getSiblingPath(treeId, getSiblingIndex); + expect(siblingPath).toEqual(wsSiblingPath.toFields()); + }); + + it('Should fork a prefilled tree and check consistency', async () => { + // Insert half of our note hashes into the copyState (forked from worldStateDB) + const preInserted = noteHashes.slice(0, 32); + await copyState.appendLeaves(treeId, preInserted); + // The tree container has a DB with the first 32 note hashes + const treeContainer = await AvmEphemeralForest.create(copyState); + + // Append the remaining note hashes within the container + const postInserted = noteHashes.slice(32); + for (const noteHash of postInserted) { + treeContainer.appendNoteHash(noteHash); + } + + // Build a worldstateDB with all the note hashes + await worldStateTrees.appendLeaves(treeId, preInserted.concat(postInserted)); + + // Check that the roots are consistent + const wsRoot = await getWorldStateRoot(treeId); + const computedRoot = treeContainer.treeMap.get(treeId)!.getRoot(); + expect(computedRoot.toBuffer()).toEqual(wsRoot); + + // Check the sibling path from an index before the fork + let wsSiblingPath = await getWorldStateSiblingPath(treeId, getSiblingIndex); + let siblingPath = await treeContainer.getSiblingPath(treeId, getSiblingIndex); + expect(siblingPath).toEqual(wsSiblingPath.toFields()); + + // Check the sibling path that we inserted in the container + getSiblingIndex = 42n; + wsSiblingPath = await getWorldStateSiblingPath(treeId, getSiblingIndex); + siblingPath = await treeContainer.getSiblingPath(treeId, getSiblingIndex); + expect(siblingPath).toEqual(wsSiblingPath.toFields()); + }); +}); + +describe('Simple Public Data Consistency', () => { + const treeId = MerkleTreeId.PUBLIC_DATA_TREE as IndexedTreeId; + let containerInsertionResults: IndexedInsertionResult[] = []; + let worldStateInsertionResults: BatchInsertionResult[] = []; + + // We need to zero out between tests + afterEach(() => { + containerInsertionResults = []; + worldStateInsertionResults = []; + }); + + it('Should do a simple append and check the root starting with empty', async () => { + const treeContainer = await AvmEphemeralForest.create(copyState); + + // Store the insertion results for comparison + // Append all the leaves to the container and the world state trees + for (let i = 0; i < slots.length; i++) { + containerInsertionResults.push(await treeContainer.writePublicStorage(slots[i], values[i])); + worldStateInsertionResults.push(await publicDataInsertWorldState(slots[i], values[i])); + } + + // Compare the roots of the container and the world state trees + const wsRoot = await getWorldStateRoot(treeId); + const computedRoot = treeContainer.treeMap.get(treeId)!.getRoot(); + expect(computedRoot.toBuffer()).toEqual(wsRoot); + + // Check that all the accumulated insertion results match + checkEqualityOfInsertionResults(containerInsertionResults, worldStateInsertionResults); + }); + + it('Should fork a prefilled tree and check consistency', async () => { + const treeId = MerkleTreeId.PUBLIC_DATA_TREE as IndexedTreeId; + const preInsertIndex = 32; + + // Insert the first half of the leaves into the copyState + for (let i = 0; i < preInsertIndex; i++) { + await copyState.batchInsert(treeId, [new PublicDataTreeLeaf(slots[i], values[i]).toBuffer()], 0); + } + const treeContainer = await AvmEphemeralForest.create(copyState); + + // Insert the second half of the leaves into the container + for (let i = preInsertIndex; i < slots.length; i++) { + await treeContainer.writePublicStorage(slots[i], values[i]); + } + + // Insert all the leaves into the world state trees + for (let i = 0; i < slots.length; i++) { + await publicDataInsertWorldState(slots[i], values[i]); + } + + // Compare the roots of the container and the world state trees + const wsRoot = await getWorldStateRoot(treeId); + const computedRoot = treeContainer.treeMap.get(treeId)!.getRoot(); + expect(computedRoot.toBuffer()).toEqual(wsRoot); + + // Get a sibling path from a random index and check it is consistent + const wsSiblingPath = await getWorldStateSiblingPath(treeId, getSiblingIndex); + const siblingPath = await treeContainer.getSiblingPath(treeId, getSiblingIndex); + expect(siblingPath).toEqual(wsSiblingPath.toFields()); + }); + + it('Should update a leaf and check consistency', async () => { + // This test does the following: + // 1. Create a tree with the leaf (slot = 128, value = 256) + // 2. Update the leaf to (slot = 128, value = 258) + // 3. Append a new leaf (slot = 129, value = 257) + // 4. Update the new leaf to (slot = 129, value = 259) + // 5. Check that the roots and resulting sibling paths are consistent after each operation + + // Step 1 + const treeId = MerkleTreeId.PUBLIC_DATA_TREE as IndexedTreeId; + // Add a single element of (slot = 128, value = 256) to the tree before forking into our ephemeral state + await copyState.batchInsert(treeId, [new PublicDataTreeLeaf(new Fr(128), new Fr(256)).toBuffer()], 0); + const treeContainer = await AvmEphemeralForest.create(copyState); + + await publicDataInsertWorldState(new Fr(128), new Fr(256)); + + // Step 2 + containerInsertionResults.push(await treeContainer.writePublicStorage(new Fr(128), new Fr(258))); + worldStateInsertionResults.push(await publicDataInsertWorldState(new Fr(128), new Fr(258))); + + // Step 3 + containerInsertionResults.push(await treeContainer.writePublicStorage(new Fr(129), new Fr(257))); + worldStateInsertionResults.push(await publicDataInsertWorldState(new Fr(129), new Fr(257))); + + // Step 4 + containerInsertionResults.push(await treeContainer.writePublicStorage(new Fr(129), new Fr(259))); + worldStateInsertionResults.push(await publicDataInsertWorldState(new Fr(129), new Fr(259))); + + // Check the roots are consistent + const wsRoot = await getWorldStateRoot(treeId); + const computedRoot = treeContainer.treeMap.get(treeId)!.getRoot(); + expect(computedRoot.toBuffer()).toEqual(wsRoot); + + // Check the insertion results match + checkEqualityOfInsertionResults(containerInsertionResults, worldStateInsertionResults); + }); +}); + +describe('Simple Nullifier Consistency', () => { + const treeId = MerkleTreeId.NULLIFIER_TREE as IndexedTreeId; + let containerInsertionResults: IndexedInsertionResult[] = []; + let worldStateInsertionResults: BatchInsertionResult[] = []; + + // We need to zero out between tests + afterEach(() => { + containerInsertionResults = []; + worldStateInsertionResults = []; + }); + + it('Should do a simple append and check the root starting with empty', async () => { + const treeContainer = await AvmEphemeralForest.create(copyState); + + // Insert all the leaves to the container and the world state trees + for (let i = 0; i < indexedHashes.length; i++) { + containerInsertionResults.push(await treeContainer.appendNullifier(indexedHashes[i])); + worldStateInsertionResults.push(await nullifierInsertWorldState(indexedHashes[i])); + } + + // Compare the roots of the container and the world state + const wsRoot = await getWorldStateRoot(treeId); + const computedRoot = treeContainer.treeMap.get(treeId)!.getRoot(); + expect(computedRoot.toBuffer()).toEqual(wsRoot); + + // Check that all the accumulated insertion results match + checkEqualityOfInsertionResults(containerInsertionResults, worldStateInsertionResults); + + // Check a sibling path from a random index is consistent + const wsSiblingPath = await getWorldStateSiblingPath(treeId, getSiblingIndex); + const siblingPath = await treeContainer.getSiblingPath(treeId, getSiblingIndex); + expect(siblingPath).toEqual(wsSiblingPath.toFields()); + }); + + it('Should fork a prefilled tree and check consistency', async () => { + const preInsertIndex = 32; + for (let i = 0; i < preInsertIndex; i++) { + await copyState.batchInsert(treeId, [indexedHashes[i].toBuffer()], 0); + } + const treeContainer = await AvmEphemeralForest.create(copyState); + + for (let i = preInsertIndex; i < indexedHashes.length; i++) { + containerInsertionResults.push(await treeContainer.appendNullifier(indexedHashes[i])); + } + for (let i = 0; i < indexedHashes.length; i++) { + worldStateInsertionResults.push(await nullifierInsertWorldState(indexedHashes[i])); + } + + // Compare the roots of the container and the world state + const wsRoot = await getWorldStateRoot(treeId); + const computedRoot = treeContainer.treeMap.get(treeId)!.getRoot(); + expect(computedRoot.toBuffer()).toEqual(wsRoot); + + // Check insertion results - note we can only compare against the post-insertion results + checkEqualityOfInsertionResults(containerInsertionResults, worldStateInsertionResults.slice(preInsertIndex)); + }); +}); + +describe('Big Random Avm Ephemeral Container Test', () => { + it('Should do a big random test', async () => { + const shuffleArray = (array: Fr[]) => { + for (let i = array.length - 1; i > 0; i--) { + const j = Math.floor(Math.random() * (i + 1)); + const temp = array[i]; + array[i] = array[j]; + array[j] = temp; + } + }; + + // Can be up to 64 + const ENTRY_COUNT = 50; + shuffleArray(noteHashes); + shuffleArray(indexedHashes); + shuffleArray(slots); + shuffleArray(values); + + const treeContainer = await AvmEphemeralForest.create(copyState); + + // Insert values ino merkleTrees + // Note Hash + await worldStateTrees.appendLeaves(MerkleTreeId.NOTE_HASH_TREE, noteHashes.slice(0, ENTRY_COUNT)); + // Everything else + for (let i = 0; i < ENTRY_COUNT; i++) { + await nullifierInsertWorldState(indexedHashes[i]); + await publicDataInsertWorldState(slots[i], values[i]); + treeContainer.appendNoteHash(noteHashes[i]); + await treeContainer.appendNullifier(indexedHashes[i]); + await treeContainer.writePublicStorage(slots[i], values[i]); + } + + const wsRoots = []; + const computedRoots = []; + for (const treeId of [MerkleTreeId.NOTE_HASH_TREE, MerkleTreeId.NULLIFIER_TREE, MerkleTreeId.PUBLIC_DATA_TREE]) { + wsRoots.push(await getWorldStateRoot(treeId)); + computedRoots.push(treeContainer.treeMap.get(treeId)!.getRoot().toBuffer()); + } + + // All the roots should match + for (let i = 0; i < wsRoots.length; i++) { + expect(computedRoots[i]).toEqual(wsRoots[i]); + } + + // Get a sibling path from each tree and check it is consistent + let wsSiblingPath = await getWorldStateSiblingPath(MerkleTreeId.NOTE_HASH_TREE, getSiblingIndex); + let siblingPath = await treeContainer.getSiblingPath(MerkleTreeId.NOTE_HASH_TREE, getSiblingIndex); + expect(siblingPath).toEqual(wsSiblingPath.toFields()); + + wsSiblingPath = await getWorldStateSiblingPath(MerkleTreeId.NULLIFIER_TREE, getSiblingIndex); + siblingPath = await treeContainer.getSiblingPath(MerkleTreeId.NULLIFIER_TREE, getSiblingIndex); + expect(siblingPath).toEqual(wsSiblingPath.toFields()); + + wsSiblingPath = await getWorldStateSiblingPath(MerkleTreeId.PUBLIC_DATA_TREE, getSiblingIndex); + siblingPath = await treeContainer.getSiblingPath(MerkleTreeId.PUBLIC_DATA_TREE, getSiblingIndex); + expect(siblingPath).toEqual(wsSiblingPath.toFields()); + }); +}); + +describe('Checking forking and merging', () => { + const treeId = MerkleTreeId.PUBLIC_DATA_TREE as IndexedTreeId; + + it('Should check forked results are eventually consistent', async () => { + const treeContainer = await AvmEphemeralForest.create(copyState); + // Write all but the last value to the container + for (let i = 0; i < slots.length - 1; i++) { + await treeContainer.writePublicStorage(slots[i], values[i]); + } + // Fork the ephemeral container + const forkedContainer = treeContainer.fork(); + + // Write the last element to the forked container + await forkedContainer.writePublicStorage(slots[slots.length - 1], values[slots.length - 1]); + const forkedRoot = forkedContainer.treeMap.get(treeId)!.getRoot(); + let originalRoot = treeContainer.treeMap.get(treeId)!.getRoot(); + + // The roots should NOT match since we have an extra element + expect(forkedRoot.toBuffer()).not.toEqual(originalRoot.toBuffer()); + + // Write the last element to original container + await treeContainer.writePublicStorage(slots[slots.length - 1], values[slots.length - 1]); + originalRoot = treeContainer.treeMap.get(treeId)!.getRoot(); + + // We should be consistent now + expect(forkedRoot.toBuffer()).toEqual(originalRoot.toBuffer()); + }); + + it('Fork-Rollback-Fork-Merge should be consistent', async () => { + // To store results + const wsInsertionResults: BatchInsertionResult[] = []; + const containerInsertionResults = []; + + const treeContainer = await AvmEphemeralForest.create(copyState); + // Write the first element to the container + containerInsertionResults.push(await treeContainer.writePublicStorage(slots[0], values[0])); + + // Fork the ephemeral container (simulating a nested call) + const forkedContainer = treeContainer.fork(); + // Write to the fork + containerInsertionResults.push(await forkedContainer.writePublicStorage(slots[1], values[1])); + + // We add to this fork but we check it doesnt impact the initial fork (i.e. Rollback) + const nestedCallFork = forkedContainer.fork(); + await nestedCallFork.writePublicStorage(slots[2], values[2]); + + // We write to the original fork (i.e. the parent of nestedCallFork), e.g. if nestedCallFork reverts + containerInsertionResults.push(await forkedContainer.writePublicStorage(slots[3], values[3])); + + // Build the original worldState with elements of inde 0, 1 and 3 (note that we skip 2) + wsInsertionResults.push(await publicDataInsertWorldState(slots[0], values[0])); + wsInsertionResults.push(await publicDataInsertWorldState(slots[1], values[1])); + wsInsertionResults.push(await publicDataInsertWorldState(slots[3], values[3])); + + const containerRoot = forkedContainer.treeMap.get(treeId)!.getRoot(); + const wsRoot = await getWorldStateRoot(treeId); + expect(containerRoot.toBuffer()).toEqual(wsRoot); + + // Check that all the accumulated insertion results + checkEqualityOfInsertionResults(containerInsertionResults, wsInsertionResults); + }); +}); + +/** + * This is a small test that acts like a sanity check that the inner Tree class is working as expected. + * This is a good canary if things are going wrong, whether the problem is in the Tree class or the EphemeralForest class. + */ +describe('AVM Ephemeral Tree Sanity Test', () => { + it('Should calculate the frontier correctly', async () => { + const store = openTmpStore(true); + const worldStateTrees = await MerkleTrees.new(store, new NoopTelemetryClient()); + const leaves = []; + const numLeaves = 6; + for (let i = 0; i < numLeaves; i++) { + const leaf = new Fr(i); + leaves.push(leaf); + } + const copyState = await worldStateTrees.fork(); + await copyState.appendLeaves(MerkleTreeId.NOTE_HASH_TREE, leaves); + const tree = await EphemeralAvmTree.create( + BigInt(numLeaves), + NOTE_HASH_TREE_HEIGHT, + copyState, + MerkleTreeId.NOTE_HASH_TREE, + ); + + const expectedFrontier0 = new Fr(4); + const exepctedFrontier1 = poseidon2Hash([new Fr(4), new Fr(5)]); + const expectedFrontier2 = poseidon2Hash([ + poseidon2Hash([new Fr(0), new Fr(1)]), + poseidon2Hash([new Fr(2), new Fr(3)]), + ]); + const expectedFrontier = [expectedFrontier0, exepctedFrontier1, expectedFrontier2]; + expect(tree.frontier).toEqual(expectedFrontier); + // Check root + await worldStateTrees.appendLeaves(MerkleTreeId.NOTE_HASH_TREE, leaves); + const treeInfo = await worldStateTrees.getTreeInfo(MerkleTreeId.NOTE_HASH_TREE, true); + const localRoot = tree.getRoot(); + expect(localRoot.toBuffer()).toEqual(treeInfo.root); + }); +}); + +/* eslint no-console: ["error", { allow: ["time", "timeEnd"] }] */ +describe('A basic benchmark', () => { + it('Should benchmark writes', async () => { + // This random is fine for now, since the entry leaves are between 0-127 + // We just want to make sure there are minimal "updates" from this set + const leaves = Array.from({ length: 64 }, _ => Fr.random()); + const slots = leaves.map((_, i) => new Fr(i + 128)); + + const container = await AvmEphemeralForest.create(copyState); + + // Updating the first slot, triggers the index 0 to be added to the minimum stored key in the container + await container.writePublicStorage(new Fr(0), new Fr(128)); + console.time('benchmark'); + // These writes are all new leaves and should be impacted by the key sorted algorithm of the tree. + for (let i = 0; i < leaves.length; i++) { + await container.writePublicStorage(slots[i], leaves[i]); + } + console.timeEnd('benchmark'); + }); +}); diff --git a/yarn-project/simulator/src/avm/avm_tree.ts b/yarn-project/simulator/src/avm/avm_tree.ts new file mode 100644 index 00000000000..6c67bbd0565 --- /dev/null +++ b/yarn-project/simulator/src/avm/avm_tree.ts @@ -0,0 +1,809 @@ +import { type IndexedTreeId, MerkleTreeId, type MerkleTreeReadOperations, getTreeHeight } from '@aztec/circuit-types'; +import { NullifierLeafPreimage, PublicDataTreeLeafPreimage } from '@aztec/circuits.js'; +import { poseidon2Hash } from '@aztec/foundation/crypto'; +import { Fr } from '@aztec/foundation/fields'; +import { type IndexedTreeLeafPreimage, type TreeLeafPreimage } from '@aztec/foundation/trees'; + +import cloneDeep from 'lodash.clonedeep'; + +/****************************************************/ +/****** Structs Used by the AvmEphemeralForest ******/ +/****************************************************/ + +/** + * The preimage and the leaf index of the Low Leaf (Low Nullifier or Low Public Data Leaf) + */ +type PreimageWitness = { + preimage: T; + index: bigint; + update: boolean; +}; + +/** + * Contains the low sibling path and a boolean if the next index pointed to + * by the low leaf is an update or an insertion (i.e. exists or not). + */ +type LeafWitness = PreimageWitness & { + siblingPath: Fr[]; +}; + +/** + * The result of an indexed insertion in an indexed merkle tree. + * This will be used to hint the circuit + */ +export type IndexedInsertionResult = { + leafIndex: bigint; + insertionPath: Fr[]; + newOrElementToUpdate: { update: boolean; element: T }; + lowWitness: LeafWitness; +}; + +/****************************************************/ +/****** The AvmEphemeralForest Class ****************/ +/****************************************************/ + +/** + * This provides a forkable abstraction over the EphemeralAvmTree class + * It contains the logic to look up into a read-only MerkleTreeDb to discover + * the sibling paths and low witnesses that weren't inserted as part of this tx + */ +export class AvmEphemeralForest { + constructor( + public treeDb: MerkleTreeReadOperations, + public treeMap: Map, + // This contains the [leaf index,indexed leaf preimages] tuple that were updated or inserted in the ephemeral tree + // This is needed since we have a sparse collection of keys sorted leaves in the ephemeral tree + public indexedUpdates: Map>, + public indexedSortedKeys: Map, + ) {} + + static async create(treeDb: MerkleTreeReadOperations): Promise { + const treeMap = new Map(); + for (const treeType of [MerkleTreeId.NULLIFIER_TREE, MerkleTreeId.NOTE_HASH_TREE, MerkleTreeId.PUBLIC_DATA_TREE]) { + const treeInfo = await treeDb.getTreeInfo(treeType); + const tree = await EphemeralAvmTree.create(treeInfo.size, treeInfo.depth, treeDb, treeType); + treeMap.set(treeType, tree); + } + const indexedSortedKeys = new Map(); + indexedSortedKeys.set(MerkleTreeId.NULLIFIER_TREE as IndexedTreeId, []); + indexedSortedKeys.set(MerkleTreeId.PUBLIC_DATA_TREE as IndexedTreeId, []); + return new AvmEphemeralForest(treeDb, treeMap, new Map(), indexedSortedKeys); + } + + fork(): AvmEphemeralForest { + return new AvmEphemeralForest( + this.treeDb, + cloneDeep(this.treeMap), + cloneDeep(this.indexedUpdates), + cloneDeep(this.indexedSortedKeys), + ); + } + + /** + * Gets sibling path for a leaf - if the sibling path is not found in the tree, it is fetched from the DB + * @param treeId - The tree to be queried for a sibling path. + * @param index - The index of the leaf for which a sibling path should be returned. + * @returns The sibling path of the leaf. + */ + async getSiblingPath(treeId: MerkleTreeId, index: bigint): Promise { + const tree = this.treeMap.get(treeId)!; + let path = tree.getSiblingPath(index); + if (path === undefined) { + // We dont have the sibling path in our tree - we have to get it from the DB + path = (await this.treeDb.getSiblingPath(treeId, index)).toFields(); + // Since the sibling path could be outdated, we compare it with nodes in our tree + // if we encounter a mismatch, we replace it with the node we found in our tree. + for (let i = 0; i < path.length; i++) { + const siblingIndex = index ^ 1n; + const node = tree.getNode(siblingIndex, tree.depth - i); + if (node !== undefined) { + const nodeHash = tree.hashTree(node, i + 1); + if (!nodeHash.equals(path[i])) { + path[i] = nodeHash; + } + } + index >>= 1n; + } + } + return path; + } + + /** + * This does the work of appending the new leaf and updating the low witness + * @param treeId - The tree to be queried for a sibling path. + * @param lowWitnessIndex - The index of the low leaf in the tree. + * @param lowWitness - The preimage of the low leaf. + * @param newLeafPreimage - The preimage of the new leaf to be inserted. + * @returns The sibling path of the new leaf (i.e. the insertion path) + */ + appendIndexedTree( + treeId: ID, + lowLeafIndex: bigint, + lowLeafPreimage: T, + newLeafPreimage: T, + ): Fr[] { + const tree = this.treeMap.get(treeId)!; + const newLeaf = this.hashPreimage(newLeafPreimage); + const insertIndex = tree.leafCount; + + const lowLeaf = this.hashPreimage(lowLeafPreimage); + // Update the low nullifier hash + this.setIndexedUpdates(treeId, lowLeafIndex, lowLeafPreimage); + tree.updateLeaf(lowLeaf, lowLeafIndex); + // Append the new leaf + tree.appendLeaf(newLeaf); + this.setIndexedUpdates(treeId, insertIndex, newLeafPreimage); + + return tree.getSiblingPath(insertIndex)!; + } + + /** + * This writes or updates a slot in the public data tree with a value + * @param slot - The slot to be written to. + * @param newValue - The value to be written or updated to + * @returns The insertion result which contains the insertion path, low leaf and the new leaf index + */ + async writePublicStorage(slot: Fr, newValue: Fr): Promise> { + // This only works for the public data tree + const treeId = MerkleTreeId.PUBLIC_DATA_TREE; + const tree = this.treeMap.get(treeId)!; + const { preimage, index, update }: PreimageWitness = await this.getLeafOrLowLeafInfo( + treeId, + slot, + ); + const siblingPath = await this.getSiblingPath(treeId, index); + if (update) { + const updatedPreimage = cloneDeep(preimage); + const existingPublicDataSiblingPath = siblingPath; + updatedPreimage.value = newValue; + + // It is really unintuitive that by updating, we are also appending a Zero Leaf to the tree + // Additionally, this leaf preimage does not seem to factor into further appends + const emptyLeaf = new PublicDataTreeLeafPreimage(Fr.ZERO, Fr.ZERO, Fr.ZERO, 0n); + const insertionIndex = tree.leafCount; + tree.updateLeaf(this.hashPreimage(updatedPreimage), index); + tree.appendLeaf(Fr.ZERO); + this.setIndexedUpdates(treeId, index, updatedPreimage); + this.setIndexedUpdates(treeId, insertionIndex, emptyLeaf); + const insertionPath = tree.getSiblingPath(insertionIndex)!; + + // Even though we append an empty leaf into the tree as a part of update - it doesnt seem to impact future inserts... + this._updateSortedKeys(treeId, [updatedPreimage.slot], [index]); + + return { + leafIndex: insertionIndex, + insertionPath, + newOrElementToUpdate: { update: true, element: updatedPreimage }, + lowWitness: { + preimage: preimage, + index: index, + update: true, + siblingPath: existingPublicDataSiblingPath, + }, + }; + } + // We are writing to a new slot, so our preimage is a lowNullifier + const insertionIndex = tree.leafCount; + const updatedLowLeaf = cloneDeep(preimage); + updatedLowLeaf.nextSlot = slot; + updatedLowLeaf.nextIndex = insertionIndex; + + const newPublicDataLeaf = new PublicDataTreeLeafPreimage( + slot, + newValue, + new Fr(preimage.getNextKey()), + preimage.getNextIndex(), + ); + const insertionPath = this.appendIndexedTree(treeId, index, updatedLowLeaf, newPublicDataLeaf); + + // Even though the low leaf key is not updated, we still need to update the sorted keys in case we have + // not seen the low leaf before + this._updateSortedKeys(treeId, [newPublicDataLeaf.slot, updatedLowLeaf.slot], [insertionIndex, index]); + + return { + leafIndex: insertionIndex, + insertionPath: insertionPath, + newOrElementToUpdate: { update: false, element: newPublicDataLeaf }, + lowWitness: { + preimage, + index: index, + update: false, + siblingPath, + }, + }; + } + + private _updateSortedKeys(treeId: IndexedTreeId, keys: Fr[], index: bigint[]): void { + // This is a reference + const existingKeyVector = this.indexedSortedKeys.get(treeId)!; + // Should already be sorted so not need to re-sort if we just update or splice + for (let i = 0; i < keys.length; i++) { + const foundIndex = existingKeyVector.findIndex(x => x[1] === index[i]); + if (foundIndex === -1) { + // New element, we splice it into the correct location + const spliceIndex = + this.searchForKey( + keys[i], + existingKeyVector.map(x => x[0]), + ) + 1; + existingKeyVector.splice(spliceIndex, 0, [keys[i], index[i]]); + } else { + // Update the existing element + existingKeyVector[foundIndex][0] = keys[i]; + } + } + } + + /** + * This appends a nullifier to the nullifier tree, and throws if the nullifier already exists + * @param value - The nullifier to be appended + * @returns The insertion result which contains the insertion path, low leaf and the new leaf index + */ + async appendNullifier(nullifier: Fr): Promise> { + const treeId = MerkleTreeId.NULLIFIER_TREE; + const tree = this.treeMap.get(treeId)!; + const { preimage, index, update }: PreimageWitness = await this.getLeafOrLowLeafInfo( + treeId, + nullifier, + ); + const siblingPath = await this.getSiblingPath(treeId, index); + + if (update) { + throw new Error('Not allowed to update a nullifier'); + } + // We are writing a new entry + const insertionIndex = tree.leafCount; + const updatedLowNullifier = cloneDeep(preimage); + updatedLowNullifier.nextNullifier = nullifier; + updatedLowNullifier.nextIndex = insertionIndex; + + const newNullifierLeaf = new NullifierLeafPreimage(nullifier, preimage.nextNullifier, preimage.nextIndex); + const insertionPath = this.appendIndexedTree(treeId, index, updatedLowNullifier, newNullifierLeaf); + + // Even though the low nullifier key is not updated, we still need to update the sorted keys in case we have + // not seen the low nullifier before + this._updateSortedKeys( + treeId, + [newNullifierLeaf.nullifier, updatedLowNullifier.nullifier], + [insertionIndex, index], + ); + + return { + leafIndex: insertionIndex, + insertionPath: insertionPath, + newOrElementToUpdate: { update: false, element: newNullifierLeaf }, + lowWitness: { + preimage, + index, + update, + siblingPath, + }, + }; + } + + /** + * This appends a note hash to the note hash tree + * @param value - The note hash to be appended + * @returns The insertion result which contains the insertion path + */ + appendNoteHash(noteHash: Fr): Fr[] { + const tree = this.treeMap.get(MerkleTreeId.NOTE_HASH_TREE)!; + tree.appendLeaf(noteHash); + // We use leafCount - 1 here because we would have just appended a leaf + const insertionPath = tree.getSiblingPath(tree.leafCount - 1n); + return insertionPath!; + } + + /** + * This is wrapper around treeId to set values in the indexedUpdates map + */ + private setIndexedUpdates( + treeId: ID, + index: bigint, + preimage: T, + ): void { + let updates = this.indexedUpdates.get(treeId); + if (updates === undefined) { + updates = new Map(); + this.indexedUpdates.set(treeId, updates); + } + updates.set(index, preimage); + } + + /** + * This is wrapper around treeId to get values in the indexedUpdates map + */ + private getIndexedUpdates(treeId: ID, index: bigint): T { + const updates = this.indexedUpdates.get(treeId); + if (updates === undefined) { + throw new Error('No updates found'); + } + const preimage = updates.get(index); + if (preimage === undefined) { + throw new Error('No updates found'); + } + return preimage as T; + } + + /** + * This is wrapper around treeId to check membership (i.e. has()) of index in the indexedUpdates map + */ + private hasLocalUpdates(treeId: ID, index: bigint): boolean { + const updates = this.indexedUpdates.get(treeId); + if (updates === undefined) { + return false; + } + return updates.has(index); + } + + private searchForKey(key: Fr, arr: Fr[]): number { + // We are looking for the index of the largest element in the array that is less than the key + let start = 0; + let end = arr.length; + // Note that the easiest way is to increment the search key by 1 and then do a binary search + const searchKey = key.add(Fr.ONE); + while (start < end) { + const mid = Math.floor((start + end) / 2); + if (arr[mid].cmp(searchKey) < 0) { + // The key + 1 is greater than the arr element, so we can continue searching the top half + start = mid + 1; + } else { + // The key + 1 is LT or EQ the arr element, so we can continue searching the bottom half + end = mid; + } + } + // We either found key + 1 or start is now at the index of the largest element that we would have inserted key + 1 + // Therefore start - 1 is the index of the element just below - note it can be -1 if the first element in the array is + // greater than the key + return start - 1; + } + + /** + * This gets the low leaf preimage and the index of the low leaf in the indexed tree given a value (slot or nullifier value) + * If the value is not found in the tree, it does an external lookup to the merkleDB + * @param treeId - The tree we are looking up in + * @param key - The key for which we are look up the low leaf for. + * @param T - The type of the preimage (PublicData or Nullifier) + * @returns The low leaf preimage and the index of the low leaf in the indexed tree + */ + async getLeafOrLowLeafInfo( + treeId: ID, + key: Fr, + ): Promise> { + const keyOrderedVector = this.indexedSortedKeys.get(treeId)!; + + const vectorIndex = this.searchForKey( + key, + keyOrderedVector.map(x => x[0]), + ); + // We have a match in our local updates + let minPreimage = undefined; + + if (vectorIndex !== -1) { + const [_, leafIndex] = keyOrderedVector[vectorIndex]; + minPreimage = { + preimage: this.getIndexedUpdates(treeId, leafIndex) as T, + index: leafIndex, + }; + } + // This can probably be done better, we want to say if the minInfo is undefined (because this is our first operation) we do the external lookup + const start = minPreimage?.preimage; + const bigIntKey = key.toBigInt(); + + // If we don't have a first element or if that first element is already greater than the target key, we need to do an external lookup + // The low public data witness is in the previous tree + if (start === undefined || start.getKey() > key.toBigInt()) { + // This function returns the leaf index to the actual element if it exists or the leaf index to the low leaf otherwise + const { index, alreadyPresent } = (await this.treeDb.getPreviousValueIndex(treeId, bigIntKey))!; + const preimage = await this.treeDb.getLeafPreimage(treeId, index); + + // Since we have never seen this before - we should insert it into our tree, as we know we will modify this leaf node + const siblingPath = await this.getSiblingPath(treeId, index); + // const siblingPath = (await this.treeDb.getSiblingPath(treeId, index)).toFields(); + + // Is it enough to just insert the sibling path without inserting the leaf? - now probably since we will update this low nullifier index in append + this.treeMap.get(treeId)!.insertSiblingPath(index, siblingPath); + + const lowPublicDataPreimage = preimage as T; + + return { preimage: lowPublicDataPreimage, index: index, update: alreadyPresent }; + } + + // We look for the low element by bouncing between our local indexedUpdates map or the external DB + // The conditions we are looking for are: + // (1) Exact Match: curr.nextKey == key (this is only valid for public data tree) + // (2) Sandwich Match: curr.nextKey > key and curr.key < key + // (3) Max Condition: curr.next_index == 0 and curr.key < key + // Note the min condition does not need to be handled since indexed trees are prefilled with at least the 0 element + let found = false; + let curr = minPreimage!.preimage as T; + let result: PreimageWitness | undefined = undefined; + // Temp to avoid infinite loops - the limit is the number of leaves we may have to read + const LIMIT = 2n ** BigInt(getTreeHeight(treeId)) - 1n; + let counter = 0n; + let lowPublicDataIndex = minPreimage!.index; + while (!found && counter < LIMIT) { + if (curr.getKey() === bigIntKey) { + // We found an exact match - therefore this is an update + found = true; + result = { preimage: curr, index: lowPublicDataIndex, update: true }; + } else if (curr.getKey() < bigIntKey && (curr.getNextIndex() === 0n || curr.getNextKey() > bigIntKey)) { + // We found it via sandwich or max condition, this is a low nullifier + found = true; + result = { preimage: curr, index: lowPublicDataIndex, update: false }; + } + // Update the the values for the next iteration + else { + lowPublicDataIndex = curr.getNextIndex(); + if (this.hasLocalUpdates(treeId, lowPublicDataIndex)) { + curr = this.getIndexedUpdates(treeId, lowPublicDataIndex)!; + } else { + const preimage: IndexedTreeLeafPreimage = (await this.treeDb.getLeafPreimage(treeId, lowPublicDataIndex))!; + curr = preimage as T; + } + } + counter++; + } + // We did not find it - this is unexpected + if (result === undefined) { + throw new Error('No previous value found or ran out of iterations'); + } + return result; + } + + /** + * This hashes the preimage to a field element + */ + hashPreimage(preimage: T): Fr { + // Watch for this edge-case, we are hashing the key=0 leaf to 0. + // This is for backward compatibility with the world state implementation + if (preimage.getKey() === 0n) { + return Fr.zero(); + } + const input = preimage.toHashInputs().map(x => Fr.fromBuffer(x)); + return poseidon2Hash(input); + } +} + +/****************************************************/ +/****** Some useful Structs and Enums **************/ +/****************************************************/ +enum TreeType { + LEAF, + NODE, +} + +type Leaf = { + tag: TreeType.LEAF; + value: Fr; +}; +type Node = { + tag: TreeType.NODE; + leftTree: Tree; + rightTree: Tree; +}; + +type Tree = Leaf | Node; + +enum SiblingStatus { + MEMBER, + NONMEMBER, + ERROR, +} + +type AccumulatedSiblingPath = { + path: Fr[]; + status: SiblingStatus; +}; + +/****************************************************/ +/****** Some Helpful Constructors for Trees ********/ +/****************************************************/ +const Node = (left: Tree, right: Tree): Node => ({ + tag: TreeType.NODE, + leftTree: left, + rightTree: right, +}); + +const Leaf = (value: Fr): Leaf => ({ + tag: TreeType.LEAF, + value, +}); + +/****************************************************/ +/****** The EphemeralAvmTree Class *****************/ +/****************************************************/ + +/** + * This class contains a recursively defined tree that has leaves at different heights + * It is seeded by an existing merkle treeDb for which it derives a frontier + * It is intended to be a lightweight tree that contains only the necessary information to suppport appends or updates + */ +export class EphemeralAvmTree { + private tree: Tree; + private readonly zeroHashes: Fr[]; + public frontier: Fr[]; + + private constructor(public leafCount: bigint, public depth: number) { + let zeroHash = Fr.zero(); + // Can probably cache this elsewhere + const zeroHashes = []; + for (let i = 0; i < this.depth; i++) { + zeroHashes.push(zeroHash); + zeroHash = poseidon2Hash([zeroHash, zeroHash]); + } + this.tree = Leaf(zeroHash); + this.zeroHashes = zeroHashes; + this.frontier = []; + } + + static async create( + forkedLeafCount: bigint, + depth: number, + treeDb: MerkleTreeReadOperations, + merkleId: MerkleTreeId, + ): Promise { + const tree = new EphemeralAvmTree(forkedLeafCount, depth); + await tree.initializeFrontier(treeDb, merkleId); + return tree; + } + + /** + * This is a recursive function that inserts a leaf into the tree + * @param value - The value of the leaf to be inserted + */ + appendLeaf(value: Fr): void { + const insertPath = this._derivePathLE(this.leafCount); + this.tree = this._insertLeaf(value, insertPath, this.depth, this.tree); + this.leafCount++; + } + + /** + * This is a recursive function that upserts a leaf into the tree at a index and depth + * @param value - The value of the leaf to be inserted + * @param index - The index of the leaf to be inserted + * @param depth - The depth of the leaf to be inserted (defaults to the bottom of the tree) + */ + updateLeaf(value: Fr, index: bigint, depth = this.depth): void { + const insertPath = this._derivePathLE(index, depth); + this.tree = this._insertLeaf(value, insertPath, depth, this.tree); + } + + /** + * Get the sibling path of a leaf in the tree + * @param index - The index of the leaf for which a sibling path should be returned. + * @returns The sibling path of the leaf, can fail if the path is not found + */ + getSiblingPath(index: bigint): Fr[] | undefined { + const searchPath = this._derivePathLE(index); + // Handle cases where we error out + const { path, status } = this._getSiblingPath(searchPath, this.tree, []); + if (status === SiblingStatus.ERROR) { + return undefined; + } + return path; + } + + /** + * This upserts the nodes of the sibling path into the tree + * @param index - The index of the leaf that the sibling path is derived from + * @param siblingPath - The sibling path of the index + */ + insertSiblingPath(index: bigint, siblingPath: Fr[]): void { + for (let i = 0; i < siblingPath.length; i++) { + // Flip(XOR) the last bit because we are inserting siblings of the leaf + const sibIndex = index ^ 1n; + this.updateLeaf(siblingPath[i], sibIndex, this.depth - i); + index >>= 1n; + } + } + + /** + * This is a helper function that computes the index of the frontier nodes at each depth + * @param leafCount - The number of leaves in the tree + * @returns An array of frontier indices at each depth, sorted from leaf to root + */ + // Do we really need LeafCount to be a bigint - log2 is on numbers only + static computeFrontierLeafIndices(leafCount: number): number[] { + const numFrontierEntries = Math.floor(Math.log2(leafCount)) + 1; + const frontierIndices = []; + for (let i = 0; i < numFrontierEntries; i++) { + if (leafCount === 0) { + frontierIndices.push(0); + } else if (leafCount % 2 === 0) { + frontierIndices.push(leafCount - 2); + } else { + frontierIndices.push(leafCount - 1); + } + leafCount >>= 1; + } + return frontierIndices; + } + + /** + * This derives the frontier and inserts them into the tree + * @param treeDb - The treeDb to be queried for sibling paths + * @param merkleId - The treeId of the tree to be queried for sibling paths + */ + async initializeFrontier(treeDb: MerkleTreeReadOperations, merkleId: MerkleTreeId): Promise { + // The frontier indices are sorted from the leaf to root + const frontierIndices = EphemeralAvmTree.computeFrontierLeafIndices(Number(this.leafCount)); + // The frontier indices are level-based - i.e. index N at level L. + // Since we can only ask the DB for paths from the root to the leaf, we do the following complicated calculations + // 1) The goal is to insert the frontier node N at level L into the tree. + // 2) We get the path to a leaf that passes through the frontier node we want (there are multiple paths so we just pick one) + // 3) We can only get sibling paths from the root to the leaf, so we get the sibling path of the leaf from (2) + // NOTE: This is terribly inefficient and we should probably change the DB API to allow for getting paths to a node + + const frontierValues = []; + // These are leaf indexes that pass through the frontier nodes + for (let i = 0; i < frontierIndices.length; i++) { + // Given the index to a frontier, we first xor it so we can get its sibling index at depth L + // We then extend the path to that sibling index by shifting left the requisite number of times (for simplicity we just go left down the tree - it doesnt matter) + // This provides us the leaf index such that if we ask for this leafIndex's sibling path, it will pass through the frontier node + const index = BigInt(frontierIndices[i] ^ 1) << BigInt(i); + // This path passes through our frontier node at depth - i + const path = await treeDb.getSiblingPath(merkleId, index); + + // We derive the path that we can walk and truncate it so that it terminates exactly at the frontier node + const frontierPath = this._derivePathLE(BigInt(frontierIndices[i]), this.depth - i); + // The value of the frontier is the at the i-th index of the sibling path + const frontierValue = path.toFields()[i]; + frontierValues.push(frontierValue); + // We insert it at depth - i (the truncated position) + // Note this is a leaf node that wont necessarily be at the bottom of the tree (besides the first frontier) + this.tree = this._insertLeaf(frontierValue, frontierPath, this.depth - i, this.tree); + } + this.frontier = frontierValues; + } + + /** + * Computes the root of the tree + */ + public getRoot(): Fr { + return this.hashTree(this.tree, this.depth); + } + + /** + * Recursively hashes the subtree + * @param tree - The tree to be hashed + * @param depth - The depth of the tree + */ + public hashTree(tree: Tree, depth: number): Fr { + switch (tree.tag) { + case TreeType.NODE: { + return poseidon2Hash([this.hashTree(tree.leftTree, depth - 1), this.hashTree(tree.rightTree, depth - 1)]); + } + case TreeType.LEAF: { + return tree.value; + } + } + } + + /** + * Extracts the subtree from a given index and depth + * @param index - The index of the node to be extracted + * @param depth - The depth of the node to be extracted + * @returns The subtree rooted at the index and depth + */ + public getNode(index: bigint, depth: number): Tree | undefined { + const path = this._derivePathBE(index, depth); + const truncatedPath = path.slice(0, depth); + truncatedPath.reverse(); + try { + return this._getNode(truncatedPath, this.tree); + } catch (e) { + return undefined; + } + } + + /** + * This is the recursive helper for getNode + */ + private _getNode(nodePath: number[], tree: Tree): Tree { + if (nodePath.length === 0) { + return tree; + } + switch (tree.tag) { + case TreeType.NODE: + return nodePath.pop() === 0 ? this._getNode(nodePath, tree.leftTree) : this._getNode(nodePath, tree.rightTree); + + case TreeType.LEAF: + throw new Error('Node not found'); + } + } + + /** Our tree traversal uses an array of 1s and 0s to represent the path to a leaf and expects them to be in LE order + * This helps with deriving it given an index and (optionally a depth) + * @param index - The index to derive a path to within the tree, does not have to terminate at a leaf + * @param depth - The depth to traverse, if not provided it will traverse to the bottom of the tree + * @returns The path to the leaf in LE order + */ + private _derivePathLE(index: bigint, depth = this.depth): number[] { + return this._derivePathBE(index, depth).reverse(); + } + + /** Sometimes we want it in BE order, to make truncating easier + * @param index - The index to derive a path to within the tree, does not have to terminate at a leaf + * @param depth - The depth to traverse, if not provided it will traverse to the bottom of the tree + * @returns The path to the leaf in LE order + */ + private _derivePathBE(index: bigint, depth = this.depth): number[] { + return index + .toString(2) + .padStart(depth, '0') + .split('') + .map(x => parseInt(x)); + } + + /** + * This is a recursive function that upserts a leaf into the tree given a path + * @param value - The value of the leaf to be upserted + * @param insertPath - The path to the leaf, this should be ordered from leaf to root (i.e. LE encoded) + * @param depth - The depth of the tree + * @param tree - The current tree + * @param appendMode - If true we append the relevant zeroHashes to the tree as we traverse + */ + private _insertLeaf(value: Fr, insertPath: number[], depth: number, tree: Tree): Tree { + if (insertPath.length > this.depth || depth > this.depth) { + throw new Error('PATH EXCEEDS DEPTH'); + } + if (depth === 0 || insertPath.length === 0) { + return Leaf(value); + } + switch (tree.tag) { + case TreeType.NODE: { + return insertPath.pop() === 0 + ? Node(this._insertLeaf(value, insertPath, depth - 1, tree.leftTree), tree.rightTree) + : Node(tree.leftTree, this._insertLeaf(value, insertPath, depth - 1, tree.rightTree)); + } + case TreeType.LEAF: { + const zeroLeaf = Leaf(this.zeroHashes[depth - 1]); + return insertPath.pop() === 0 + ? Node(this._insertLeaf(value, insertPath, depth - 1, zeroLeaf), zeroLeaf) + : Node(zeroLeaf, this._insertLeaf(value, insertPath, depth - 1, zeroLeaf)); + } + } + } + + /* Recursive helper for getSiblingPath, this only looks inside the tree and does not resolve using + * a DB. If a path is not found, it returns an error status that is expected to be handled by the caller + * @param searchPath - The path to the leaf for which we would like the sibling pathin LE order + * @param tree - The current tree + * @param acc - The accumulated sibling path + */ + private _getSiblingPath(searchPath: number[], tree: Tree, acc: Fr[]): AccumulatedSiblingPath { + // If we have reached the end of the path, we should be at a leaf or empty node + // If it is a leaf, we check if the value is equal to the leaf value + // If it is empty we check if the value is equal to zero + if (searchPath.length === 0) { + switch (tree.tag) { + case TreeType.LEAF: + return { path: acc, status: SiblingStatus.MEMBER }; + case TreeType.NODE: + return { path: [], status: SiblingStatus.ERROR }; + } + } + // Keep exploring here + switch (tree.tag) { + case TreeType.NODE: { + // Look at the next element of the path to decided if we go left or right, note this mutates! + return searchPath.pop() === 0 + ? this._getSiblingPath( + searchPath, + tree.leftTree, + [this.hashTree(tree.rightTree, searchPath.length)].concat(acc), + ) + : this._getSiblingPath( + searchPath, + tree.rightTree, + [this.hashTree(tree.leftTree, searchPath.length)].concat(acc), + ); + } + // In these two situations we are exploring a subtree we dont have information about + // We should return an error and look inside the DB + case TreeType.LEAF: + return { path: [], status: SiblingStatus.ERROR }; + } + } +} diff --git a/yarn-project/simulator/src/avm/errors.ts b/yarn-project/simulator/src/avm/errors.ts index f9c5d819a77..8df7a7d7303 100644 --- a/yarn-project/simulator/src/avm/errors.ts +++ b/yarn-project/simulator/src/avm/errors.ts @@ -67,6 +67,17 @@ export class TagCheckError extends AvmExecutionError { } } +/** + * Error is thrown when a relative memory address resolved to an offset which + * is out of range, i.e, greater than maxUint32. + */ +export class AddressOutOfRangeError extends AvmExecutionError { + constructor(baseAddr: number, relOffset: number) { + super(`Address out of range. Base address ${baseAddr}, relative offset ${relOffset}`); + this.name = 'AddressOutOfRangeError'; + } +} + /** Error thrown when out of gas. */ export class OutOfGasError extends AvmExecutionError { constructor(dimensions: string[]) { diff --git a/yarn-project/simulator/src/avm/fixtures/index.ts b/yarn-project/simulator/src/avm/fixtures/index.ts index 3a8d594e4b5..5711c9d9dc6 100644 --- a/yarn-project/simulator/src/avm/fixtures/index.ts +++ b/yarn-project/simulator/src/avm/fixtures/index.ts @@ -1,4 +1,4 @@ -import { type MerkleTreeWriteOperations, isNoirCallStackUnresolved } from '@aztec/circuit-types'; +import { isNoirCallStackUnresolved } from '@aztec/circuit-types'; import { GasFees, GlobalVariables, MAX_L2_GAS_PER_ENQUEUED_CALL } from '@aztec/circuits.js'; import { type FunctionArtifact, FunctionSelector } from '@aztec/foundation/abi'; import { AztecAddress } from '@aztec/foundation/aztec-address'; @@ -16,6 +16,7 @@ import { AvmContext } from '../avm_context.js'; import { AvmExecutionEnvironment } from '../avm_execution_environment.js'; import { AvmMachineState } from '../avm_machine_state.js'; import { Field, Uint8, Uint32, Uint64 } from '../avm_memory_types.js'; +import { type AvmEphemeralForest } from '../avm_tree.js'; import { type AvmRevertReason } from '../errors.js'; import { AvmPersistableStateManager } from '../journal/journal.js'; import { NullifierManager } from '../journal/nullifiers.js'; @@ -43,7 +44,7 @@ export function initPersistableStateManager(overrides?: { publicStorage?: PublicStorage; nullifiers?: NullifierManager; doMerkleOperations?: boolean; - merkleTrees?: MerkleTreeWriteOperations; + merkleTrees?: AvmEphemeralForest; }): AvmPersistableStateManager { const worldStateDB = overrides?.worldStateDB || mock(); return new AvmPersistableStateManager( @@ -52,7 +53,7 @@ export function initPersistableStateManager(overrides?: { overrides?.publicStorage || new PublicStorage(worldStateDB), overrides?.nullifiers || new NullifierManager(worldStateDB), overrides?.doMerkleOperations || false, - overrides?.merkleTrees || mock(), + overrides?.merkleTrees || mock(), ); } diff --git a/yarn-project/simulator/src/avm/index.ts b/yarn-project/simulator/src/avm/index.ts index 611d697850c..4beb53410cb 100644 --- a/yarn-project/simulator/src/avm/index.ts +++ b/yarn-project/simulator/src/avm/index.ts @@ -1,2 +1,3 @@ export * from './avm_simulator.js'; export * from './journal/index.js'; +export * from './avm_tree.js'; diff --git a/yarn-project/simulator/src/avm/journal/journal.test.ts b/yarn-project/simulator/src/avm/journal/journal.test.ts index 68c009a8afe..976f5f748ea 100644 --- a/yarn-project/simulator/src/avm/journal/journal.test.ts +++ b/yarn-project/simulator/src/avm/journal/journal.test.ts @@ -75,8 +75,8 @@ describe('journal', () => { expect(trace.traceNoteHashCheck).toHaveBeenCalledWith(address, utxo, leafIndex, exists); }); - it('writeNoteHash works', async () => { - await persistableState.writeNoteHash(address, utxo); + it('writeNoteHash works', () => { + persistableState.writeNoteHash(address, utxo); expect(trace.traceNewNoteHash).toHaveBeenCalledTimes(1); expect(trace.traceNewNoteHash).toHaveBeenCalledWith(expect.objectContaining(address), /*noteHash=*/ utxo); }); diff --git a/yarn-project/simulator/src/avm/journal/journal.ts b/yarn-project/simulator/src/avm/journal/journal.ts index ac5e8810474..23e4d3e8e94 100644 --- a/yarn-project/simulator/src/avm/journal/journal.ts +++ b/yarn-project/simulator/src/avm/journal/journal.ts @@ -1,18 +1,16 @@ -import { type IndexedTreeId, MerkleTreeId, type MerkleTreeWriteOperations } from '@aztec/circuit-types'; +import { MerkleTreeId } from '@aztec/circuit-types'; import { type AztecAddress, type Gas, type NullifierLeafPreimage, type PublicCallRequest, - PublicDataTreeLeaf, - PublicDataTreeLeafPreimage, + type PublicDataTreeLeafPreimage, SerializableContractInstance, computePublicBytecodeCommitment, } from '@aztec/circuits.js'; import { computePublicDataTreeLeafSlot, siloNoteHash, siloNullifier } from '@aztec/circuits.js/hash'; import { Fr } from '@aztec/foundation/fields'; import { createDebugLogger } from '@aztec/foundation/log'; -import { type IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; import assert from 'assert'; @@ -21,7 +19,8 @@ import { type WorldStateDB } from '../../public/public_db_sources.js'; import { type PublicSideEffectTraceInterface } from '../../public/side_effect_trace_interface.js'; import { type AvmContractCallResult } from '../avm_contract_call_result.js'; import { type AvmExecutionEnvironment } from '../avm_execution_environment.js'; -import { NullifierManager } from './nullifiers.js'; +import { AvmEphemeralForest } from '../avm_tree.js'; +import { NullifierCollisionError, NullifierManager } from './nullifiers.js'; import { PublicStorage } from './public_storage.js'; /** @@ -36,8 +35,8 @@ import { PublicStorage } from './public_storage.js'; export class AvmPersistableStateManager { private readonly log = createDebugLogger('aztec:avm_simulator:state_manager'); - /** Interface to perform merkle tree operations */ - public merkleTrees: MerkleTreeWriteOperations; + /** Make sure a forked state is never merged twice. */ + private alreadyMergedIntoParent = false; constructor( /** Reference to node storage */ @@ -50,43 +49,92 @@ export class AvmPersistableStateManager { /** Nullifier set, including cached/recently-emitted nullifiers */ private readonly nullifiers: NullifierManager = new NullifierManager(worldStateDB), private readonly doMerkleOperations: boolean = false, - merkleTrees?: MerkleTreeWriteOperations, - ) { - if (merkleTrees) { - this.merkleTrees = merkleTrees; - } else { - this.merkleTrees = worldStateDB.getMerkleInterface(); - } - } + /** Ephmeral forest for merkle tree operations */ + public readonly merkleTrees: AvmEphemeralForest, + ) {} /** * Create a new state manager with some preloaded pending siloed nullifiers */ - public static newWithPendingSiloedNullifiers( + public static async newWithPendingSiloedNullifiers( worldStateDB: WorldStateDB, trace: PublicSideEffectTraceInterface, pendingSiloedNullifiers: Fr[], + doMerkleOperations: boolean = false, ) { const parentNullifiers = NullifierManager.newWithPendingSiloedNullifiers(worldStateDB, pendingSiloedNullifiers); + const ephemeralForest = await AvmEphemeralForest.create(worldStateDB.getMerkleInterface()); return new AvmPersistableStateManager( worldStateDB, trace, /*publicStorage=*/ new PublicStorage(worldStateDB), /*nullifiers=*/ parentNullifiers.fork(), + doMerkleOperations, + ephemeralForest, + ); + } + + /** + * Create a new state manager + */ + public static async create(worldStateDB: WorldStateDB, trace: PublicSideEffectTraceInterface) { + const ephemeralForest = await AvmEphemeralForest.create(worldStateDB.getMerkleInterface()); + return new AvmPersistableStateManager( + worldStateDB, + trace, + /*publicStorage=*/ new PublicStorage(worldStateDB), + /*nullifiers=*/ new NullifierManager(worldStateDB), + /*doMerkleOperations=*/ true, + ephemeralForest, ); } /** * Create a new state manager forked from this one */ - public fork(incrementSideEffectCounter: boolean = false) { + public fork() { return new AvmPersistableStateManager( this.worldStateDB, - this.trace.fork(incrementSideEffectCounter), + this.trace.fork(), this.publicStorage.fork(), this.nullifiers.fork(), this.doMerkleOperations, + this.merkleTrees.fork(), + ); + } + + /** + * Accept forked world state modifications & traced side effects / hints + */ + public merge(forkedState: AvmPersistableStateManager) { + this._merge(forkedState, /*reverted=*/ false); + } + + /** + * Reject forked world state modifications & traced side effects, keep traced hints + */ + public reject(forkedState: AvmPersistableStateManager) { + this._merge(forkedState, /*reverted=*/ true); + } + + /** + * Commit cached storage writes to the DB. + * Keeps public storage up to date from tx to tx within a block. + */ + public async commitStorageWritesToDB() { + await this.publicStorage.commitToDB(); + } + + private _merge(forkedState: AvmPersistableStateManager, reverted: boolean) { + // sanity check to avoid merging the same forked trace twice + assert( + !forkedState.alreadyMergedIntoParent, + 'Cannot merge forked state that has already been merged into its parent!', ); + forkedState.alreadyMergedIntoParent = true; + this.publicStorage.acceptAndMerge(forkedState.publicStorage); + this.nullifiers.acceptAndMerge(forkedState.nullifiers); + this.trace.merge(forkedState.trace, reverted); } /** @@ -102,27 +150,18 @@ export class AvmPersistableStateManager { this.publicStorage.write(contractAddress, slot, value); const leafSlot = computePublicDataTreeLeafSlot(contractAddress, slot); if (this.doMerkleOperations) { - const result = await this.merkleTrees.batchInsert( - MerkleTreeId.PUBLIC_DATA_TREE, - [new PublicDataTreeLeaf(leafSlot, value).toBuffer()], - 0, - ); + const result = await this.merkleTrees.writePublicStorage(leafSlot, value); assert(result !== undefined, 'Public data tree insertion error. You might want to disable skipMerkleOperations.'); this.log.debug(`Inserted public data tree leaf at leafSlot ${leafSlot}, value: ${value}`); - const lowLeafInfo = result.lowLeavesWitnessData![0]; - const lowLeafPreimage = lowLeafInfo.leafPreimage as PublicDataTreeLeafPreimage; + const lowLeafInfo = result.lowWitness; + const lowLeafPreimage = result.lowWitness.preimage as PublicDataTreeLeafPreimage; const lowLeafIndex = lowLeafInfo.index; - const lowLeafPath = lowLeafInfo.siblingPath.toFields(); + const lowLeafPath = lowLeafInfo.siblingPath; + + const insertionPath = result.insertionPath; + const newLeafPreimage = result.newOrElementToUpdate.element as PublicDataTreeLeafPreimage; - const insertionPath = result.newSubtreeSiblingPath.toFields(); - const newLeafPreimage = new PublicDataTreeLeafPreimage( - leafSlot, - value, - lowLeafPreimage.nextSlot, - lowLeafPreimage.nextIndex, - ); - // FIXME: Why do we need to hint both preimages for public data writes, but not for nullifier insertions? this.trace.tracePublicStorageWrite( contractAddress, slot, @@ -146,23 +185,24 @@ export class AvmPersistableStateManager { * @returns the latest value written to slot, or 0 if never written to before */ public async readStorage(contractAddress: AztecAddress, slot: Fr): Promise { - const { value, exists, cached } = await this.publicStorage.read(contractAddress, slot); - this.log.debug( - `Storage read (address=${contractAddress}, slot=${slot}): value=${value}, exists=${exists}, cached=${cached}`, - ); + const { value, cached } = await this.publicStorage.read(contractAddress, slot); + this.log.debug(`Storage read (address=${contractAddress}, slot=${slot}): value=${value}, cached=${cached}`); const leafSlot = computePublicDataTreeLeafSlot(contractAddress, slot); if (this.doMerkleOperations) { // Get leaf if present, low leaf if absent // If leaf is present, hint/trace it. Otherwise, hint/trace the low leaf. - const [leafIndex, leafPreimage, leafPath, _alreadyPresent] = await getLeafOrLowLeaf( - MerkleTreeId.PUBLIC_DATA_TREE, - leafSlot.toBigInt(), - this.merkleTrees, - ); - // FIXME: cannot have this assertion until "caching" is done via ephemeral merkle writes - //assert(alreadyPresent == exists, 'WorldStateDB contains public data leaf, but merkle tree does not.... This is a bug!'); + const { + preimage, + index: leafIndex, + update: exists, + } = await this.merkleTrees.getLeafOrLowLeafInfo(MerkleTreeId.PUBLIC_DATA_TREE, leafSlot); + // The index and preimage here is either the low leaf or the leaf itself (depending on the value of update flag) + // In either case, we just want the sibling path to this leaf - it's up to the avm to distinguish if it's a low leaf or not + const leafPath = await this.merkleTrees.getSiblingPath(MerkleTreeId.PUBLIC_DATA_TREE, leafIndex); + const leafPreimage = preimage as PublicDataTreeLeafPreimage; + this.log.debug( `leafPreimage.nextSlot: ${leafPreimage.nextSlot}, leafPreimage.nextIndex: ${Number(leafPreimage.nextIndex)}`, ); @@ -171,15 +211,16 @@ export class AvmPersistableStateManager { if (!exists) { // Sanity check that the leaf slot is skipped by low leaf when it doesn't exist assert( - leafSlot.toBigInt() > leafPreimage.slot.toBigInt() && leafSlot.toBigInt() < leafPreimage.nextSlot.toBigInt(), - 'Public data tree low leaf should skip the target leaf slot when the target leaf does not exist.', + leafPreimage.slot.toBigInt() < leafSlot.toBigInt() && + (leafPreimage.nextIndex === 0n || leafPreimage.nextSlot.toBigInt() > leafSlot.toBigInt()), + 'Public data tree low leaf should skip the target leaf slot when the target leaf does not exist or is the max value.', ); } this.log.debug( `Tracing storage leaf preimage slot=${slot}, leafSlot=${leafSlot}, value=${value}, nextKey=${leafPreimage.nextSlot}, nextIndex=${leafPreimage.nextIndex}`, ); // On non-existence, AVM circuit will need to recognize that leafPreimage.slot != leafSlot, - // prove that this is a low leaf that skips leafSlot, and then prove memebership of the leaf. + // prove that this is a low leaf that skips leafSlot, and then prove membership of the leaf. this.trace.tracePublicStorageRead(contractAddress, slot, value, leafPreimage, new Fr(leafIndex), leafPath); } else { this.trace.tracePublicStorageRead(contractAddress, slot, value); @@ -196,10 +237,8 @@ export class AvmPersistableStateManager { * @returns the latest value written to slot, or 0 if never written to before */ public async peekStorage(contractAddress: AztecAddress, slot: Fr): Promise { - const { value, exists, cached } = await this.publicStorage.read(contractAddress, slot); - this.log.debug( - `Storage peek (address=${contractAddress}, slot=${slot}): value=${value}, exists=${exists}, cached=${cached}`, - ); + const { value, cached } = await this.publicStorage.read(contractAddress, slot); + this.log.debug(`Storage peek (address=${contractAddress}, slot=${slot}): value=${value}, cached=${cached}`); return Promise.resolve(value); } @@ -222,7 +261,7 @@ export class AvmPersistableStateManager { // TODO(8287): We still return exists here, but we need to transmit both the requested noteHash and the gotLeafValue // such that the VM can constrain the equality and decide on exists based on that. const path = await this.merkleTrees.getSiblingPath(MerkleTreeId.NOTE_HASH_TREE, leafIndex.toBigInt()); - this.trace.traceNoteHashCheck(contractAddress, gotLeafValue, leafIndex, exists, path.toFields()); + this.trace.traceNoteHashCheck(contractAddress, gotLeafValue, leafIndex, exists, path); } else { this.trace.traceNoteHashCheck(contractAddress, gotLeafValue, leafIndex, exists); } @@ -233,19 +272,15 @@ export class AvmPersistableStateManager { * Write a note hash, trace the write. * @param noteHash - the unsiloed note hash to write */ - public async writeNoteHash(contractAddress: AztecAddress, noteHash: Fr): Promise { + public writeNoteHash(contractAddress: AztecAddress, noteHash: Fr): void { this.log.debug(`noteHashes(${contractAddress}) += @${noteHash}.`); if (this.doMerkleOperations) { - // TODO: We should track this globally here in the state manager - const info = await this.merkleTrees.getTreeInfo(MerkleTreeId.NOTE_HASH_TREE); - const leafIndex = new Fr(info.size + 1n); - - const path = await this.merkleTrees.getSiblingPath(MerkleTreeId.NOTE_HASH_TREE, leafIndex.toBigInt()); + // Should write a helper for this + const leafIndex = new Fr(this.merkleTrees.treeMap.get(MerkleTreeId.NOTE_HASH_TREE)!.leafCount); const siloedNoteHash = siloNoteHash(contractAddress, noteHash); - - await this.merkleTrees.appendLeaves(MerkleTreeId.NOTE_HASH_TREE, [siloedNoteHash]); - this.trace.traceNewNoteHash(contractAddress, noteHash, leafIndex, path.toFields()); + const insertionPath = this.merkleTrees.appendNoteHash(siloedNoteHash); + this.trace.traceNewNoteHash(contractAddress, noteHash, leafIndex, insertionPath); } else { this.trace.traceNewNoteHash(contractAddress, noteHash); } @@ -265,15 +300,15 @@ export class AvmPersistableStateManager { if (this.doMerkleOperations) { // Get leaf if present, low leaf if absent // If leaf is present, hint/trace it. Otherwise, hint/trace the low leaf. - const [leafIndex, leafPreimage, leafPath, alreadyPresent] = await getLeafOrLowLeaf( - MerkleTreeId.NULLIFIER_TREE, - siloedNullifier.toBigInt(), - this.merkleTrees, - ); - assert( - alreadyPresent == exists, - 'WorldStateDB contains nullifier leaf, but merkle tree does not.... This is a bug!', - ); + const { + preimage, + index: leafIndex, + update, + } = await this.merkleTrees.getLeafOrLowLeafInfo(MerkleTreeId.NULLIFIER_TREE, siloedNullifier); + const leafPreimage = preimage as NullifierLeafPreimage; + const leafPath = await this.merkleTrees.getSiblingPath(MerkleTreeId.NULLIFIER_TREE, leafIndex); + + assert(update == exists, 'WorldStateDB contains nullifier leaf, but merkle tree does not.... This is a bug!'); this.log.debug( `nullifiers(${contractAddress})@${nullifier} ?? leafIndex: ${leafIndex}, exists: ${exists}, pending: ${isPending}.`, @@ -313,40 +348,55 @@ export class AvmPersistableStateManager { */ public async writeNullifier(contractAddress: AztecAddress, nullifier: Fr) { this.log.debug(`nullifiers(${contractAddress}) += ${nullifier}.`); - // Cache pending nullifiers for later access - await this.nullifiers.append(contractAddress, nullifier); const siloedNullifier = siloNullifier(contractAddress, nullifier); if (this.doMerkleOperations) { + // Maybe overkill, but we should check if the nullifier is already present in the tree before attempting to insert + // It might be better to catch the error from the insert operation // Trace all nullifier creations, even duplicate insertions that fail - const alreadyPresent = await this.merkleTrees.getPreviousValueIndex( + const { preimage, index, update } = await this.merkleTrees.getLeafOrLowLeafInfo( MerkleTreeId.NULLIFIER_TREE, - siloedNullifier.toBigInt(), + siloedNullifier, ); - if (alreadyPresent) { - this.log.verbose(`Nullifier already present in tree: ${nullifier} at index ${alreadyPresent.index}.`); + if (update) { + this.log.verbose(`Nullifier already present in tree: ${nullifier} at index ${index}.`); + // If the nullifier is already present, we should not insert it again + // instead we provide the direct membership path + const path = await this.merkleTrees.getSiblingPath(MerkleTreeId.NULLIFIER_TREE, index); + // This just becomes a nullifier read hint + this.trace.traceNullifierCheck( + contractAddress, + nullifier, + /*exists=*/ update, + preimage as NullifierLeafPreimage, + new Fr(index), + path, + ); + throw new NullifierCollisionError( + `Nullifier ${nullifier} at contract ${contractAddress} already exists in parent cache or host.`, + ); + } else { + // Cache pending nullifiers for later access + await this.nullifiers.append(contractAddress, nullifier); + // We append the new nullifier + const appendResult = await this.merkleTrees.appendNullifier(siloedNullifier); + const lowLeafPreimage = appendResult.lowWitness.preimage as NullifierLeafPreimage; + const lowLeafIndex = appendResult.lowWitness.index; + const lowLeafPath = appendResult.lowWitness.siblingPath; + const insertionPath = appendResult.insertionPath; + this.trace.traceNewNullifier( + contractAddress, + nullifier, + lowLeafPreimage, + new Fr(lowLeafIndex), + lowLeafPath, + insertionPath, + ); } - const insertionResult = await this.merkleTrees.batchInsert( - MerkleTreeId.NULLIFIER_TREE, - [siloedNullifier.toBuffer()], - 0, - ); - const lowLeafInfo = insertionResult.lowLeavesWitnessData![0]; - const lowLeafPreimage = lowLeafInfo.leafPreimage as NullifierLeafPreimage; - const lowLeafIndex = lowLeafInfo.index; - const lowLeafPath = lowLeafInfo.siblingPath.toFields(); - const insertionPath = insertionResult.newSubtreeSiblingPath.toFields(); - - this.trace.traceNewNullifier( - contractAddress, - nullifier, - lowLeafPreimage, - new Fr(lowLeafIndex), - lowLeafPath, - insertionPath, - ); } else { + // Cache pending nullifiers for later access + await this.nullifiers.append(contractAddress, nullifier); this.trace.traceNewNullifier(contractAddress, nullifier); } } @@ -371,7 +421,11 @@ export class AvmPersistableStateManager { if (this.doMerkleOperations) { // TODO(8287): We still return exists here, but we need to transmit both the requested msgHash and the value // such that the VM can constrain the equality and decide on exists based on that. - const path = await this.merkleTrees.getSiblingPath(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, msgLeafIndex.toBigInt()); + // We should defintely add a helper here + const path = await this.merkleTrees.treeDb.getSiblingPath( + MerkleTreeId.L1_TO_L2_MESSAGE_TREE, + msgLeafIndex.toBigInt(), + ); this.trace.traceL1ToL2MessageCheck(contractAddress, valueAtIndex, msgLeafIndex, exists, path.toFields()); } else { this.trace.traceL1ToL2MessageCheck(contractAddress, valueAtIndex, msgLeafIndex, exists); @@ -427,21 +481,6 @@ export class AvmPersistableStateManager { } } - /** - * Accept nested world state modifications - */ - public mergeForkedState(forkedState: AvmPersistableStateManager) { - this.publicStorage.acceptAndMerge(forkedState.publicStorage); - this.nullifiers.acceptAndMerge(forkedState.nullifiers); - this.trace.mergeSuccessfulForkedTrace(forkedState.trace); - } - - public rejectForkedState(forkedState: AvmPersistableStateManager) { - this.publicStorage.acceptAndMerge(forkedState.publicStorage); - this.nullifiers.acceptAndMerge(forkedState.nullifiers); - this.trace.mergeRevertedForkedTrace(forkedState.trace); - } - /** * Get a contract's bytecode from the contracts DB, also trace the contract class and instance */ @@ -486,7 +525,6 @@ export class AvmPersistableStateManager { forkedState: AvmPersistableStateManager, nestedEnvironment: AvmExecutionEnvironment, startGasLeft: Gas, - endGasLeft: Gas, bytecode: Buffer, avmCallResults: AvmContractCallResult, ) { @@ -503,7 +541,6 @@ export class AvmPersistableStateManager { forkedState.trace, nestedEnvironment, startGasLeft, - endGasLeft, bytecode, avmCallResults, functionName, @@ -514,30 +551,3 @@ export class AvmPersistableStateManager { this.trace.traceEnqueuedCall(publicCallRequest, calldata, reverted); } } - -/** - * Get leaf if present, low leaf if absent - */ -export async function getLeafOrLowLeaf( - treeId: IndexedTreeId, - key: bigint, - merkleTrees: MerkleTreeWriteOperations, -) { - // "key" is siloed slot (leafSlot) or siloed nullifier - const previousValueIndex = await merkleTrees.getPreviousValueIndex(treeId, key); - assert( - previousValueIndex !== undefined, - `${MerkleTreeId[treeId]} low leaf index should always be found (even if target leaf does not exist)`, - ); - const { index: leafIndex, alreadyPresent } = previousValueIndex; - - const leafPreimage = await merkleTrees.getLeafPreimage(treeId, leafIndex); - assert( - leafPreimage !== undefined, - `${MerkleTreeId[treeId]} low leaf preimage should never be undefined (even if target leaf does not exist)`, - ); - - const leafPath = await merkleTrees.getSiblingPath(treeId, leafIndex); - - return [leafIndex, leafPreimage as TreePreimageType, leafPath.toFields(), alreadyPresent] as const; -} diff --git a/yarn-project/simulator/src/avm/journal/public_storage.test.ts b/yarn-project/simulator/src/avm/journal/public_storage.test.ts index 442761b6786..6db7f7e7bbf 100644 --- a/yarn-project/simulator/src/avm/journal/public_storage.test.ts +++ b/yarn-project/simulator/src/avm/journal/public_storage.test.ts @@ -20,9 +20,9 @@ describe('avm public storage', () => { const contractAddress = AztecAddress.fromNumber(1); const slot = new Fr(2); // never written! - const { exists, value: gotValue, cached } = await publicStorage.read(contractAddress, slot); + publicDb.storageRead.mockResolvedValue(Fr.ZERO); + const { value: gotValue, cached } = await publicStorage.read(contractAddress, slot); // doesn't exist, value is zero - expect(exists).toEqual(false); expect(gotValue).toEqual(Fr.ZERO); expect(cached).toEqual(false); }); @@ -33,9 +33,8 @@ describe('avm public storage', () => { const value = new Fr(3); // Write to cache publicStorage.write(contractAddress, slot, value); - const { exists, value: gotValue, cached } = await publicStorage.read(contractAddress, slot); + const { value: gotValue, cached } = await publicStorage.read(contractAddress, slot); // exists because it was previously written - expect(exists).toEqual(true); expect(gotValue).toEqual(value); expect(cached).toEqual(true); }); @@ -47,9 +46,8 @@ describe('avm public storage', () => { // ensure that fallback to host gets a value publicDb.storageRead.mockResolvedValue(storedValue); - const { exists, value: gotValue, cached } = await publicStorage.read(contractAddress, slot); + const { value: gotValue, cached } = await publicStorage.read(contractAddress, slot); // it exists in the host, so it must've been written before - expect(exists).toEqual(true); expect(gotValue).toEqual(storedValue); expect(cached).toEqual(false); }); @@ -61,9 +59,8 @@ describe('avm public storage', () => { const childStorage = new PublicStorage(publicDb, publicStorage); publicStorage.write(contractAddress, slot, value); - const { exists, value: gotValue, cached } = await childStorage.read(contractAddress, slot); + const { value: gotValue, cached } = await childStorage.read(contractAddress, slot); // exists because it was previously written! - expect(exists).toEqual(true); expect(gotValue).toEqual(value); expect(cached).toEqual(true); }); @@ -76,9 +73,8 @@ describe('avm public storage', () => { const grandChildStorage = new PublicStorage(publicDb, childStorage); publicStorage.write(contractAddress, slot, value); - const { exists, value: gotValue, cached } = await grandChildStorage.read(contractAddress, slot); + const { value: gotValue, cached } = await grandChildStorage.read(contractAddress, slot); // exists because it was previously written! - expect(exists).toEqual(true); expect(gotValue).toEqual(value); expect(cached).toEqual(true); }); @@ -136,8 +132,7 @@ describe('avm public storage', () => { publicStorage.acceptAndMerge(childStorage); // Read from parent gives latest value written in child before merge (valueT1) - const { exists, value: result } = await publicStorage.read(contractAddress, slot); - expect(exists).toEqual(true); + const { value: result } = await publicStorage.read(contractAddress, slot); expect(result).toEqual(valueT1); }); }); diff --git a/yarn-project/simulator/src/avm/journal/public_storage.ts b/yarn-project/simulator/src/avm/journal/public_storage.ts index 794cb745048..da2565e8d2f 100644 --- a/yarn-project/simulator/src/avm/journal/public_storage.ts +++ b/yarn-project/simulator/src/avm/journal/public_storage.ts @@ -5,7 +5,6 @@ import type { PublicStateDB } from '../../index.js'; type PublicStorageReadResult = { value: Fr; - exists: boolean; cached: boolean; }; @@ -77,17 +76,17 @@ export class PublicStorage { let value = this.readHereOrParent(contractAddress, slot); // Finally try the host's Aztec state (a trip to the database) if (!value) { - value = await this.hostPublicStorage.storageRead(contractAddress, slot); + // This functions returns Fr.ZERO if it has never been written to before + // we explicity coalesce to Fr.ZERO in case we have some implementations that cause this to return undefined + value = (await this.hostPublicStorage.storageRead(contractAddress, slot)) ?? Fr.ZERO; // TODO(dbanks12): if value retrieved from host storage, we can cache it here // any future reads to the same slot can read from cache instead of more expensive // DB access } else { cached = true; } - // if value is undefined, that means this slot has never been written to! - const exists = value !== undefined; - const valueOrZero = exists ? value : Fr.ZERO; - return Promise.resolve({ value: valueOrZero, exists, cached }); + // if value is Fr.ZERO here, it that means this slot has never been written to! + return Promise.resolve({ value, cached }); } /** diff --git a/yarn-project/simulator/src/avm/opcodes/accrued_substate.ts b/yarn-project/simulator/src/avm/opcodes/accrued_substate.ts index b93bfa40de7..0a8bd01f140 100644 --- a/yarn-project/simulator/src/avm/opcodes/accrued_substate.ts +++ b/yarn-project/simulator/src/avm/opcodes/accrued_substate.ts @@ -70,7 +70,7 @@ export class EmitNoteHash extends Instruction { } const noteHash = memory.get(noteHashOffset).toFr(); - await context.persistableState.writeNoteHash(context.environment.address, noteHash); + context.persistableState.writeNoteHash(context.environment.address, noteHash); memory.assert({ reads: 1, addressing }); } diff --git a/yarn-project/simulator/src/avm/opcodes/addressing_mode.ts b/yarn-project/simulator/src/avm/opcodes/addressing_mode.ts index 2d054f7a299..89107196cf9 100644 --- a/yarn-project/simulator/src/avm/opcodes/addressing_mode.ts +++ b/yarn-project/simulator/src/avm/opcodes/addressing_mode.ts @@ -1,6 +1,7 @@ import { strict as assert } from 'assert'; -import { type TaggedMemoryInterface } from '../avm_memory_types.js'; +import { TaggedMemory, type TaggedMemoryInterface } from '../avm_memory_types.js'; +import { AddressOutOfRangeError } from '../errors.js'; export enum AddressingMode { DIRECT = 0, @@ -63,7 +64,11 @@ export class Addressing { resolved[i] = offset; if (mode & AddressingMode.RELATIVE) { mem.checkIsValidMemoryOffsetTag(0); - resolved[i] += Number(mem.get(0).toBigInt()); + const baseAddr = Number(mem.get(0).toBigInt()); + resolved[i] += baseAddr; + if (resolved[i] >= TaggedMemory.MAX_MEMORY_SIZE) { + throw new AddressOutOfRangeError(baseAddr, offset); + } } if (mode & AddressingMode.INDIRECT) { mem.checkIsValidMemoryOffsetTag(resolved[i]); diff --git a/yarn-project/simulator/src/avm/opcodes/external_calls.test.ts b/yarn-project/simulator/src/avm/opcodes/external_calls.test.ts index 534a45102b7..24605b642da 100644 --- a/yarn-project/simulator/src/avm/opcodes/external_calls.test.ts +++ b/yarn-project/simulator/src/avm/opcodes/external_calls.test.ts @@ -58,6 +58,41 @@ describe('External Calls', () => { expect(inst.serialize()).toEqual(buf); }); + it('Call to non-existent bytecode returns failure', async () => { + const gasOffset = 0; + const l2Gas = 2e6; + const daGas = 3e6; + const addrOffset = 2; + const addr = new Fr(123456n); + const argsOffset = 3; + const args = [new Field(1), new Field(2), new Field(3)]; + const argsSize = args.length; + const argsSizeOffset = 20; + const successOffset = 6; + + const { l2GasLeft: initialL2Gas, daGasLeft: initialDaGas } = context.machineState; + + context.machineState.memory.set(0, new Field(l2Gas)); + context.machineState.memory.set(1, new Field(daGas)); + context.machineState.memory.set(2, new Field(addr)); + context.machineState.memory.set(argsSizeOffset, new Uint32(argsSize)); + context.machineState.memory.setSlice(3, args); + + const instruction = new Call(/*indirect=*/ 0, gasOffset, addrOffset, argsOffset, argsSizeOffset, successOffset); + await instruction.execute(context); + + const successValue = context.machineState.memory.get(successOffset); + expect(successValue).toEqual(new Uint1(0n)); // failure, contract non-existent! + + const retValue = context.machineState.nestedReturndata; + expect(retValue).toEqual([]); + + // should charge for the CALL instruction itself, and all allocated gas should be consumed + expect(context.machineState.l2GasLeft).toBeLessThan(initialL2Gas - l2Gas); + expect(context.machineState.daGasLeft).toEqual(initialDaGas - daGas); + expect(context.machineState.collectedRevertInfo?.recursiveRevertReason?.message).toMatch(/No bytecode found/); + }); + it('Should execute a call correctly', async () => { const gasOffset = 0; const l2Gas = 2e6; diff --git a/yarn-project/simulator/src/avm/opcodes/external_calls.ts b/yarn-project/simulator/src/avm/opcodes/external_calls.ts index 7bfbe5e8c71..a5cb97ebc57 100644 --- a/yarn-project/simulator/src/avm/opcodes/external_calls.ts +++ b/yarn-project/simulator/src/avm/opcodes/external_calls.ts @@ -2,7 +2,6 @@ import { Fr, FunctionSelector, Gas, PUBLIC_DISPATCH_SELECTOR } from '@aztec/circ import type { AvmContext } from '../avm_context.js'; import { type AvmContractCallResult } from '../avm_contract_call_result.js'; -import { gasLeftToGas } from '../avm_gas.js'; import { type Field, TypeTag, Uint1 } from '../avm_memory_types.js'; import { AvmSimulator } from '../avm_simulator.js'; import { Opcode, OperandType } from '../serialization/instruction_serialization.js'; @@ -95,17 +94,18 @@ abstract class ExternalCall extends Instruction { memory.set(successOffset, new Uint1(success ? 1 : 0)); // Refund unused gas - context.machineState.refundGas(gasLeftToGas(nestedContext.machineState)); + context.machineState.refundGas(nestedCallResults.gasLeft); - // Accept the nested call's state and trace the nested call + // Merge nested call's state and trace based on whether it succeeded. if (success) { - context.persistableState.mergeForkedState(nestedContext.persistableState); + context.persistableState.merge(nestedContext.persistableState); + } else { + context.persistableState.reject(nestedContext.persistableState); } await context.persistableState.traceNestedCall( /*nestedState=*/ nestedContext.persistableState, /*nestedEnvironment=*/ nestedContext.environment, /*startGasLeft=*/ Gas.from(allocatedGas), - /*endGasLeft=*/ Gas.from(nestedContext.machineState.gasLeft), /*bytecode=*/ simulator.getBytecode()!, /*avmCallResults=*/ nestedCallResults, ); diff --git a/yarn-project/simulator/src/avm/opcodes/instruction.ts b/yarn-project/simulator/src/avm/opcodes/instruction.ts index c7ae85a5d84..2b7a0e8f551 100644 --- a/yarn-project/simulator/src/avm/opcodes/instruction.ts +++ b/yarn-project/simulator/src/avm/opcodes/instruction.ts @@ -94,7 +94,7 @@ export abstract class Instruction { * Computes gas cost for the instruction based on its base cost and memory operations. * @returns Gas cost. */ - protected gasCost(dynMultiplier: number = 0): Gas { + public gasCost(dynMultiplier: number = 0): Gas { const baseGasCost = getBaseGasCost(this.opcode); const dynGasCost = mulGas(getDynamicGasCost(this.opcode), dynMultiplier); return sumGas(baseGasCost, dynGasCost); diff --git a/yarn-project/simulator/src/index.ts b/yarn-project/simulator/src/index.ts index fc5a4653413..f8095f6baf7 100644 --- a/yarn-project/simulator/src/index.ts +++ b/yarn-project/simulator/src/index.ts @@ -4,5 +4,4 @@ export * from './client/index.js'; export * from './common/index.js'; export * from './public/index.js'; export * from './providers/index.js'; -export * from './mocks/index.js'; export * from './stats/index.js'; diff --git a/yarn-project/simulator/src/mocks/fixtures.ts b/yarn-project/simulator/src/mocks/fixtures.ts deleted file mode 100644 index 9c5bd74e06d..00000000000 --- a/yarn-project/simulator/src/mocks/fixtures.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { SimulationError } from '@aztec/circuit-types'; -import { ARGS_LENGTH, Fr, Gas } from '@aztec/circuits.js'; -import { makeAztecAddress, makeSelector } from '@aztec/circuits.js/testing'; -import { FunctionType } from '@aztec/foundation/abi'; -import { padArrayEnd } from '@aztec/foundation/collection'; - -import { type EnqueuedPublicCallExecutionResult } from '../public/execution.js'; - -export class PublicExecutionResultBuilder { - private _returnValues: Fr[] = []; - private _reverted = false; - private _revertReason: SimulationError | undefined = undefined; - - constructor() {} - - static empty(basicRevert = false) { - const builder = new PublicExecutionResultBuilder(); - if (basicRevert) { - builder.withReverted(new SimulationError('Simulation failed', [])); - } - return builder; - } - - static fromPublicExecutionRequest({ - returnValues = [new Fr(1n)], - revertReason = undefined, - }: { - returnValues?: Fr[]; - revertReason?: SimulationError; - }): PublicExecutionResultBuilder { - const builder = new PublicExecutionResultBuilder(); - - builder.withReturnValues(...returnValues); - if (revertReason) { - builder.withReverted(revertReason); - } - - return builder; - } - - withReturnValues(...values: Fr[]): PublicExecutionResultBuilder { - this._returnValues.push(...values); - return this; - } - - withReverted(reason: SimulationError): PublicExecutionResultBuilder { - this._reverted = true; - this._revertReason = reason; - return this; - } - - build(overrides: Partial = {}): EnqueuedPublicCallExecutionResult { - return { - endGasLeft: Gas.empty(), - endSideEffectCounter: Fr.ZERO, - returnValues: padArrayEnd(this._returnValues, Fr.ZERO, 4), // TODO(#5450) Need to use the proper return values here - reverted: this._reverted, - revertReason: this._revertReason, - ...overrides, - }; - } -} - -export const makeFunctionCall = ( - name = 'function', - to = makeAztecAddress(30), - selector = makeSelector(5), - type = FunctionType.PUBLIC, - args = new Array(ARGS_LENGTH).fill(Fr.ZERO), - isStatic = false, - returnTypes = [], -) => ({ name, to, selector, type, args, isStatic, returnTypes }); diff --git a/yarn-project/simulator/src/mocks/index.ts b/yarn-project/simulator/src/mocks/index.ts deleted file mode 100644 index dd1a464237b..00000000000 --- a/yarn-project/simulator/src/mocks/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './fixtures.js'; diff --git a/yarn-project/simulator/src/public/dual_side_effect_trace.ts b/yarn-project/simulator/src/public/dual_side_effect_trace.ts index 15fa4b76456..ff396d68496 100644 --- a/yarn-project/simulator/src/public/dual_side_effect_trace.ts +++ b/yarn-project/simulator/src/public/dual_side_effect_trace.ts @@ -1,20 +1,18 @@ import { type UnencryptedL2Log } from '@aztec/circuit-types'; import { - type CombinedConstantData, type ContractClassIdPreimage, type Gas, type NullifierLeafPreimage, type PublicCallRequest, type PublicDataTreeLeafPreimage, type SerializableContractInstance, - type VMCircuitPublicInputs, } from '@aztec/circuits.js'; import { type AztecAddress } from '@aztec/foundation/aztec-address'; import { type Fr } from '@aztec/foundation/fields'; import { assert } from 'console'; -import { type AvmContractCallResult } from '../avm/avm_contract_call_result.js'; +import { type AvmContractCallResult, type AvmFinalizedCallResult } from '../avm/avm_contract_call_result.js'; import { type AvmExecutionEnvironment } from '../avm/avm_execution_environment.js'; import { type PublicEnqueuedCallSideEffectTrace } from './enqueued_call_side_effect_trace.js'; import { type EnqueuedPublicCallExecutionResultWithSideEffects, type PublicFunctionCallResult } from './execution.js'; @@ -27,11 +25,12 @@ export class DualSideEffectTrace implements PublicSideEffectTraceInterface { public readonly enqueuedCallTrace: PublicEnqueuedCallSideEffectTrace, ) {} - public fork(incrementSideEffectCounter: boolean = false) { - return new DualSideEffectTrace( - this.innerCallTrace.fork(incrementSideEffectCounter), - this.enqueuedCallTrace.fork(incrementSideEffectCounter), - ); + public fork() { + return new DualSideEffectTrace(this.innerCallTrace.fork(), this.enqueuedCallTrace.fork()); + } + + public merge(nestedTrace: this, reverted: boolean = false) { + this.enqueuedCallTrace.merge(nestedTrace.enqueuedCallTrace, reverted); } public getCounter() { @@ -192,8 +191,6 @@ export class DualSideEffectTrace implements PublicSideEffectTraceInterface { nestedEnvironment: AvmExecutionEnvironment, /** How much gas was available for this public execution. */ startGasLeft: Gas, - /** How much gas was left after this public execution. */ - endGasLeft: Gas, /** Bytecode used for this execution. */ bytecode: Buffer, /** The call's results */ @@ -205,7 +202,6 @@ export class DualSideEffectTrace implements PublicSideEffectTraceInterface { nestedCallTrace.innerCallTrace, nestedEnvironment, startGasLeft, - endGasLeft, bytecode, avmCallResults, functionName, @@ -214,7 +210,6 @@ export class DualSideEffectTrace implements PublicSideEffectTraceInterface { nestedCallTrace.enqueuedCallTrace, nestedEnvironment, startGasLeft, - endGasLeft, bytecode, avmCallResults, functionName, @@ -232,24 +227,14 @@ export class DualSideEffectTrace implements PublicSideEffectTraceInterface { this.enqueuedCallTrace.traceEnqueuedCall(publicCallRequest, calldata, reverted); } - public mergeSuccessfulForkedTrace(nestedTrace: this) { - this.enqueuedCallTrace.mergeSuccessfulForkedTrace(nestedTrace.enqueuedCallTrace); - } - - public mergeRevertedForkedTrace(nestedTrace: this) { - this.enqueuedCallTrace.mergeRevertedForkedTrace(nestedTrace.enqueuedCallTrace); - } - /** * Convert this trace to a PublicExecutionResult for use externally to the simulator. */ public toPublicEnqueuedCallExecutionResult( - /** How much gas was left after this public execution. */ - endGasLeft: Gas, /** The call's results */ - avmCallResults: AvmContractCallResult, + avmCallResults: AvmFinalizedCallResult, ): EnqueuedPublicCallExecutionResultWithSideEffects { - return this.enqueuedCallTrace.toPublicEnqueuedCallExecutionResult(endGasLeft, avmCallResults); + return this.enqueuedCallTrace.toPublicEnqueuedCallExecutionResult(avmCallResults); } /** * Convert this trace to a PublicExecutionResult for use externally to the simulator. @@ -259,49 +244,22 @@ export class DualSideEffectTrace implements PublicSideEffectTraceInterface { avmEnvironment: AvmExecutionEnvironment, /** How much gas was available for this public execution. */ startGasLeft: Gas, - /** How much gas was left after this public execution. */ - endGasLeft: Gas, /** Bytecode used for this execution. */ bytecode: Buffer, /** The call's results */ - avmCallResults: AvmContractCallResult, + avmCallResults: AvmFinalizedCallResult, /** Function name for logging */ functionName: string = 'unknown', ): PublicFunctionCallResult { return this.innerCallTrace.toPublicFunctionCallResult( avmEnvironment, startGasLeft, - endGasLeft, bytecode, avmCallResults, functionName, ); } - public toVMCircuitPublicInputs( - /** Constants */ - constants: CombinedConstantData, - /** The call request that triggered public execution. */ - callRequest: PublicCallRequest, - /** How much gas was available for this public execution. */ - startGasLeft: Gas, - /** How much gas was left after this public execution. */ - endGasLeft: Gas, - /** Transaction fee. */ - transactionFee: Fr, - /** The call's results */ - avmCallResults: AvmContractCallResult, - ): VMCircuitPublicInputs { - return this.enqueuedCallTrace.toVMCircuitPublicInputs( - constants, - callRequest, - startGasLeft, - endGasLeft, - transactionFee, - avmCallResults, - ); - } - public getUnencryptedLogs(): UnencryptedL2Log[] { return this.enqueuedCallTrace.getUnencryptedLogs(); } diff --git a/yarn-project/simulator/src/public/enqueued_call_side_effect_trace.test.ts b/yarn-project/simulator/src/public/enqueued_call_side_effect_trace.test.ts index dc1ccb6bc45..96b415a2e04 100644 --- a/yarn-project/simulator/src/public/enqueued_call_side_effect_trace.test.ts +++ b/yarn-project/simulator/src/public/enqueued_call_side_effect_trace.test.ts @@ -1,9 +1,7 @@ import { UnencryptedL2Log } from '@aztec/circuit-types'; import { AztecAddress, - CombinedConstantData, EthAddress, - Gas, L2ToL1Message, LogHash, MAX_L1_TO_L2_MSG_READ_REQUESTS_PER_TX, @@ -19,42 +17,21 @@ import { NoteHash, Nullifier, NullifierLeafPreimage, - PublicAccumulatedData, - PublicAccumulatedDataArrayLengths, - PublicCallRequest, PublicDataRead, PublicDataTreeLeafPreimage, PublicDataUpdateRequest, - PublicValidationRequestArrayLengths, - PublicValidationRequests, ReadRequest, SerializableContractInstance, TreeLeafReadRequest, } from '@aztec/circuits.js'; -import { computePublicDataTreeLeafSlot, computeVarArgsHash, siloNullifier } from '@aztec/circuits.js/hash'; +import { computePublicDataTreeLeafSlot, siloNullifier } from '@aztec/circuits.js/hash'; import { Fr } from '@aztec/foundation/fields'; import { randomInt } from 'crypto'; -import { AvmContractCallResult } from '../avm/avm_contract_call_result.js'; -import { type AvmExecutionEnvironment } from '../avm/avm_execution_environment.js'; -import { initExecutionEnvironment } from '../avm/fixtures/index.js'; -import { PublicEnqueuedCallSideEffectTrace } from './enqueued_call_side_effect_trace.js'; +import { PublicEnqueuedCallSideEffectTrace, SideEffectArrayLengths } from './enqueued_call_side_effect_trace.js'; import { SideEffectLimitReachedError } from './side_effect_errors.js'; -/** - * Helper function to create a public execution request from an AVM execution environment - */ -function createPublicCallRequest(avmEnvironment: AvmExecutionEnvironment): PublicCallRequest { - return new PublicCallRequest( - avmEnvironment.sender, - avmEnvironment.address, - avmEnvironment.functionSelector, - avmEnvironment.isStaticCall, - computeVarArgsHash(avmEnvironment.calldata), - ); -} - describe('Enqueued-call Side Effect Trace', () => { const address = AztecAddress.random(); const utxo = Fr.random(); @@ -66,22 +43,6 @@ describe('Enqueued-call Side Effect Trace', () => { const log = [Fr.random(), Fr.random(), Fr.random()]; const contractInstance = SerializableContractInstance.default(); - const startGasLeft = Gas.fromFields([new Fr(randomInt(10000)), new Fr(randomInt(10000))]); - const endGasLeft = Gas.fromFields([new Fr(randomInt(10000)), new Fr(randomInt(10000))]); - const transactionFee = Fr.random(); - const calldata = [Fr.random(), Fr.random(), Fr.random(), Fr.random()]; - const returnValues = [Fr.random(), Fr.random()]; - - const constants = CombinedConstantData.empty(); - const avmEnvironment = initExecutionEnvironment({ - address, - calldata, - transactionFee, - }); - const avmCallResults = new AvmContractCallResult(/*reverted=*/ false, returnValues); - - const emptyValidationRequests = PublicValidationRequests.empty(); - let startCounter: number; let startCounterFr: Fr; let startCounterPlus1: number; @@ -94,28 +55,15 @@ describe('Enqueued-call Side Effect Trace', () => { trace = new PublicEnqueuedCallSideEffectTrace(startCounter); }); - const toVMCircuitPublicInputs = (trc: PublicEnqueuedCallSideEffectTrace) => { - return trc.toVMCircuitPublicInputs( - constants, - createPublicCallRequest(avmEnvironment), - startGasLeft, - endGasLeft, - transactionFee, - avmCallResults, - ); - }; - it('Should trace storage reads', () => { const leafPreimage = new PublicDataTreeLeafPreimage(slot, value, Fr.ZERO, 0n); trace.tracePublicStorageRead(address, slot, value, leafPreimage, Fr.ZERO, []); expect(trace.getCounter()).toBe(startCounterPlus1); - const expectedArray = PublicValidationRequests.empty().publicDataReads; const leafSlot = computePublicDataTreeLeafSlot(address, slot); - expectedArray[0] = new PublicDataRead(leafSlot, value, startCounter /*contractAddress*/); + const expected = [new PublicDataRead(leafSlot, value, startCounter /*contractAddress*/)]; + expect(trace.getSideEffects().publicDataReads).toEqual(expected); - const circuitPublicInputs = toVMCircuitPublicInputs(trace); - expect(circuitPublicInputs.validationRequests.publicDataReads).toEqual(expectedArray); expect(trace.getAvmCircuitHints().storageValues.items).toEqual([{ key: startCounterFr, value }]); }); @@ -126,23 +74,17 @@ describe('Enqueued-call Side Effect Trace', () => { trace.tracePublicStorageWrite(address, slot, value, lowLeafPreimage, Fr.ZERO, [], newLeafPreimage, []); expect(trace.getCounter()).toBe(startCounterPlus1); - const expectedArray = PublicAccumulatedData.empty().publicDataUpdateRequests; const leafSlot = computePublicDataTreeLeafSlot(address, slot); - expectedArray[0] = new PublicDataUpdateRequest(leafSlot, value, startCounter /*contractAddress*/); - - const circuitPublicInputs = toVMCircuitPublicInputs(trace); - expect(circuitPublicInputs.accumulatedData.publicDataUpdateRequests).toEqual(expectedArray); + const expected = [new PublicDataUpdateRequest(leafSlot, value, startCounter /*contractAddress*/)]; + expect(trace.getSideEffects().publicDataWrites).toEqual(expected); }); it('Should trace note hash checks', () => { const exists = true; trace.traceNoteHashCheck(address, utxo, leafIndex, exists, []); - const expectedArray = PublicValidationRequests.empty().noteHashReadRequests; - expectedArray[0] = new TreeLeafReadRequest(utxo, leafIndex); - - const circuitPublicInputs = toVMCircuitPublicInputs(trace); - expect(circuitPublicInputs.validationRequests.noteHashReadRequests).toEqual(expectedArray); + const expected = [new TreeLeafReadRequest(utxo, leafIndex)]; + expect(trace.getSideEffects().noteHashReadRequests).toEqual(expected); expect(trace.getAvmCircuitHints().noteHashExists.items).toEqual([{ key: leafIndex, value: new Fr(exists) }]); }); @@ -151,11 +93,8 @@ describe('Enqueued-call Side Effect Trace', () => { trace.traceNewNoteHash(address, utxo, Fr.ZERO, []); expect(trace.getCounter()).toBe(startCounterPlus1); - const expectedArray = PublicAccumulatedData.empty().noteHashes; - expectedArray[0] = new NoteHash(utxo, startCounter).scope(address); - - const circuitPublicInputs = toVMCircuitPublicInputs(trace); - expect(circuitPublicInputs.accumulatedData.noteHashes).toEqual(expectedArray); + const expected = [new NoteHash(utxo, startCounter).scope(address)]; + expect(trace.getSideEffects().noteHashes).toEqual(expected); }); it('Should trace nullifier checks', () => { @@ -164,14 +103,10 @@ describe('Enqueued-call Side Effect Trace', () => { trace.traceNullifierCheck(address, utxo, exists, lowLeafPreimage, Fr.ZERO, []); expect(trace.getCounter()).toBe(startCounterPlus1); - const expectedArray = PublicValidationRequests.empty().nullifierReadRequests; - expectedArray[0] = new ReadRequest(utxo, startCounter).scope(address); - - const circuitPublicInputs = toVMCircuitPublicInputs(trace); - expect(circuitPublicInputs.validationRequests.nullifierReadRequests).toEqual(expectedArray); - expect(circuitPublicInputs.validationRequests.nullifierNonExistentReadRequests).toEqual( - emptyValidationRequests.nullifierNonExistentReadRequests, - ); + const { nullifierReadRequests, nullifierNonExistentReadRequests } = trace.getSideEffects(); + const expected = [new ReadRequest(utxo, startCounter).scope(address)]; + expect(nullifierReadRequests).toEqual(expected); + expect(nullifierNonExistentReadRequests).toEqual([]); expect(trace.getAvmCircuitHints().nullifierExists.items).toEqual([{ key: startCounterFr, value: new Fr(exists) }]); }); @@ -182,14 +117,11 @@ describe('Enqueued-call Side Effect Trace', () => { trace.traceNullifierCheck(address, utxo, exists, lowLeafPreimage, Fr.ZERO, []); expect(trace.getCounter()).toBe(startCounterPlus1); - const expectedArray = PublicValidationRequests.empty().nullifierNonExistentReadRequests; - expectedArray[0] = new ReadRequest(utxo, startCounter).scope(address); + const { nullifierReadRequests, nullifierNonExistentReadRequests } = trace.getSideEffects(); + expect(nullifierReadRequests).toEqual([]); - const circuitPublicInputs = toVMCircuitPublicInputs(trace); - expect(circuitPublicInputs.validationRequests.nullifierReadRequests).toEqual( - emptyValidationRequests.nullifierReadRequests, - ); - expect(circuitPublicInputs.validationRequests.nullifierNonExistentReadRequests).toEqual(expectedArray); + const expected = [new ReadRequest(utxo, startCounter).scope(address)]; + expect(nullifierNonExistentReadRequests).toEqual(expected); expect(trace.getAvmCircuitHints().nullifierExists.items).toEqual([{ key: startCounterFr, value: new Fr(exists) }]); }); @@ -199,22 +131,16 @@ describe('Enqueued-call Side Effect Trace', () => { trace.traceNewNullifier(address, utxo, lowLeafPreimage, Fr.ZERO, [], []); expect(trace.getCounter()).toBe(startCounterPlus1); - const expectedArray = PublicAccumulatedData.empty().nullifiers; - expectedArray[0] = new Nullifier(siloNullifier(address, utxo), startCounter, Fr.ZERO); - - const circuitPublicInputs = toVMCircuitPublicInputs(trace); - expect(circuitPublicInputs.accumulatedData.nullifiers).toEqual(expectedArray); + const expected = [new Nullifier(siloNullifier(address, utxo), startCounter, Fr.ZERO)]; + expect(trace.getSideEffects().nullifiers).toEqual(expected); }); it('Should trace L1ToL2 Message checks', () => { const exists = true; trace.traceL1ToL2MessageCheck(address, utxo, leafIndex, exists, []); - const expectedArray = PublicValidationRequests.empty().l1ToL2MsgReadRequests; - expectedArray[0] = new TreeLeafReadRequest(utxo, leafIndex); - - const circuitPublicInputs = toVMCircuitPublicInputs(trace); - expect(circuitPublicInputs.validationRequests.l1ToL2MsgReadRequests).toEqual(expectedArray); + const expected = [new TreeLeafReadRequest(utxo, leafIndex)]; + expect(trace.getSideEffects().l1ToL2MsgReadRequests).toEqual(expected); expect(trace.getAvmCircuitHints().l1ToL2MessageExists.items).toEqual([ { @@ -228,11 +154,8 @@ describe('Enqueued-call Side Effect Trace', () => { trace.traceNewL2ToL1Message(address, recipient, content); expect(trace.getCounter()).toBe(startCounterPlus1); - const expectedArray = PublicAccumulatedData.empty().l2ToL1Msgs; - expectedArray[0] = new L2ToL1Message(EthAddress.fromField(recipient), content, startCounter).scope(address); - - const circuitPublicInputs = toVMCircuitPublicInputs(trace); - expect(circuitPublicInputs.accumulatedData.l2ToL1Msgs).toEqual(expectedArray); + const expected = [new L2ToL1Message(EthAddress.fromField(recipient), content, startCounter).scope(address)]; + expect(trace.getSideEffects().l2ToL1Msgs).toEqual(expected); }); it('Should trace new unencrypted logs', () => { @@ -240,16 +163,12 @@ describe('Enqueued-call Side Effect Trace', () => { expect(trace.getCounter()).toBe(startCounterPlus1); const expectedLog = new UnencryptedL2Log(address, Buffer.concat(log.map(f => f.toBuffer()))); - const expectedArray = PublicAccumulatedData.empty().unencryptedLogsHashes; - expectedArray[0] = new LogHash( - Fr.fromBuffer(expectedLog.hash()), - startCounter, - new Fr(expectedLog.length + 4), - ).scope(address); - - const circuitPublicInputs = toVMCircuitPublicInputs(trace); + const expectedHashes = [ + new LogHash(Fr.fromBuffer(expectedLog.hash()), startCounter, new Fr(expectedLog.length + 4)).scope(address), + ]; + expect(trace.getUnencryptedLogs()).toEqual([expectedLog]); - expect(circuitPublicInputs.accumulatedData.unencryptedLogsHashes).toEqual(expectedArray); + expect(trace.getSideEffects().unencryptedLogsHashes).toEqual(expectedHashes); }); it('Should trace get contract instance', () => { @@ -259,7 +178,6 @@ describe('Enqueued-call Side Effect Trace', () => { trace.traceGetContractInstance(address, exists, instance); expect(trace.getCounter()).toBe(startCounterPlus1); - //const circuitPublicInputs = toVMCircuitPublicInputs(trace); expect(trace.getAvmCircuitHints().contractInstances.items).toEqual([ { address, @@ -418,22 +336,17 @@ describe('Enqueued-call Side Effect Trace', () => { it('PreviousValidationRequestArrayLengths and PreviousAccumulatedDataArrayLengths contribute to limits', () => { trace = new PublicEnqueuedCallSideEffectTrace( 0, - new PublicValidationRequestArrayLengths( + new SideEffectArrayLengths( + MAX_PUBLIC_DATA_READS_PER_TX, + MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, MAX_NOTE_HASH_READ_REQUESTS_PER_TX, + MAX_NOTE_HASHES_PER_TX, MAX_NULLIFIER_READ_REQUESTS_PER_TX, MAX_NULLIFIER_NON_EXISTENT_READ_REQUESTS_PER_TX, - MAX_L1_TO_L2_MSG_READ_REQUESTS_PER_TX, - MAX_PUBLIC_DATA_READS_PER_TX, - ), - new PublicAccumulatedDataArrayLengths( - MAX_NOTE_HASHES_PER_TX, MAX_NULLIFIERS_PER_TX, + MAX_L1_TO_L2_MSG_READ_REQUESTS_PER_TX, MAX_L2_TO_L1_MSGS_PER_TX, - 0, - 0, MAX_UNENCRYPTED_LOGS_PER_TX, - MAX_PUBLIC_DATA_READS_PER_TX, - 0, ), ); expect(() => trace.tracePublicStorageRead(AztecAddress.fromNumber(42), new Fr(42), new Fr(42))).toThrow( @@ -508,11 +421,7 @@ describe('Enqueued-call Side Effect Trace', () => { nestedTrace.traceGetContractInstance(address, /*exists=*/ false, contractInstance); testCounter++; - if (reverted) { - trace.mergeRevertedForkedTrace(nestedTrace); - } else { - trace.mergeSuccessfulForkedTrace(nestedTrace); - } + trace.merge(nestedTrace, reverted); // parent trace adopts nested call's counter expect(trace.getCounter()).toBe(testCounter); diff --git a/yarn-project/simulator/src/public/enqueued_call_side_effect_trace.ts b/yarn-project/simulator/src/public/enqueued_call_side_effect_trace.ts index b5f87c521b9..768243bf473 100644 --- a/yarn-project/simulator/src/public/enqueued_call_side_effect_trace.ts +++ b/yarn-project/simulator/src/public/enqueued_call_side_effect_trace.ts @@ -14,7 +14,6 @@ import { AvmPublicDataReadTreeHint, AvmPublicDataWriteTreeHint, type AztecAddress, - type CombinedConstantData, type ContractClassIdPreimage, EthAddress, Gas, @@ -23,11 +22,9 @@ import { L1_TO_L2_MSG_TREE_HEIGHT, L2ToL1Message, LogHash, - MAX_ENCRYPTED_LOGS_PER_TX, MAX_ENQUEUED_CALLS_PER_TX, MAX_L1_TO_L2_MSG_READ_REQUESTS_PER_TX, MAX_L2_TO_L1_MSGS_PER_TX, - MAX_NOTE_ENCRYPTED_LOGS_PER_TX, MAX_NOTE_HASHES_PER_TX, MAX_NOTE_HASH_READ_REQUESTS_PER_TX, MAX_NULLIFIERS_PER_TX, @@ -44,39 +41,29 @@ import { PUBLIC_DATA_TREE_HEIGHT, PrivateToAvmAccumulatedData, PrivateToAvmAccumulatedDataArrayLengths, - PublicAccumulatedData, - PublicAccumulatedDataArrayLengths, PublicCallRequest, PublicDataRead, PublicDataTreeLeafPreimage, PublicDataUpdateRequest, PublicDataWrite, - PublicInnerCallRequest, - PublicValidationRequestArrayLengths, - PublicValidationRequests, ReadRequest, - RollupValidationRequests, ScopedL2ToL1Message, ScopedLogHash, - ScopedNoteHash, - ScopedReadRequest, + type ScopedNoteHash, + type ScopedReadRequest, SerializableContractInstance, TreeLeafReadRequest, type TreeSnapshots, - VMCircuitPublicInputs, } from '@aztec/circuits.js'; import { computePublicDataTreeLeafSlot, siloNullifier } from '@aztec/circuits.js/hash'; -import { makeTuple } from '@aztec/foundation/array'; import { padArrayEnd } from '@aztec/foundation/collection'; import { Fr } from '@aztec/foundation/fields'; import { createDebugLogger } from '@aztec/foundation/log'; -import { type Tuple } from '@aztec/foundation/serialize'; import { assert } from 'console'; -import { type AvmContractCallResult } from '../avm/avm_contract_call_result.js'; +import { type AvmContractCallResult, type AvmFinalizedCallResult } from '../avm/avm_contract_call_result.js'; import { type AvmExecutionEnvironment } from '../avm/avm_execution_environment.js'; -import { createSimulationError } from '../common/errors.js'; import { type EnqueuedPublicCallExecutionResultWithSideEffects, type PublicFunctionCallResult } from './execution.js'; import { SideEffectLimitReachedError } from './side_effect_errors.js'; import { type PublicSideEffectTraceInterface } from './side_effect_trace_interface.js'; @@ -111,6 +98,29 @@ export type SideEffects = { unencryptedLogsHashes: ScopedLogHash[]; }; +export class SideEffectArrayLengths { + constructor( + public readonly publicDataReads: number, + public readonly publicDataWrites: number, + + public readonly noteHashReadRequests: number, + public readonly noteHashes: number, + + public readonly nullifierReadRequests: number, + public readonly nullifierNonExistentReadRequests: number, + public readonly nullifiers: number, + + public readonly l1ToL2MsgReadRequests: number, + public readonly l2ToL1Msgs: number, + + public readonly unencryptedLogs: number, + ) {} + + static empty() { + return new this(0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + } +} + /** * Trace side effects for an entire enqueued call. */ @@ -140,44 +150,68 @@ export class PublicEnqueuedCallSideEffectTrace implements PublicSideEffectTraceI private avmCircuitHints: AvmExecutionHints; + /** Make sure a forked trace is never merged twice. */ + private alreadyMergedIntoParent = false; + constructor( /** The counter of this trace's first side effect. */ public readonly startSideEffectCounter: number = 0, /** Track parent's (or previous kernel's) lengths so the AVM can properly enforce TX-wide limits, * otherwise the public kernel can fail to prove because TX limits are breached. */ - private readonly previousValidationRequestArrayLengths: PublicValidationRequestArrayLengths = PublicValidationRequestArrayLengths.empty(), - private readonly previousAccumulatedDataArrayLengths: PublicAccumulatedDataArrayLengths = PublicAccumulatedDataArrayLengths.empty(), + private readonly previousSideEffectArrayLengths: SideEffectArrayLengths = SideEffectArrayLengths.empty(), ) { this.log.debug(`Creating trace instance with startSideEffectCounter: ${startSideEffectCounter}`); this.sideEffectCounter = startSideEffectCounter; this.avmCircuitHints = AvmExecutionHints.empty(); } - public fork(incrementSideEffectCounter: boolean = false) { + public fork() { return new PublicEnqueuedCallSideEffectTrace( - incrementSideEffectCounter ? this.sideEffectCounter + 1 : this.sideEffectCounter, - new PublicValidationRequestArrayLengths( - this.previousValidationRequestArrayLengths.noteHashReadRequests + this.noteHashReadRequests.length, - this.previousValidationRequestArrayLengths.nullifierReadRequests + this.nullifierReadRequests.length, - this.previousValidationRequestArrayLengths.nullifierNonExistentReadRequests + + this.sideEffectCounter, + new SideEffectArrayLengths( + this.previousSideEffectArrayLengths.publicDataReads + this.publicDataReads.length, + this.previousSideEffectArrayLengths.publicDataWrites + this.publicDataWrites.length, + this.previousSideEffectArrayLengths.noteHashReadRequests + this.noteHashReadRequests.length, + this.previousSideEffectArrayLengths.noteHashes + this.noteHashes.length, + this.previousSideEffectArrayLengths.nullifierReadRequests + this.nullifierReadRequests.length, + this.previousSideEffectArrayLengths.nullifierNonExistentReadRequests + this.nullifierNonExistentReadRequests.length, - this.previousValidationRequestArrayLengths.l1ToL2MsgReadRequests + this.l1ToL2MsgReadRequests.length, - this.previousValidationRequestArrayLengths.publicDataReads + this.publicDataReads.length, - ), - new PublicAccumulatedDataArrayLengths( - this.previousAccumulatedDataArrayLengths.noteHashes + this.noteHashes.length, - this.previousAccumulatedDataArrayLengths.nullifiers + this.nullifiers.length, - this.previousAccumulatedDataArrayLengths.l2ToL1Msgs + this.l2ToL1Messages.length, - this.previousAccumulatedDataArrayLengths.noteEncryptedLogsHashes, - this.previousAccumulatedDataArrayLengths.encryptedLogsHashes, - this.previousAccumulatedDataArrayLengths.unencryptedLogsHashes + this.unencryptedLogsHashes.length, - this.previousAccumulatedDataArrayLengths.publicDataUpdateRequests + this.publicDataWrites.length, - this.previousAccumulatedDataArrayLengths.publicCallStack, + this.previousSideEffectArrayLengths.nullifiers + this.nullifiers.length, + this.previousSideEffectArrayLengths.l1ToL2MsgReadRequests + this.l1ToL2MsgReadRequests.length, + this.previousSideEffectArrayLengths.l2ToL1Msgs + this.l2ToL1Messages.length, + this.previousSideEffectArrayLengths.unencryptedLogs + this.unencryptedLogs.length, ), ); } + public merge(forkedTrace: this, reverted: boolean = false) { + // sanity check to avoid merging the same forked trace twice + assert( + !forkedTrace.alreadyMergedIntoParent, + 'Cannot merge a forked trace that has already been merged into its parent!', + ); + forkedTrace.alreadyMergedIntoParent = true; + + // TODO(dbanks12): accept & merge forked trace's hints! + this.sideEffectCounter = forkedTrace.sideEffectCounter; + this.enqueuedCalls.push(...forkedTrace.enqueuedCalls); + + if (!reverted) { + this.publicDataReads.push(...forkedTrace.publicDataReads); + this.publicDataWrites.push(...forkedTrace.publicDataWrites); + this.noteHashReadRequests.push(...forkedTrace.noteHashReadRequests); + this.noteHashes.push(...forkedTrace.noteHashes); + this.nullifierReadRequests.push(...forkedTrace.nullifierReadRequests); + this.nullifierNonExistentReadRequests.push(...forkedTrace.nullifierNonExistentReadRequests); + this.nullifiers.push(...forkedTrace.nullifiers); + this.l1ToL2MsgReadRequests.push(...forkedTrace.l1ToL2MsgReadRequests); + this.l2ToL1Messages.push(...forkedTrace.l2ToL1Messages); + this.unencryptedLogs.push(...forkedTrace.unencryptedLogs); + this.unencryptedLogsHashes.push(...forkedTrace.unencryptedLogsHashes); + } + } + public getCounter() { return this.sideEffectCounter; } @@ -200,7 +234,7 @@ export class PublicEnqueuedCallSideEffectTrace implements PublicSideEffectTraceI } // NOTE: exists and cached are unused for now but may be used for optimizations or kernel hints later if ( - this.publicDataReads.length + this.previousValidationRequestArrayLengths.publicDataReads >= + this.publicDataReads.length + this.previousSideEffectArrayLengths.publicDataReads >= MAX_PUBLIC_DATA_READS_PER_TX ) { throw new SideEffectLimitReachedError('public data (contract storage) read', MAX_PUBLIC_DATA_READS_PER_TX); @@ -233,7 +267,7 @@ export class PublicEnqueuedCallSideEffectTrace implements PublicSideEffectTraceI assert(newLeafPreimage.value.equals(value), 'Value mismatch when tracing in public data read'); } if ( - this.publicDataWrites.length + this.previousAccumulatedDataArrayLengths.publicDataUpdateRequests >= + this.publicDataWrites.length + this.previousSideEffectArrayLengths.publicDataWrites >= MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX ) { throw new SideEffectLimitReachedError( @@ -267,7 +301,7 @@ export class PublicEnqueuedCallSideEffectTrace implements PublicSideEffectTraceI ) { // NOTE: contractAddress is unused because noteHash is an already-siloed leaf if ( - this.noteHashReadRequests.length + this.previousValidationRequestArrayLengths.noteHashReadRequests >= + this.noteHashReadRequests.length + this.previousSideEffectArrayLengths.noteHashReadRequests >= MAX_NOTE_HASH_READ_REQUESTS_PER_TX ) { throw new SideEffectLimitReachedError('note hash read request', MAX_NOTE_HASH_READ_REQUESTS_PER_TX); @@ -286,10 +320,10 @@ export class PublicEnqueuedCallSideEffectTrace implements PublicSideEffectTraceI public traceNewNoteHash( contractAddress: AztecAddress, noteHash: Fr, - leafIndex: Fr, + leafIndex: Fr = Fr.zero(), path: Fr[] = emptyNoteHashPath(), ) { - if (this.noteHashes.length + this.previousAccumulatedDataArrayLengths.noteHashes >= MAX_NOTE_HASHES_PER_TX) { + if (this.noteHashes.length + this.previousSideEffectArrayLengths.noteHashes >= MAX_NOTE_HASHES_PER_TX) { throw new SideEffectLimitReachedError('note hash', MAX_NOTE_HASHES_PER_TX); } @@ -340,7 +374,7 @@ export class PublicEnqueuedCallSideEffectTrace implements PublicSideEffectTraceI lowLeafPath: Fr[] = emptyNullifierPath(), insertionPath: Fr[] = emptyNullifierPath(), ) { - if (this.nullifiers.length + this.previousAccumulatedDataArrayLengths.nullifiers >= MAX_NULLIFIERS_PER_TX) { + if (this.nullifiers.length + this.previousSideEffectArrayLengths.nullifiers >= MAX_NULLIFIERS_PER_TX) { throw new SideEffectLimitReachedError('nullifier', MAX_NULLIFIERS_PER_TX); } @@ -364,7 +398,7 @@ export class PublicEnqueuedCallSideEffectTrace implements PublicSideEffectTraceI ) { // NOTE: contractAddress is unused because msgHash is an already-siloed leaf if ( - this.l1ToL2MsgReadRequests.length + this.previousValidationRequestArrayLengths.l1ToL2MsgReadRequests >= + this.l1ToL2MsgReadRequests.length + this.previousSideEffectArrayLengths.l1ToL2MsgReadRequests >= MAX_L1_TO_L2_MSG_READ_REQUESTS_PER_TX ) { throw new SideEffectLimitReachedError('l1 to l2 message read request', MAX_L1_TO_L2_MSG_READ_REQUESTS_PER_TX); @@ -379,7 +413,7 @@ export class PublicEnqueuedCallSideEffectTrace implements PublicSideEffectTraceI } public traceNewL2ToL1Message(contractAddress: AztecAddress, recipient: Fr, content: Fr) { - if (this.l2ToL1Messages.length + this.previousAccumulatedDataArrayLengths.l2ToL1Msgs >= MAX_L2_TO_L1_MSGS_PER_TX) { + if (this.l2ToL1Messages.length + this.previousSideEffectArrayLengths.l2ToL1Msgs >= MAX_L2_TO_L1_MSGS_PER_TX) { throw new SideEffectLimitReachedError('l2 to l1 message', MAX_L2_TO_L1_MSGS_PER_TX); } @@ -393,7 +427,7 @@ export class PublicEnqueuedCallSideEffectTrace implements PublicSideEffectTraceI public traceUnencryptedLog(contractAddress: AztecAddress, log: Fr[]) { if ( - this.unencryptedLogs.length + this.previousAccumulatedDataArrayLengths.unencryptedLogsHashes >= + this.unencryptedLogs.length + this.previousSideEffectArrayLengths.unencryptedLogs >= MAX_UNENCRYPTED_LOGS_PER_TX ) { throw new SideEffectLimitReachedError('unencrypted log', MAX_UNENCRYPTED_LOGS_PER_TX); @@ -434,7 +468,7 @@ export class PublicEnqueuedCallSideEffectTrace implements PublicSideEffectTraceI } // This tracing function gets called everytime we start simulation/execution. - // This happens both when starting a new top-level trace and the start of every nested trace + // This happens both when starting a new top-level trace and the start of every forked trace // We use this to collect the AvmContractBytecodeHints public traceGetBytecode( contractAddress: AztecAddress, @@ -473,13 +507,11 @@ export class PublicEnqueuedCallSideEffectTrace implements PublicSideEffectTraceI */ public traceNestedCall( /** The trace of the nested call. */ - nestedCallTrace: this, + _nestedCallTrace: this, /** The execution environment of the nested call. */ nestedEnvironment: AvmExecutionEnvironment, /** How much gas was available for this public execution. */ startGasLeft: Gas, - /** How much gas was left after this public execution. */ - endGasLeft: Gas, /** Bytecode used for this execution. */ _bytecode: Buffer, /** The call's results */ @@ -492,7 +524,10 @@ export class PublicEnqueuedCallSideEffectTrace implements PublicSideEffectTraceI // Store end side effect counter before it gets updated by absorbing nested call trace const endSideEffectCounter = new Fr(this.sideEffectCounter); - const gasUsed = new Gas(startGasLeft.daGas - endGasLeft.daGas, startGasLeft.l2Gas - endGasLeft.l2Gas); + const gasUsed = new Gas( + startGasLeft.daGas - avmCallResults.gasLeft.daGas, + startGasLeft.l2Gas - avmCallResults.gasLeft.l2Gas, + ); this.avmCircuitHints.externalCalls.items.push( new AvmExternalCallHint( @@ -523,38 +558,6 @@ export class PublicEnqueuedCallSideEffectTrace implements PublicSideEffectTraceI this.avmCircuitHints.enqueuedCalls.items.push(new AvmEnqueuedCallHint(publicCallRequest.contractAddress, calldata)); } - public mergeSuccessfulForkedTrace(nestedTrace: this) { - // TODO(dbanks12): accept & merge nested trace's hints! - this.sideEffectCounter = nestedTrace.sideEffectCounter; - - this.enqueuedCalls.push(...nestedTrace.enqueuedCalls); - - this.publicDataReads.push(...nestedTrace.publicDataReads); - this.publicDataWrites.push(...nestedTrace.publicDataWrites); - this.noteHashReadRequests.push(...nestedTrace.noteHashReadRequests); - this.noteHashes.push(...nestedTrace.noteHashes); - this.nullifierReadRequests.push(...nestedTrace.nullifierReadRequests); - this.nullifierNonExistentReadRequests.push(...nestedTrace.nullifierNonExistentReadRequests); - this.log.debug(`Merging nullifiers: ${nestedTrace.nullifiers.length}`); - this.log.debug(`Into parent nullifiers: ${this.nullifiers.length}`); - this.nullifiers.push(...nestedTrace.nullifiers); - this.log.debug(`After merge: ${JSON.stringify(this.nullifiers)}`); - this.l1ToL2MsgReadRequests.push(...nestedTrace.l1ToL2MsgReadRequests); - this.l2ToL1Messages.push(...nestedTrace.l2ToL1Messages); - this.unencryptedLogs.push(...nestedTrace.unencryptedLogs); - this.unencryptedLogsHashes.push(...nestedTrace.unencryptedLogsHashes); - } - - /** - * Discard accumulated side effects, but keep hints. - */ - public mergeRevertedForkedTrace(nestedTrace: this) { - // TODO(dbanks12): accept & merge nested trace's hints! - this.sideEffectCounter = nestedTrace.sideEffectCounter; - - this.enqueuedCalls.push(...nestedTrace.enqueuedCalls); - } - public getSideEffects(): SideEffects { return { enqueuedCalls: this.enqueuedCalls, @@ -576,19 +579,15 @@ export class PublicEnqueuedCallSideEffectTrace implements PublicSideEffectTraceI * Get the results of public execution. */ public toPublicEnqueuedCallExecutionResult( - /** How much gas was left after this public execution. */ - endGasLeft: Gas, /** The call's results */ - avmCallResults: AvmContractCallResult, + avmCallResults: AvmFinalizedCallResult, ): EnqueuedPublicCallExecutionResultWithSideEffects { return { - endGasLeft, + endGasLeft: Gas.from(avmCallResults.gasLeft), endSideEffectCounter: new Fr(this.sideEffectCounter), returnValues: avmCallResults.output, reverted: avmCallResults.reverted, - revertReason: avmCallResults.revertReason - ? createSimulationError(avmCallResults.revertReason, avmCallResults.output) - : undefined, + revertReason: avmCallResults.revertReason, sideEffects: { publicDataWrites: this.publicDataWrites, noteHashes: this.noteHashes, @@ -600,40 +599,6 @@ export class PublicEnqueuedCallSideEffectTrace implements PublicSideEffectTraceI }; } - /** - * Construct AVM circuit public inputs based on traced contents. - */ - public toVMCircuitPublicInputs( - /** Constants. */ - constants: CombinedConstantData, - /** The call request that triggered public execution. */ - callRequest: PublicCallRequest, - /** How much gas was available for this public execution. */ - startGasLeft: Gas, - /** How much gas was left after this public execution. */ - endGasLeft: Gas, - /** Transaction fee. */ - transactionFee: Fr, - /** The call's results */ - avmCallResults: AvmContractCallResult, - ): VMCircuitPublicInputs { - this.log.debug(`Creating public inputs with call result: ${avmCallResults.reverted}`); - return new VMCircuitPublicInputs( - /*constants=*/ constants, - /*callRequest=*/ callRequest, - /*publicCallStack=*/ makeTuple(MAX_ENQUEUED_CALLS_PER_TX, PublicInnerCallRequest.empty), - /*previousValidationRequestArrayLengths=*/ this.previousValidationRequestArrayLengths, - /*validationRequests=*/ this.getValidationRequests(), - /*previousAccumulatedDataArrayLengths=*/ this.previousAccumulatedDataArrayLengths, - /*accumulatedData=*/ this.getAccumulatedData(startGasLeft.sub(endGasLeft)), - /*startSideEffectCounter=*/ this.startSideEffectCounter, - /*endSideEffectCounter=*/ this.sideEffectCounter, - /*startGasLeft=*/ startGasLeft, - /*transactionFee=*/ transactionFee, - /*reverted=*/ avmCallResults.reverted, - ); - } - public toAvmCircuitPublicInputs( /** Globals. */ globalVariables: GlobalVariables, @@ -644,9 +609,9 @@ export class PublicEnqueuedCallSideEffectTrace implements PublicSideEffectTraceI /** How much gas was available for this public execution. */ gasLimits: GasSettings, /** Call requests for setup phase. */ - publicSetupCallRequests: Tuple, + publicSetupCallRequests: PublicCallRequest[], /** Call requests for app logic phase. */ - publicAppLogicCallRequests: Tuple, + publicAppLogicCallRequests: PublicCallRequest[], /** Call request for teardown phase. */ publicTeardownCallRequest: PublicCallRequest, /** End tree snapshots. */ @@ -666,8 +631,8 @@ export class PublicEnqueuedCallSideEffectTrace implements PublicSideEffectTraceI startTreeSnapshots, startGasUsed, gasLimits, - publicSetupCallRequests, - publicAppLogicCallRequests, + padArrayEnd(publicSetupCallRequests, PublicCallRequest.empty(), MAX_ENQUEUED_CALLS_PER_TX), + padArrayEnd(publicAppLogicCallRequests, PublicCallRequest.empty(), MAX_ENQUEUED_CALLS_PER_TX), publicTeardownCallRequest, /*previousNonRevertibleAccumulatedDataArrayLengths=*/ PrivateToAvmAccumulatedDataArrayLengths.empty(), /*previousRevertibleAccumulatedDataArrayLengths=*/ PrivateToAvmAccumulatedDataArrayLengths.empty(), @@ -686,12 +651,10 @@ export class PublicEnqueuedCallSideEffectTrace implements PublicSideEffectTraceI _avmEnvironment: AvmExecutionEnvironment, /** How much gas was available for this public execution. */ _startGasLeft: Gas, - /** How much gas was left after this public execution. */ - _endGasLeft: Gas, /** Bytecode used for this execution. */ _bytecode: Buffer, /** The call's results */ - _avmCallResults: AvmContractCallResult, + _avmCallResults: AvmFinalizedCallResult, /** Function name for logging */ _functionName: string = 'unknown', ): PublicFunctionCallResult { @@ -706,21 +669,6 @@ export class PublicEnqueuedCallSideEffectTrace implements PublicSideEffectTraceI return this.avmCircuitHints; } - private getValidationRequests() { - return new PublicValidationRequests( - RollupValidationRequests.empty(), - padArrayEnd(this.noteHashReadRequests, TreeLeafReadRequest.empty(), MAX_NOTE_HASH_READ_REQUESTS_PER_TX), - padArrayEnd(this.nullifierReadRequests, ScopedReadRequest.empty(), MAX_NULLIFIER_READ_REQUESTS_PER_TX), - padArrayEnd( - this.nullifierNonExistentReadRequests, - ScopedReadRequest.empty(), - MAX_NULLIFIER_NON_EXISTENT_READ_REQUESTS_PER_TX, - ), - padArrayEnd(this.l1ToL2MsgReadRequests, TreeLeafReadRequest.empty(), MAX_L1_TO_L2_MSG_READ_REQUESTS_PER_TX), - padArrayEnd(this.publicDataReads, PublicDataRead.empty(), MAX_PUBLIC_DATA_READS_PER_TX), - ); - } - private getAvmAccumulatedData() { return new AvmAccumulatedData( padArrayEnd( @@ -743,20 +691,6 @@ export class PublicEnqueuedCallSideEffectTrace implements PublicSideEffectTraceI ); } - private getAccumulatedData(gasUsed: Gas) { - return new PublicAccumulatedData( - padArrayEnd(this.noteHashes, ScopedNoteHash.empty(), MAX_NOTE_HASHES_PER_TX), - padArrayEnd(this.nullifiers, Nullifier.empty(), MAX_NULLIFIERS_PER_TX), - padArrayEnd(this.l2ToL1Messages, ScopedL2ToL1Message.empty(), MAX_L2_TO_L1_MSGS_PER_TX), - /*noteEncryptedLogsHashes=*/ makeTuple(MAX_NOTE_ENCRYPTED_LOGS_PER_TX, LogHash.empty), - /*encryptedLogsHashes=*/ makeTuple(MAX_ENCRYPTED_LOGS_PER_TX, ScopedLogHash.empty), - padArrayEnd(this.unencryptedLogsHashes, ScopedLogHash.empty(), MAX_UNENCRYPTED_LOGS_PER_TX), - padArrayEnd(this.publicDataWrites, PublicDataUpdateRequest.empty(), MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX), - /*publicCallStack=*/ makeTuple(MAX_ENQUEUED_CALLS_PER_TX, PublicCallRequest.empty), - /*gasUsed=*/ gasUsed, - ); - } - private enforceLimitOnNullifierChecks(errorMsgOrigin: string = '') { // NOTE: Why error if _either_ limit was reached? If user code emits either an existent or non-existent // nullifier read request (NULLIFIEREXISTS, GETCONTRACTINSTANCE, *CALL), and one of the limits has been @@ -765,7 +699,7 @@ export class PublicEnqueuedCallSideEffectTrace implements PublicSideEffectTraceI // going to skip the read request and just revert instead" when the nullifier actually doesn't exist // (or vice versa). So, if either maximum has been reached, any nullifier-reading operation must error. if ( - this.nullifierReadRequests.length + this.previousValidationRequestArrayLengths.nullifierReadRequests >= + this.nullifierReadRequests.length + this.previousSideEffectArrayLengths.nullifierReadRequests >= MAX_NULLIFIER_READ_REQUESTS_PER_TX ) { throw new SideEffectLimitReachedError( @@ -775,7 +709,7 @@ export class PublicEnqueuedCallSideEffectTrace implements PublicSideEffectTraceI } if ( this.nullifierNonExistentReadRequests.length + - this.previousValidationRequestArrayLengths.nullifierNonExistentReadRequests >= + this.previousSideEffectArrayLengths.nullifierNonExistentReadRequests >= MAX_NULLIFIER_NON_EXISTENT_READ_REQUESTS_PER_TX ) { throw new SideEffectLimitReachedError( diff --git a/yarn-project/simulator/src/public/enqueued_call_simulator.ts b/yarn-project/simulator/src/public/enqueued_call_simulator.ts deleted file mode 100644 index 62b48402d3f..00000000000 --- a/yarn-project/simulator/src/public/enqueued_call_simulator.ts +++ /dev/null @@ -1,268 +0,0 @@ -import { - type AvmProvingRequest, - MerkleTreeId, - NestedProcessReturnValues, - ProvingRequestType, - type PublicExecutionRequest, - type SimulationError, - UnencryptedFunctionL2Logs, -} from '@aztec/circuit-types'; -import { - AvmCircuitInputs, - AvmCircuitPublicInputs, - AztecAddress, - type CombinedConstantData, - ContractStorageRead, - ContractStorageUpdateRequest, - Fr, - Gas, - type GlobalVariables, - type Header, - L2ToL1Message, - LogHash, - MAX_ENQUEUED_CALLS_PER_CALL, - MAX_L1_TO_L2_MSG_READ_REQUESTS_PER_CALL, - MAX_L2_GAS_PER_ENQUEUED_CALL, - MAX_L2_TO_L1_MSGS_PER_CALL, - MAX_NOTE_HASHES_PER_CALL, - MAX_NOTE_HASH_READ_REQUESTS_PER_CALL, - MAX_NULLIFIERS_PER_CALL, - MAX_NULLIFIER_NON_EXISTENT_READ_REQUESTS_PER_CALL, - MAX_NULLIFIER_READ_REQUESTS_PER_CALL, - MAX_PUBLIC_DATA_READS_PER_CALL, - MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_CALL, - MAX_UNENCRYPTED_LOGS_PER_CALL, - NoteHash, - Nullifier, - type PublicCallRequest, - PublicCircuitPublicInputs, - PublicInnerCallRequest, - ReadRequest, - RevertCode, - TreeLeafReadRequest, - type VMCircuitPublicInputs, -} from '@aztec/circuits.js'; -import { computeVarArgsHash } from '@aztec/circuits.js/hash'; -import { padArrayEnd } from '@aztec/foundation/collection'; -import { type DebugLogger, createDebugLogger } from '@aztec/foundation/log'; -import { type MerkleTreeReadOperations } from '@aztec/world-state'; - -import { AvmContractCallResult } from '../avm/avm_contract_call_result.js'; -import { type AvmPersistableStateManager } from '../avm/journal/journal.js'; -import { getPublicFunctionDebugName } from '../common/debug_fn_name.js'; -import { type EnqueuedPublicCallExecutionResult, type PublicFunctionCallResult } from './execution.js'; -import { type PublicExecutor, createAvmExecutionEnvironment } from './executor.js'; -import { type WorldStateDB } from './public_db_sources.js'; - -function emptyAvmProvingRequest(): AvmProvingRequest { - return { - type: ProvingRequestType.PUBLIC_VM, - inputs: AvmCircuitInputs.empty(), - }; -} -function makeAvmProvingRequest(inputs: PublicCircuitPublicInputs, result: PublicFunctionCallResult): AvmProvingRequest { - return { - type: ProvingRequestType.PUBLIC_VM, - inputs: new AvmCircuitInputs( - result.functionName, - result.calldata, - inputs, - result.avmCircuitHints, - AvmCircuitPublicInputs.empty(), - ), - }; -} - -export type EnqueuedCallResult = { - /** Inputs to be used for proving */ - avmProvingRequest: AvmProvingRequest; - /** The public kernel output at the end of the enqueued call */ - kernelOutput: VMCircuitPublicInputs; - /** Unencrypted logs generated during the execution of this enqueued call */ - newUnencryptedLogs: UnencryptedFunctionL2Logs; - /** Return values of simulating complete callstack */ - returnValues: NestedProcessReturnValues; - /** Gas used during the execution of this enqueued call */ - gasUsed: Gas; - /** Did call revert? */ - reverted: boolean; - /** Revert reason, if any */ - revertReason?: SimulationError; -}; - -export class EnqueuedCallSimulator { - private log: DebugLogger; - constructor( - private db: MerkleTreeReadOperations, - private worldStateDB: WorldStateDB, - private publicExecutor: PublicExecutor, - private globalVariables: GlobalVariables, - private historicalHeader: Header, - private realAvmProvingRequests: boolean, - ) { - this.log = createDebugLogger(`aztec:sequencer`); - } - - async simulate( - callRequest: PublicCallRequest, - executionRequest: PublicExecutionRequest, - constants: CombinedConstantData, - availableGas: Gas, - transactionFee: Fr, - stateManager: AvmPersistableStateManager, - ): Promise { - // Gas allocated to an enqueued call can be different from the available gas - // if there is more gas available than the max allocation per enqueued call. - const allocatedGas = new Gas( - /*daGas=*/ availableGas.daGas, - /*l2Gas=*/ Math.min(availableGas.l2Gas, MAX_L2_GAS_PER_ENQUEUED_CALL), - ); - - const result = (await this.publicExecutor.simulate( - stateManager, - executionRequest, - this.globalVariables, - allocatedGas, - transactionFee, - )) as EnqueuedPublicCallExecutionResult; - - /////////////////////////////////////////////////////////////////////////// - // ALL TEMPORARY - const fnName = await getPublicFunctionDebugName( - this.worldStateDB, - executionRequest.callContext.contractAddress, - executionRequest.callContext.functionSelector, - executionRequest.args, - ); - - const avmExecutionEnv = createAvmExecutionEnvironment(executionRequest, this.globalVariables, transactionFee); - const avmCallResult = new AvmContractCallResult(result.reverted, result.returnValues); - - // Generate an AVM proving request - let avmProvingRequest: AvmProvingRequest; - if (this.realAvmProvingRequests) { - const deprecatedFunctionCallResult = stateManager.trace.toPublicFunctionCallResult( - avmExecutionEnv, - /*startGasLeft=*/ allocatedGas, - /*endGasLeft=*/ Gas.from(result.endGasLeft), - Buffer.alloc(0), - avmCallResult, - fnName, - ); - const publicInputs = await this.getPublicCircuitPublicInputs(deprecatedFunctionCallResult); - avmProvingRequest = makeAvmProvingRequest(publicInputs, deprecatedFunctionCallResult); - } else { - avmProvingRequest = emptyAvmProvingRequest(); - } - - // TODO(dbanks12): Since AVM circuit will be at the level of all enqueued calls in a TX, - // this public inputs generation will move up to the enqueued calls processor. - const vmCircuitPublicInputs = stateManager.trace.toVMCircuitPublicInputs( - constants, - callRequest, - /*startGasLeft=*/ allocatedGas, - /*endGasLeft=*/ Gas.from(result.endGasLeft), - transactionFee, - avmCallResult, - ); - - const gasUsed = allocatedGas.sub(Gas.from(result.endGasLeft)); - - return { - avmProvingRequest, - kernelOutput: vmCircuitPublicInputs, - newUnencryptedLogs: new UnencryptedFunctionL2Logs(stateManager!.trace.getUnencryptedLogs()), - returnValues: new NestedProcessReturnValues(result.returnValues), - gasUsed, - reverted: result.reverted, - revertReason: result.revertReason, - }; - } - - private async getPublicCircuitPublicInputs(result: PublicFunctionCallResult) { - const publicDataTreeInfo = await this.db.getTreeInfo(MerkleTreeId.PUBLIC_DATA_TREE); - this.historicalHeader.state.partial.publicDataTree.root = Fr.fromBuffer(publicDataTreeInfo.root); - - return PublicCircuitPublicInputs.from({ - callContext: result.executionRequest.callContext, - proverAddress: AztecAddress.ZERO, - argsHash: computeVarArgsHash(result.executionRequest.args), - noteHashes: padArrayEnd( - result.noteHashes, - NoteHash.empty(), - MAX_NOTE_HASHES_PER_CALL, - `Too many note hashes. Got ${result.noteHashes.length} with max being ${MAX_NOTE_HASHES_PER_CALL}`, - ), - nullifiers: padArrayEnd( - result.nullifiers, - Nullifier.empty(), - MAX_NULLIFIERS_PER_CALL, - `Too many nullifiers. Got ${result.nullifiers.length} with max being ${MAX_NULLIFIERS_PER_CALL}`, - ), - l2ToL1Msgs: padArrayEnd( - result.l2ToL1Messages, - L2ToL1Message.empty(), - MAX_L2_TO_L1_MSGS_PER_CALL, - `Too many L2 to L1 messages. Got ${result.l2ToL1Messages.length} with max being ${MAX_L2_TO_L1_MSGS_PER_CALL}`, - ), - startSideEffectCounter: result.startSideEffectCounter, - endSideEffectCounter: result.endSideEffectCounter, - returnsHash: computeVarArgsHash(result.returnValues), - noteHashReadRequests: padArrayEnd( - result.noteHashReadRequests, - TreeLeafReadRequest.empty(), - MAX_NOTE_HASH_READ_REQUESTS_PER_CALL, - `Too many note hash read requests. Got ${result.noteHashReadRequests.length} with max being ${MAX_NOTE_HASH_READ_REQUESTS_PER_CALL}`, - ), - nullifierReadRequests: padArrayEnd( - result.nullifierReadRequests, - ReadRequest.empty(), - MAX_NULLIFIER_READ_REQUESTS_PER_CALL, - `Too many nullifier read requests. Got ${result.nullifierReadRequests.length} with max being ${MAX_NULLIFIER_READ_REQUESTS_PER_CALL}`, - ), - nullifierNonExistentReadRequests: padArrayEnd( - result.nullifierNonExistentReadRequests, - ReadRequest.empty(), - MAX_NULLIFIER_NON_EXISTENT_READ_REQUESTS_PER_CALL, - `Too many nullifier non-existent read requests. Got ${result.nullifierNonExistentReadRequests.length} with max being ${MAX_NULLIFIER_NON_EXISTENT_READ_REQUESTS_PER_CALL}`, - ), - l1ToL2MsgReadRequests: padArrayEnd( - result.l1ToL2MsgReadRequests, - TreeLeafReadRequest.empty(), - MAX_L1_TO_L2_MSG_READ_REQUESTS_PER_CALL, - `Too many L1 to L2 message read requests. Got ${result.l1ToL2MsgReadRequests.length} with max being ${MAX_L1_TO_L2_MSG_READ_REQUESTS_PER_CALL}`, - ), - contractStorageReads: padArrayEnd( - result.contractStorageReads, - ContractStorageRead.empty(), - MAX_PUBLIC_DATA_READS_PER_CALL, - `Too many public data reads. Got ${result.contractStorageReads.length} with max being ${MAX_PUBLIC_DATA_READS_PER_CALL}`, - ), - contractStorageUpdateRequests: padArrayEnd( - result.contractStorageUpdateRequests, - ContractStorageUpdateRequest.empty(), - MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_CALL, - `Too many public data update requests. Got ${result.contractStorageUpdateRequests.length} with max being ${MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_CALL}`, - ), - publicCallRequests: padArrayEnd( - result.publicCallRequests, - PublicInnerCallRequest.empty(), - MAX_ENQUEUED_CALLS_PER_CALL, - `Too many public call requests. Got ${result.publicCallRequests.length} with max being ${MAX_ENQUEUED_CALLS_PER_CALL}`, - ), - unencryptedLogsHashes: padArrayEnd( - result.unencryptedLogsHashes, - LogHash.empty(), - MAX_UNENCRYPTED_LOGS_PER_CALL, - `Too many unencrypted logs. Got ${result.unencryptedLogsHashes.length} with max being ${MAX_UNENCRYPTED_LOGS_PER_CALL}`, - ), - historicalHeader: this.historicalHeader, - globalVariables: this.globalVariables, - startGasLeft: Gas.from(result.startGasLeft), - endGasLeft: Gas.from(result.endGasLeft), - transactionFee: result.transactionFee, - // TODO(@just-mitch): need better mapping from simulator to revert code. - revertCode: result.reverted ? RevertCode.APP_LOGIC_REVERTED : RevertCode.OK, - }); - } -} diff --git a/yarn-project/simulator/src/public/executor.ts b/yarn-project/simulator/src/public/executor.ts deleted file mode 100644 index 3a90b863bdb..00000000000 --- a/yarn-project/simulator/src/public/executor.ts +++ /dev/null @@ -1,152 +0,0 @@ -import { type PublicExecutionRequest } from '@aztec/circuit-types'; -import { type AvmSimulationStats } from '@aztec/circuit-types/stats'; -import { Fr, Gas, type GlobalVariables } from '@aztec/circuits.js'; -import { createDebugLogger } from '@aztec/foundation/log'; -import { Timer } from '@aztec/foundation/timer'; -import { type TelemetryClient } from '@aztec/telemetry-client'; - -import { AvmContext } from '../avm/avm_context.js'; -import { AvmExecutionEnvironment } from '../avm/avm_execution_environment.js'; -import { AvmMachineState } from '../avm/avm_machine_state.js'; -import { AvmSimulator } from '../avm/avm_simulator.js'; -import { AvmPersistableStateManager } from '../avm/journal/index.js'; -import { getPublicFunctionDebugName } from '../common/debug_fn_name.js'; -import { DualSideEffectTrace } from './dual_side_effect_trace.js'; -import { PublicEnqueuedCallSideEffectTrace } from './enqueued_call_side_effect_trace.js'; -import { - type EnqueuedPublicCallExecutionResult, - type EnqueuedPublicCallExecutionResultWithSideEffects, -} from './execution.js'; -import { ExecutorMetrics } from './executor_metrics.js'; -import { type WorldStateDB } from './public_db_sources.js'; -import { PublicSideEffectTrace } from './side_effect_trace.js'; - -/** - * Handles execution of public functions. - */ -export class PublicExecutor { - metrics: ExecutorMetrics; - - constructor(private readonly worldStateDB: WorldStateDB, client: TelemetryClient) { - this.metrics = new ExecutorMetrics(client, 'PublicExecutor'); - } - - static readonly log = createDebugLogger('aztec:simulator:public_executor'); - - /** - * Simulate a public execution request. - * @param executionRequest - The execution to run. - * @param globalVariables - The global variables to use. - * @param allocatedGas - The gas available at the start of this enqueued call. - * @param transactionFee - Fee offered for this TX. - * @returns The result of execution. - */ - public async simulate( - stateManager: AvmPersistableStateManager, - executionRequest: PublicExecutionRequest, // TODO(dbanks12): CallRequest instead? - globalVariables: GlobalVariables, - allocatedGas: Gas, - transactionFee: Fr = Fr.ZERO, - ): Promise { - const address = executionRequest.callContext.contractAddress; - const selector = executionRequest.callContext.functionSelector; - const fnName = await getPublicFunctionDebugName(this.worldStateDB, address, selector, executionRequest.args); - - PublicExecutor.log.verbose( - `[AVM] Executing public external function ${fnName}@${address} with ${allocatedGas.l2Gas} allocated L2 gas.`, - ); - const timer = new Timer(); - - const avmExecutionEnv = createAvmExecutionEnvironment(executionRequest, globalVariables, transactionFee); - - const avmMachineState = new AvmMachineState(allocatedGas); - const avmContext = new AvmContext(stateManager, avmExecutionEnv, avmMachineState); - const simulator = new AvmSimulator(avmContext); - const avmCallResult = await simulator.execute(); - const bytecode = simulator.getBytecode()!; - - PublicExecutor.log.verbose( - `[AVM] ${fnName} returned, reverted: ${avmCallResult.reverted}${ - avmCallResult.reverted ? ', reason: ' + avmCallResult.revertReason : '' - }.`, - { - eventName: 'avm-simulation', - appCircuitName: fnName, - duration: timer.ms(), - bytecodeSize: bytecode!.length, - } satisfies AvmSimulationStats, - ); - - const publicExecutionResult = stateManager.trace.toPublicEnqueuedCallExecutionResult( - /*endGasLeft=*/ Gas.from(avmContext.machineState.gasLeft), - avmCallResult, - ); - - if (avmCallResult.reverted) { - this.metrics.recordFunctionSimulationFailure(); - } else { - this.metrics.recordFunctionSimulation(bytecode.length, timer.ms()); - } - - PublicExecutor.log.verbose( - `[AVM] ${fnName} simulation complete. Reverted=${avmCallResult.reverted}. Consumed ${ - allocatedGas.l2Gas - avmContext.machineState.gasLeft.l2Gas - } L2 gas, ending with ${avmContext.machineState.gasLeft.l2Gas} L2 gas left.`, - ); - - return publicExecutionResult; - } - - /** - * Simulate an enqueued call on its own, and include its side effects in its results. - * @param executionRequest - The execution to run. - * @param globalVariables - The global variables to use. - * @param allocatedGas - The gas available at the start of this enqueued call. - * @param transactionFee - Fee offered for this TX. - * @param startSideEffectCounter - The start counter to initialize the side effect trace with. - * @returns The result of execution including side effect vectors. - */ - public async simulateIsolatedEnqueuedCall( - executionRequest: PublicExecutionRequest, - globalVariables: GlobalVariables, - allocatedGas: Gas, - transactionFee: Fr = Fr.ONE, - startSideEffectCounter: number = 0, - ): Promise { - const innerCallTrace = new PublicSideEffectTrace(startSideEffectCounter); - const enqueuedCallTrace = new PublicEnqueuedCallSideEffectTrace(startSideEffectCounter); - const trace = new DualSideEffectTrace(innerCallTrace, enqueuedCallTrace); - const stateManager = new AvmPersistableStateManager(this.worldStateDB, trace); - return (await this.simulate( - stateManager, - executionRequest, - globalVariables, - allocatedGas, - transactionFee, - )) as EnqueuedPublicCallExecutionResultWithSideEffects; - } -} - -/** - * Convert a PublicExecutionRequest object to an AvmExecutionEnvironment - * - * @param executionRequest - * @param globalVariables - * @returns - */ -export function createAvmExecutionEnvironment( - executionRequest: PublicExecutionRequest, - globalVariables: GlobalVariables, - transactionFee: Fr, -): AvmExecutionEnvironment { - return new AvmExecutionEnvironment( - executionRequest.callContext.contractAddress, - executionRequest.callContext.msgSender, - executionRequest.callContext.functionSelector, - /*contractCallDepth=*/ Fr.zero(), - transactionFee, - globalVariables, - executionRequest.callContext.isStaticCall, - executionRequest.args, - ); -} diff --git a/yarn-project/simulator/src/public/executor_metrics.ts b/yarn-project/simulator/src/public/executor_metrics.ts index 1d4a05566a8..4b648dfe4ef 100644 --- a/yarn-project/simulator/src/public/executor_metrics.ts +++ b/yarn-project/simulator/src/public/executor_metrics.ts @@ -10,7 +10,6 @@ import { export class ExecutorMetrics { private fnCount: UpDownCounter; private fnDuration: Histogram; - private bytecodeSize: Histogram; constructor(client: TelemetryClient, name = 'PublicExecutor') { const meter = client.getMeter(name); @@ -24,19 +23,12 @@ export class ExecutorMetrics { unit: 'ms', valueType: ValueType.INT, }); - - this.bytecodeSize = meter.createHistogram(Metrics.PUBLIC_EXECUTION_SIMULATION_BYTECODE_SIZE, { - description: 'Size of the function bytecode', - unit: 'By', - valueType: ValueType.INT, - }); } - recordFunctionSimulation(bytecodeSize: number, durationMs: number) { + recordFunctionSimulation(durationMs: number) { this.fnCount.add(1, { [Attributes.OK]: true, }); - this.bytecodeSize.record(bytecodeSize); this.fnDuration.record(Math.ceil(durationMs)); } diff --git a/yarn-project/simulator/src/public/fixtures/index.ts b/yarn-project/simulator/src/public/fixtures/index.ts new file mode 100644 index 00000000000..8c0f53fab32 --- /dev/null +++ b/yarn-project/simulator/src/public/fixtures/index.ts @@ -0,0 +1,158 @@ +import { PublicExecutionRequest, Tx } from '@aztec/circuit-types'; +import { + type AvmCircuitInputs, + CallContext, + DEFAULT_GAS_LIMIT, + Gas, + GasFees, + GasSettings, + GlobalVariables, + Header, + MAX_L2_GAS_PER_ENQUEUED_CALL, + PartialPrivateTailPublicInputsForPublic, + PrivateKernelTailCircuitPublicInputs, + type PublicFunction, + PublicKeys, + RollupValidationRequests, + SerializableContractInstance, + TxConstantData, + TxContext, +} from '@aztec/circuits.js'; +import { makeContractClassPublic, makeContractInstanceFromClassId } from '@aztec/circuits.js/testing'; +import { AztecAddress } from '@aztec/foundation/aztec-address'; +import { Fr, Point } from '@aztec/foundation/fields'; +import { openTmpStore } from '@aztec/kv-store/utils'; +import { PublicTxSimulator, type WorldStateDB } from '@aztec/simulator'; +import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; +import { MerkleTrees } from '@aztec/world-state'; + +import { mock } from 'jest-mock-extended'; + +import { getAvmTestContractBytecode, getAvmTestContractFunctionSelector } from '../../avm/fixtures/index.js'; + +const TIMESTAMP = new Fr(99833); + +/** + * If assertionErrString is set, we expect a (non exceptional halting) revert due to a failing assertion and + * we check that the revert reason error contains this string. However, the circuit must correctly prove the + * execution. + */ +export async function simulateAvmTestContractGenerateCircuitInputs( + functionName: string, + calldata: Fr[] = [], + assertionErrString?: string, +): Promise { + const sender = AztecAddress.random(); + const functionSelector = getAvmTestContractFunctionSelector(functionName); + calldata = [functionSelector.toField(), ...calldata]; + + const globalVariables = GlobalVariables.empty(); + globalVariables.gasFees = GasFees.default(); + globalVariables.timestamp = TIMESTAMP; + + const worldStateDB = mock(); + const telemetry = new NoopTelemetryClient(); + const merkleTrees = await (await MerkleTrees.new(openTmpStore(), telemetry)).fork(); + worldStateDB.getMerkleInterface.mockReturnValue(merkleTrees); + + // Top level contract call + const bytecode = getAvmTestContractBytecode('public_dispatch'); + const dispatchSelector = getAvmTestContractFunctionSelector('public_dispatch'); + const publicFn: PublicFunction = { bytecode, selector: dispatchSelector }; + const contractClass = makeContractClassPublic(0, publicFn); + const contractInstance = makeContractInstanceFromClassId(contractClass.id); + + // The values here should match those in `avm_simulator.test.ts` + const instanceGet = new SerializableContractInstance({ + version: 1, + salt: new Fr(0x123), + deployer: new AztecAddress(new Fr(0x456)), + contractClassId: new Fr(0x789), + initializationHash: new Fr(0x101112), + publicKeys: new PublicKeys( + new Point(new Fr(0x131415), new Fr(0x161718), false), + new Point(new Fr(0x192021), new Fr(0x222324), false), + new Point(new Fr(0x252627), new Fr(0x282930), false), + new Point(new Fr(0x313233), new Fr(0x343536), false), + ), + }).withAddress(contractInstance.address); + worldStateDB.getContractInstance + .mockResolvedValueOnce(contractInstance) + .mockResolvedValueOnce(instanceGet) // test gets deployer + .mockResolvedValueOnce(instanceGet) // test gets class id + .mockResolvedValueOnce(instanceGet) // test gets init hash + .mockResolvedValue(contractInstance); + worldStateDB.getContractClass.mockResolvedValue(contractClass); + worldStateDB.getBytecode.mockResolvedValue(bytecode); + + const storageValue = new Fr(5); + worldStateDB.storageRead.mockResolvedValue(Promise.resolve(storageValue)); + + const simulator = new PublicTxSimulator( + merkleTrees, + worldStateDB, + new NoopTelemetryClient(), + globalVariables, + /*doMerkleOperations=*/ true, + ); + + const callContext = new CallContext(sender, contractInstance.address, dispatchSelector, /*isStaticCall=*/ false); + const executionRequest = new PublicExecutionRequest(callContext, calldata); + + const tx: Tx = createTxForPublicCall(executionRequest); + + const avmResult = await simulator.simulate(tx); + + if (assertionErrString == undefined) { + expect(avmResult.revertCode.isOK()).toBe(true); + } else { + // Explicit revert when an assertion failed. + expect(avmResult.revertCode.isOK()).toBe(false); + expect(avmResult.revertReason).toBeDefined(); + expect(avmResult.revertReason?.getMessage()).toContain(assertionErrString); + } + + const avmCircuitInputs: AvmCircuitInputs = avmResult.avmProvingRequest.inputs; + return avmCircuitInputs; +} + +/** + * Craft a carrier transaction for a public call for simulation by PublicTxSimulator. + */ +export function createTxForPublicCall( + executionRequest: PublicExecutionRequest, + gasUsedByPrivate: Gas = Gas.empty(), + isTeardown: boolean = false, +): Tx { + const callRequest = executionRequest.toCallRequest(); + // use max limits + const gasLimits = new Gas(DEFAULT_GAS_LIMIT, MAX_L2_GAS_PER_ENQUEUED_CALL); + + const forPublic = PartialPrivateTailPublicInputsForPublic.empty(); + // TODO(#9269): Remove this fake nullifier method as we move away from 1st nullifier as hash. + forPublic.nonRevertibleAccumulatedData.nullifiers[0] = Fr.random(); // fake tx nullifier + if (isTeardown) { + forPublic.publicTeardownCallRequest = callRequest; + } else { + forPublic.revertibleAccumulatedData.publicCallRequests[0] = callRequest; + } + + const teardownGasLimits = isTeardown ? gasLimits : Gas.empty(); + const gasSettings = new GasSettings(gasLimits, teardownGasLimits, GasFees.empty()); + const txContext = new TxContext(Fr.zero(), Fr.zero(), gasSettings); + const constantData = new TxConstantData(Header.empty(), txContext, Fr.zero(), Fr.zero()); + + const txData = new PrivateKernelTailCircuitPublicInputs( + constantData, + RollupValidationRequests.empty(), + /*gasUsed=*/ gasUsedByPrivate, + AztecAddress.zero(), + forPublic, + ); + const tx = isTeardown ? Tx.newWithTxData(txData, executionRequest) : Tx.newWithTxData(txData); + if (!isTeardown) { + tx.enqueuedPublicFunctionCalls[0] = executionRequest; + } + + return tx; +} diff --git a/yarn-project/simulator/src/public/hints_builder.ts b/yarn-project/simulator/src/public/hints_builder.ts deleted file mode 100644 index 2674545f091..00000000000 --- a/yarn-project/simulator/src/public/hints_builder.ts +++ /dev/null @@ -1,168 +0,0 @@ -import { type IndexedTreeId, MerkleTreeId } from '@aztec/circuit-types'; -import { - type Fr, - L1_TO_L2_MSG_TREE_HEIGHT, - MAX_L1_TO_L2_MSG_READ_REQUESTS_PER_TX, - MAX_NOTE_HASH_READ_REQUESTS_PER_TX, - type MAX_NULLIFIERS_PER_TX, - type MAX_NULLIFIER_NON_EXISTENT_READ_REQUESTS_PER_TX, - MAX_NULLIFIER_READ_REQUESTS_PER_TX, - type MAX_PUBLIC_DATA_READS_PER_TX, - type MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, - MembershipWitness, - NOTE_HASH_TREE_HEIGHT, - NULLIFIER_TREE_HEIGHT, - type Nullifier, - type NullifierLeafPreimage, - PUBLIC_DATA_TREE_HEIGHT, - type PublicDataRead, - type PublicDataTreeLeafPreimage, - type PublicDataUpdateRequest, - type ScopedReadRequest, - type TreeLeafReadRequest, - TreeLeafReadRequestHint, - buildNullifierNonExistentReadRequestHints, - buildPublicDataHint, - buildPublicDataHints, - buildSiloedNullifierReadRequestHints, -} from '@aztec/circuits.js'; -import { makeTuple } from '@aztec/foundation/array'; -import { type Tuple } from '@aztec/foundation/serialize'; -import { type IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; -import { type MerkleTreeReadOperations } from '@aztec/world-state'; - -export class HintsBuilder { - constructor(private db: MerkleTreeReadOperations) {} - - async getNoteHashReadRequestsHints( - readRequests: Tuple, - ) { - return await this.getTreeLeafReadRequestsHints( - readRequests, - MAX_NOTE_HASH_READ_REQUESTS_PER_TX, - NOTE_HASH_TREE_HEIGHT, - MerkleTreeId.NOTE_HASH_TREE, - ); - } - - async getNullifierReadRequestHints( - nullifierReadRequests: Tuple, - pendingNullifiers: Tuple, - ) { - return await buildSiloedNullifierReadRequestHints( - this, - nullifierReadRequests, - pendingNullifiers, - MAX_NULLIFIER_READ_REQUESTS_PER_TX, - MAX_NULLIFIER_READ_REQUESTS_PER_TX, - ); - } - - getNullifierNonExistentReadRequestHints( - nullifierNonExistentReadRequests: Tuple, - pendingNullifiers: Tuple, - ) { - return buildNullifierNonExistentReadRequestHints(this, nullifierNonExistentReadRequests, pendingNullifiers); - } - - async getL1ToL2MsgReadRequestsHints( - readRequests: Tuple, - ) { - return await this.getTreeLeafReadRequestsHints( - readRequests, - MAX_L1_TO_L2_MSG_READ_REQUESTS_PER_TX, - L1_TO_L2_MSG_TREE_HEIGHT, - MerkleTreeId.L1_TO_L2_MESSAGE_TREE, - ); - } - - getPublicDataHints( - publicDataReads: Tuple, - publicDataUpdateRequests: Tuple, - ) { - return buildPublicDataHints(this, publicDataReads, publicDataUpdateRequests); - } - - getPublicDataHint(dataAction: PublicDataRead | PublicDataUpdateRequest | bigint) { - const slot = typeof dataAction === 'bigint' ? dataAction : dataAction.leafSlot.toBigInt(); - return buildPublicDataHint(this, slot); - } - - async getNullifierMembershipWitness(nullifier: Fr) { - const index = await this.db.findLeafIndex(MerkleTreeId.NULLIFIER_TREE, nullifier.toBuffer()); - if (index === undefined) { - throw new Error(`Cannot find the leaf for nullifier ${nullifier.toBigInt()}.`); - } - - return this.getMembershipWitnessWithPreimage( - MerkleTreeId.NULLIFIER_TREE, - NULLIFIER_TREE_HEIGHT, - index, - ); - } - - async getLowNullifierMembershipWitness(nullifier: Fr) { - const res = await this.db.getPreviousValueIndex(MerkleTreeId.NULLIFIER_TREE, nullifier.toBigInt()); - if (!res) { - throw new Error(`Cannot find the low leaf for nullifier ${nullifier.toBigInt()}.`); - } - - const { index, alreadyPresent } = res; - if (alreadyPresent) { - throw new Error(`Nullifier ${nullifier.toBigInt()} already exists in the tree.`); - } - - // Should find a way to stop casting IndexedTreeLeafPreimage as NullifierLeafPreimage - return this.getMembershipWitnessWithPreimage( - MerkleTreeId.NULLIFIER_TREE, - NULLIFIER_TREE_HEIGHT, - index, - ); - } - - async getMatchOrLowPublicDataMembershipWitness(leafSlot: bigint) { - const res = await this.db.getPreviousValueIndex(MerkleTreeId.PUBLIC_DATA_TREE, leafSlot); - if (!res) { - throw new Error(`Cannot find the previous value index for public data ${leafSlot}.`); - } - - // Should find a way to stop casting IndexedTreeLeafPreimage as PublicDataTreeLeafPreimage everywhere. - return this.getMembershipWitnessWithPreimage( - MerkleTreeId.PUBLIC_DATA_TREE, - PUBLIC_DATA_TREE_HEIGHT, - res.index, - ); - } - - private async getMembershipWitnessWithPreimage< - TREE_HEIGHT extends number, - LEAF_PREIMAGE extends IndexedTreeLeafPreimage = IndexedTreeLeafPreimage, - >(treeId: IndexedTreeId, treeHeight: TREE_HEIGHT, index: bigint) { - const siblingPath = await this.db.getSiblingPath(treeId, index); - const membershipWitness = new MembershipWitness(treeHeight, index, siblingPath.toTuple()); - - const leafPreimage = (await this.db.getLeafPreimage(treeId, index)) as LEAF_PREIMAGE; - if (!leafPreimage) { - throw new Error(`Cannot find the leaf preimage for tree ${treeId} at index ${index}.`); - } - - return { membershipWitness, leafPreimage }; - } - - private async getTreeLeafReadRequestsHints( - readRequests: Tuple, - size: N, - treeHeight: TREE_HEIGHT, - treeId: MerkleTreeId, - ): Promise, N>> { - const hints = makeTuple(size, () => TreeLeafReadRequestHint.empty(treeHeight)); - for (let i = 0; i < readRequests.length; i++) { - const request = readRequests[i]; - if (!request.isEmpty()) { - const siblingPath = await this.db.getSiblingPath(treeId, request.leafIndex.toBigInt()); - hints[i] = new TreeLeafReadRequestHint(treeHeight, siblingPath.toTuple()); - } - } - return hints; - } -} diff --git a/yarn-project/simulator/src/public/index.ts b/yarn-project/simulator/src/public/index.ts index 1735b437ef1..b7a3a5929f5 100644 --- a/yarn-project/simulator/src/public/index.ts +++ b/yarn-project/simulator/src/public/index.ts @@ -1,18 +1,9 @@ export * from './db_interfaces.js'; -export { EnqueuedCallSimulator } from './enqueued_call_simulator.js'; export * from './public_tx_simulator.js'; -export { - type EnqueuedPublicCallExecutionResult as PublicExecutionResult, - type PublicFunctionCallResult, -} from './execution.js'; -export { PublicExecutor } from './executor.js'; +export { type EnqueuedPublicCallExecutionResult, type PublicFunctionCallResult } from './execution.js'; export * from './fee_payment.js'; -export { HintsBuilder } from './hints_builder.js'; export * from './public_db_sources.js'; -export * from './public_kernel.js'; -export * from './public_kernel_circuit_simulator.js'; export { PublicProcessor, PublicProcessorFactory } from './public_processor.js'; export { PublicSideEffectTrace } from './side_effect_trace.js'; export { PublicEnqueuedCallSideEffectTrace } from './enqueued_call_side_effect_trace.js'; -export { DualSideEffectTrace } from './dual_side_effect_trace.js'; export { getExecutionRequestsByPhase } from './utils.js'; diff --git a/yarn-project/simulator/src/public/public_kernel.ts b/yarn-project/simulator/src/public/public_kernel.ts deleted file mode 100644 index 146c3441f23..00000000000 --- a/yarn-project/simulator/src/public/public_kernel.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { - type KernelCircuitPublicInputs, - type PublicKernelCircuitPrivateInputs, - type PublicKernelCircuitPublicInputs, - type PublicKernelTailCircuitPrivateInputs, -} from '@aztec/circuits.js'; -import { createDebugLogger } from '@aztec/foundation/log'; -import { elapsed } from '@aztec/foundation/timer'; -import { - SimulatedPublicKernelMergeArtifact, - SimulatedPublicKernelTailArtifact, - convertSimulatedPublicMergeInputsToWitnessMap, - convertSimulatedPublicMergeOutputFromWitnessMap, - convertSimulatedPublicTailInputsToWitnessMap, - convertSimulatedPublicTailOutputFromWitnessMap, -} from '@aztec/noir-protocol-circuits-types'; - -import { type SimulationProvider } from '../providers/simulation_provider.js'; -import { type PublicKernelCircuitSimulator } from './public_kernel_circuit_simulator.js'; - -/** - * Implements the PublicKernelCircuitSimulator. - */ -export class RealPublicKernelCircuitSimulator implements PublicKernelCircuitSimulator { - private log = createDebugLogger('aztec:public-kernel-simulator'); - - constructor(private simulator: SimulationProvider) {} - - /** - * Simulates the public kernel app logic circuit from its inputs. - * @param input - Inputs to the circuit. - * @returns The public inputs as outputs of the simulation. - */ - public async publicKernelCircuitMerge( - input: PublicKernelCircuitPrivateInputs, - ): Promise { - const inputWitness = convertSimulatedPublicMergeInputsToWitnessMap(input); - const [duration, witness] = await elapsed(() => - this.simulator.simulateCircuit(inputWitness, SimulatedPublicKernelMergeArtifact), - ); - const result = convertSimulatedPublicMergeOutputFromWitnessMap(witness); - this.log.debug(`Simulated public kernel merge circuit`, { - eventName: 'circuit-simulation', - circuitName: 'public-kernel-merge', - duration, - inputSize: input.toBuffer().length, - outputSize: result.toBuffer().length, - }); - return result; - } - - /** - * Simulates the public kernel tail circuit from its inputs. - * @param input - Inputs to the circuit. - * @returns The public inputs as outputs of the simulation. - */ - public async publicKernelCircuitTail( - input: PublicKernelTailCircuitPrivateInputs, - ): Promise { - const inputWitness = convertSimulatedPublicTailInputsToWitnessMap(input); - const [duration, witness] = await elapsed(() => - this.simulator.simulateCircuit(inputWitness, SimulatedPublicKernelTailArtifact), - ); - const result = convertSimulatedPublicTailOutputFromWitnessMap(witness); - this.log.debug(`Simulated public kernel tail circuit`, { - eventName: 'circuit-simulation', - circuitName: 'public-kernel-tail', - duration, - inputSize: input.toBuffer().length, - outputSize: result.toBuffer().length, - }); - return result; - } -} diff --git a/yarn-project/simulator/src/public/public_kernel_circuit_simulator.ts b/yarn-project/simulator/src/public/public_kernel_circuit_simulator.ts deleted file mode 100644 index 36f26b36c4a..00000000000 --- a/yarn-project/simulator/src/public/public_kernel_circuit_simulator.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { - type KernelCircuitPublicInputs, - type PublicKernelCircuitPrivateInputs, - type PublicKernelCircuitPublicInputs, - type PublicKernelTailCircuitPrivateInputs, -} from '@aztec/circuits.js'; - -/** - * Circuit simulator for the public kernel circuits. - */ -export interface PublicKernelCircuitSimulator { - /** - * Simulates the public kernel merge circuit from its inputs. - * @param inputs - Inputs to the circuit. - * @returns The public inputs as outputs of the simulation. - */ - publicKernelCircuitMerge(inputs: PublicKernelCircuitPrivateInputs): Promise; - /** - * Simulates the public kernel tail circuit from its inputs. - * @param inputs - Inputs to the circuit. - * @returns The public inputs as outputs of the simulation. - */ - publicKernelCircuitTail(inputs: PublicKernelTailCircuitPrivateInputs): Promise; -} diff --git a/yarn-project/simulator/src/public/public_kernel_tail_simulator.ts b/yarn-project/simulator/src/public/public_kernel_tail_simulator.ts deleted file mode 100644 index 6143f5b316d..00000000000 --- a/yarn-project/simulator/src/public/public_kernel_tail_simulator.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { - type KernelCircuitPublicInputs, - MAX_NULLIFIERS_PER_TX, - MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, - NESTED_RECURSIVE_PROOF_LENGTH, - type PublicKernelCircuitPublicInputs, - PublicKernelData, - PublicKernelTailCircuitPrivateInputs, - VerificationKeyData, - makeEmptyRecursiveProof, - mergeAccumulatedData, -} from '@aztec/circuits.js'; -import { getVKSiblingPath } from '@aztec/noir-protocol-circuits-types'; -import { type MerkleTreeReadOperations } from '@aztec/world-state'; - -import { HintsBuilder } from './hints_builder.js'; -import { type PublicKernelCircuitSimulator } from './public_kernel_circuit_simulator.js'; - -export class PublicKernelTailSimulator { - constructor( - private db: MerkleTreeReadOperations, - private publicKernelSimulator: PublicKernelCircuitSimulator, - private hintsBuilder: HintsBuilder, - ) {} - - static create(db: MerkleTreeReadOperations, publicKernelSimulator: PublicKernelCircuitSimulator) { - const hintsBuilder = new HintsBuilder(db); - return new PublicKernelTailSimulator(db, publicKernelSimulator, hintsBuilder); - } - - async simulate(previousOutput: PublicKernelCircuitPublicInputs): Promise { - const inputs = await this.buildPrivateInputs(previousOutput); - - return await this.publicKernelSimulator.publicKernelCircuitTail(inputs); - } - - private async buildPrivateInputs(previousOutput: PublicKernelCircuitPublicInputs) { - const previousKernel = this.getPreviousKernelData(previousOutput); - - const { validationRequests, endNonRevertibleData: nonRevertibleData, end: revertibleData } = previousOutput; - - const noteHashReadRequestHints = await this.hintsBuilder.getNoteHashReadRequestsHints( - validationRequests.noteHashReadRequests, - ); - - const pendingNullifiers = mergeAccumulatedData( - nonRevertibleData.nullifiers, - revertibleData.nullifiers, - MAX_NULLIFIERS_PER_TX, - ); - - const nullifierReadRequestHints = await this.hintsBuilder.getNullifierReadRequestHints( - validationRequests.nullifierReadRequests, - pendingNullifiers, - ); - - const nullifierNonExistentReadRequestHints = await this.hintsBuilder.getNullifierNonExistentReadRequestHints( - validationRequests.nullifierNonExistentReadRequests, - pendingNullifiers, - ); - - const l1ToL2MsgReadRequestHints = await this.hintsBuilder.getL1ToL2MsgReadRequestsHints( - validationRequests.l1ToL2MsgReadRequests, - ); - - const pendingPublicDataWrites = mergeAccumulatedData( - nonRevertibleData.publicDataUpdateRequests, - revertibleData.publicDataUpdateRequests, - MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, - ); - - const publicDataHints = await this.hintsBuilder.getPublicDataHints( - validationRequests.publicDataReads, - pendingPublicDataWrites, - ); - - const currentState = await this.db.getStateReference(); - - return new PublicKernelTailCircuitPrivateInputs( - previousKernel, - noteHashReadRequestHints, - nullifierReadRequestHints, - nullifierNonExistentReadRequestHints, - l1ToL2MsgReadRequestHints, - publicDataHints, - currentState.partial, - ); - } - - private getPreviousKernelData(previousOutput: PublicKernelCircuitPublicInputs): PublicKernelData { - const proof = makeEmptyRecursiveProof(NESTED_RECURSIVE_PROOF_LENGTH); - const vk = VerificationKeyData.makeFakeHonk(); - const vkIndex = 0; - const siblingPath = getVKSiblingPath(vkIndex); - return new PublicKernelData(previousOutput, proof, vk, vkIndex, siblingPath); - } -} diff --git a/yarn-project/simulator/src/public/public_processor.test.ts b/yarn-project/simulator/src/public/public_processor.test.ts index 621a833af3b..7db6b283dc8 100644 --- a/yarn-project/simulator/src/public/public_processor.test.ts +++ b/yarn-project/simulator/src/public/public_processor.test.ts @@ -79,7 +79,7 @@ describe('public_processor', () => { worldStateDB.storageRead.mockResolvedValue(Fr.ZERO); - publicTxProcessor.process.mockImplementation(() => { + publicTxProcessor.simulate.mockImplementation(() => { return Promise.resolve(mockedEnqueuedCallsResult); }); @@ -136,7 +136,7 @@ describe('public_processor', () => { }); it('returns failed txs without aborting entire operation', async function () { - publicTxProcessor.process.mockRejectedValue(new SimulationError(`Failed`, [])); + publicTxProcessor.simulate.mockRejectedValue(new SimulationError(`Failed`, [])); const tx = mockTxWithPublicCalls(); const [processed, failed] = await processor.process([tx], 1, handler); diff --git a/yarn-project/simulator/src/public/public_processor.ts b/yarn-project/simulator/src/public/public_processor.ts index 55fce164cfb..7e1e35b8710 100644 --- a/yarn-project/simulator/src/public/public_processor.ts +++ b/yarn-project/simulator/src/public/public_processor.ts @@ -20,9 +20,7 @@ import { type Header, MAX_NOTE_HASHES_PER_TX, MAX_NULLIFIERS_PER_TX, - MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, NULLIFIER_SUBTREE_HEIGHT, - PUBLIC_DATA_SUBTREE_HEIGHT, PublicDataWrite, } from '@aztec/circuits.js'; import { padArrayEnd } from '@aztec/foundation/collection'; @@ -31,12 +29,8 @@ import { Timer } from '@aztec/foundation/timer'; import { ProtocolContractAddress } from '@aztec/protocol-contracts'; import { Attributes, type TelemetryClient, type Tracer, trackSpan } from '@aztec/telemetry-client'; -import { type SimulationProvider } from '../providers/index.js'; -import { PublicExecutor } from './executor.js'; import { computeFeePayerBalanceLeafSlot, computeFeePayerBalanceStorageSlot } from './fee_payment.js'; import { WorldStateDB } from './public_db_sources.js'; -import { RealPublicKernelCircuitSimulator } from './public_kernel.js'; -import { type PublicKernelCircuitSimulator } from './public_kernel_circuit_simulator.js'; import { PublicProcessorMetrics } from './public_processor_metrics.js'; import { PublicTxSimulator } from './public_tx_simulator.js'; @@ -44,11 +38,7 @@ import { PublicTxSimulator } from './public_tx_simulator.js'; * Creates new instances of PublicProcessor given the provided merkle tree db and contract data source. */ export class PublicProcessorFactory { - constructor( - private contractDataSource: ContractDataSource, - private simulator: SimulationProvider, - private telemetryClient: TelemetryClient, - ) {} + constructor(private contractDataSource: ContractDataSource, private telemetryClient: TelemetryClient) {} /** * Creates a new instance of a PublicProcessor. @@ -61,20 +51,17 @@ export class PublicProcessorFactory { maybeHistoricalHeader: Header | undefined, globalVariables: GlobalVariables, ): PublicProcessor { - const { telemetryClient } = this; const historicalHeader = maybeHistoricalHeader ?? merkleTree.getInitialHeader(); const worldStateDB = new WorldStateDB(merkleTree, this.contractDataSource); - const publicExecutor = new PublicExecutor(worldStateDB, telemetryClient); - const publicKernelSimulator = new RealPublicKernelCircuitSimulator(this.simulator); + const publicTxSimulator = new PublicTxSimulator(merkleTree, worldStateDB, this.telemetryClient, globalVariables); - return PublicProcessor.create( + return new PublicProcessor( merkleTree, - publicExecutor, - publicKernelSimulator, globalVariables, historicalHeader, worldStateDB, + publicTxSimulator, this.telemetryClient, ); } @@ -91,41 +78,13 @@ export class PublicProcessor { protected globalVariables: GlobalVariables, protected historicalHeader: Header, protected worldStateDB: WorldStateDB, - protected enqueuedCallsProcessor: PublicTxSimulator, + protected publicTxSimulator: PublicTxSimulator, telemetryClient: TelemetryClient, private log = createDebugLogger('aztec:sequencer:public-processor'), ) { this.metrics = new PublicProcessorMetrics(telemetryClient, 'PublicProcessor'); } - static create( - db: MerkleTreeWriteOperations, - publicExecutor: PublicExecutor, - publicKernelSimulator: PublicKernelCircuitSimulator, - globalVariables: GlobalVariables, - historicalHeader: Header, - worldStateDB: WorldStateDB, - telemetryClient: TelemetryClient, - ) { - const enqueuedCallsProcessor = PublicTxSimulator.create( - db, - publicExecutor, - publicKernelSimulator, - globalVariables, - historicalHeader, - worldStateDB, - ); - - return new PublicProcessor( - db, - globalVariables, - historicalHeader, - worldStateDB, - enqueuedCallsProcessor, - telemetryClient, - ); - } - get tracer(): Tracer { return this.metrics.tracer; } @@ -208,15 +167,10 @@ export class PublicProcessor { } } - const allPublicDataWrites = padArrayEnd( - processedTx.txEffect.publicDataWrites, - PublicDataWrite.empty(), - MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, - ); await this.db.batchInsert( MerkleTreeId.PUBLIC_DATA_TREE, - allPublicDataWrites.map(x => x.toBuffer()), - PUBLIC_DATA_SUBTREE_HEIGHT, + processedTx.txEffect.publicDataWrites.map(x => x.toBuffer()), + 0, ); result.push(processedTx); returns = returns.concat(returnValues ?? []); @@ -299,7 +253,7 @@ export class PublicProcessor { const timer = new Timer(); const { avmProvingRequest, gasUsed, revertCode, revertReason, processedPhases } = - await this.enqueuedCallsProcessor.process(tx); + await this.publicTxSimulator.simulate(tx); if (!avmProvingRequest) { this.metrics.recordFailedTx(); diff --git a/yarn-project/simulator/src/public/public_tx_context.ts b/yarn-project/simulator/src/public/public_tx_context.ts index 8e2a898ffbb..fd4c860a35c 100644 --- a/yarn-project/simulator/src/public/public_tx_context.ts +++ b/yarn-project/simulator/src/public/public_tx_context.ts @@ -5,87 +5,61 @@ import { type SimulationError, type Tx, TxExecutionPhase, + TxHash, } from '@aztec/circuit-types'; import { - CombinedConstantData, + type AvmCircuitPublicInputs, Fr, Gas, type GasSettings, type GlobalVariables, - PublicAccumulatedDataArrayLengths, + type Header, + type PrivateToPublicAccumulatedData, type PublicCallRequest, - type PublicKernelCircuitPublicInputs, - PublicValidationRequestArrayLengths, RevertCode, type StateReference, + countAccumulatedItems, } from '@aztec/circuits.js'; import { type DebugLogger, createDebugLogger } from '@aztec/foundation/log'; -import { assert } from 'console'; +import { strict as assert } from 'assert'; import { inspect } from 'util'; +import { type AvmFinalizedCallResult } from '../avm/avm_contract_call_result.js'; import { AvmPersistableStateManager } from '../avm/index.js'; import { DualSideEffectTrace } from './dual_side_effect_trace.js'; -import { PublicEnqueuedCallSideEffectTrace } from './enqueued_call_side_effect_trace.js'; +import { PublicEnqueuedCallSideEffectTrace, SideEffectArrayLengths } from './enqueued_call_side_effect_trace.js'; import { type WorldStateDB } from './public_db_sources.js'; import { PublicSideEffectTrace } from './side_effect_trace.js'; -import { getCallRequestsByPhase, getExecutionRequestsByPhase, getPublicKernelCircuitPublicInputs } from './utils.js'; - -class PhaseStateManager { - private currentlyActiveStateManager: AvmPersistableStateManager | undefined; - - constructor(private readonly txStateManager: AvmPersistableStateManager) {} - - fork() { - assert(!this.currentlyActiveStateManager, 'Cannot fork when already forked'); - this.currentlyActiveStateManager = this.txStateManager.fork(); - } - - getActiveStateManager() { - return this.currentlyActiveStateManager || this.txStateManager; - } - - isForked() { - return !!this.currentlyActiveStateManager; - } - - mergeForkedState() { - assert(this.currentlyActiveStateManager, 'No forked state to merge'); - this.txStateManager.mergeForkedState(this.currentlyActiveStateManager!); - // Drop the forked state manager now that it is merged - this.currentlyActiveStateManager = undefined; - } - - discardForkedState() { - assert(this.currentlyActiveStateManager, 'No forked state to discard'); - this.txStateManager.rejectForkedState(this.currentlyActiveStateManager!); - // Drop the forked state manager. We don't want it! - this.currentlyActiveStateManager = undefined; - } -} +import { generateAvmCircuitPublicInputs, generateAvmProvingRequest } from './transitional_adapters.js'; +import { getCallRequestsByPhase, getExecutionRequestsByPhase } from './utils.js'; +/** + * The transaction-level context for public execution. + */ export class PublicTxContext { private log: DebugLogger; - private currentPhase: TxExecutionPhase = TxExecutionPhase.SETUP; - /* Gas used including private, teardown gas _limit_, setup and app logic */ private gasUsed: Gas; /* Gas actually used during teardown (different from limit) */ public teardownGasUsed: Gas = Gas.empty(); - public revertCode: RevertCode = RevertCode.OK; + /* Entire transaction execution is done. */ + private halted = false; + /* Where did reverts happen (if at all)? */ + private revertCode: RevertCode = RevertCode.OK; + /* What caused a revert (if one occurred)? */ public revertReason: SimulationError | undefined; - public avmProvingRequest: AvmProvingRequest | undefined; // tmp hack + public avmProvingRequest: AvmProvingRequest | undefined; // FIXME(dbanks12): remove constructor( public readonly state: PhaseStateManager, - public readonly tx: Tx, // tmp hack - public readonly globalVariables: GlobalVariables, - public readonly constants: CombinedConstantData, // tmp hack - public readonly startStateReference: StateReference, - startGasUsed: Gas, + private readonly globalVariables: GlobalVariables, + private readonly historicalHeader: Header, // FIXME(dbanks12): remove + private readonly startStateReference: StateReference, + private readonly startGasUsed: Gas, private readonly gasSettings: GasSettings, private readonly setupCallRequests: PublicCallRequest[], private readonly appLogicCallRequests: PublicCallRequest[], @@ -93,8 +67,10 @@ export class PublicTxContext { private readonly setupExecutionRequests: PublicExecutionRequest[], private readonly appLogicExecutionRequests: PublicExecutionRequest[], private readonly teardownExecutionRequests: PublicExecutionRequest[], - public latestPublicKernelOutput: PublicKernelCircuitPublicInputs, - public trace: PublicEnqueuedCallSideEffectTrace, + private readonly nonRevertibleAccumulatedDataFromPrivate: PrivateToPublicAccumulatedData, + private readonly revertibleAccumulatedDataFromPrivate: PrivateToPublicAccumulatedData, + public trace: PublicEnqueuedCallSideEffectTrace, // FIXME(dbanks12): should be private + private doMerkleOperations: boolean, ) { this.log = createDebugLogger(`aztec:public_tx_context`); this.gasUsed = startGasUsed; @@ -105,45 +81,47 @@ export class PublicTxContext { worldStateDB: WorldStateDB, tx: Tx, globalVariables: GlobalVariables, + doMerkleOperations: boolean, ) { - const privateKernelOutput = tx.data; - const latestPublicKernelOutput = getPublicKernelCircuitPublicInputs(privateKernelOutput, globalVariables); - - const nonRevertibleNullifiersFromPrivate = latestPublicKernelOutput.endNonRevertibleData.nullifiers - .filter(n => !n.isEmpty()) - .map(n => n.value); - const _revertibleNullifiersFromPrivate = latestPublicKernelOutput.end.nullifiers - .filter(n => !n.isEmpty()) - .map(n => n.value); - - // During SETUP, non revertible side effects from private are our "previous data" - const prevAccumulatedData = latestPublicKernelOutput.endNonRevertibleData; - const previousValidationRequestArrayLengths = PublicValidationRequestArrayLengths.new( - latestPublicKernelOutput.validationRequests, + const nonRevertibleAccumulatedDataFromPrivate = tx.data.forPublic!.nonRevertibleAccumulatedData; + const revertibleAccumulatedDataFromPrivate = tx.data.forPublic!.revertibleAccumulatedData; + const nonRevertibleNullifiersFromPrivate = nonRevertibleAccumulatedDataFromPrivate.nullifiers.filter( + n => !n.isEmpty(), ); - - const previousAccumulatedDataArrayLengths = PublicAccumulatedDataArrayLengths.new(prevAccumulatedData); + const _revertibleNullifiersFromPrivate = revertibleAccumulatedDataFromPrivate.nullifiers.filter(n => !n.isEmpty()); const innerCallTrace = new PublicSideEffectTrace(); + + const previousAccumulatedDataArrayLengths = new SideEffectArrayLengths( + /*publicDataReads*/ 0, + /*publicDataWrites*/ 0, + /*noteHashReadRequests*/ 0, + countAccumulatedItems(nonRevertibleAccumulatedDataFromPrivate.noteHashes), + /*nullifierReadRequests*/ 0, + /*nullifierNonExistentReadRequests*/ 0, + countAccumulatedItems(nonRevertibleAccumulatedDataFromPrivate.nullifiers), + /*l1ToL2MsgReadRequests*/ 0, + countAccumulatedItems(nonRevertibleAccumulatedDataFromPrivate.l2ToL1Msgs), + /*unencryptedLogsHashes*/ 0, + ); const enqueuedCallTrace = new PublicEnqueuedCallSideEffectTrace( /*startSideEffectCounter=*/ 0, - previousValidationRequestArrayLengths, previousAccumulatedDataArrayLengths, ); const trace = new DualSideEffectTrace(innerCallTrace, enqueuedCallTrace); // Transaction level state manager that will be forked for revertible phases. - const txStateManager = AvmPersistableStateManager.newWithPendingSiloedNullifiers( + const txStateManager = await AvmPersistableStateManager.newWithPendingSiloedNullifiers( worldStateDB, trace, nonRevertibleNullifiersFromPrivate, + doMerkleOperations, ); return new PublicTxContext( new PhaseStateManager(txStateManager), - tx, globalVariables, - CombinedConstantData.combine(tx.data.constants, globalVariables), + tx.data.constants.historicalHeader, await db.getStateReference(), tx.data.gasUsed, tx.data.constants.txContext.gasSettings, @@ -153,54 +131,45 @@ export class PublicTxContext { getExecutionRequestsByPhase(tx, TxExecutionPhase.SETUP), getExecutionRequestsByPhase(tx, TxExecutionPhase.APP_LOGIC), getExecutionRequestsByPhase(tx, TxExecutionPhase.TEARDOWN), - latestPublicKernelOutput, + tx.data.forPublic!.nonRevertibleAccumulatedData, + tx.data.forPublic!.revertibleAccumulatedData, enqueuedCallTrace, + doMerkleOperations, ); } - getCurrentPhase(): TxExecutionPhase { - return this.currentPhase; - } - - hasPhase(phase: TxExecutionPhase = this.currentPhase): boolean { - if (phase === TxExecutionPhase.SETUP) { - return this.setupCallRequests.length > 0; - } else if (phase === TxExecutionPhase.APP_LOGIC) { - return this.appLogicCallRequests.length > 0; - } else { - // phase === TxExecutionPhase.TEARDOWN - return this.teardownCallRequests.length > 0; - } + /** + * Signal that the entire transaction execution is done. + * All phases have been processed. + * Actual transaction fee and actual total consumed gas can now be queried. + */ + halt() { + this.halted = true; } - progressToNextPhase() { - assert(this.currentPhase !== TxExecutionPhase.TEARDOWN, 'Cannot progress past teardown'); - if (this.currentPhase === TxExecutionPhase.SETUP) { - this.currentPhase = TxExecutionPhase.APP_LOGIC; - } else { - this.currentPhase = TxExecutionPhase.TEARDOWN; - } - } + /** + * Revert execution a phase. Populate revertReason & revertCode. + * If in setup, throw an error (transaction will be thrown out). + * NOTE: this does not "halt" the entire transaction execution. + */ + revert(phase: TxExecutionPhase, revertReason: SimulationError | undefined = undefined, culprit = '') { + this.log.debug(`${TxExecutionPhase[phase]} phase reverted! ${culprit} failed with reason: ${revertReason}`); - revert(revertReason: SimulationError | undefined = undefined, culprit = '') { - this.log.debug( - `${TxExecutionPhase[this.currentPhase]} phase reverted! ${culprit} failed with reason: ${revertReason}`, - ); if (revertReason && !this.revertReason) { // don't override revertReason // (if app logic and teardown both revert, we want app logic's reason) this.revertReason = revertReason; } - if (this.currentPhase === TxExecutionPhase.SETUP) { + if (phase === TxExecutionPhase.SETUP) { this.log.debug(`Setup phase reverted! The transaction will be thrown out.`); if (revertReason) { throw revertReason; } else { throw new Error(`Setup phase reverted! The transaction will be thrown out. ${culprit} failed`); } - } else if (this.currentPhase === TxExecutionPhase.APP_LOGIC) { + } else if (phase === TxExecutionPhase.APP_LOGIC) { this.revertCode = RevertCode.APP_LOGIC_REVERTED; - } else if (this.currentPhase === TxExecutionPhase.TEARDOWN) { + } else if (phase === TxExecutionPhase.TEARDOWN) { if (this.revertCode.equals(RevertCode.APP_LOGIC_REVERTED)) { this.revertCode = RevertCode.BOTH_REVERTED; } else { @@ -209,8 +178,47 @@ export class PublicTxContext { } } - getCallRequestsForCurrentPhase(): PublicCallRequest[] { - switch (this.currentPhase) { + /** + * Get the revert code. + * @returns The revert code. + */ + getFinalRevertCode(): RevertCode { + assert(this.halted, 'Cannot know the final revert code until tx execution ends'); + return this.revertCode; + } + + /** + * Construct & return transaction hash. + * @returns The transaction's hash. + */ + getTxHash(): TxHash { + // Private kernel functions are executed client side and for this reason tx hash is already set as first nullifier + const firstNullifier = this.nonRevertibleAccumulatedDataFromPrivate.nullifiers[0]; + if (!firstNullifier || firstNullifier.isZero()) { + throw new Error(`Cannot get tx hash since first nullifier is missing`); + } + return new TxHash(firstNullifier.toBuffer()); + } + + /** + * Are there any call requests for the speciiied phase? + */ + hasPhase(phase: TxExecutionPhase): boolean { + if (phase === TxExecutionPhase.SETUP) { + return this.setupCallRequests.length > 0; + } else if (phase === TxExecutionPhase.APP_LOGIC) { + return this.appLogicCallRequests.length > 0; + } else { + // phase === TxExecutionPhase.TEARDOWN + return this.teardownCallRequests.length > 0; + } + } + + /** + * Get the call requests for the specified phase (including args hashes). + */ + getCallRequestsForPhase(phase: TxExecutionPhase): PublicCallRequest[] { + switch (phase) { case TxExecutionPhase.SETUP: return this.setupCallRequests; case TxExecutionPhase.APP_LOGIC: @@ -220,8 +228,11 @@ export class PublicTxContext { } } - getExecutionRequestsForCurrentPhase(): PublicExecutionRequest[] { - switch (this.currentPhase) { + /** + * Get the call requests for the specified phase (including actual args). + */ + getExecutionRequestsForPhase(phase: TxExecutionPhase): PublicExecutionRequest[] { + switch (phase) { case TxExecutionPhase.SETUP: return this.setupExecutionRequests; case TxExecutionPhase.APP_LOGIC: @@ -231,17 +242,22 @@ export class PublicTxContext { } } - getGasLeftForCurrentPhase(): Gas { - if (this.currentPhase === TxExecutionPhase.TEARDOWN) { + /** + * How much gas is left for the specified phase? + */ + getGasLeftForPhase(phase: TxExecutionPhase): Gas { + if (phase === TxExecutionPhase.TEARDOWN) { return this.gasSettings.teardownGasLimits; } else { return this.gasSettings.gasLimits.sub(this.gasUsed); } } - consumeGas(gas: Gas) { - if (this.currentPhase === TxExecutionPhase.TEARDOWN) { - // track teardown gas used separately + /** + * Consume gas. Track gas for teardown phase separately. + */ + consumeGas(phase: TxExecutionPhase, gas: Gas) { + if (phase === TxExecutionPhase.TEARDOWN) { this.teardownGasUsed = this.teardownGasUsed.add(gas); } else { this.gasUsed = this.gasUsed.add(gas); @@ -251,33 +267,39 @@ export class PublicTxContext { /** * Compute the gas used using the actual gas used during teardown instead * of the teardown gas limit. - * Note that this.startGasUsed comes from private and private includes - * teardown gas limit in its output gasUsed. + * Note that this.gasUsed is initialized from private's gasUsed which includes + * teardown gas limit. */ getActualGasUsed(): Gas { - assert(this.currentPhase === TxExecutionPhase.TEARDOWN, 'Can only compute actual gas used after app logic'); + assert(this.halted, 'Can only compute actual gas used after tx execution ends'); const requireTeardown = this.teardownCallRequests.length > 0; const teardownGasLimits = requireTeardown ? this.gasSettings.teardownGasLimits : Gas.empty(); return this.gasUsed.sub(teardownGasLimits).add(this.teardownGasUsed); } + /** + * The gasUsed as if the entire teardown gas limit was consumed. + */ getGasUsedForFee(): Gas { return this.gasUsed; } - getTransactionFeeAtCurrentPhase(): Fr { - if (this.currentPhase === TxExecutionPhase.TEARDOWN) { + /** + * Get the transaction fee as is available to the specified phase. + * Only teardown should have access to the actual transaction fee. + */ + getTransactionFee(phase: TxExecutionPhase): Fr { + if (phase === TxExecutionPhase.TEARDOWN) { return this.getTransactionFeeUnsafe(); } else { return Fr.zero(); } } - getTransactionFee(): Fr { - assert(this.currentPhase === TxExecutionPhase.TEARDOWN, 'Transaction fee is only known during/after teardown'); - return this.getTransactionFeeUnsafe(); - } - + /** + * Compute the transaction fee. + * Should only be called during or after teardown. + */ private getTransactionFeeUnsafe(): Fr { const txFee = this.gasUsed.computeFee(this.globalVariables.gasFees); this.log.debug(`Computed tx fee`, { @@ -287,4 +309,108 @@ export class PublicTxContext { }); return txFee; } + + /** + * Generate the public inputs for the AVM circuit. + */ + private generateAvmCircuitPublicInputs(endStateReference: StateReference): AvmCircuitPublicInputs { + assert(this.halted, 'Can only get AvmCircuitPublicInputs after tx execution ends'); + return generateAvmCircuitPublicInputs( + this.trace, + this.globalVariables, + this.startStateReference, + this.startGasUsed, + this.gasSettings, + this.setupCallRequests, + this.appLogicCallRequests, + this.teardownCallRequests, + this.nonRevertibleAccumulatedDataFromPrivate, + this.revertibleAccumulatedDataFromPrivate, + endStateReference, + /*endGasUsed=*/ this.gasUsed, + this.getTransactionFeeUnsafe(), + this.revertCode, + ); + } + + /** + * Generate the proving request for the AVM circuit. + */ + generateProvingRequest(endStateReference: StateReference): AvmProvingRequest { + // TODO(dbanks12): Once we actually have tx-level proving, this will generate the entire + // proving request for the first time + this.avmProvingRequest!.inputs.output = this.generateAvmCircuitPublicInputs(endStateReference); + return this.avmProvingRequest!; + } + + // TODO(dbanks12): remove once AVM proves entire public tx + updateProvingRequest( + real: boolean, + phase: TxExecutionPhase, + fnName: string, + stateManager: AvmPersistableStateManager, + executionRequest: PublicExecutionRequest, + result: AvmFinalizedCallResult, + allocatedGas: Gas, + ) { + if (this.avmProvingRequest === undefined) { + // Propagate the very first avmProvingRequest of the tx for now. + // Eventually this will be the proof for the entire public portion of the transaction. + this.avmProvingRequest = generateAvmProvingRequest( + real, + fnName, + stateManager, + this.historicalHeader, + this.globalVariables, + executionRequest, + // TODO(dbanks12): do we need this return type unless we are doing an isolated call? + stateManager.trace.toPublicEnqueuedCallExecutionResult(result), + allocatedGas, + this.getTransactionFee(phase), + ); + } + } +} + +/** + * Thin wrapper around the state manager to handle forking and merging for phases. + * + * This lets us keep track of whether the state has already been forked + * so that we can conditionally fork at the start of a phase. + * + * There is a state manager that lives at the level of the entire transaction, + * but for setup and teardown the active state manager will be a fork of the + * transaction level one. + */ +class PhaseStateManager { + private currentlyActiveStateManager: AvmPersistableStateManager | undefined; + + constructor(private readonly txStateManager: AvmPersistableStateManager) {} + + fork() { + assert(!this.currentlyActiveStateManager, 'Cannot fork when already forked'); + this.currentlyActiveStateManager = this.txStateManager.fork(); + } + + getActiveStateManager() { + return this.currentlyActiveStateManager || this.txStateManager; + } + + isForked() { + return !!this.currentlyActiveStateManager; + } + + mergeForkedState() { + assert(this.currentlyActiveStateManager, 'No forked state to merge'); + this.txStateManager.merge(this.currentlyActiveStateManager!); + // Drop the forked state manager now that it is merged + this.currentlyActiveStateManager = undefined; + } + + discardForkedState() { + assert(this.currentlyActiveStateManager, 'No forked state to discard'); + this.txStateManager.reject(this.currentlyActiveStateManager!); + // Drop the forked state manager. We don't want it! + this.currentlyActiveStateManager = undefined; + } } diff --git a/yarn-project/simulator/src/public/public_tx_simulator.test.ts b/yarn-project/simulator/src/public/public_tx_simulator.test.ts index 08780d7008c..a530981a7b6 100644 --- a/yarn-project/simulator/src/public/public_tx_simulator.test.ts +++ b/yarn-project/simulator/src/public/public_tx_simulator.test.ts @@ -1,10 +1,4 @@ -import { - type MerkleTreeWriteOperations, - SimulationError, - type TreeInfo, - TxExecutionPhase, - mockTx, -} from '@aztec/circuit-types'; +import { type MerkleTreeWriteOperations, SimulationError, TxExecutionPhase, mockTx } from '@aztec/circuit-types'; import { AppendOnlyTreeSnapshot, AztecAddress, @@ -16,7 +10,6 @@ import { Header, PUBLIC_DATA_TREE_HEIGHT, PartialStateReference, - PublicDataTreeLeafPreimage, PublicDataWrite, RevertCode, StateReference, @@ -24,18 +17,18 @@ import { } from '@aztec/circuits.js'; import { computePublicDataTreeLeafSlot, siloNullifier } from '@aztec/circuits.js/hash'; import { fr } from '@aztec/circuits.js/testing'; +import { type AztecKVStore } from '@aztec/kv-store'; import { openTmpStore } from '@aztec/kv-store/utils'; import { type AppendOnlyTree, Poseidon, StandardTree, newTree } from '@aztec/merkle-tree'; +import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; +import { MerkleTrees } from '@aztec/world-state'; +import { jest } from '@jest/globals'; import { type MockProxy, mock } from 'jest-mock-extended'; +import { AvmFinalizedCallResult } from '../avm/avm_contract_call_result.js'; import { type AvmPersistableStateManager } from '../avm/journal/journal.js'; -import { PublicExecutionResultBuilder } from '../mocks/fixtures.js'; -import { WASMSimulator } from '../providers/acvm_wasm.js'; -import { type PublicExecutor } from './executor.js'; import { type WorldStateDB } from './public_db_sources.js'; -import { RealPublicKernelCircuitSimulator } from './public_kernel.js'; -import { type PublicKernelCircuitSimulator } from './public_kernel_circuit_simulator.js'; import { PublicTxSimulator } from './public_tx_simulator.js'; describe('public_tx_simulator', () => { @@ -51,15 +44,22 @@ describe('public_tx_simulator', () => { // gasUsed for each enqueued call. const enqueuedCallGasUsed = new Gas(12, 34); - let db: MockProxy; - let publicExecutor: MockProxy; - let publicKernel: PublicKernelCircuitSimulator; + let db: MerkleTreeWriteOperations; let worldStateDB: MockProxy; - let root: Buffer; let publicDataTree: AppendOnlyTree; - let processor: PublicTxSimulator; + let treeStore: AztecKVStore; + let simulator: PublicTxSimulator; + let simulateInternal: jest.SpiedFunction< + ( + stateManager: AvmPersistableStateManager, + executionResult: any, + allocatedGas: Gas, + transactionFee: any, + fnName: any, + ) => Promise + >; const mockTxWithPublicCalls = ({ numberOfSetupCalls = 0, @@ -91,54 +91,66 @@ describe('public_tx_simulator', () => { }; const mockPublicExecutor = ( - mockedSimulatorExecutions: (( - stateManager: AvmPersistableStateManager, - resultBuilder: PublicExecutionResultBuilder, - ) => Promise)[], + mockedSimulatorExecutions: ((stateManager: AvmPersistableStateManager) => Promise)[], ) => { for (const executeSimulator of mockedSimulatorExecutions) { - publicExecutor.simulate.mockImplementationOnce( - async (stateManager: AvmPersistableStateManager, _executionResult, _globalVariables, availableGas) => { - const builder = PublicExecutionResultBuilder.empty(); - await executeSimulator(stateManager, builder); - return builder.build({ - endGasLeft: availableGas.sub(enqueuedCallGasUsed), - }); + simulateInternal.mockImplementationOnce( + async ( + stateManager: AvmPersistableStateManager, + _executionResult: any, + allocatedGas: Gas, + _transactionFee: any, + _fnName: any, + ) => { + const revertReason = await executeSimulator(stateManager); + if (revertReason === undefined) { + return Promise.resolve( + new AvmFinalizedCallResult( + /*reverted=*/ false, + /*output=*/ [], + /*gasLeft=*/ allocatedGas.sub(enqueuedCallGasUsed), + ), + ); + } else { + return Promise.resolve( + new AvmFinalizedCallResult( + /*reverted=*/ true, + /*output=*/ [], + /*gasLeft=*/ allocatedGas.sub(enqueuedCallGasUsed), + revertReason, + ), + ); + } }, ); } }; const expectAvailableGasForCalls = (availableGases: Gas[]) => { - expect(publicExecutor.simulate).toHaveBeenCalledTimes(availableGases.length); + expect(simulateInternal).toHaveBeenCalledTimes(availableGases.length); availableGases.forEach((availableGas, i) => { - expect(publicExecutor.simulate).toHaveBeenNthCalledWith( + expect(simulateInternal).toHaveBeenNthCalledWith( i + 1, expect.anything(), // AvmPersistableStateManager expect.anything(), // publicExecutionRequest - expect.anything(), // globalVariables Gas.from(availableGas), expect.anything(), // txFee + expect.anything(), // fnName ); }); }; beforeEach(async () => { - db = mock(); + const tmp = openTmpStore(); + const telemetryClient = new NoopTelemetryClient(); + db = await (await MerkleTrees.new(tmp, telemetryClient)).fork(); worldStateDB = mock(); - root = Buffer.alloc(32, 5); - - publicExecutor = mock(); - publicExecutor.simulate.mockImplementation((_stateManager, _executionResult, _globalVariables, availableGas) => { - const result = PublicExecutionResultBuilder.empty().build({ - endGasLeft: availableGas.sub(enqueuedCallGasUsed), - }); - return Promise.resolve(result); - }); + + treeStore = openTmpStore(); publicDataTree = await newTree( StandardTree, - openTmpStore(), + treeStore, new Poseidon(), 'PublicData', Fr, @@ -157,23 +169,43 @@ describe('public_tx_simulator', () => { // Clone the whole state because somewhere down the line (AbstractPhaseManager) the public data root is modified in the referenced header directly :/ header.state = StateReference.fromBuffer(stateReference.toBuffer()); - db.getTreeInfo.mockResolvedValue({ root } as TreeInfo); - db.getStateReference.mockResolvedValue(stateReference); - db.getSiblingPath.mockResolvedValue(publicDataTree.getSiblingPath(0n, false)); - db.getPreviousValueIndex.mockResolvedValue({ index: 0n, alreadyPresent: true }); - db.getLeafPreimage.mockResolvedValue(new PublicDataTreeLeafPreimage(new Fr(0), new Fr(0), new Fr(0), 0n)); + worldStateDB.getMerkleInterface.mockReturnValue(db); - publicKernel = new RealPublicKernelCircuitSimulator(new WASMSimulator()); - - processor = PublicTxSimulator.create( + simulator = new PublicTxSimulator( db, - publicExecutor, - publicKernel, - GlobalVariables.from({ ...GlobalVariables.empty(), gasFees }), - Header.empty(), worldStateDB, + new NoopTelemetryClient(), + GlobalVariables.from({ ...GlobalVariables.empty(), gasFees }), /*realAvmProvingRequest=*/ false, ); + + // Mock the internal private function. Borrowed from https://stackoverflow.com/a/71033167 + simulateInternal = jest.spyOn( + simulator as unknown as { simulateEnqueuedCallInternal: PublicTxSimulator['simulateEnqueuedCallInternal'] }, + 'simulateEnqueuedCallInternal', + ); + simulateInternal.mockImplementation( + ( + _stateManager: AvmPersistableStateManager, + _executionResult: any, + allocatedGas: Gas, + _transactionFee: any, + _fnName: any, + ) => { + return Promise.resolve( + new AvmFinalizedCallResult( + /*reverted=*/ false, + /*output=*/ [], + /*gasLeft=*/ allocatedGas.sub(enqueuedCallGasUsed), + /*revertReason=*/ undefined, + ), + ); + }, + ); + }); + + afterEach(async () => { + await treeStore.delete(); }); it('runs a tx with enqueued public calls in setup phase only', async () => { @@ -181,7 +213,7 @@ describe('public_tx_simulator', () => { numberOfSetupCalls: 2, }); - const txResult = await processor.process(tx); + const txResult = await simulator.simulate(tx); expect(txResult.processedPhases).toEqual([ expect.objectContaining({ phase: TxExecutionPhase.SETUP, revertReason: undefined }), @@ -216,7 +248,7 @@ describe('public_tx_simulator', () => { numberOfAppLogicCalls: 2, }); - const txResult = await processor.process(tx); + const txResult = await simulator.simulate(tx); expect(txResult.processedPhases).toEqual([ expect.objectContaining({ phase: TxExecutionPhase.APP_LOGIC, revertReason: undefined }), @@ -251,7 +283,7 @@ describe('public_tx_simulator', () => { hasPublicTeardownCall: true, }); - const txResult = await processor.process(tx); + const txResult = await simulator.simulate(tx); expect(txResult.processedPhases).toEqual([ expect.objectContaining({ phase: TxExecutionPhase.TEARDOWN, revertReason: undefined }), @@ -286,7 +318,7 @@ describe('public_tx_simulator', () => { hasPublicTeardownCall: true, }); - const txResult = await processor.process(tx); + const txResult = await simulator.simulate(tx); expect(txResult.processedPhases).toHaveLength(3); expect(txResult.processedPhases).toEqual([ @@ -364,9 +396,9 @@ describe('public_tx_simulator', () => { }, ]); - const txResult = await processor.process(tx); + const txResult = await simulator.simulate(tx); - expect(publicExecutor.simulate).toHaveBeenCalledTimes(3); + expect(simulateInternal).toHaveBeenCalledTimes(3); const output = txResult.avmProvingRequest!.inputs.output; @@ -394,13 +426,18 @@ describe('public_tx_simulator', () => { const setupFailureMsg = 'Simulation Failed in setup'; const setupFailure = new SimulationError(setupFailureMsg, []); - publicExecutor.simulate.mockResolvedValueOnce( - PublicExecutionResultBuilder.empty().withReverted(setupFailure).build(), + simulateInternal.mockResolvedValueOnce( + new AvmFinalizedCallResult( + /*reverted=*/ true, + /*output=*/ [], + /*gasLeft=*/ Gas.empty(), + /*revertReason=*/ setupFailure, + ), ); - await expect(processor.process(tx)).rejects.toThrow(setupFailureMsg); + await expect(simulator.simulate(tx)).rejects.toThrow(setupFailureMsg); - expect(publicExecutor.simulate).toHaveBeenCalledTimes(1); + expect(simulateInternal).toHaveBeenCalledTimes(1); }); it('includes a transaction that reverts in app logic only', async function () { @@ -423,9 +460,9 @@ describe('public_tx_simulator', () => { await stateManager.writeNullifier(contractAddress, new Fr(2)); await stateManager.writeNullifier(contractAddress, new Fr(3)); }, - async (stateManager: AvmPersistableStateManager, resultBuilder: PublicExecutionResultBuilder) => { + async (stateManager: AvmPersistableStateManager) => { await stateManager.writeNullifier(contractAddress, new Fr(4)); - resultBuilder.withReverted(appLogicFailure); + return Promise.resolve(appLogicFailure); }, // TEARDOWN async (stateManager: AvmPersistableStateManager) => { @@ -433,7 +470,7 @@ describe('public_tx_simulator', () => { }, ]); - const txResult = await processor.process(tx); + const txResult = await simulator.simulate(tx); expect(txResult.processedPhases).toHaveLength(3); expect(txResult.processedPhases).toEqual([ @@ -505,13 +542,13 @@ describe('public_tx_simulator', () => { await stateManager.writeNullifier(contractAddress, new Fr(4)); }, // TEARDOWN - async (stateManager: AvmPersistableStateManager, resultBuilder: PublicExecutionResultBuilder) => { + async (stateManager: AvmPersistableStateManager) => { await stateManager.writeNullifier(contractAddress, new Fr(5)); - resultBuilder.withReverted(teardownFailure); + return Promise.resolve(teardownFailure); }, ]); - const txResult = await processor.process(tx); + const txResult = await simulator.simulate(tx); expect(txResult.processedPhases).toEqual([ expect.objectContaining({ phase: TxExecutionPhase.SETUP, revertReason: undefined }), @@ -582,18 +619,18 @@ describe('public_tx_simulator', () => { await stateManager.writeNullifier(contractAddress, new Fr(2)); await stateManager.writeNullifier(contractAddress, new Fr(3)); }, - async (stateManager: AvmPersistableStateManager, resultBuilder: PublicExecutionResultBuilder) => { + async (stateManager: AvmPersistableStateManager) => { await stateManager.writeNullifier(contractAddress, new Fr(4)); - resultBuilder.withReverted(appLogicFailure); + return Promise.resolve(appLogicFailure); }, // TEARDOWN - async (stateManager: AvmPersistableStateManager, resultBuilder: PublicExecutionResultBuilder) => { + async (stateManager: AvmPersistableStateManager) => { await stateManager.writeNullifier(contractAddress, new Fr(5)); - resultBuilder.withReverted(teardownFailure); + return Promise.resolve(teardownFailure); }, ]); - const txResult = await processor.process(tx); + const txResult = await simulator.simulate(tx); expect(txResult.processedPhases).toHaveLength(3); expect(txResult.processedPhases).toEqual([ diff --git a/yarn-project/simulator/src/public/public_tx_simulator.ts b/yarn-project/simulator/src/public/public_tx_simulator.ts index e694b45dc9c..44801eff13f 100644 --- a/yarn-project/simulator/src/public/public_tx_simulator.ts +++ b/yarn-project/simulator/src/public/public_tx_simulator.ts @@ -2,22 +2,32 @@ import { type AvmProvingRequest, type GasUsed, type MerkleTreeReadOperations, - type NestedProcessReturnValues, + NestedProcessReturnValues, + type PublicExecutionRequest, type SimulationError, type Tx, TxExecutionPhase, + UnencryptedFunctionL2Logs, } from '@aztec/circuit-types'; -import { type GlobalVariables, type Header, type RevertCode } from '@aztec/circuits.js'; +import { type AvmSimulationStats } from '@aztec/circuit-types/stats'; +import { + type Fr, + Gas, + type GlobalVariables, + MAX_L2_GAS_PER_ENQUEUED_CALL, + type PublicCallRequest, + type RevertCode, +} from '@aztec/circuits.js'; import { type DebugLogger, createDebugLogger } from '@aztec/foundation/log'; import { Timer } from '@aztec/foundation/timer'; +import { type TelemetryClient } from '@aztec/telemetry-client'; -import { EnqueuedCallSimulator } from './enqueued_call_simulator.js'; -import { type PublicExecutor } from './executor.js'; +import { type AvmFinalizedCallResult } from '../avm/avm_contract_call_result.js'; +import { type AvmPersistableStateManager, AvmSimulator } from '../avm/index.js'; +import { getPublicFunctionDebugName } from '../common/debug_fn_name.js'; +import { ExecutorMetrics } from './executor_metrics.js'; import { type WorldStateDB } from './public_db_sources.js'; -import { type PublicKernelCircuitSimulator } from './public_kernel_circuit_simulator.js'; -import { PublicKernelTailSimulator } from './public_kernel_tail_simulator.js'; import { PublicTxContext } from './public_tx_context.js'; -import { generateAvmCircuitPublicInputs, runMergeKernelCircuit } from './utils.js'; export type ProcessedPhase = { phase: TxExecutionPhase; @@ -38,149 +48,162 @@ export type PublicTxResult = { }; export class PublicTxSimulator { + metrics: ExecutorMetrics; + private log: DebugLogger; constructor( private db: MerkleTreeReadOperations, - private publicKernelSimulator: PublicKernelCircuitSimulator, - private globalVariables: GlobalVariables, private worldStateDB: WorldStateDB, - private enqueuedCallSimulator: EnqueuedCallSimulator, - private publicKernelTailSimulator: PublicKernelTailSimulator, + client: TelemetryClient, + private globalVariables: GlobalVariables, + private realAvmProvingRequests: boolean = true, + private doMerkleOperations: boolean = false, ) { - this.log = createDebugLogger(`aztec:sequencer`); + this.log = createDebugLogger(`aztec:public_tx_simulator`); + this.metrics = new ExecutorMetrics(client, 'PublicTxSimulator'); } - static create( - db: MerkleTreeReadOperations, - publicExecutor: PublicExecutor, - publicKernelSimulator: PublicKernelCircuitSimulator, - globalVariables: GlobalVariables, - historicalHeader: Header, - worldStateDB: WorldStateDB, - realAvmProvingRequests: boolean = true, - ) { - const enqueuedCallSimulator = new EnqueuedCallSimulator( - db, - worldStateDB, - publicExecutor, - globalVariables, - historicalHeader, - realAvmProvingRequests, - ); - - const publicKernelTailSimulator = PublicKernelTailSimulator.create(db, publicKernelSimulator); - - return new PublicTxSimulator( - db, - publicKernelSimulator, - globalVariables, - worldStateDB, - enqueuedCallSimulator, - publicKernelTailSimulator, - ); - } - - async process(tx: Tx): Promise { + /** + * Simulate a transaction's public portion including all of its phases. + * @param tx - The transaction to simulate. + * @returns The result of the transaction's public execution. + */ + async simulate(tx: Tx): Promise { this.log.verbose(`Processing tx ${tx.getTxHash()}`); - const context = await PublicTxContext.create(this.db, this.worldStateDB, tx, this.globalVariables); - - const setupResult = await this.processSetupPhase(context); - const appLogicResult = await this.processAppLogicPhase(context); - const teardownResult = await this.processTeardownPhase(context); + const context = await PublicTxContext.create( + this.db, + this.worldStateDB, + tx, + this.globalVariables, + this.doMerkleOperations, + ); - const processedPhases = [setupResult, appLogicResult, teardownResult].filter( - result => result !== undefined, - ) as ProcessedPhase[]; + // add new contracts to the contracts db so that their functions may be found and called + // TODO(#4073): This is catching only private deployments, when we add public ones, we'll + // have to capture contracts emitted in that phase as well. + // TODO(@spalladino): Should we allow emitting contracts in the fee preparation phase? + // TODO(#6464): Should we allow emitting contracts in the private setup phase? + // if so, this should only add contracts that were deployed during private app logic. + // FIXME: we shouldn't need to directly modify worldStateDb here! + await this.worldStateDB.addNewContracts(tx); + + const processedPhases: ProcessedPhase[] = []; + if (context.hasPhase(TxExecutionPhase.SETUP)) { + const setupResult: ProcessedPhase = await this.simulateSetupPhase(context); + processedPhases.push(setupResult); + } + if (context.hasPhase(TxExecutionPhase.APP_LOGIC)) { + const appLogicResult: ProcessedPhase = await this.simulateAppLogicPhase(context); + processedPhases.push(appLogicResult); + } + if (context.hasPhase(TxExecutionPhase.TEARDOWN)) { + const teardownResult: ProcessedPhase = await this.simulateTeardownPhase(context); + processedPhases.push(teardownResult); + } + context.halt(); - const _endStateReference = await this.db.getStateReference(); - const transactionFee = context.getTransactionFee(); + const endStateReference = await this.db.getStateReference(); - const tailKernelOutput = await this.publicKernelTailSimulator.simulate(context.latestPublicKernelOutput); + const avmProvingRequest = context.generateProvingRequest(endStateReference); + const avmCircuitPublicInputs = avmProvingRequest.inputs.output!; - context.avmProvingRequest!.inputs.output = generateAvmCircuitPublicInputs( - tx, - tailKernelOutput, - context.getGasUsedForFee(), - transactionFee, - ); + const revertCode = context.getFinalRevertCode(); + if (!revertCode.isOK()) { + // TODO(#6464): Should we allow emitting contracts in the private setup phase? + // if so, this is removing contracts deployed in private setup + // You can't submit contracts in public, so this is only relevant for private-created side effects + // FIXME: we shouldn't need to directly modify worldStateDb here! + await this.worldStateDB.removeNewContracts(tx); + // FIXME(dbanks12): should not be changing immutable tx + tx.filterRevertedLogs( + tx.data.forPublic!.nonRevertibleAccumulatedData, + avmCircuitPublicInputs.accumulatedData.unencryptedLogsHashes, + ); + } + // FIXME(dbanks12): should not be changing immutable tx + tx.unencryptedLogs.addFunctionLogs([new UnencryptedFunctionL2Logs(context.trace.getUnencryptedLogs())]); - const gasUsed = { - totalGas: context.getActualGasUsed(), - teardownGas: context.teardownGasUsed, - }; return { - avmProvingRequest: context.avmProvingRequest!, - gasUsed, - revertCode: context.revertCode, + avmProvingRequest, + gasUsed: { totalGas: context.getActualGasUsed(), teardownGas: context.teardownGasUsed }, + revertCode, revertReason: context.revertReason, processedPhases: processedPhases, }; } - private async processSetupPhase(context: PublicTxContext): Promise { - // Start in phase TxExecutionPhase.SETUP; - if (context.hasPhase()) { - return await this.processPhase(context); - } + /** + * Simulate the setup phase of a transaction's public execution. + * @param context - WILL BE MUTATED. The context of the currently executing public transaction portion + * @returns The phase result. + */ + private async simulateSetupPhase(context: PublicTxContext): Promise { + return await this.simulatePhase(TxExecutionPhase.SETUP, context); } - private async processAppLogicPhase(context: PublicTxContext): Promise { - context.progressToNextPhase(); // to app logic - if (context.hasPhase()) { - // Fork the state manager so that we can rollback state if app logic or teardown reverts. - // Don't need to fork for setup since it's non-revertible (if setup fails, transaction is thrown out). - context.state.fork(); - - const result = await this.processPhase(context); - - if (result.reverted) { - // Drop the currently active forked state manager and rollback to end of setup. - // Fork again for teardown so that if teardown fails we can again rollback to end of setup. - context.state.discardForkedState(); - } else { - if (!context.hasPhase(TxExecutionPhase.TEARDOWN)) { - // Nothing to do after this (no teardown), so merge state in now instead of letting teardown handle it. - context.state.mergeForkedState(); - } + /** + * Simulate the app logic phase of a transaction's public execution. + * @param context - WILL BE MUTATED. The context of the currently executing public transaction portion + * @returns The phase result. + */ + private async simulateAppLogicPhase(context: PublicTxContext): Promise { + // Fork the state manager so that we can rollback state if app logic or teardown reverts. + // Don't need to fork for setup since it's non-revertible (if setup fails, transaction is thrown out). + context.state.fork(); + + const result = await this.simulatePhase(TxExecutionPhase.APP_LOGIC, context); + + if (result.reverted) { + // Drop the currently active forked state manager and rollback to end of setup. + context.state.discardForkedState(); + } else { + if (!context.hasPhase(TxExecutionPhase.TEARDOWN)) { + // Nothing to do after this (no teardown), so merge state updates now instead of letting teardown handle it. + context.state.mergeForkedState(); } - - return result; } - } - private async processTeardownPhase(context: PublicTxContext): Promise { - context.progressToNextPhase(); // to teardown - if (context.hasPhase()) { - if (!context.state.isForked()) { - // if state isn't forked (app logic was empty or reverted), fork now - // so we can rollback to the end of setup on teardown revert - context.state.fork(); - } + return result; + } - const result = await this.processPhase(context); + /** + * Simulate the teardown phase of a transaction's public execution. + * @param context - WILL BE MUTATED. The context of the currently executing public transaction portion + * @returns The phase result. + */ + private async simulateTeardownPhase(context: PublicTxContext): Promise { + if (!context.state.isForked()) { + // If state isn't forked (app logic was empty or reverted), fork now + // so we can rollback to the end of setup if teardown reverts. + context.state.fork(); + } - if (result.reverted) { - // Drop the currently active forked state manager and rollback to end of setup. - context.state.discardForkedState(); - } else { - context.state.mergeForkedState(); - } + const result = await this.simulatePhase(TxExecutionPhase.TEARDOWN, context); - return result; + if (result.reverted) { + // Drop the currently active forked state manager and rollback to end of setup. + context.state.discardForkedState(); + } else { + // Merge state updates from teardown, + context.state.mergeForkedState(); } + + return result; } - private async processPhase(context: PublicTxContext): Promise { - const tx = context.tx; - const callRequests = context.getCallRequestsForCurrentPhase(); - const executionRequests = context.getExecutionRequestsForCurrentPhase(); - const txStateManager = context.state.getActiveStateManager(); + /** + * Simulate a phase of a transaction's public execution. + * @param phase - The current phase + * @param context - WILL BE MUTATED. The context of the currently executing public transaction portion + * @returns The phase result. + */ + private async simulatePhase(phase: TxExecutionPhase, context: PublicTxContext): Promise { + const callRequests = context.getCallRequestsForPhase(phase); + const executionRequests = context.getExecutionRequestsForPhase(phase); - this.log.debug( - `Beginning processing in phase ${TxExecutionPhase[context.getCurrentPhase()]} for tx ${tx.getTxHash()}`, - ); + this.log.debug(`Beginning processing in phase ${TxExecutionPhase[phase]} for tx ${context.getTxHash()}`); const returnValues: NestedProcessReturnValues[] = []; let reverted = false; @@ -194,70 +217,148 @@ export class PublicTxSimulator { const callRequest = callRequests[i]; const executionRequest = executionRequests[i]; - // add new contracts to the contracts db so that their functions may be found and called - // TODO(#4073): This is catching only private deployments, when we add public ones, we'll - // have to capture contracts emitted in that phase as well. - // TODO(@spalladino): Should we allow emitting contracts in the fee preparation phase? - // TODO(#6464): Should we allow emitting contracts in the private setup phase? - // if so, this should only add contracts that were deployed during private app logic. - // FIXME: we shouldn't need to directly modify worldStateDb here! - await this.worldStateDB.addNewContracts(tx); - - // each enqueued call starts with an incremented side effect counter - // FIXME: should be able to stop forking here and just trace the enqueued call (for hinting) - // and proceed with the same state manager for the entire phase - const enqueuedCallStateManager = txStateManager.fork(/*incrementSideEffectCounter=*/ true); - const enqueuedCallResult = await this.enqueuedCallSimulator.simulate( - callRequest, - executionRequest, - context.constants, - /*availableGas=*/ context.getGasLeftForCurrentPhase(), - /*transactionFee=*/ context.getTransactionFeeAtCurrentPhase(), - enqueuedCallStateManager, - ); + const enqueuedCallResult = await this.simulateEnqueuedCall(phase, context, callRequest, executionRequest); - txStateManager.traceEnqueuedCall(callRequest, executionRequest.args, enqueuedCallResult.reverted!); + returnValues.push(new NestedProcessReturnValues(enqueuedCallResult.output)); - context.consumeGas(enqueuedCallResult.gasUsed); - returnValues.push(enqueuedCallResult.returnValues); - // Propagate only one avmProvingRequest of a function call for now, so that we know it's still provable. - // Eventually this will be the proof for the entire public portion of the transaction. - context.avmProvingRequest = enqueuedCallResult.avmProvingRequest; if (enqueuedCallResult.reverted) { reverted = true; - const culprit = `${executionRequest.callContext.contractAddress}:${executionRequest.callContext.functionSelector}`; revertReason = enqueuedCallResult.revertReason; - context.revert(enqueuedCallResult.revertReason, culprit); // throws if in setup (non-revertible) phase - - // TODO(#6464): Should we allow emitting contracts in the private setup phase? - // if so, this is removing contracts deployed in private setup - // You can't submit contracts in public, so this is only relevant for private-created side effects - // FIXME: we shouldn't need to directly modify worldStateDb here! - await this.worldStateDB.removeNewContracts(tx); - // FIXME: we shouldn't be modifying the transaction here! - tx.filterRevertedLogs(context.latestPublicKernelOutput); - // Enqueeud call reverted. Discard state updates and accumulated side effects, but keep hints traced for the circuit. - txStateManager.rejectForkedState(enqueuedCallStateManager); - } else { - // FIXME: we shouldn't be modifying the transaction here! - tx.unencryptedLogs.addFunctionLogs([enqueuedCallResult.newUnencryptedLogs]); - // Enqueued call succeeded! Merge in any state updates made in the forked state manager. - txStateManager.mergeForkedState(enqueuedCallStateManager); } - - context.latestPublicKernelOutput = await runMergeKernelCircuit( - context.latestPublicKernelOutput, - enqueuedCallResult.kernelOutput, - this.publicKernelSimulator, - ); } return { - phase: context.getCurrentPhase(), + phase, durationMs: phaseTimer.ms(), returnValues, reverted, revertReason, }; } + + /** + * Simulate an enqueued public call. + * @param phase - The current phase of public execution + * @param context - WILL BE MUTATED. The context of the currently executing public transaction portion + * @param callRequest - The enqueued call to execute + * @param executionRequest - The execution request (includes args) + * @returns The result of execution. + */ + private async simulateEnqueuedCall( + phase: TxExecutionPhase, + context: PublicTxContext, + callRequest: PublicCallRequest, + executionRequest: PublicExecutionRequest, + ): Promise { + const stateManager = context.state.getActiveStateManager(); + const address = executionRequest.callContext.contractAddress; + const selector = executionRequest.callContext.functionSelector; + const fnName = await getPublicFunctionDebugName(this.worldStateDB, address, selector, executionRequest.args); + + const availableGas = context.getGasLeftForPhase(phase); + // Gas allocated to an enqueued call can be different from the available gas + // if there is more gas available than the max allocation per enqueued call. + const allocatedGas = new Gas( + /*daGas=*/ availableGas.daGas, + /*l2Gas=*/ Math.min(availableGas.l2Gas, MAX_L2_GAS_PER_ENQUEUED_CALL), + ); + + const result = await this.simulateEnqueuedCallInternal( + context.state.getActiveStateManager(), + executionRequest, + allocatedGas, + context.getTransactionFee(phase), + fnName, + ); + + const gasUsed = allocatedGas.sub(result.gasLeft); + context.consumeGas(phase, gasUsed); + this.log.verbose( + `[AVM] Enqueued public call consumed ${gasUsed.l2Gas} L2 gas ending with ${result.gasLeft.l2Gas} L2 gas left.`, + ); + + // TODO(dbanks12): remove once AVM proves entire public tx + context.updateProvingRequest( + this.realAvmProvingRequests, + phase, + fnName, + stateManager, + executionRequest, + result, + allocatedGas, + ); + + stateManager.traceEnqueuedCall(callRequest, executionRequest.args, result.reverted); + + if (result.reverted) { + const culprit = `${executionRequest.callContext.contractAddress}:${executionRequest.callContext.functionSelector}`; + context.revert(phase, result.revertReason, culprit); // throws if in setup (non-revertible) phase + } + + return result; + } + + /** + * Simulate an enqueued public call, without modifying the context (PublicTxContext). + * Resulting modifcations to the context can be applied by the caller. + * + * This function can be mocked for testing to skip actual AVM simulation + * while still simulating phases and generating a proving request. + * + * @param stateManager - The state manager for AvmSimulation + * @param context - The context of the currently executing public transaction portion + * @param executionRequest - The execution request (includes args) + * @param allocatedGas - The gas allocated to the enqueued call + * @param fnName - The name of the function + * @returns The result of execution. + */ + private async simulateEnqueuedCallInternal( + stateManager: AvmPersistableStateManager, + executionRequest: PublicExecutionRequest, + allocatedGas: Gas, + transactionFee: Fr, + fnName: string, + ): Promise { + const address = executionRequest.callContext.contractAddress; + const sender = executionRequest.callContext.msgSender; + const selector = executionRequest.callContext.functionSelector; + + this.log.verbose( + `[AVM] Executing enqueued public call to external function ${fnName}@${address} with ${allocatedGas.l2Gas} allocated L2 gas.`, + ); + const timer = new Timer(); + + const simulator = AvmSimulator.create( + stateManager, + address, + sender, + selector, + transactionFee, + this.globalVariables, + executionRequest.callContext.isStaticCall, + executionRequest.args, + allocatedGas, + ); + const avmCallResult = await simulator.execute(); + const result = avmCallResult.finalize(); + + this.log.verbose( + `[AVM] Simulation of enqueued public call ${fnName} completed. reverted: ${result.reverted}${ + result.reverted ? ', reason: ' + result.revertReason : '' + }.`, + { + eventName: 'avm-simulation', + appCircuitName: fnName, + duration: timer.ms(), + } satisfies AvmSimulationStats, + ); + + if (result.reverted) { + this.metrics.recordFunctionSimulationFailure(); + } else { + this.metrics.recordFunctionSimulation(timer.ms()); + } + + return result; + } } diff --git a/yarn-project/simulator/src/public/side_effect_trace.test.ts b/yarn-project/simulator/src/public/side_effect_trace.test.ts index 2a57e7df088..ccbcea0267c 100644 --- a/yarn-project/simulator/src/public/side_effect_trace.test.ts +++ b/yarn-project/simulator/src/public/side_effect_trace.test.ts @@ -51,7 +51,7 @@ describe('Side Effect Trace', () => { transactionFee, }); const reverted = false; - const avmCallResults = new AvmContractCallResult(reverted, returnValues); + const avmCallResults = new AvmContractCallResult(reverted, returnValues, endGasLeft); let startCounter: number; let startCounterFr: Fr; @@ -66,7 +66,7 @@ describe('Side Effect Trace', () => { }); const toPxResult = (trc: PublicSideEffectTrace) => { - return trc.toPublicFunctionCallResult(avmEnvironment, startGasLeft, endGasLeft, bytecode, avmCallResults); + return trc.toPublicFunctionCallResult(avmEnvironment, startGasLeft, bytecode, avmCallResults.finalize()); }; it('Should trace storage reads', () => { @@ -427,7 +427,7 @@ describe('Side Effect Trace', () => { nestedTrace.traceGetContractInstance(address, /*exists=*/ false, contractInstance); testCounter++; - trace.traceNestedCall(nestedTrace, avmEnvironment, startGasLeft, endGasLeft, bytecode, avmCallResults); + trace.traceNestedCall(nestedTrace, avmEnvironment, startGasLeft, bytecode, avmCallResults); // parent trace adopts nested call's counter expect(trace.getCounter()).toBe(testCounter); diff --git a/yarn-project/simulator/src/public/side_effect_trace.ts b/yarn-project/simulator/src/public/side_effect_trace.ts index efe7a3afa6b..ac1f4a98f16 100644 --- a/yarn-project/simulator/src/public/side_effect_trace.ts +++ b/yarn-project/simulator/src/public/side_effect_trace.ts @@ -12,7 +12,6 @@ import { AvmPublicDataWriteTreeHint, type AztecAddress, CallContext, - type CombinedConstantData, type ContractClassIdPreimage, type ContractInstanceWithAddress, ContractStorageRead, @@ -44,16 +43,14 @@ import { ReadRequest, SerializableContractInstance, TreeLeafReadRequest, - type VMCircuitPublicInputs, } from '@aztec/circuits.js'; import { Fr } from '@aztec/foundation/fields'; import { createDebugLogger } from '@aztec/foundation/log'; import { assert } from 'console'; -import { type AvmContractCallResult } from '../avm/avm_contract_call_result.js'; +import { type AvmContractCallResult, type AvmFinalizedCallResult } from '../avm/avm_contract_call_result.js'; import { type AvmExecutionEnvironment } from '../avm/avm_execution_environment.js'; -import { createSimulationError } from '../common/errors.js'; import { type EnqueuedPublicCallExecutionResultWithSideEffects, type PublicFunctionCallResult, @@ -106,8 +103,8 @@ export class PublicSideEffectTrace implements PublicSideEffectTraceInterface { this.avmCircuitHints = AvmExecutionHints.empty(); } - public fork(incrementSideEffectCounter: boolean = false) { - return new PublicSideEffectTrace(incrementSideEffectCounter ? this.sideEffectCounter + 1 : this.sideEffectCounter); + public fork() { + return new PublicSideEffectTrace(this.sideEffectCounter); } public getCounter() { @@ -202,7 +199,7 @@ export class PublicSideEffectTrace implements PublicSideEffectTraceInterface { public traceNewNoteHash( _contractAddress: AztecAddress, noteHash: Fr, - leafIndex: Fr, + leafIndex: Fr = Fr.zero(), path: Fr[] = emptyNoteHashPath(), ) { if (this.noteHashes.length >= MAX_NOTE_HASHES_PER_TX) { @@ -382,8 +379,6 @@ export class PublicSideEffectTrace implements PublicSideEffectTraceInterface { nestedEnvironment: AvmExecutionEnvironment, /** How much gas was available for this public execution. */ startGasLeft: Gas, - /** How much gas was left after this public execution. */ - endGasLeft: Gas, /** Bytecode used for this execution. */ bytecode: Buffer, /** The call's results */ @@ -399,9 +394,8 @@ export class PublicSideEffectTrace implements PublicSideEffectTraceInterface { const result = nestedCallTrace.toPublicFunctionCallResult( nestedEnvironment, startGasLeft, - endGasLeft, bytecode, - avmCallResults, + avmCallResults.finalize(), functionName, ); this.sideEffectCounter = result.endSideEffectCounter.toNumber(); @@ -412,8 +406,8 @@ export class PublicSideEffectTrace implements PublicSideEffectTraceInterface { this.nestedExecutions.push(result); const gasUsed = new Gas( - result.startGasLeft.daGas - result.endGasLeft.daGas, - result.startGasLeft.l2Gas - result.endGasLeft.l2Gas, + result.startGasLeft.daGas - avmCallResults.gasLeft.daGas, + result.startGasLeft.l2Gas - avmCallResults.gasLeft.l2Gas, ); this.publicCallRequests.push(resultToPublicCallRequest(result)); @@ -440,11 +434,7 @@ export class PublicSideEffectTrace implements PublicSideEffectTraceInterface { throw new Error('Not implemented'); } - public mergeSuccessfulForkedTrace(_nestedTrace: this) { - throw new Error('Not implemented'); - } - - public mergeRevertedForkedTrace(_nestedTrace: this) { + public merge(_nestedTrace: this, _reverted: boolean = false) { throw new Error('Not implemented'); } @@ -456,12 +446,10 @@ export class PublicSideEffectTrace implements PublicSideEffectTraceInterface { avmEnvironment: AvmExecutionEnvironment, /** How much gas was available for this public execution. */ startGasLeft: Gas, - /** How much gas was left after this public execution. */ - endGasLeft: Gas, /** Bytecode used for this execution. */ bytecode: Buffer, /** The call's results */ - avmCallResults: AvmContractCallResult, + avmCallResults: AvmFinalizedCallResult, /** Function name for logging */ functionName: string = 'unknown', ): PublicFunctionCallResult { @@ -471,16 +459,14 @@ export class PublicSideEffectTrace implements PublicSideEffectTraceInterface { startSideEffectCounter: new Fr(this.startSideEffectCounter), endSideEffectCounter: new Fr(this.sideEffectCounter), startGasLeft, - endGasLeft, + endGasLeft: avmCallResults.gasLeft, transactionFee: avmEnvironment.transactionFee, bytecode, calldata: avmEnvironment.calldata, returnValues: avmCallResults.output, reverted: avmCallResults.reverted, - revertReason: avmCallResults.revertReason - ? createSimulationError(avmCallResults.revertReason, avmCallResults.output) - : undefined, + revertReason: avmCallResults.revertReason, contractStorageReads: this.contractStorageReads, contractStorageUpdateRequests: this.contractStorageUpdateRequests, @@ -506,31 +492,12 @@ export class PublicSideEffectTrace implements PublicSideEffectTraceInterface { } public toPublicEnqueuedCallExecutionResult( - /** How much gas was left after this public execution. */ - _endGasLeft: Gas, /** The call's results */ - _avmCallResults: AvmContractCallResult, + _avmCallResults: AvmFinalizedCallResult, ): EnqueuedPublicCallExecutionResultWithSideEffects { throw new Error('Not implemented'); } - public toVMCircuitPublicInputs( - /** Constants. */ - _constants: CombinedConstantData, - /** The call request that triggered public execution. */ - _callRequest: PublicCallRequest, - /** How much gas was available for this public execution. */ - _startGasLeft: Gas, - /** How much gas was left after this public execution. */ - _endGasLeft: Gas, - /** Transaction fee. */ - _transactionFee: Fr, - /** The call's results */ - _avmCallResults: AvmContractCallResult, - ): VMCircuitPublicInputs { - throw new Error('Not implemented'); - } - private enforceLimitOnNullifierChecks(errorMsgOrigin: string = '') { // NOTE: Why error if _either_ limit was reached? If user code emits either an existent or non-existent // nullifier read request (NULLIFIEREXISTS, GETCONTRACTINSTANCE, *CALL), and one of the limits has been diff --git a/yarn-project/simulator/src/public/side_effect_trace_interface.ts b/yarn-project/simulator/src/public/side_effect_trace_interface.ts index 02179157071..98bdd9f809e 100644 --- a/yarn-project/simulator/src/public/side_effect_trace_interface.ts +++ b/yarn-project/simulator/src/public/side_effect_trace_interface.ts @@ -1,23 +1,22 @@ import { type UnencryptedL2Log } from '@aztec/circuit-types'; import { - type CombinedConstantData, type ContractClassIdPreimage, type Gas, type NullifierLeafPreimage, type PublicCallRequest, type PublicDataTreeLeafPreimage, type SerializableContractInstance, - type VMCircuitPublicInputs, } from '@aztec/circuits.js'; import { type AztecAddress } from '@aztec/foundation/aztec-address'; import { type Fr } from '@aztec/foundation/fields'; -import { type AvmContractCallResult } from '../avm/avm_contract_call_result.js'; +import { type AvmContractCallResult, type AvmFinalizedCallResult } from '../avm/avm_contract_call_result.js'; import { type AvmExecutionEnvironment } from '../avm/avm_execution_environment.js'; import { type EnqueuedPublicCallExecutionResultWithSideEffects, type PublicFunctionCallResult } from './execution.js'; export interface PublicSideEffectTraceInterface { - fork(incrementSideEffectCounter?: boolean): PublicSideEffectTraceInterface; + fork(): PublicSideEffectTraceInterface; + merge(nestedTrace: PublicSideEffectTraceInterface, reverted?: boolean): void; getCounter(): number; // all "trace*" functions can throw SideEffectLimitReachedError tracePublicStorageRead( @@ -84,8 +83,6 @@ export interface PublicSideEffectTraceInterface { nestedEnvironment: AvmExecutionEnvironment, /** How much gas was available for this public execution. */ startGasLeft: Gas, - /** How much gas was left after this public execution. */ - endGasLeft: Gas, /** Bytecode used for this execution. */ bytecode: Buffer, /** The call's results */ @@ -101,41 +98,21 @@ export interface PublicSideEffectTraceInterface { /** Did the call revert? */ reverted: boolean, ): void; - mergeSuccessfulForkedTrace(nestedTrace: PublicSideEffectTraceInterface): void; - mergeRevertedForkedTrace(nestedTrace: PublicSideEffectTraceInterface): void; toPublicEnqueuedCallExecutionResult( - /** How much gas was left after this public execution. */ - endGasLeft: Gas, /** The call's results */ - avmCallResults: AvmContractCallResult, + avmCallResults: AvmFinalizedCallResult, ): EnqueuedPublicCallExecutionResultWithSideEffects; toPublicFunctionCallResult( /** The execution environment of the nested call. */ avmEnvironment: AvmExecutionEnvironment, /** How much gas was available for this public execution. */ startGasLeft: Gas, - /** How much gas was left after this public execution. */ - endGasLeft: Gas, /** Bytecode used for this execution. */ bytecode: Buffer, /** The call's results */ - avmCallResults: AvmContractCallResult, + avmCallResults: AvmFinalizedCallResult, /** Function name for logging */ functionName: string, ): PublicFunctionCallResult; - toVMCircuitPublicInputs( - /** Constants. */ - constants: CombinedConstantData, - /** The call request that triggered public execution. */ - callRequest: PublicCallRequest, - /** How much gas was available for this public execution. */ - startGasLeft: Gas, - /** How much gas was left after this public execution. */ - endGasLeft: Gas, - /** Transaction fee. */ - transactionFee: Fr, - /** The call's results */ - avmCallResults: AvmContractCallResult, - ): VMCircuitPublicInputs; getUnencryptedLogs(): UnencryptedL2Log[]; } diff --git a/yarn-project/simulator/src/public/transitional_adapters.ts b/yarn-project/simulator/src/public/transitional_adapters.ts new file mode 100644 index 00000000000..a088f5fa000 --- /dev/null +++ b/yarn-project/simulator/src/public/transitional_adapters.ts @@ -0,0 +1,347 @@ +import { type AvmProvingRequest, ProvingRequestType, type PublicExecutionRequest } from '@aztec/circuit-types'; +import { + AvmCircuitInputs, + AvmCircuitPublicInputs, + AztecAddress, + ContractStorageRead, + ContractStorageUpdateRequest, + Fr, + Gas, + type GasSettings, + type GlobalVariables, + type Header, + L2ToL1Message, + LogHash, + MAX_ENQUEUED_CALLS_PER_CALL, + MAX_L1_TO_L2_MSG_READ_REQUESTS_PER_CALL, + MAX_L2_TO_L1_MSGS_PER_CALL, + MAX_L2_TO_L1_MSGS_PER_TX, + MAX_NOTE_HASHES_PER_CALL, + MAX_NOTE_HASHES_PER_TX, + MAX_NOTE_HASH_READ_REQUESTS_PER_CALL, + MAX_NULLIFIERS_PER_CALL, + MAX_NULLIFIERS_PER_TX, + MAX_NULLIFIER_NON_EXISTENT_READ_REQUESTS_PER_CALL, + MAX_NULLIFIER_READ_REQUESTS_PER_CALL, + MAX_PUBLIC_DATA_READS_PER_CALL, + MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_CALL, + MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, + MAX_UNENCRYPTED_LOGS_PER_CALL, + NoteHash, + Nullifier, + PrivateToAvmAccumulatedData, + PrivateToAvmAccumulatedDataArrayLengths, + type PrivateToPublicAccumulatedData, + PublicCallRequest, + PublicCircuitPublicInputs, + PublicDataWrite, + PublicInnerCallRequest, + ReadRequest, + RevertCode, + type StateReference, + TreeLeafReadRequest, + TreeSnapshots, + countAccumulatedItems, + mergeAccumulatedData, +} from '@aztec/circuits.js'; +import { computeNoteHashNonce, computeUniqueNoteHash, computeVarArgsHash, siloNoteHash } from '@aztec/circuits.js/hash'; +import { padArrayEnd } from '@aztec/foundation/collection'; +import { assertLength } from '@aztec/foundation/serialize'; + +import { AvmFinalizedCallResult } from '../avm/avm_contract_call_result.js'; +import { AvmExecutionEnvironment } from '../avm/avm_execution_environment.js'; +import { type AvmPersistableStateManager } from '../avm/journal/journal.js'; +import { type PublicEnqueuedCallSideEffectTrace } from './enqueued_call_side_effect_trace.js'; +import { type EnqueuedPublicCallExecutionResult, type PublicFunctionCallResult } from './execution.js'; + +export function generateAvmCircuitPublicInputs( + trace: PublicEnqueuedCallSideEffectTrace, + globalVariables: GlobalVariables, + startStateReference: StateReference, + startGasUsed: Gas, + gasSettings: GasSettings, + setupCallRequests: PublicCallRequest[], + appLogicCallRequests: PublicCallRequest[], + teardownCallRequests: PublicCallRequest[], + nonRevertibleAccumulatedDataFromPrivate: PrivateToPublicAccumulatedData, + revertibleAccumulatedDataFromPrivate: PrivateToPublicAccumulatedData, + endStateReference: StateReference, + endGasUsed: Gas, + transactionFee: Fr, + revertCode: RevertCode, +): AvmCircuitPublicInputs { + const startTreeSnapshots = new TreeSnapshots( + startStateReference.l1ToL2MessageTree, + startStateReference.partial.noteHashTree, + startStateReference.partial.nullifierTree, + startStateReference.partial.publicDataTree, + ); + const endTreeSnapshots = new TreeSnapshots( + endStateReference.l1ToL2MessageTree, + endStateReference.partial.noteHashTree, + endStateReference.partial.nullifierTree, + endStateReference.partial.publicDataTree, + ); + + const avmCircuitPublicInputs = trace.toAvmCircuitPublicInputs( + globalVariables, + startTreeSnapshots, + startGasUsed, + gasSettings, + setupCallRequests, + appLogicCallRequests, + teardownCallRequests.length ? teardownCallRequests[0] : PublicCallRequest.empty(), + endTreeSnapshots, + endGasUsed, + transactionFee, + !revertCode.isOK(), + ); + + const getArrayLengths = (from: PrivateToPublicAccumulatedData) => + new PrivateToAvmAccumulatedDataArrayLengths( + countAccumulatedItems(from.noteHashes), + countAccumulatedItems(from.nullifiers), + countAccumulatedItems(from.l2ToL1Msgs), + ); + const convertAccumulatedData = (from: PrivateToPublicAccumulatedData) => + new PrivateToAvmAccumulatedData(from.noteHashes, from.nullifiers, from.l2ToL1Msgs); + // Temporary overrides as these entries aren't yet populated in trace + avmCircuitPublicInputs.previousNonRevertibleAccumulatedDataArrayLengths = getArrayLengths( + nonRevertibleAccumulatedDataFromPrivate, + ); + avmCircuitPublicInputs.previousRevertibleAccumulatedDataArrayLengths = getArrayLengths( + revertibleAccumulatedDataFromPrivate, + ); + avmCircuitPublicInputs.previousNonRevertibleAccumulatedData = convertAccumulatedData( + nonRevertibleAccumulatedDataFromPrivate, + ); + avmCircuitPublicInputs.previousRevertibleAccumulatedData = convertAccumulatedData( + revertibleAccumulatedDataFromPrivate, + ); + + // merge all revertible & non-revertible side effects into output accumulated data + const noteHashesFromPrivate = revertCode.isOK() + ? mergeAccumulatedData( + avmCircuitPublicInputs.previousNonRevertibleAccumulatedData.noteHashes, + avmCircuitPublicInputs.previousRevertibleAccumulatedData.noteHashes, + ) + : avmCircuitPublicInputs.previousNonRevertibleAccumulatedData.noteHashes; + avmCircuitPublicInputs.accumulatedData.noteHashes = assertLength( + mergeAccumulatedData(noteHashesFromPrivate, avmCircuitPublicInputs.accumulatedData.noteHashes), + MAX_NOTE_HASHES_PER_TX, + ); + + const txHash = avmCircuitPublicInputs.previousNonRevertibleAccumulatedData.nullifiers[0]; + + const scopedNoteHashesFromPublic = trace.getSideEffects().noteHashes; + for (let i = 0; i < scopedNoteHashesFromPublic.length; i++) { + const scopedNoteHash = scopedNoteHashesFromPublic[i]; + const noteHash = scopedNoteHash.value; + if (!noteHash.isZero()) { + const noteHashIndexInTx = i + countAccumulatedItems(noteHashesFromPrivate); + const nonce = computeNoteHashNonce(txHash, noteHashIndexInTx); + const uniqueNoteHash = computeUniqueNoteHash(nonce, noteHash); + const siloedNoteHash = siloNoteHash(scopedNoteHash.contractAddress, uniqueNoteHash); + avmCircuitPublicInputs.accumulatedData.noteHashes[noteHashIndexInTx] = siloedNoteHash; + } + } + + const nullifiersFromPrivate = revertCode.isOK() + ? mergeAccumulatedData( + avmCircuitPublicInputs.previousNonRevertibleAccumulatedData.nullifiers, + avmCircuitPublicInputs.previousRevertibleAccumulatedData.nullifiers, + ) + : avmCircuitPublicInputs.previousNonRevertibleAccumulatedData.nullifiers; + avmCircuitPublicInputs.accumulatedData.nullifiers = assertLength( + mergeAccumulatedData(nullifiersFromPrivate, avmCircuitPublicInputs.accumulatedData.nullifiers), + MAX_NULLIFIERS_PER_TX, + ); + const msgsFromPrivate = revertCode.isOK() + ? mergeAccumulatedData( + avmCircuitPublicInputs.previousNonRevertibleAccumulatedData.l2ToL1Msgs, + avmCircuitPublicInputs.previousRevertibleAccumulatedData.l2ToL1Msgs, + ) + : avmCircuitPublicInputs.previousNonRevertibleAccumulatedData.l2ToL1Msgs; + avmCircuitPublicInputs.accumulatedData.l2ToL1Msgs = assertLength( + mergeAccumulatedData(msgsFromPrivate, avmCircuitPublicInputs.accumulatedData.l2ToL1Msgs), + MAX_L2_TO_L1_MSGS_PER_TX, + ); + + const dedupedPublicDataWrites: Array = []; + const leafSlotOccurences: Map = new Map(); + for (const publicDataWrite of avmCircuitPublicInputs.accumulatedData.publicDataWrites) { + const slot = publicDataWrite.leafSlot.toBigInt(); + const prevOccurrences = leafSlotOccurences.get(slot) || 0; + leafSlotOccurences.set(slot, prevOccurrences + 1); + } + + for (const publicDataWrite of avmCircuitPublicInputs.accumulatedData.publicDataWrites) { + const slot = publicDataWrite.leafSlot.toBigInt(); + const prevOccurrences = leafSlotOccurences.get(slot) || 0; + if (prevOccurrences === 1) { + dedupedPublicDataWrites.push(publicDataWrite); + } else { + leafSlotOccurences.set(slot, prevOccurrences - 1); + } + } + + avmCircuitPublicInputs.accumulatedData.publicDataWrites = padArrayEnd( + dedupedPublicDataWrites, + PublicDataWrite.empty(), + MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, + ); + //console.log(`AvmCircuitPublicInputs:\n${inspect(avmCircuitPublicInputs)}`); + return avmCircuitPublicInputs; +} + +export function generateAvmProvingRequest( + real: boolean, + fnName: string, + stateManager: AvmPersistableStateManager, + historicalHeader: Header, + globalVariables: GlobalVariables, + executionRequest: PublicExecutionRequest, + result: EnqueuedPublicCallExecutionResult, + allocatedGas: Gas, + transactionFee: Fr, +): AvmProvingRequest { + const avmExecutionEnv = new AvmExecutionEnvironment( + executionRequest.callContext.contractAddress, + executionRequest.callContext.msgSender, + executionRequest.callContext.functionSelector, + /*contractCallDepth=*/ Fr.zero(), + transactionFee, + globalVariables, + executionRequest.callContext.isStaticCall, + executionRequest.args, + ); + + const avmCallResult = new AvmFinalizedCallResult(result.reverted, result.returnValues, result.endGasLeft); + + // Generate an AVM proving request + let avmProvingRequest: AvmProvingRequest; + if (real) { + const deprecatedFunctionCallResult = stateManager.trace.toPublicFunctionCallResult( + avmExecutionEnv, + /*startGasLeft=*/ allocatedGas, + Buffer.alloc(0), + avmCallResult, + fnName, + ); + const publicInputs = getPublicCircuitPublicInputs(historicalHeader, globalVariables, deprecatedFunctionCallResult); + avmProvingRequest = makeAvmProvingRequest(publicInputs, deprecatedFunctionCallResult); + } else { + avmProvingRequest = emptyAvmProvingRequest(); + } + return avmProvingRequest; +} + +function emptyAvmProvingRequest(): AvmProvingRequest { + return { + type: ProvingRequestType.PUBLIC_VM, + inputs: AvmCircuitInputs.empty(), + }; +} +function makeAvmProvingRequest(inputs: PublicCircuitPublicInputs, result: PublicFunctionCallResult): AvmProvingRequest { + return { + type: ProvingRequestType.PUBLIC_VM, + inputs: new AvmCircuitInputs( + result.functionName, + result.calldata, + inputs, + result.avmCircuitHints, + AvmCircuitPublicInputs.empty(), + ), + }; +} + +function getPublicCircuitPublicInputs( + historicalHeader: Header, + globalVariables: GlobalVariables, + result: PublicFunctionCallResult, +) { + const header = historicalHeader.clone(); // don't modify the original + header.state.partial.publicDataTree.root = Fr.zero(); // AVM doesn't check this yet + + return PublicCircuitPublicInputs.from({ + callContext: result.executionRequest.callContext, + proverAddress: AztecAddress.ZERO, + argsHash: computeVarArgsHash(result.executionRequest.args), + noteHashes: padArrayEnd( + result.noteHashes, + NoteHash.empty(), + MAX_NOTE_HASHES_PER_CALL, + `Too many note hashes. Got ${result.noteHashes.length} with max being ${MAX_NOTE_HASHES_PER_CALL}`, + ), + nullifiers: padArrayEnd( + result.nullifiers, + Nullifier.empty(), + MAX_NULLIFIERS_PER_CALL, + `Too many nullifiers. Got ${result.nullifiers.length} with max being ${MAX_NULLIFIERS_PER_CALL}`, + ), + l2ToL1Msgs: padArrayEnd( + result.l2ToL1Messages, + L2ToL1Message.empty(), + MAX_L2_TO_L1_MSGS_PER_CALL, + `Too many L2 to L1 messages. Got ${result.l2ToL1Messages.length} with max being ${MAX_L2_TO_L1_MSGS_PER_CALL}`, + ), + startSideEffectCounter: result.startSideEffectCounter, + endSideEffectCounter: result.endSideEffectCounter, + returnsHash: computeVarArgsHash(result.returnValues), + noteHashReadRequests: padArrayEnd( + result.noteHashReadRequests, + TreeLeafReadRequest.empty(), + MAX_NOTE_HASH_READ_REQUESTS_PER_CALL, + `Too many note hash read requests. Got ${result.noteHashReadRequests.length} with max being ${MAX_NOTE_HASH_READ_REQUESTS_PER_CALL}`, + ), + nullifierReadRequests: padArrayEnd( + result.nullifierReadRequests, + ReadRequest.empty(), + MAX_NULLIFIER_READ_REQUESTS_PER_CALL, + `Too many nullifier read requests. Got ${result.nullifierReadRequests.length} with max being ${MAX_NULLIFIER_READ_REQUESTS_PER_CALL}`, + ), + nullifierNonExistentReadRequests: padArrayEnd( + result.nullifierNonExistentReadRequests, + ReadRequest.empty(), + MAX_NULLIFIER_NON_EXISTENT_READ_REQUESTS_PER_CALL, + `Too many nullifier non-existent read requests. Got ${result.nullifierNonExistentReadRequests.length} with max being ${MAX_NULLIFIER_NON_EXISTENT_READ_REQUESTS_PER_CALL}`, + ), + l1ToL2MsgReadRequests: padArrayEnd( + result.l1ToL2MsgReadRequests, + TreeLeafReadRequest.empty(), + MAX_L1_TO_L2_MSG_READ_REQUESTS_PER_CALL, + `Too many L1 to L2 message read requests. Got ${result.l1ToL2MsgReadRequests.length} with max being ${MAX_L1_TO_L2_MSG_READ_REQUESTS_PER_CALL}`, + ), + contractStorageReads: padArrayEnd( + result.contractStorageReads, + ContractStorageRead.empty(), + MAX_PUBLIC_DATA_READS_PER_CALL, + `Too many public data reads. Got ${result.contractStorageReads.length} with max being ${MAX_PUBLIC_DATA_READS_PER_CALL}`, + ), + contractStorageUpdateRequests: padArrayEnd( + result.contractStorageUpdateRequests, + ContractStorageUpdateRequest.empty(), + MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_CALL, + `Too many public data update requests. Got ${result.contractStorageUpdateRequests.length} with max being ${MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_CALL}`, + ), + publicCallRequests: padArrayEnd( + result.publicCallRequests, + PublicInnerCallRequest.empty(), + MAX_ENQUEUED_CALLS_PER_CALL, + `Too many public call requests. Got ${result.publicCallRequests.length} with max being ${MAX_ENQUEUED_CALLS_PER_CALL}`, + ), + unencryptedLogsHashes: padArrayEnd( + result.unencryptedLogsHashes, + LogHash.empty(), + MAX_UNENCRYPTED_LOGS_PER_CALL, + `Too many unencrypted logs. Got ${result.unencryptedLogsHashes.length} with max being ${MAX_UNENCRYPTED_LOGS_PER_CALL}`, + ), + historicalHeader: header, + globalVariables: globalVariables, + startGasLeft: Gas.from(result.startGasLeft), + endGasLeft: Gas.from(result.endGasLeft), + transactionFee: result.transactionFee, + // TODO(@just-mitch): need better mapping from simulator to revert code. + revertCode: result.reverted ? RevertCode.APP_LOGIC_REVERTED : RevertCode.OK, + }); +} diff --git a/yarn-project/simulator/src/public/utils.ts b/yarn-project/simulator/src/public/utils.ts index 8d3f6093f75..664fb1d2311 100644 --- a/yarn-project/simulator/src/public/utils.ts +++ b/yarn-project/simulator/src/public/utils.ts @@ -1,36 +1,5 @@ import { type PublicExecutionRequest, type Tx, TxExecutionPhase } from '@aztec/circuit-types'; -import { - AvmAccumulatedData, - AvmCircuitPublicInputs, - type CombinedAccumulatedData, - CombinedConstantData, - EnqueuedCallData, - type Fr, - type Gas, - type GlobalVariables, - type KernelCircuitPublicInputs, - NESTED_RECURSIVE_PROOF_LENGTH, - type PrivateKernelTailCircuitPublicInputs, - PrivateToAvmAccumulatedData, - PrivateToAvmAccumulatedDataArrayLengths, - type PrivateToPublicAccumulatedData, - PublicAccumulatedData, - type PublicCallRequest, - PublicKernelCircuitPrivateInputs, - PublicKernelCircuitPublicInputs, - PublicKernelData, - PublicValidationRequests, - RevertCode, - TreeSnapshots, - type VMCircuitPublicInputs, - VerificationKeyData, - countAccumulatedItems, - makeEmptyProof, - makeEmptyRecursiveProof, -} from '@aztec/circuits.js'; -import { getVKSiblingPath } from '@aztec/noir-protocol-circuits-types'; - -import { type PublicKernelCircuitSimulator } from './public_kernel_circuit_simulator.js'; +import { type PublicCallRequest } from '@aztec/circuits.js'; export function getExecutionRequestsByPhase(tx: Tx, phase: TxExecutionPhase): PublicExecutionRequest[] { switch (phase) { @@ -61,122 +30,3 @@ export function getCallRequestsByPhase(tx: Tx, phase: TxExecutionPhase): PublicC throw new Error(`Unknown phase: ${phase}`); } } - -// Temporary hack to create PublicKernelCircuitPublicInputs from PrivateKernelTailCircuitPublicInputs. -export function getPublicKernelCircuitPublicInputs( - data: PrivateKernelTailCircuitPublicInputs, - globalVariables: GlobalVariables, -) { - const constants = CombinedConstantData.combine(data.constants, globalVariables); - - const validationRequest = PublicValidationRequests.empty(); - validationRequest.forRollup = data.rollupValidationRequests; - - const convertAccumulatedData = (from: PrivateToPublicAccumulatedData) => { - const to = PublicAccumulatedData.empty(); - to.noteHashes.forEach((_, i) => (to.noteHashes[i].noteHash.value = from.noteHashes[i])); - to.nullifiers.forEach((_, i) => (to.nullifiers[i].value = from.nullifiers[i])); - to.l2ToL1Msgs.forEach((_, i) => (to.l2ToL1Msgs[i] = from.l2ToL1Msgs[i])); - to.noteEncryptedLogsHashes.forEach((_, i) => (to.noteEncryptedLogsHashes[i] = from.noteEncryptedLogsHashes[i])); - to.encryptedLogsHashes.forEach((_, i) => (to.encryptedLogsHashes[i] = from.encryptedLogsHashes[i])); - to.publicCallStack.forEach((_, i) => (to.publicCallStack[i] = from.publicCallRequests[i])); - return to; - }; - - return new PublicKernelCircuitPublicInputs( - constants, - validationRequest, - convertAccumulatedData(data.forPublic!.nonRevertibleAccumulatedData), - convertAccumulatedData(data.forPublic!.revertibleAccumulatedData), - 0, - data.forPublic!.publicTeardownCallRequest, - data.feePayer, - RevertCode.OK, - ); -} - -// Temporary hack to create the AvmCircuitPublicInputs from public tail's public inputs. -export function generateAvmCircuitPublicInputs( - tx: Tx, - tailOutput: KernelCircuitPublicInputs, - gasUsedForFee: Gas, - transactionFee: Fr, -) { - const startTreeSnapshots = new TreeSnapshots( - tailOutput.constants.historicalHeader.state.l1ToL2MessageTree, - tailOutput.startState.noteHashTree, - tailOutput.startState.nullifierTree, - tailOutput.startState.publicDataTree, - ); - - const getArrayLengths = (from: PrivateToPublicAccumulatedData) => - new PrivateToAvmAccumulatedDataArrayLengths( - countAccumulatedItems(from.noteHashes), - countAccumulatedItems(from.nullifiers), - countAccumulatedItems(from.l2ToL1Msgs), - ); - - const convertAccumulatedData = (from: PrivateToPublicAccumulatedData) => - new PrivateToAvmAccumulatedData(from.noteHashes, from.nullifiers, from.l2ToL1Msgs); - - const convertAvmAccumulatedData = (from: CombinedAccumulatedData) => - new AvmAccumulatedData( - from.noteHashes, - from.nullifiers, - from.l2ToL1Msgs, - from.unencryptedLogsHashes, - from.publicDataWrites, - ); - - // This is wrong. But this is not used or checked in the rollup at the moment. - // Should fetch the updated roots from db. - const endTreeSnapshots = startTreeSnapshots; - - const avmCircuitpublicInputs = new AvmCircuitPublicInputs( - tailOutput.constants.globalVariables, - startTreeSnapshots, - tx.data.gasUsed, - tx.data.constants.txContext.gasSettings, - tx.data.forPublic!.nonRevertibleAccumulatedData.publicCallRequests, - tx.data.forPublic!.revertibleAccumulatedData.publicCallRequests, - tx.data.forPublic!.publicTeardownCallRequest, - getArrayLengths(tx.data.forPublic!.nonRevertibleAccumulatedData), - getArrayLengths(tx.data.forPublic!.revertibleAccumulatedData), - convertAccumulatedData(tx.data.forPublic!.nonRevertibleAccumulatedData), - convertAccumulatedData(tx.data.forPublic!.revertibleAccumulatedData), - endTreeSnapshots, - gasUsedForFee, - convertAvmAccumulatedData(tailOutput.end), - transactionFee, - !tailOutput.revertCode.equals(RevertCode.OK), - ); - //console.log(`[FROM TAIL] AVM: ${inspect(avmCircuitpublicInputs, { depth: 5 })}`); - return avmCircuitpublicInputs; -} - -function getPreviousKernelData(previousOutput: PublicKernelCircuitPublicInputs): PublicKernelData { - // The proof is not used in simulation. - const proof = makeEmptyRecursiveProof(NESTED_RECURSIVE_PROOF_LENGTH); - - const vk = VerificationKeyData.makeFakeHonk(); - const vkIndex = 0; - const siblingPath = getVKSiblingPath(vkIndex); - - return new PublicKernelData(previousOutput, proof, vk, vkIndex, siblingPath); -} - -export async function runMergeKernelCircuit( - previousOutput: PublicKernelCircuitPublicInputs, - enqueuedCallData: VMCircuitPublicInputs, - publicKernelSimulator: PublicKernelCircuitSimulator, -): Promise { - const previousKernel = getPreviousKernelData(previousOutput); - - // The proof is not used in simulation. - const vmProof = makeEmptyProof(); - const callData = new EnqueuedCallData(enqueuedCallData, vmProof); - - const inputs = new PublicKernelCircuitPrivateInputs(previousKernel, callData); - - return await publicKernelSimulator.publicKernelCircuitMerge(inputs); -} diff --git a/yarn-project/telemetry-client/src/attributes.ts b/yarn-project/telemetry-client/src/attributes.ts index a1a2a21a550..32ce263917a 100644 --- a/yarn-project/telemetry-client/src/attributes.ts +++ b/yarn-project/telemetry-client/src/attributes.ts @@ -54,6 +54,8 @@ export const BLOCK_TXS_COUNT = 'aztec.block.txs_count'; export const BLOCK_SIZE = 'aztec.block.size'; /** How many blocks are included in this epoch */ export const EPOCH_SIZE = 'aztec.epoch.size'; +/** The proposer of a block */ +export const BLOCK_PROPOSER = 'aztec.block.proposer'; /** The epoch number */ export const EPOCH_NUMBER = 'aztec.epoch.number'; /** The tx hash */ diff --git a/yarn-project/telemetry-client/src/metrics.ts b/yarn-project/telemetry-client/src/metrics.ts index 4ff83135235..6e097754e03 100644 --- a/yarn-project/telemetry-client/src/metrics.ts +++ b/yarn-project/telemetry-client/src/metrics.ts @@ -166,3 +166,6 @@ export const WORLD_STATE_LEAF_INDICES_DB_NUM_ITEMS_MESSAGE = 'aztec.world_state. export const WORLD_STATE_LEAF_INDICES_DB_NUM_ITEMS_NOTE_HASH = 'aztec.world_state.db_num_items.leaf_indices.note_hash'; export const PROOF_VERIFIER_COUNT = 'aztec.proof_verifier.count'; + +export const VALIDATOR_RE_EXECUTION_TIME = 'aztec.validator.re_execution_time'; +export const VALIDATOR_FAILED_REEXECUTION_COUNT = 'aztec.validator.failed_reexecution_count'; diff --git a/yarn-project/txe/src/oracle/txe_oracle.ts b/yarn-project/txe/src/oracle/txe_oracle.ts index b6c0f55b7ec..de40b91e8f4 100644 --- a/yarn-project/txe/src/oracle/txe_oracle.ts +++ b/yarn-project/txe/src/oracle/txe_oracle.ts @@ -13,27 +13,25 @@ import { import { type CircuitWitnessGenerationStats } from '@aztec/circuit-types/stats'; import { CallContext, - CombinedConstantData, type ContractInstance, type ContractInstanceWithAddress, - DEFAULT_GAS_LIMIT, Gas, + GasFees, + GlobalVariables, Header, IndexedTaggingSecret, type KeyValidationRequest, type L1_TO_L2_MSG_TREE_HEIGHT, - MAX_L2_GAS_PER_ENQUEUED_CALL, NULLIFIER_SUBTREE_HEIGHT, type NULLIFIER_TREE_HEIGHT, type NullifierLeafPreimage, PRIVATE_CONTEXT_INPUTS_LENGTH, - PUBLIC_DATA_SUBTREE_HEIGHT, type PUBLIC_DATA_TREE_HEIGHT, PUBLIC_DISPATCH_SELECTOR, PrivateContextInputs, PublicDataTreeLeaf, type PublicDataTreeLeafPreimage, - type PublicDataUpdateRequest, + type PublicDataWrite, computeContractClassId, computeTaggingSecret, deriveKeys, @@ -62,7 +60,8 @@ import { type NoteData, Oracle, type PackedValuesCache, - PublicExecutor, + type PublicTxResult, + PublicTxSimulator, type TypedOracle, acvm, createSimulationError, @@ -73,10 +72,10 @@ import { toACVMWitness, witnessMapToFields, } from '@aztec/simulator'; +import { createTxForPublicCall } from '@aztec/simulator/public/fixtures'; import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { MerkleTreeSnapshotOperationsFacade, type MerkleTrees } from '@aztec/world-state'; -import { type EnqueuedPublicCallExecutionResultWithSideEffects } from '../../../simulator/src/public/execution.js'; import { type TXEDatabase } from '../util/txe_database.js'; import { TXEPublicContractDataSource } from '../util/txe_public_contract_data_source.js'; import { TXEWorldStateDB } from '../util/txe_world_state_db.js'; @@ -222,26 +221,17 @@ export class TXE implements TypedOracle { return this.txeDatabase.addAuthWitness(authWitness.requestHash, authWitness.witness); } - async addPublicDataWrites(writes: PublicDataUpdateRequest[]) { + async addPublicDataWrites(writes: PublicDataWrite[]) { const db = await this.trees.getLatest(); - - // only insert the last write to a given slot - const uniqueWritesSet = new Map(); - for (const write of writes) { - uniqueWritesSet.set(write.leafSlot.toBigInt(), write); - } - const uniqueWrites = Array.from(uniqueWritesSet.values()); - await db.batchInsert( MerkleTreeId.PUBLIC_DATA_TREE, - uniqueWrites.map(w => new PublicDataTreeLeaf(w.leafSlot, w.newValue).toBuffer()), + writes.map(w => new PublicDataTreeLeaf(w.leafSlot, w.value).toBuffer()), 0, ); } async addSiloedNullifiers(siloedNullifiers: Fr[]) { const db = await this.trees.getLatest(); - await db.batchInsert( MerkleTreeId.NULLIFIER_TREE, siloedNullifiers.map(n => n.toBuffer()), @@ -254,11 +244,14 @@ export class TXE implements TypedOracle { await this.addSiloedNullifiers(siloedNullifiers); } - async addNoteHashes(contractAddress: AztecAddress, noteHashes: Fr[]) { + async addSiloedNoteHashes(siloedNoteHashes: Fr[]) { const db = await this.trees.getLatest(); - const siloedNoteHashes = noteHashes.map(noteHash => siloNoteHash(contractAddress, noteHash)); await db.appendLeaves(MerkleTreeId.NOTE_HASH_TREE, siloedNoteHashes); } + async addNoteHashes(contractAddress: AztecAddress, noteHashes: Fr[]) { + const siloedNoteHashes = noteHashes.map(noteHash => siloNoteHash(contractAddress, noteHash)); + await this.addSiloedNoteHashes(siloedNoteHashes); + } // TypedOracle @@ -511,7 +504,7 @@ export class TXE implements TypedOracle { await db.batchInsert( MerkleTreeId.PUBLIC_DATA_TREE, publicDataWrites.map(write => write.toBuffer()), - PUBLIC_DATA_SUBTREE_HEIGHT, + 0, ); return publicDataWrites.map(write => write.value); } @@ -655,39 +648,32 @@ export class TXE implements TypedOracle { return `${artifact.name}:${f.name}`; } - private async executePublicFunction(args: Fr[], callContext: CallContext) { - const execution = new PublicExecutionRequest(callContext, args); + private async executePublicFunction(args: Fr[], callContext: CallContext, isTeardown: boolean = false) { + const executionRequest = new PublicExecutionRequest(callContext, args); const db = await this.trees.getLatest(); - const previousBlockState = await this.#getTreesAt(this.blockNumber - 1); - - const combinedConstantData = CombinedConstantData.empty(); - combinedConstantData.globalVariables.chainId = this.chainId; - combinedConstantData.globalVariables.version = this.version; - combinedConstantData.globalVariables.blockNumber = new Fr(this.blockNumber); - combinedConstantData.historicalHeader.globalVariables.chainId = this.chainId; - combinedConstantData.historicalHeader.globalVariables.version = this.version; - combinedConstantData.historicalHeader.globalVariables.blockNumber = new Fr(this.blockNumber - 1); - combinedConstantData.historicalHeader.state = await db.getStateReference(); - combinedConstantData.historicalHeader.lastArchive.root = Fr.fromBuffer( - (await previousBlockState.getTreeInfo(MerkleTreeId.ARCHIVE)).root, - ); - combinedConstantData.txContext.chainId = this.chainId; - combinedConstantData.txContext.version = this.version; + const globalVariables = GlobalVariables.empty(); + globalVariables.chainId = this.chainId; + globalVariables.version = this.version; + globalVariables.blockNumber = new Fr(this.blockNumber); + globalVariables.gasFees = GasFees.default(); - const simulator = new PublicExecutor( + const simulator = new PublicTxSimulator( + db, new TXEWorldStateDB(db, new TXEPublicContractDataSource(this)), new NoopTelemetryClient(), + globalVariables, + /*realAvmProvingRequests=*/ false, ); - const executionResult = await simulator.simulateIsolatedEnqueuedCall( - execution, - combinedConstantData.globalVariables, - /*allocatedGas*/ new Gas(DEFAULT_GAS_LIMIT, MAX_L2_GAS_PER_ENQUEUED_CALL), - /*transactionFee=*/ Fr.ONE, - /*startSideEffectCounter=*/ this.sideEffectCounter, - ); - return Promise.resolve(executionResult); + + // When setting up a teardown call, we tell it that + // private execution used Gas(1, 1) so it can compute a tx fee. + const gasUsedByPrivate = isTeardown ? new Gas(1, 1) : Gas.empty(); + const tx = createTxForPublicCall(executionRequest, gasUsedByPrivate, isTeardown); + + const result = await simulator.simulate(tx); + return Promise.resolve(result); } async enqueuePublicFunctionCall( @@ -696,6 +682,7 @@ export class TXE implements TypedOracle { argsHash: Fr, _sideEffectCounter: number, isStaticCall: boolean, + isTeardown = false, ): Promise { // Store and modify env const currentContractAddress = this.contractAddress; @@ -715,10 +702,10 @@ export class TXE implements TypedOracle { const args = [this.functionSelector.toField(), ...this.packedValuesCache.unpack(argsHash)]; const newArgsHash = this.packedValuesCache.pack(args); - const executionResult = await this.executePublicFunction(args, callContext); + const executionResult = await this.executePublicFunction(args, callContext, isTeardown); // Poor man's revert handling - if (executionResult.reverted) { + if (!executionResult.revertCode.isOK()) { if (executionResult.revertReason && executionResult.revertReason instanceof SimulationError) { await enrichPublicSimulationError( executionResult.revertReason, @@ -733,13 +720,13 @@ export class TXE implements TypedOracle { } // Apply side effects - this.sideEffectCounter = executionResult.endSideEffectCounter.toNumber(); - await this.addPublicDataWrites(executionResult.sideEffects.publicDataWrites); - await this.addNoteHashes( - targetContractAddress, - executionResult.sideEffects.noteHashes.map(noteHash => noteHash.value), - ); - await this.addSiloedNullifiers(executionResult.sideEffects.nullifiers.map(nullifier => nullifier.value)); + const sideEffects = executionResult.avmProvingRequest.inputs.output.accumulatedData; + const publicDataWrites = sideEffects.publicDataWrites.filter(s => !s.isEmpty()); + const noteHashes = sideEffects.noteHashes.filter(s => !s.isEmpty()); + const nullifiers = sideEffects.nullifiers.filter(s => !s.isEmpty()); + await this.addPublicDataWrites(publicDataWrites); + await this.addSiloedNoteHashes(noteHashes); + await this.addSiloedNullifiers(nullifiers); this.setContractAddress(currentContractAddress); this.setMsgSender(currentMessageSender); @@ -763,6 +750,7 @@ export class TXE implements TypedOracle { argsHash, sideEffectCounter, isStaticCall, + /*isTeardown=*/ true, ); } @@ -785,8 +773,9 @@ export class TXE implements TypedOracle { } async incrementAppTaggingSecretIndexAsSender(sender: AztecAddress, recipient: AztecAddress): Promise { - const directionalSecret = await this.#calculateTaggingSecret(this.contractAddress, sender, recipient); - await this.txeDatabase.incrementTaggingSecretsIndexesAsSender([directionalSecret]); + const appSecret = await this.#calculateTaggingSecret(this.contractAddress, sender, recipient); + const [index] = await this.txeDatabase.getTaggingSecretsIndexesAsSender([appSecret]); + await this.txeDatabase.setTaggingSecretsIndexesAsSender([new IndexedTaggingSecret(appSecret, index + 1)]); } async getAppTaggingSecretAsSender(sender: AztecAddress, recipient: AztecAddress): Promise { @@ -811,11 +800,7 @@ export class TXE implements TypedOracle { // AVM oracles - async avmOpcodeCall( - targetContractAddress: AztecAddress, - args: Fr[], - isStaticCall: boolean, - ): Promise { + async avmOpcodeCall(targetContractAddress: AztecAddress, args: Fr[], isStaticCall: boolean): Promise { // Store and modify env const currentContractAddress = this.contractAddress; const currentMessageSender = this.msgSender; @@ -831,17 +816,17 @@ export class TXE implements TypedOracle { const executionResult = await this.executePublicFunction(args, callContext); // Save return/revert data for later. - this.nestedCallReturndata = executionResult.returnValues; + this.nestedCallReturndata = executionResult.processedPhases[0]!.returnValues[0].values!; // Apply side effects - if (!executionResult.reverted) { - this.sideEffectCounter = executionResult.endSideEffectCounter.toNumber(); - await this.addPublicDataWrites(executionResult.sideEffects.publicDataWrites); - await this.addNoteHashes( - targetContractAddress, - executionResult.sideEffects.noteHashes.map(noteHash => noteHash.value), - ); - await this.addSiloedNullifiers(executionResult.sideEffects.nullifiers.map(nullifier => nullifier.value)); + if (executionResult.revertCode.isOK()) { + const sideEffects = executionResult.avmProvingRequest.inputs.output.accumulatedData; + const publicDataWrites = sideEffects.publicDataWrites.filter(s => !s.isEmpty()); + const noteHashes = sideEffects.noteHashes.filter(s => !s.isEmpty()); + const nullifiers = sideEffects.nullifiers.filter(s => !s.isEmpty()); + await this.addPublicDataWrites(publicDataWrites); + await this.addSiloedNoteHashes(noteHashes); + await this.addSiloedNullifiers(nullifiers); } this.setContractAddress(currentContractAddress); diff --git a/yarn-project/txe/src/txe_service/txe_service.ts b/yarn-project/txe/src/txe_service/txe_service.ts index 75c7f5b5a20..5c7cb4d6c60 100644 --- a/yarn-project/txe/src/txe_service/txe_service.ts +++ b/yarn-project/txe/src/txe_service/txe_service.ts @@ -4,7 +4,6 @@ import { Fr, FunctionSelector, Header, - PUBLIC_DATA_SUBTREE_HEIGHT, PublicDataTreeLeaf, PublicKeys, computePartialAddress, @@ -157,7 +156,7 @@ export class TXEService { await db.batchInsert( MerkleTreeId.PUBLIC_DATA_TREE, publicDataWrites.map(write => write.toBuffer()), - PUBLIC_DATA_SUBTREE_HEIGHT, + 0, ); return toForeignCallResult([toArray(publicDataWrites.map(write => write.value))]); } @@ -222,7 +221,7 @@ export class TXEService { const parsedSelector = fromSingle(functionSelector); const extendedArgs = [parsedSelector, ...fromArray(args)]; const result = await (this.typedOracle as TXE).avmOpcodeCall(parsedAddress, extendedArgs, false); - if (!result.reverted) { + if (result.revertCode.isOK()) { throw new ExpectedFailureError('Public call did not revert'); } @@ -738,7 +737,7 @@ export class TXEService { ); // Poor man's revert handling - if (result.reverted) { + if (!result.revertCode.isOK()) { if (result.revertReason && result.revertReason instanceof SimulationError) { await enrichPublicSimulationError( result.revertReason, @@ -752,7 +751,7 @@ export class TXEService { } } - return toForeignCallResult([toSingle(new Fr(!result.reverted))]); + return toForeignCallResult([toSingle(new Fr(result.revertCode.isOK()))]); } async avmOpcodeStaticCall( @@ -768,7 +767,7 @@ export class TXEService { ); // Poor man's revert handling - if (result.reverted) { + if (!result.revertCode.isOK()) { if (result.revertReason && result.revertReason instanceof SimulationError) { await enrichPublicSimulationError( result.revertReason, @@ -782,6 +781,6 @@ export class TXEService { } } - return toForeignCallResult([toSingle(new Fr(!result.reverted))]); + return toForeignCallResult([toSingle(new Fr(result.revertCode.isOK()))]); } } diff --git a/yarn-project/txe/src/util/txe_public_contract_data_source.ts b/yarn-project/txe/src/util/txe_public_contract_data_source.ts index 437bed249cf..1ae75b00a60 100644 --- a/yarn-project/txe/src/util/txe_public_contract_data_source.ts +++ b/yarn-project/txe/src/util/txe_public_contract_data_source.ts @@ -71,4 +71,10 @@ export class TXEPublicContractDataSource implements ContractDataSource { addContractArtifact(address: AztecAddress, contract: ContractArtifact): Promise { return this.txeOracle.addContractArtifact(contract); } + + // TODO(#10007): Remove this method. + addContractClass(_contractClass: ContractClassPublic): Promise { + // We don't really need to do anything for the txe here + return Promise.resolve(); + } } diff --git a/yarn-project/txe/src/util/txe_world_state_db.ts b/yarn-project/txe/src/util/txe_world_state_db.ts index f5545c6f8ba..d1a2b6f1737 100644 --- a/yarn-project/txe/src/util/txe_world_state_db.ts +++ b/yarn-project/txe/src/util/txe_world_state_db.ts @@ -3,7 +3,6 @@ import { type AztecAddress, type ContractDataSource, Fr, - PUBLIC_DATA_SUBTREE_HEIGHT, PublicDataTreeLeaf, type PublicDataTreeLeafPreimage, } from '@aztec/circuits.js'; @@ -35,7 +34,7 @@ export class TXEWorldStateDB extends WorldStateDB { await this.merkleDb.batchInsert( MerkleTreeId.PUBLIC_DATA_TREE, [new PublicDataTreeLeaf(computePublicDataTreeLeafSlot(contract, slot), newValue).toBuffer()], - PUBLIC_DATA_SUBTREE_HEIGHT, + 0, ); return newValue.toBigInt(); } diff --git a/yarn-project/types/src/abi/contract_artifact.ts b/yarn-project/types/src/abi/contract_artifact.ts index 4fc0616e78c..0141145e7b3 100644 --- a/yarn-project/types/src/abi/contract_artifact.ts +++ b/yarn-project/types/src/abi/contract_artifact.ts @@ -4,16 +4,15 @@ import { type AbiType, type BasicValue, type ContractArtifact, + ContractArtifactSchema, type ContractNote, type FieldLayout, type FunctionArtifact, FunctionType, type IntegerValue, - NoteSelector, type StructValue, type TypedStructFieldValue, } from '@aztec/foundation/abi'; -import { Fr } from '@aztec/foundation/fields'; import { AZTEC_INITIALIZER_ATTRIBUTE, @@ -53,18 +52,7 @@ export function contractArtifactToBuffer(artifact: ContractArtifact): Buffer { * @returns Deserialized artifact. */ export function contractArtifactFromBuffer(buffer: Buffer): ContractArtifact { - return JSON.parse(buffer.toString('utf-8'), (key, value) => { - if (key === 'bytecode' && typeof value === 'string') { - return Buffer.from(value, 'base64'); - } - if (typeof value === 'object' && value !== null && value.type === 'NoteSelector') { - return new NoteSelector(Number(value.value)); - } - if (typeof value === 'object' && value !== null && value.type === 'Fr') { - return new Fr(BigInt(value.value)); - } - return value; - }); + return ContractArtifactSchema.parse(JSON.parse(buffer.toString('utf-8'))); } /** @@ -133,7 +121,10 @@ type NoirCompiledContractFunction = NoirCompiledContract['functions'][number]; * @param contract - Parent contract. * @returns Function artifact. */ -function generateFunctionArtifact(fn: NoirCompiledContractFunction, contract: NoirCompiledContract): FunctionArtifact { +function generateFunctionArtifact( + fn: NoirCompiledContractFunction, + contract: NoirCompiledContract, +): Omit & { bytecode: string } { if (fn.custom_attributes === undefined) { throw new Error( `No custom attributes found for contract function ${fn.name}. Try rebuilding the contract with the latest nargo version.`, @@ -178,7 +169,7 @@ function generateFunctionArtifact(fn: NoirCompiledContractFunction, contract: No isInitializer: fn.custom_attributes.includes(AZTEC_INITIALIZER_ATTRIBUTE), parameters, returnTypes, - bytecode: Buffer.from(fn.bytecode, 'base64'), + bytecode: fn.bytecode, debugSymbols: fn.debug_symbols, errorTypes: fn.abi.error_types, ...(fn.assert_messages ? { assertMessages: fn.assert_messages } : undefined), @@ -238,11 +229,11 @@ function getStorageLayout(input: NoirCompiledContract) { return {}; } - return storageFields.reduce((acc: Record, field) => { + return storageFields.reduce((acc: Record & { slot: string }>, field) => { const name = field.name; const slot = field.value.fields[0].value as IntegerValue; acc[name] = { - slot: Fr.fromString(slot.value), + slot: slot.value, }; return acc; }, {}); @@ -262,7 +253,7 @@ function getNoteTypes(input: NoirCompiledContract) { return {}; } - return notes.reduce((acc: Record, note) => { + return notes.reduce((acc: Record & { id: string }>, note) => { const noteFields = note.fields; // We find note type id by looking for respective kinds as each of them is unique @@ -274,7 +265,7 @@ function getNoteTypes(input: NoirCompiledContract) { throw new Error(`Could not find note type id, name or fields for note ${note}`); } - const noteTypeId = NoteSelector.fromField(Fr.fromString(rawNoteTypeId.value)); + const noteTypeId = rawNoteTypeId.value as string; const name = rawName.value as string; // Note type id is encoded as a hex string @@ -301,7 +292,7 @@ function getNoteTypes(input: NoirCompiledContract) { */ function generateContractArtifact(contract: NoirCompiledContract, aztecNrVersion?: string): ContractArtifact { try { - return { + return ContractArtifactSchema.parse({ name: contract.name, functions: contract.functions.map(f => generateFunctionArtifact(f, contract)), outputs: contract.outputs, @@ -309,7 +300,7 @@ function generateContractArtifact(contract: NoirCompiledContract, aztecNrVersion notes: getNoteTypes(contract), fileMap: contract.file_map, ...(aztecNrVersion ? { aztecNrVersion } : {}), - }; + }); } catch (err) { throw new Error(`Could not generate contract artifact for ${contract.name}: ${err}`); } diff --git a/yarn-project/validator-client/src/config.ts b/yarn-project/validator-client/src/config.ts index 4adb17a82e6..075f6249920 100644 --- a/yarn-project/validator-client/src/config.ts +++ b/yarn-project/validator-client/src/config.ts @@ -22,6 +22,9 @@ export interface ValidatorClientConfig { /** Wait for attestations timeout */ attestationWaitTimeoutMs: number; + + /** Re-execute transactions before attesting */ + validatorReexecute: boolean; } export const validatorClientConfigMappings: ConfigMappingsType = { @@ -48,6 +51,11 @@ export const validatorClientConfigMappings: ConfigMappingsType void { + const start = performance.now(); + return () => { + const end = performance.now(); + this.recordReExecutionTime(end - start); + }; + } + + public recordReExecutionTime(time: number) { + this.reExecutionTime.record(time); + } + + public recordFailedReexecution(proposal: BlockProposal) { + this.failedReexecutionCounter.add(1, { + [Attributes.STATUS]: 'failed', + [Attributes.BLOCK_NUMBER]: proposal.payload.header.globalVariables.blockNumber.toString(), + [Attributes.BLOCK_PROPOSER]: proposal.getSender()?.toString(), + }); + } +} diff --git a/yarn-project/validator-client/src/validator.test.ts b/yarn-project/validator-client/src/validator.test.ts index ffce55a4d21..d60937f2fcc 100644 --- a/yarn-project/validator-client/src/validator.test.ts +++ b/yarn-project/validator-client/src/validator.test.ts @@ -1,7 +1,7 @@ /** * Validation logic unit tests */ -import { TxHash } from '@aztec/circuit-types'; +import { TxHash, mockTx } from '@aztec/circuit-types'; import { makeHeader } from '@aztec/circuits.js/testing'; import { Secp256k1Signer } from '@aztec/foundation/crypto'; import { EthAddress } from '@aztec/foundation/eth-address'; @@ -17,6 +17,7 @@ import { makeBlockAttestation, makeBlockProposal } from '../../circuit-types/src import { type ValidatorClientConfig } from './config.js'; import { AttestationTimeoutError, + BlockBuilderNotProvidedError, InvalidValidatorPrivateKeyError, TransactionsNotAvailableError, } from './errors/validator.error.js'; @@ -40,6 +41,7 @@ describe('ValidationService', () => { attestationPollingIntervalMs: 1000, attestationWaitTimeoutMs: 1000, disableValidator: false, + validatorReexecute: false, }; validatorClient = ValidatorClient.new(config, p2pClient, new NoopTelemetryClient()); }); @@ -51,6 +53,13 @@ describe('ValidationService', () => { ); }); + it('Should throw an error if re-execution is enabled but no block builder is provided', async () => { + config.validatorReexecute = true; + p2pClient.getTxByHash.mockImplementation(() => Promise.resolve(mockTx())); + const val = ValidatorClient.new(config, p2pClient); + await expect(val.reExecuteTransactions(makeBlockProposal())).rejects.toThrow(BlockBuilderNotProvidedError); + }); + it('Should create a valid block proposal', async () => { const header = makeHeader(); const archive = Fr.random(); @@ -83,6 +92,21 @@ describe('ValidationService', () => { ); }); + it('Should not return an attestation if re-execution fails', async () => { + const proposal = makeBlockProposal(); + + // mock the p2pClient.getTxStatus to return undefined for all transactions + p2pClient.getTxStatus.mockImplementation(() => undefined); + + const val = ValidatorClient.new(config, p2pClient, new NoopTelemetryClient()); + val.registerBlockBuilder(() => { + throw new Error('Failed to build block'); + }); + + const attestation = await val.attestToProposal(proposal); + expect(attestation).toBeUndefined(); + }); + it('Should collect attestations for a proposal', async () => { const signer = Secp256k1Signer.random(); const attestor1 = Secp256k1Signer.random(); diff --git a/yarn-project/validator-client/src/validator.ts b/yarn-project/validator-client/src/validator.ts index bf8efbe13c5..7ced2639ab1 100644 --- a/yarn-project/validator-client/src/validator.ts +++ b/yarn-project/validator-client/src/validator.ts @@ -1,25 +1,50 @@ -import { type BlockAttestation, type BlockProposal, type TxHash } from '@aztec/circuit-types'; -import { type Header } from '@aztec/circuits.js'; +import { + type BlockAttestation, + type BlockProposal, + type L2Block, + type ProcessedTx, + type Tx, + type TxHash, +} from '@aztec/circuit-types'; +import { type GlobalVariables, type Header } from '@aztec/circuits.js'; import { Buffer32 } from '@aztec/foundation/buffer'; import { type Fr } from '@aztec/foundation/fields'; -import { attachedFixedDataToLogger, createDebugLogger } from '@aztec/foundation/log'; +import { createDebugLogger } from '@aztec/foundation/log'; import { sleep } from '@aztec/foundation/sleep'; +import { type Timer } from '@aztec/foundation/timer'; import { type P2P } from '@aztec/p2p'; import { type TelemetryClient, WithTracer } from '@aztec/telemetry-client'; +import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { type ValidatorClientConfig } from './config.js'; import { ValidationService } from './duties/validation_service.js'; import { AttestationTimeoutError, + BlockBuilderNotProvidedError, InvalidValidatorPrivateKeyError, + ReExStateMismatchError, TransactionsNotAvailableError, } from './errors/validator.error.js'; import { type ValidatorKeyStore } from './key_store/interface.js'; import { LocalKeyStore } from './key_store/local_key_store.js'; +import { ValidatorMetrics } from './metrics.js'; + +/** + * Callback function for building a block + * + * We reuse the sequencer's block building functionality for re-execution + */ +type BlockBuilderCallback = ( + txs: Tx[], + globalVariables: GlobalVariables, + historicalHeader?: Header, + interrupt?: (processedTxs: ProcessedTx[]) => Promise, +) => Promise<{ block: L2Block; publicProcessorDuration: number; numProcessedTxs: number; blockBuildingTimer: Timer }>; export interface Validator { start(): Promise; registerBlockProposalHandler(): void; + registerBlockBuilder(blockBuilder: BlockBuilderCallback): void; // Block validation responsiblities createBlockProposal(header: Header, archive: Fr, txs: TxHash[]): Promise; @@ -29,30 +54,33 @@ export interface Validator { collectAttestations(proposal: BlockProposal, numberOfRequiredAttestations: number): Promise; } -/** Validator Client +/** + * Validator Client */ export class ValidatorClient extends WithTracer implements Validator { private validationService: ValidationService; + private metrics: ValidatorMetrics; + + // Callback registered to: sequencer.buildBlock + private blockBuilder?: BlockBuilderCallback = undefined; constructor( keyStore: ValidatorKeyStore, private p2pClient: P2P, - private attestationPollingIntervalMs: number, - private attestationWaitTimeoutMs: number, - telemetry: TelemetryClient, - private log = attachedFixedDataToLogger(createDebugLogger('aztec:validator'), { - validatorAddress: keyStore.getAddress().toString(), - }), + private config: ValidatorClientConfig, + telemetry: TelemetryClient = new NoopTelemetryClient(), + private log = createDebugLogger('aztec:validator'), ) { // Instantiate tracer super(telemetry, 'Validator'); + this.metrics = new ValidatorMetrics(telemetry); //TODO: We need to setup and store all of the currently active validators https://github.com/AztecProtocol/aztec-packages/issues/7962 this.validationService = new ValidationService(keyStore); this.log.verbose('Initialized validator'); } - static new(config: ValidatorClientConfig, p2pClient: P2P, telemetry: TelemetryClient) { + static new(config: ValidatorClientConfig, p2pClient: P2P, telemetry: TelemetryClient = new NoopTelemetryClient()) { if (!config.validatorPrivateKey) { throw new InvalidValidatorPrivateKeyError(); } @@ -60,13 +88,7 @@ export class ValidatorClient extends WithTracer implements Validator { const privateKey = validatePrivateKey(config.validatorPrivateKey); const localKeyStore = new LocalKeyStore(privateKey); - const validator = new ValidatorClient( - localKeyStore, - p2pClient, - config.attestationPollingIntervalMs, - config.attestationWaitTimeoutMs, - telemetry, - ); + const validator = new ValidatorClient(localKeyStore, p2pClient, config, telemetry); validator.registerBlockProposalHandler(); return validator; } @@ -86,17 +108,34 @@ export class ValidatorClient extends WithTracer implements Validator { this.p2pClient.registerBlockProposalHandler(handler); } + /** + * Register a callback function for building a block + * + * We reuse the sequencer's block building functionality for re-execution + */ + public registerBlockBuilder(blockBuilder: BlockBuilderCallback) { + this.blockBuilder = blockBuilder; + } + async attestToProposal(proposal: BlockProposal): Promise { // Check that all of the tranasctions in the proposal are available in the tx pool before attesting this.log.verbose(`request to attest`, { archive: proposal.payload.archive.toString(), - txHashes: proposal.payload.txHashes, + txHashes: proposal.payload.txHashes.map(txHash => txHash.toString()), }); try { await this.ensureTransactionsAreAvailable(proposal); + + if (this.config.validatorReexecute) { + this.log.verbose(`Re-executing transactions in the proposal before attesting`); + await this.reExecuteTransactions(proposal); + } } catch (error: any) { if (error instanceof TransactionsNotAvailableError) { this.log.error(`Transactions not available, skipping attestation ${error.message}`); + } else { + // Catch all error handler + this.log.error(`Failed to attest to proposal: ${error.message}`); } return undefined; } @@ -108,6 +147,42 @@ export class ValidatorClient extends WithTracer implements Validator { return this.validationService.attestToProposal(proposal); } + /** + * Re-execute the transactions in the proposal and check that the state updates match the header state + * @param proposal - The proposal to re-execute + */ + async reExecuteTransactions(proposal: BlockProposal) { + const { header, txHashes } = proposal.payload; + + const txs = (await Promise.all(txHashes.map(tx => this.p2pClient.getTxByHash(tx)))).filter( + tx => tx !== undefined, + ) as Tx[]; + + // If we cannot request all of the transactions, then we should fail + if (txs.length !== txHashes.length) { + this.log.error(`Failed to get transactions from the network: ${txHashes.join(', ')}`); + throw new TransactionsNotAvailableError(txHashes); + } + + // Assertion: This check will fail if re-execution is not enabled + if (this.blockBuilder === undefined) { + throw new BlockBuilderNotProvidedError(); + } + + // Use the sequencer's block building logic to re-execute the transactions + const stopTimer = this.metrics.reExecutionTimer(); + const { block } = await this.blockBuilder(txs, header.globalVariables); + stopTimer(); + + this.log.verbose(`Re-ex: Re-execution complete`); + + // This function will throw an error if state updates do not match + if (!block.archive.root.equals(proposal.archive)) { + this.metrics.recordFailedReexecution(proposal); + throw new ReExStateMismatchError(); + } + } + /** * Ensure that all of the transactions in the proposal are available in the tx pool before attesting * @@ -166,15 +241,15 @@ export class ValidatorClient extends WithTracer implements Validator { } const elapsedTime = Date.now() - startTime; - if (elapsedTime > this.attestationWaitTimeoutMs) { + if (elapsedTime > this.config.attestationWaitTimeoutMs) { this.log.error(`Timeout waiting for ${numberOfRequiredAttestations} attestations for slot, ${slot}`); throw new AttestationTimeoutError(numberOfRequiredAttestations, slot); } this.log.verbose( - `Collected ${attestations.length} attestations so far, waiting ${this.attestationPollingIntervalMs}ms for more...`, + `Collected ${attestations.length} attestations so far, waiting ${this.config.attestationPollingIntervalMs}ms for more...`, ); - await sleep(this.attestationPollingIntervalMs); + await sleep(this.config.attestationPollingIntervalMs); } } } diff --git a/yarn-project/world-state/src/native/message.ts b/yarn-project/world-state/src/native/message.ts index e40ef6011e1..8013e81e772 100644 --- a/yarn-project/world-state/src/native/message.ts +++ b/yarn-project/world-state/src/native/message.ts @@ -397,7 +397,7 @@ interface SyncBlockRequest { paddedNoteHashes: readonly SerializedLeafValue[]; paddedL1ToL2Messages: readonly SerializedLeafValue[]; paddedNullifiers: readonly SerializedLeafValue[]; - batchesOfPaddedPublicDataWrites: readonly SerializedLeafValue[][]; + batchesOfPublicDataWrites: readonly SerializedLeafValue[][]; } interface CreateForkRequest { diff --git a/yarn-project/world-state/src/native/native_world_state.test.ts b/yarn-project/world-state/src/native/native_world_state.test.ts index 069c081eb3c..dbfd92a0b24 100644 --- a/yarn-project/world-state/src/native/native_world_state.test.ts +++ b/yarn-project/world-state/src/native/native_world_state.test.ts @@ -23,7 +23,8 @@ import { join } from 'path'; import { assertSameState, compareChains, mockBlock } from '../test/utils.js'; import { INITIAL_NULLIFIER_TREE_SIZE, INITIAL_PUBLIC_DATA_TREE_SIZE } from '../world-state-db/merkle_tree_db.js'; import { type WorldStateStatusSummary } from './message.js'; -import { NativeWorldStateService } from './native_world_state.js'; +import { NativeWorldStateService, WORLD_STATE_VERSION_FILE } from './native_world_state.js'; +import { WorldStateVersion } from './world_state_version.js'; describe('NativeWorldState', () => { let dataDir: string; @@ -58,6 +59,8 @@ describe('NativeWorldState', () => { await expect( ws.getCommitted().findLeafIndex(MerkleTreeId.NOTE_HASH_TREE, block.body.txEffects[0].noteHashes[0]), ).resolves.toBeDefined(); + const status = await ws.getStatusSummary(); + expect(status.unfinalisedBlockNumber).toBe(1n); await ws.close(); }); @@ -77,6 +80,41 @@ describe('NativeWorldState', () => { await expect( ws.getCommitted().findLeafIndex(MerkleTreeId.NOTE_HASH_TREE, block.body.txEffects[0].noteHashes[0]), ).resolves.toBeUndefined(); + const status = await ws.getStatusSummary(); + expect(status.unfinalisedBlockNumber).toBe(0n); + await ws.close(); + }); + + it('clears the database if the world state version is different', async () => { + // open ws against the data again + let ws = await NativeWorldStateService.new(rollupAddress, dataDir, defaultDBMapSize); + // db should be empty + let emptyStatus = await ws.getStatusSummary(); + expect(emptyStatus.unfinalisedBlockNumber).toBe(0n); + + // populate it and then close it + const fork = await ws.fork(); + ({ block, messages } = await mockBlock(1, 2, fork)); + await fork.close(); + + const status = await ws.handleL2BlockAndMessages(block, messages); + expect(status.summary.unfinalisedBlockNumber).toBe(1n); + await ws.close(); + // we open up the version file that was created and modify the version to be older + const fullPath = join(dataDir, 'world_state', WORLD_STATE_VERSION_FILE); + const storedWorldStateVersion = await WorldStateVersion.readVersion(fullPath); + expect(storedWorldStateVersion).toBeDefined(); + const modifiedVersion = new WorldStateVersion( + storedWorldStateVersion!.version - 1, + storedWorldStateVersion!.rollupAddress, + ); + await modifiedVersion.writeVersionFile(fullPath); + + // Open the world state again and it should be empty + ws = await NativeWorldStateService.new(rollupAddress, dataDir, defaultDBMapSize); + // db should be empty + emptyStatus = await ws.getStatusSummary(); + expect(emptyStatus.unfinalisedBlockNumber).toBe(0n); await ws.close(); }); diff --git a/yarn-project/world-state/src/native/native_world_state.ts b/yarn-project/world-state/src/native/native_world_state.ts index 8dc112fecfc..8e159ba5606 100644 --- a/yarn-project/world-state/src/native/native_world_state.ts +++ b/yarn-project/world-state/src/native/native_world_state.ts @@ -12,7 +12,6 @@ import { Header, MAX_NOTE_HASHES_PER_TX, MAX_NULLIFIERS_PER_TX, - MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, NullifierLeaf, type NullifierLeafPreimage, @@ -24,7 +23,7 @@ import { padArrayEnd } from '@aztec/foundation/collection'; import { createDebugLogger } from '@aztec/foundation/log'; import assert from 'assert/strict'; -import { mkdir, mkdtemp, readFile, rm, writeFile } from 'fs/promises'; +import { mkdir, mkdtemp, rm } from 'fs/promises'; import { tmpdir } from 'os'; import { join } from 'path'; @@ -40,8 +39,16 @@ import { worldStateRevision, } from './message.js'; import { NativeWorldState } from './native_world_state_instance.js'; +import { WorldStateVersion } from './world_state_version.js'; -const ROLLUP_ADDRESS_FILE = 'rollup_address'; +export const WORLD_STATE_VERSION_FILE = 'version'; + +// A crude way of maintaining DB versioning +// We don't currently have any method of performing data migrations +// should the world state db structure change +// For now we will track versions using this hardcoded value and delete +// the state if a change is detected +export const WORLD_STATE_DB_VERSION = 1; // The initial version export class NativeWorldStateService implements MerkleTreeDatabase { protected initialHeader: Header | undefined; @@ -60,17 +67,24 @@ export class NativeWorldStateService implements MerkleTreeDatabase { cleanup = () => Promise.resolve(), ): Promise { const worldStateDirectory = join(dataDir, 'world_state'); - const rollupAddressFile = join(worldStateDirectory, ROLLUP_ADDRESS_FILE); - const currentRollupStr = await readFile(rollupAddressFile, 'utf8').catch(() => undefined); - const currentRollupAddress = currentRollupStr ? EthAddress.fromString(currentRollupStr.trim()) : undefined; + const versionFile = join(worldStateDirectory, WORLD_STATE_VERSION_FILE); + const storedWorldStateVersion = await WorldStateVersion.readVersion(versionFile); - if (currentRollupAddress && !rollupAddress.equals(currentRollupAddress)) { - log.warn('Rollup address changed, deleting database'); + if (!storedWorldStateVersion) { + log.warn('No world state version found, deleting world state directory'); + await rm(worldStateDirectory, { recursive: true, force: true }); + } else if (!rollupAddress.equals(storedWorldStateVersion.rollupAddress)) { + log.warn('Rollup address changed, deleting world state directory'); + await rm(worldStateDirectory, { recursive: true, force: true }); + } else if (storedWorldStateVersion.version != WORLD_STATE_DB_VERSION) { + log.warn('World state version change detected, deleting world state directory'); await rm(worldStateDirectory, { recursive: true, force: true }); } + const newWorldStateVersion = new WorldStateVersion(WORLD_STATE_DB_VERSION, rollupAddress); + await mkdir(worldStateDirectory, { recursive: true }); - await writeFile(rollupAddressFile, rollupAddress.toString(), 'utf8'); + await newWorldStateVersion.writeVersionFile(versionFile); const instance = new NativeWorldState(worldStateDirectory, dbMapSizeKb); const worldState = new this(instance, log, cleanup); @@ -162,16 +176,16 @@ export class NativeWorldStateService implements MerkleTreeDatabase { .map(nullifier => new NullifierLeaf(nullifier)); // We insert the public data tree leaves with one batch per tx to avoid updating the same key twice - const batchesOfPaddedPublicDataWrites: PublicDataTreeLeaf[][] = []; + const batchesOfPublicDataWrites: PublicDataTreeLeaf[][] = []; for (const txEffect of paddedTxEffects) { - const batch: PublicDataTreeLeaf[] = Array(MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX).fill( - PublicDataTreeLeaf.empty(), + batchesOfPublicDataWrites.push( + txEffect.publicDataWrites.map(write => { + if (write.isEmpty()) { + throw new Error('Public data write must not be empty when syncing'); + } + return new PublicDataTreeLeaf(write.leafSlot, write.value); + }), ); - for (const [i, write] of txEffect.publicDataWrites.entries()) { - batch[i] = new PublicDataTreeLeaf(write.leafSlot, write.value); - } - - batchesOfPaddedPublicDataWrites.push(batch); } const response = await this.instance.call(WorldStateMessageType.SYNC_BLOCK, { @@ -180,7 +194,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase { paddedL1ToL2Messages: paddedL1ToL2Messages.map(serializeLeaf), paddedNoteHashes: paddedNoteHashes.map(serializeLeaf), paddedNullifiers: paddedNullifiers.map(serializeLeaf), - batchesOfPaddedPublicDataWrites: batchesOfPaddedPublicDataWrites.map(batch => batch.map(serializeLeaf)), + batchesOfPublicDataWrites: batchesOfPublicDataWrites.map(batch => batch.map(serializeLeaf)), blockStateRef: blockStateReference(l2Block.header.state), }); return sanitiseFullStatus(response); diff --git a/yarn-project/world-state/src/native/native_world_state_cmp.test.ts b/yarn-project/world-state/src/native/native_world_state_cmp.test.ts index 32e5a77e01e..1e18bc0ca06 100644 --- a/yarn-project/world-state/src/native/native_world_state_cmp.test.ts +++ b/yarn-project/world-state/src/native/native_world_state_cmp.test.ts @@ -8,6 +8,7 @@ import { import { EthAddress, Fr, GENESIS_ARCHIVE_ROOT, NullifierLeaf, PublicDataTreeLeaf } from '@aztec/circuits.js'; import { type DebugLogger, createDebugLogger } from '@aztec/foundation/log'; import { elapsed } from '@aztec/foundation/timer'; +import { type AztecKVStore } from '@aztec/kv-store'; import { AztecLmdbStore } from '@aztec/kv-store/lmdb'; import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; @@ -31,6 +32,8 @@ describe('NativeWorldState', () => { let log: DebugLogger; + let legacyStore: AztecKVStore; + const allTrees = Object.values(MerkleTreeId) .filter((x): x is MerkleTreeId => typeof x === 'number') .map(x => [MerkleTreeId[x], x] as const); @@ -43,13 +46,14 @@ describe('NativeWorldState', () => { }); afterAll(async () => { + await legacyStore.delete(); await rm(nativeDataDir, { recursive: true }); - await rm(legacyDataDir, { recursive: true }); }); beforeAll(async () => { + legacyStore = AztecLmdbStore.open(legacyDataDir); nativeWS = await NativeWorldStateService.new(EthAddress.random(), nativeDataDir, 1024 * 1024); - legacyWS = await MerkleTrees.new(AztecLmdbStore.open(legacyDataDir), new NoopTelemetryClient()); + legacyWS = await MerkleTrees.new(legacyStore, new NoopTelemetryClient()); }); it('has to expected genesis archive tree root', async () => { diff --git a/yarn-project/world-state/src/native/native_world_state_instance.ts b/yarn-project/world-state/src/native/native_world_state_instance.ts index 671baaa4fe6..ab63e2d6e5a 100644 --- a/yarn-project/world-state/src/native/native_world_state_instance.ts +++ b/yarn-project/world-state/src/native/native_world_state_instance.ts @@ -188,10 +188,7 @@ export class NativeWorldState implements NativeWorldStateInstance { data['notesCount'] = body.paddedNoteHashes.length; data['nullifiersCount'] = body.paddedNullifiers.length; data['l1ToL2MessagesCount'] = body.paddedL1ToL2Messages.length; - data['publicDataWritesCount'] = body.batchesOfPaddedPublicDataWrites.reduce( - (acc, batch) => acc + batch.length, - 0, - ); + data['publicDataWritesCount'] = body.batchesOfPublicDataWrites.reduce((acc, batch) => acc + batch.length, 0); } this.log.debug(`Calling messageId=${messageId} ${WorldStateMessageType[messageType]} with ${fmtLogData(data)}`); diff --git a/yarn-project/world-state/src/native/world_state_version.ts b/yarn-project/world-state/src/native/world_state_version.ts new file mode 100644 index 00000000000..20707d354ab --- /dev/null +++ b/yarn-project/world-state/src/native/world_state_version.ts @@ -0,0 +1,41 @@ +import { EthAddress } from '@aztec/circuits.js'; + +import { readFile, writeFile } from 'fs/promises'; + +export class WorldStateVersion { + constructor(readonly version: number, readonly rollupAddress: EthAddress) {} + + static async readVersion(filename: string) { + const versionData = await readFile(filename, 'utf-8').catch(() => undefined); + if (versionData === undefined) { + return undefined; + } + const versionJSON = JSON.parse(versionData); + if (versionJSON.version === undefined || versionJSON.rollupAddress === undefined) { + return undefined; + } + return WorldStateVersion.fromJSON(versionJSON); + } + + public async writeVersionFile(filename: string) { + const data = JSON.stringify(this.toJSON()); + await writeFile(filename, data, 'utf-8'); + } + + toJSON() { + return { + version: this.version, + rollupAddress: this.rollupAddress.toChecksumString(), + }; + } + + static fromJSON(obj: any): WorldStateVersion { + const version = obj.version; + const rollupAddress = EthAddress.fromString(obj.rollupAddress); + return new WorldStateVersion(version, rollupAddress); + } + + static empty() { + return new WorldStateVersion(0, EthAddress.ZERO); + } +} diff --git a/yarn-project/world-state/src/test/utils.ts b/yarn-project/world-state/src/test/utils.ts index e6148637fef..4edd1888384 100644 --- a/yarn-project/world-state/src/test/utils.ts +++ b/yarn-project/world-state/src/test/utils.ts @@ -10,11 +10,8 @@ import { Fr, MAX_NOTE_HASHES_PER_TX, MAX_NULLIFIERS_PER_TX, - MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, NULLIFIER_SUBTREE_HEIGHT, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, - PUBLIC_DATA_SUBTREE_HEIGHT, - PublicDataWrite, } from '@aztec/circuits.js'; import { padArrayEnd } from '@aztec/foundation/collection'; @@ -45,16 +42,10 @@ export async function mockBlock(blockNum: number, size: number, fork: MerkleTree { // We insert the public data tree leaves with one batch per tx to avoid updating the same key twice for (const txEffect of paddedTxEffects) { - const publicDataWrites = padArrayEnd( - txEffect.publicDataWrites, - PublicDataWrite.empty(), - MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, - ); - await fork.batchInsert( MerkleTreeId.PUBLIC_DATA_TREE, - publicDataWrites.map(write => write.toBuffer()), - PUBLIC_DATA_SUBTREE_HEIGHT, + txEffect.publicDataWrites.map(write => write.toBuffer()), + 0, ); const nullifiersPadded = padArrayEnd(txEffect.nullifiers, Fr.ZERO, MAX_NULLIFIERS_PER_TX); diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index 1ad1e7413d6..5acda1df8ca 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -623,7 +623,9 @@ __metadata: "@jest/globals": ^29.5.0 "@types/jest": ^29.5.0 "@types/node": ^18.14.6 + "@viem/anvil": ^0.0.10 dotenv: ^16.0.3 + get-port: ^7.1.0 jest: ^29.5.0 ts-node: ^10.9.1 tslib: ^2.4.0 @@ -763,6 +765,8 @@ __metadata: version: 0.0.0-use.local resolution: "@aztec/kv-store@workspace:kv-store" dependencies: + "@aztec/circuit-types": "workspace:^" + "@aztec/circuits.js": "workspace:^" "@aztec/ethereum": "workspace:^" "@aztec/foundation": "workspace:^" "@jest/globals": ^29.5.0 @@ -1165,12 +1169,14 @@ __metadata: "@noir-lang/types": "portal:../../noir/packages/types" "@types/jest": ^29.5.0 "@types/levelup": ^5.1.3 + "@types/lodash.clonedeep": ^4.5.7 "@types/lodash.merge": ^4.6.9 "@types/memdown": ^3.0.2 "@types/node": ^18.7.23 jest: ^29.5.0 jest-mock-extended: ^3.0.4 levelup: ^5.1.1 + lodash.clonedeep: ^4.5.0 lodash.merge: ^4.6.2 memdown: ^6.1.1 ts-node: ^10.9.1 @@ -3498,7 +3504,7 @@ __metadata: version: 0.0.0-use.local resolution: "@noir-lang/noir_codegen@portal:../noir/packages/noir_codegen::locator=%40aztec%2Faztec3-packages%40workspace%3A." dependencies: - "@noir-lang/types": 0.38.0 + "@noir-lang/types": 0.39.0 glob: ^10.3.10 ts-command-line-args: ^2.5.1 bin: @@ -3507,13 +3513,13 @@ __metadata: linkType: soft "@noir-lang/noir_js@file:../noir/packages/noir_js::locator=%40aztec%2Faztec3-packages%40workspace%3A.": - version: 0.38.0 - resolution: "@noir-lang/noir_js@file:../noir/packages/noir_js#../noir/packages/noir_js::hash=9fc813&locator=%40aztec%2Faztec3-packages%40workspace%3A." + version: 0.39.0 + resolution: "@noir-lang/noir_js@file:../noir/packages/noir_js#../noir/packages/noir_js::hash=2fe976&locator=%40aztec%2Faztec3-packages%40workspace%3A." dependencies: - "@noir-lang/acvm_js": 0.54.0 - "@noir-lang/noirc_abi": 0.38.0 - "@noir-lang/types": 0.38.0 - checksum: 99cdc1f1e352d45a8968261dc7f1d68d58b5bca8177e25c610667eb60954436356c56852e6a23fbf69ddb8db9b92494736438c0852f4702d33e8d0e981e60552 + "@noir-lang/acvm_js": 0.55.0 + "@noir-lang/noirc_abi": 0.39.0 + "@noir-lang/types": 0.39.0 + checksum: bbed7618af5ffb055e7020686035fcf2d25ba0510f48654338192d2bb18cf7ca74403ea6f6ea713ff1d204436f9c2066eb5adf645f585b353d3c6f9d6ee38403 languageName: node linkType: hard @@ -3521,7 +3527,7 @@ __metadata: version: 0.0.0-use.local resolution: "@noir-lang/noirc_abi@portal:../noir/packages/noirc_abi::locator=%40aztec%2Faztec3-packages%40workspace%3A." dependencies: - "@noir-lang/types": 0.38.0 + "@noir-lang/types": 0.39.0 languageName: node linkType: soft @@ -5450,6 +5456,18 @@ __metadata: languageName: node linkType: hard +"@viem/anvil@npm:^0.0.10": + version: 0.0.10 + resolution: "@viem/anvil@npm:0.0.10" + dependencies: + execa: ^7.1.1 + get-port: ^6.1.2 + http-proxy: ^1.18.1 + ws: ^8.13.0 + checksum: fb475055f36c753cea26fa0c02a0278301dddcdc8003418576395cfc31e97ba5a236fbc66ff093bd8d39ea05286487adb86b7499308e446b6cfe90dc08089b38 + languageName: node + linkType: hard + "@viem/anvil@npm:^0.0.9": version: 0.0.9 resolution: "@viem/anvil@npm:0.0.9"