From db7074343914c185e00a0051abc7c1b745fb96d2 Mon Sep 17 00:00:00 2001 From: Guillaume Hivert Date: Wed, 11 May 2022 15:56:50 +0200 Subject: [PATCH 1/9] ARSN-201 Rename all files to TS --- lib/versioning/{Version.js => Version.ts} | 0 lib/versioning/{VersionID.js => VersionID.ts} | 0 ...ersioningRequestProcessor.js => VersioningRequestProcessor.ts} | 0 lib/versioning/{WriteCache.js => WriteCache.ts} | 0 .../{WriteGatheringManager.js => WriteGatheringManager.ts} | 0 lib/versioning/{constants.js => constants.ts} | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename lib/versioning/{Version.js => Version.ts} (100%) rename lib/versioning/{VersionID.js => VersionID.ts} (100%) rename lib/versioning/{VersioningRequestProcessor.js => VersioningRequestProcessor.ts} (100%) rename lib/versioning/{WriteCache.js => WriteCache.ts} (100%) rename lib/versioning/{WriteGatheringManager.js => WriteGatheringManager.ts} (100%) rename lib/versioning/{constants.js => constants.ts} (100%) diff --git a/lib/versioning/Version.js b/lib/versioning/Version.ts similarity index 100% rename from lib/versioning/Version.js rename to lib/versioning/Version.ts diff --git a/lib/versioning/VersionID.js b/lib/versioning/VersionID.ts similarity index 100% rename from lib/versioning/VersionID.js rename to lib/versioning/VersionID.ts diff --git a/lib/versioning/VersioningRequestProcessor.js b/lib/versioning/VersioningRequestProcessor.ts similarity index 100% rename from lib/versioning/VersioningRequestProcessor.js rename to lib/versioning/VersioningRequestProcessor.ts diff --git a/lib/versioning/WriteCache.js b/lib/versioning/WriteCache.ts similarity index 100% rename from lib/versioning/WriteCache.js rename to lib/versioning/WriteCache.ts diff --git a/lib/versioning/WriteGatheringManager.js b/lib/versioning/WriteGatheringManager.ts similarity index 100% rename from lib/versioning/WriteGatheringManager.js rename to lib/versioning/WriteGatheringManager.ts diff --git a/lib/versioning/constants.js b/lib/versioning/constants.ts similarity index 100% rename from lib/versioning/constants.js rename to lib/versioning/constants.ts From 355c54051020d3a135976e1cc9760d59aff28577 Mon Sep 17 00:00:00 2001 From: Guillaume Hivert Date: Thu, 12 May 2022 15:14:42 +0200 Subject: [PATCH 2/9] ARSN-201 Type check Version --- lib/versioning/Version.ts | 111 +++++++++++++++++++++----------------- 1 file changed, 62 insertions(+), 49 deletions(-) diff --git a/lib/versioning/Version.ts b/lib/versioning/Version.ts index 124c6b224..ff2b1113d 100644 --- a/lib/versioning/Version.ts +++ b/lib/versioning/Version.ts @@ -1,7 +1,6 @@ -'use strict'; // eslint-disable-line strict - -const VID_SEP = require('./constants').VersioningConstants.VersionId.Separator; +import { VersioningConstants } from './constants'; +const VID_SEP = VersioningConstants.VersionId.Separator; /** * Class for manipulating an object version. * The format of a version: { isNull, isDeleteMarker, versionId, otherInfo } @@ -11,36 +10,48 @@ const VID_SEP = require('./constants').VersioningConstants.VersionId.Separator; * with the full stringify/parse cycle, which is too low a number for * use with a production setup with Metadata). */ -class Version { +export class Version { + version: { + isNull?: boolean; + isDeleteMarker?: boolean; + versionId?: string; + isPHD?: boolean; + }; + /** * Create a new version instantiation from its data object. - * @param {object} version - the data object to instantiate - * @param {boolean} version.isNull - is a null version - * @param {boolean} version.isDeleteMarker - is a delete marker - * @param {string} version.versionId - the version id + * @param version - the data object to instantiate + * @param version.isNull - is a null version + * @param version.isDeleteMarker - is a delete marker + * @param version.versionId - the version id * @constructor */ - constructor(version) { + constructor(version?: { + isNull?: boolean; + isDeleteMarker?: boolean; + versionId?: string; + isPHD?: boolean; + }) { this.version = version || {}; } /** * Parse the version information from a string. * - * @param {string} value - the string to parse - * @return {Version} - the version deserialized from the input string + * @param value - the string to parse + * @return - the version deserialized from the input string */ - static from(value) { + static from(value: string) { return new Version(value ? JSON.parse(value) : undefined); } /** * [MetaData Internal] Check if a version is a place holder for deletion. * - * @param {string} value - version to check - * @return {boolean} - whether this is a PHD version + * @param value - version to check + * @return - whether this is a PHD version */ - static isPHD(value) { + static isPHD(value: string) { // check if the input is a valid version if (!value) { return false; @@ -53,7 +64,8 @@ class Version { // parse the value if it has the keyword 'isPHD' try { return Version.from(value).isPHDVersion(); - } catch (exception) { // eslint-disable-line strict + } catch (exception) { + // eslint-disable-line strict return false; // nice, Vault } } @@ -64,10 +76,10 @@ class Version { * to indicate that the master version of an object has been deleted and it * needs to be updated by the latest version if any. * - * @param {string} versionId - versionId of the PHD version - * @return {string} - the serialized version + * @param versionId - versionId of the PHD version + * @return - the serialized version */ - static generatePHDVersion(versionId) { + static generatePHDVersion(versionId: string) { return `{ "isPHD": true, "versionId": "${versionId}" }`; } @@ -75,17 +87,20 @@ class Version { * Put versionId into an object in the (cheap) way of string manipulation, * instead of the more expensive alternative parsing and stringification. * - * @param {string} value - stringification of the object to append versionId - * @param {string} versionId - the versionId to append - * @return {string} - the object with versionId appended + * @param value - stringification of the object to append versionId + * @param versionId - the versionId to append + * @return - the object with versionId appended */ - static appendVersionId(value, versionId) { + static appendVersionId(value: string, versionId: string): string { // assuming value has the format of '{...}' let index = value.length - 2; while (value.charAt(index--) === ' '); const comma = value.charAt(index + 1) !== '{'; - return `${value.slice(0, value.length - 1)}` + // eslint-disable-line - (comma ? ',' : '') + `"versionId":"${versionId}"}`; + return ( + `${value.slice(0, value.length - 1)}` + // eslint-disable-line + (comma ? ',' : '') + + `"versionId":"${versionId}"}` + ); } /** @@ -93,26 +108,26 @@ class Version { * * @return {boolean} - whether this is a PHD version */ - isPHDVersion() { + isPHDVersion(): boolean { return this.version.isPHD || false; } /** * Check if a version is a null version. * - * @return {boolean} - stating if the value is a null version + * @return - stating if the value is a null version */ - isNullVersion() { - return this.version.isNull; + isNullVersion(): boolean { + return this.version.isNull ?? false; } /** * Check if a stringified object is a delete marker. * - * @param {string} value - the stringified object to check - * @return {boolean} - if the object is a delete marker + * @param value - the stringified object to check + * @return - if the object is a delete marker */ - static isDeleteMarker(value) { + static isDeleteMarker(value: string): boolean { const index = value.indexOf('isDeleteMarker'); if (index < 0) { return false; @@ -120,7 +135,8 @@ class Version { // parse the value try { return Version.from(value).isDeleteMarkerVersion(); - } catch (exception) { // eslint-disable-line strict + } catch (exception) { + // eslint-disable-line strict return false; } } @@ -128,28 +144,28 @@ class Version { /** * Check if a version is a delete marker. * - * @return {boolean} - stating if the value is a delete marker + * @return - stating if the value is a delete marker */ - isDeleteMarkerVersion() { - return this.version.isDeleteMarker; + isDeleteMarkerVersion(): boolean { + return this.version.isDeleteMarker ?? false; } /** * Get the versionId of the version. * - * @return {string} - the versionId + * @return - the versionId */ - getVersionId() { + getVersionId(): string | undefined { return this.version.versionId; } /** * Set the versionId of the version. * - * @param {string} versionId - the versionId - * @return {Version} - the updated version + * @param versionId - the versionId + * @return - the updated version */ - setVersionId(versionId) { + setVersionId(versionId: string) { this.version.versionId = versionId; return this; } @@ -157,7 +173,7 @@ class Version { /** * Mark a version as a delete marker. * - * @return {Version} - the updated version + * @return - the updated version */ setDeleteMarker() { this.version.isDeleteMarker = true; @@ -167,7 +183,7 @@ class Version { /** * Mark a version as a null version. * - * @return {Version} - the updated version + * @return - the updated version */ setNullVersion() { this.version.isNull = true; @@ -177,16 +193,13 @@ class Version { /** * Serialize the version. * - * @return {string} - the serialized version + * @return - the serialized version */ - toString() { + toString(): string { return JSON.stringify(this.version); } } -function isMasterKey(key) { +export function isMasterKey(key: string) { return !key.includes(VID_SEP); } - - -module.exports = { Version, isMasterKey }; From 485ca38867c04f89c6fdc6a8261d7ca7b58f93c7 Mon Sep 17 00:00:00 2001 From: Guillaume Hivert Date: Thu, 12 May 2022 15:14:48 +0200 Subject: [PATCH 3/9] ARSN-201 Type check VersionID --- lib/versioning/VersionID.ts | 126 +++++++++++++++++++----------------- 1 file changed, 67 insertions(+), 59 deletions(-) diff --git a/lib/versioning/VersionID.ts b/lib/versioning/VersionID.ts index c8f0302a2..b2605da9e 100644 --- a/lib/versioning/VersionID.ts +++ b/lib/versioning/VersionID.ts @@ -6,10 +6,11 @@ // - rep_group_id 07 bytes replication group identifier // - other_information arbitrary user input, such as a unique string -const base62Integer = require('base62'); +import base62Integer from 'base62'; +import baseX from 'base-x'; +import assert from 'assert/strict'; const BASE62 = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'; -const base62String = require('base-x')(BASE62); -const assert = require('assert/strict'); +const base62String = baseX(BASE62); // the lengths of the components in bytes const LENGTH_TS = 14; // timestamp: epoch in ms @@ -25,11 +26,11 @@ const TEMPLATE_RG = new Array(LENGTH_RG + 1).join(' '); * Left-pad a string representation of a value with a given template. * For example: pad('foo', '00000') gives '00foo'. * - * @param {any} value - value to pad - * @param {string} template - padding template - * @return {string} - padded string + * @param value - value to pad + * @param template - padding template + * @return - padded string */ -function padLeft(value, template) { +function padLeft(value: any, template: string) { return `${template}${value}`.slice(-template.length); } @@ -37,11 +38,11 @@ function padLeft(value, template) { * Right-pad a string representation of a value with a given template. * For example: pad('foo', '00000') gives 'foo00'. * - * @param {any} value - value to pad - * @param {string} template - padding template - * @return {string} - padded string + * @param value - value to pad + * @param template - padding template + * @return - padded string */ -function padRight(value, template) { +function padRight(value: any, template: string) { return `${value}${template}`.slice(0, template.length); } @@ -52,13 +53,16 @@ const MAX_SEQ = Math.pow(10, LENGTH_SEQ) - 1; // good for 1 billion ops /** * Generates the earliest versionId, used for versions before versioning * - * @param {string} replicationGroupId - replication group id - * @return {string} version ID for versions before versionig + * @param replicationGroupId - replication group id + * @return version ID for versions before versionig */ -function getInfVid(replicationGroupId) { +export function getInfVid(replicationGroupId: string) { const repGroupId = padRight(replicationGroupId, TEMPLATE_RG); - return (padLeft(MAX_TS, TEMPLATE_TS) + - padLeft(MAX_SEQ, TEMPLATE_SEQ) + repGroupId); + return ( + padLeft(MAX_TS, TEMPLATE_TS) + + padLeft(MAX_SEQ, TEMPLATE_SEQ) + + repGroupId + ); } // internal state of the module @@ -69,11 +73,11 @@ let lastSeq = 0; // sequential number of the last versionId * This function ACTIVELY (wastes CPU cycles and) waits for an amount of time * before returning to the caller. This should not be used frequently. * - * @param {Number} span - time to wait in nanoseconds (1/1000000 millisecond) - * @return {Undefined} - nothing + * @param span - time to wait in nanoseconds (1/1000000 millisecond) + * @return - nothing */ -function wait(span) { - function getspan(diff) { +function wait(span: number) { + function getspan(diff: [number, number]) { return diff[0] * 1e9 + diff[1]; } const start = process.hrtime(); @@ -90,11 +94,11 @@ function wait(span) { * function is stateful which means it keeps some values in the memory and the * next call depends on the previous call. * - * @param {string} info - the additional info to ensure uniqueness if desired - * @param {string} replicationGroupId - replication group id - * @return {string} - the formated versionId string + * @param info - the additional info to ensure uniqueness if desired + * @param replicationGroupId - replication group id + * @return - the formated versionId string */ -function generateVersionId(info, replicationGroupId) { +export function generateVersionId(info: string, replicationGroupId: string): string { // replication group ID, like PARIS; will be trimmed if exceed LENGTH_RG const repGroupId = padRight(replicationGroupId, TEMPLATE_RG); @@ -114,7 +118,7 @@ function generateVersionId(info, replicationGroupId) { // in the queue for the same millisecond which is supposed to be unique. // increase the position if this request is in the same epoch - lastSeq = (lastTimestamp === ts) ? lastSeq + 1 : 0; + lastSeq = lastTimestamp === ts ? lastSeq + 1 : 0; lastTimestamp = ts; // if S3_VERSION_ID_ENCODING_TYPE is "hex", info is used. By default, it is not used. @@ -128,18 +132,22 @@ function generateVersionId(info, replicationGroupId) { // timestamps so that all versions of an object can be retrieved in the // reversed chronological order---newest versions first. This is because of // the limitation of leveldb for listing keys in the reverse order. - return padLeft(MAX_TS - lastTimestamp, TEMPLATE_TS) + - padLeft(MAX_SEQ - lastSeq, TEMPLATE_SEQ) + repGroupId + info; + return ( + padLeft(MAX_TS - lastTimestamp, TEMPLATE_TS) + + padLeft(MAX_SEQ - lastSeq, TEMPLATE_SEQ) + + repGroupId + + info + ); } /** * Encode a versionId to obscure internal information contained * in a version ID. * - * @param {string} str - the versionId to encode - * @return {string} - the encoded versionId + * @param str - the versionId to encode + * @return - the encoded versionId */ -function hexEncode(str) { +export function hexEncode(str: string): string { return Buffer.from(str, 'utf8').toString('hex'); } @@ -147,10 +155,10 @@ function hexEncode(str) { * Decode a versionId. May return an error if the input string is * invalid hex string or results in an invalid value. * - * @param {string} str - the encoded versionId to decode - * @return {(string|Error)} - the decoded versionId or an error + * @param str - the encoded versionId to decode + * @return - the decoded versionId or an error */ -function hexDecode(str) { +export function hexDecode(str: string): string | Error { try { const result = Buffer.from(str, 'hex').toString('utf8'); if (result === '') { @@ -160,7 +168,7 @@ function hexDecode(str) { } catch (err) { // Buffer.from() may throw TypeError if invalid input, e.g. non-string // or string with inappropriate charlength - return err; + return err as any; } } @@ -172,7 +180,9 @@ function hexDecode(str) { */ const B62V_TOTAL = LENGTH_TS + LENGTH_SEQ; const B62V_HALF = B62V_TOTAL / 2; -const B62V_EPAD = '0'.repeat(Math.ceil(B62V_HALF * (Math.log(10) / Math.log(62)))); +const B62V_EPAD = '0'.repeat( + Math.ceil(B62V_HALF * (Math.log(10) / Math.log(62))) +); const B62V_DPAD = '0'.repeat(B62V_HALF); const B62V_STRING_EPAD = '0'.repeat(32 - 2 * B62V_EPAD.length); @@ -180,10 +190,10 @@ const B62V_STRING_EPAD = '0'.repeat(32 - 2 * B62V_EPAD.length); * Encode a versionId to obscure internal information contained * in a version ID (equal to 32 bytes). * - * @param {string} str - the versionId to encode - * @return {string} - the encoded base62VersionId + * @param str - the versionId to encode + * @return - the encoded base62VersionId */ -function base62Encode(str) { +export function base62Encode(str: string): string { const part1 = Number(str.substring(0, B62V_HALF)); const part2 = Number(str.substring(B62V_HALF, B62V_TOTAL)); const part3 = Buffer.from(str.substring(B62V_TOTAL)); @@ -200,10 +210,10 @@ function base62Encode(str) { * Decode a base62VersionId. May return an error if the input string is * invalid hex string or results in an invalid value. * - * @param {string} str - the encoded base62VersionId to decode - * @return {(string|Error)} - the decoded versionId or an error + * @param str - the encoded base62VersionId to decode + * @return - the decoded versionId or an error */ -function base62Decode(str) { +export function base62Decode(str: string): string | Error { try { let start = 0; const enc1 = str.substring(start, start + B62V_EPAD.length); @@ -225,26 +235,28 @@ function base62Decode(str) { enc3 = enc3.slice(idx); const orig3 = base62String.decode(enc3); - return (B62V_DPAD + orig1.toString()).slice(-B62V_DPAD.length) + + return ( + (B62V_DPAD + orig1.toString()).slice(-B62V_DPAD.length) + (B62V_DPAD + orig2.toString()).slice(-B62V_DPAD.length) + - orig3.toString(); + orig3.toString() + ); } catch (err) { // in case of exceptions caused by base62 libs - return err; + return err as any; } } -const ENC_TYPE_HEX = 0; // legacy (large) encoding -const ENC_TYPE_BASE62 = 1; // new (tiny) encoding +export const ENC_TYPE_HEX = 0; // legacy (large) encoding +export const ENC_TYPE_BASE62 = 1; // new (tiny) encoding /** * Encode a versionId to obscure internal information contained * in a version ID. * - * @param {string} str - the versionId to encode - * @return {string} - the encoded versionId + * @param str - the versionId to encode + * @return - the encoded versionId */ -function encode(str) { +export function encode(str: string): string { // default format without 'info' field will always be 27 characters if (str.length === 27) { return base62Encode(str); @@ -257,14 +269,16 @@ function encode(str) { * invalid format or results in an invalid value. The function will * automatically determine the format acc/ to an heuristic. * - * @param {string} str - the encoded versionId to decode - * @return {(string|Error)} - the decoded versionId or an error + * @param str - the encoded versionId to decode + * @return - the decoded versionId or an error */ -function decode(str) { +export function decode(str: string): string | Error { // default format is exactly 32 characters when encoded if (str.length === 32) { const decoded = base62Decode(str); - assert.strictEqual(decoded.length, 27); + if (typeof decoded === 'string') { + assert.strictEqual(decoded.length, 27); + } return decoded; } // legacy format @@ -273,9 +287,3 @@ function decode(str) { } return new Error(`cannot decode str ${str.length}`); } - -module.exports = { generateVersionId, getInfVid, - hexEncode, hexDecode, - base62Encode, base62Decode, - encode, decode, - ENC_TYPE_HEX, ENC_TYPE_BASE62 }; From 938d64f48e98de9ba7a5270fee05a203aba52781 Mon Sep 17 00:00:00 2001 From: Guillaume Hivert Date: Thu, 12 May 2022 15:15:28 +0200 Subject: [PATCH 4/9] ARSN-201 Type check WriteCache --- lib/versioning/WriteCache.ts | 99 +++++++++++++++++++++--------------- 1 file changed, 58 insertions(+), 41 deletions(-) diff --git a/lib/versioning/WriteCache.ts b/lib/versioning/WriteCache.ts index f3c584dcc..b37f2dd55 100644 --- a/lib/versioning/WriteCache.ts +++ b/lib/versioning/WriteCache.ts @@ -1,8 +1,7 @@ -'use strict'; // eslint-disable-line +import errors, { ArsenalError } from '../errors'; +import WriteGatheringManager from './WriteGatheringManager'; -const errors = require('../errors').default; - -function formatCacheKey(db, key) { +function formatCacheKey(db: string, key: string) { return `${db}\0\0${key}`; } @@ -24,8 +23,14 @@ function formatCacheKey(db, key) { * the latest value of that object, thus ensuring isolation. This cached entry * remains only until the write is done and no other update is using it. */ -class WriteCache { - constructor(wgm) { +export default class WriteCache { + // TODO Fix this + wgm: WriteGatheringManager; + cache: {}; + queue: {}; + counter: number; + + constructor(wgm: WriteGatheringManager) { this.wgm = wgm; // internal state this.cache = {}; @@ -36,14 +41,18 @@ class WriteCache { /** * Get the value of an entry either in temporary cache, cache, or database. * - * @param {object} request - the request in original + * @param request - the request in original * RepdConnection format { db, key * [, value][, type], method, options } - * @param {object} logger - logger - * @param {function} callback - callback function: callback(error, value) - * @return {any} - to finish the call + * @param logger - logger + * @param callback - callback function: callback(error, value) + * @return - to finish the call */ - get(request, logger, callback) { + get( + request: any, + logger: RequestLogger, + callback: (err: ArsenalError | null, data?: any) => void + ) { const { db, key } = request; const cacheKey = formatCacheKey(db, key); @@ -69,13 +78,16 @@ class WriteCache { /** * Queue up a get request. * - * @param {string} cacheKey - key of the cache entry - * @param {function} callback - callback - * @return {number} - the signature of the request if this is the first + * @param cacheKey - key of the cache entry + * @param callback - callback + * @return - the signature of the request if this is the first * entry in the queue (which will do the get from the * database), undefined otherwise */ - _enqueue(cacheKey, callback) { + _enqueue( + cacheKey: string, + callback: (err: ArsenalError | null, data?: any) => void + ) { if (this.queue[cacheKey]) { this.queue[cacheKey].queue.push(callback); return undefined; @@ -87,14 +99,19 @@ class WriteCache { /** * Dequeue the concurrent get requests on the same object. * - * @param {string} cacheKey - key of the cache entry - * @param {number} signature - signature of the first request of the queue - * @param {object} err - the error from the get - * @param {string} value - the value of the object to seed dequeueing - * @param {boolean} force - force dequeuing even on signature mismatch - * @return {undefined} - nothing + * @param cacheKey - key of the cache entry + * @param signature - signature of the first request of the queue + * @param err - the error from the get + * @param value - the value of the object to seed dequeueing + * @param force - force dequeuing even on signature mismatch */ - _dequeue(cacheKey, signature, err, value, force = false) { + _dequeue( + cacheKey: string, + signature: number | null, + err: ArsenalError | null, + value: string, + force = false + ) { if (this.queue[cacheKey] === undefined) { return; } @@ -109,7 +126,7 @@ class WriteCache { // dequeueing will read, compute, and update the cache const dequeueSignature = this.counter++; this.cache[cacheKey] = { signature: dequeueSignature, value }; - this.queue[cacheKey].queue.forEach(callback => { + this.queue[cacheKey].queue.forEach((callback) => { // always return the value from cache, not the value that // started dequeueing, because the cache might be updated // synchronously by a dequeued request @@ -133,13 +150,16 @@ class WriteCache { /** * Replicate the latest value of an entry and cache it during replication. * - * @param {object} request - the request in format { db, + * @param request - the request in format { db, * array, options } - * @param {object} logger - logger of the operation - * @param {function} callback - asynchronous callback of the call - * @return {undefined} + * @param logger - logger of the operation + * @param callback - asynchronous callback of the call */ - batch(request, logger, callback) { + batch( + request: { db: any; array: any[]; options?: any }, + logger: RequestLogger, + callback: (err: ArsenalError | null, data?: any) => void + ) { const { db, array } = request; const signature = this._cacheWrite(db, array); this.wgm.batch(request, logger, (err, data) => { @@ -155,13 +175,13 @@ class WriteCache { * it. The newly put value is always the latest; we have to use it instead * of using the potentially more stale value in the database. * - * @param {string} db - name of the database - * @param {object} array - batch operation to apply on the database - * @return {string} - signature of the request + * @param db - name of the database + * @param array - batch operation to apply on the database + * @return - signature of the request */ - _cacheWrite(db, array) { + _cacheWrite(db: string, array: { key: string; value: any }[]) { const signature = this.counter++; - array.forEach(entry => { + array.forEach((entry) => { const cacheKey = formatCacheKey(db, entry.key); this.cache[cacheKey] = { signature, value: entry.value }; this._dequeue(cacheKey, null, null, entry.value, true); @@ -172,13 +192,12 @@ class WriteCache { /** * Clear the cached entries after a successful write. * - * @param {string} db - name of the database - * @param {object} array - batch operation to apply on the database - * @param {string} signature - signature if temporarily cached - * @return {undefined} + * @param db - name of the database + * @param array - batch operation to apply on the database + * @param signature - signature if temporarily cached */ - _cacheClear(db, array, signature) { - array.forEach(entry => { + _cacheClear(db: string, array: { key: string }[], signature: number) { + array.forEach((entry) => { const key = formatCacheKey(db, entry.key); if (this.cache[key] && this.cache[key].signature === signature) { // only clear cache when the temporarily cached entry @@ -189,5 +208,3 @@ class WriteCache { }); } } - -module.exports = WriteCache; From 8d17b69eb82ac8eb068288fb51f4aa85ff58fe72 Mon Sep 17 00:00:00 2001 From: Guillaume Hivert Date: Thu, 12 May 2022 15:15:42 +0200 Subject: [PATCH 5/9] ARSN-201 Type check WriteGatheringManager --- lib/versioning/WriteGatheringManager.ts | 63 ++++++++++++++++--------- 1 file changed, 41 insertions(+), 22 deletions(-) diff --git a/lib/versioning/WriteGatheringManager.ts b/lib/versioning/WriteGatheringManager.ts index 301dc3c6e..c98bd024c 100644 --- a/lib/versioning/WriteGatheringManager.ts +++ b/lib/versioning/WriteGatheringManager.ts @@ -1,3 +1,5 @@ +import { ArsenalError } from '../errors'; + const WG_TIMEOUT = 5; // batching period in milliseconds /** @@ -5,8 +7,12 @@ const WG_TIMEOUT = 5; // batching period in milliseconds * Because we are managing buckets in separate databases, we build a batch * from operations targeting the same database. */ -class WriteGatheringManager { - constructor(db) { +export default class WriteGatheringManager { + // TODO Fix this + db: any; + dbState: {}; + + constructor(db: any) { this.db = db; this.dbState = {}; } @@ -14,18 +20,26 @@ class WriteGatheringManager { /** * Get the value of an entry either in temporary cache, cache, or database. * - * @param {object} request - the request in original + * @param request - the request in original * RepdConnection format { db, key * [, value][, type], method, options } - * @param {object} logger - logger - * @param {function} callback - callback function: callback(error, value) - * @return {any} - to finish the call + * @param logger - logger + * @param callback - callback function: callback(error, value) + * @return - to finish the call */ - get(request, logger, callback) { + get( + request: any, + logger: RequestLogger, + callback: (error: ArsenalError | null, data?: any) => void, + ) { return this.db.get(request, logger, callback); } - list(request, logger, callback) { + list( + request: any, + logger: RequestLogger, + callback: (error: ArsenalError | null, data?: any) => void, + ) { return this.db.list(request, logger, callback); } @@ -33,13 +47,17 @@ class WriteGatheringManager { * Append a request to the write gathering batch. * Replicate and commit the batch on timeout or oversize. * - * @param {object} request - the request in format { db, + * @param request - the request in format { db, * array, options } - * @param {object} logger - logger of the request - * @param {function} callback - callback(err) - * @return {WriteGatheringManager} - return this + * @param logger - logger of the request + * @param callback - callback(err) + * @return - return this */ - batch(request, logger, callback) { + batch( + request: any, + logger: RequestLogger, + callback: (error: ArsenalError | null, data?: any) => void, + ) { const { db, array } = request; if (this.dbState[db] === undefined) { this.dbState[db] = { db, isCommitting: false }; @@ -66,10 +84,10 @@ class WriteGatheringManager { /** * Commit a batch of operations on a database. * - * @param {string} db - Name of the database - * @return {any} - to finish the call + * @param db - Name of the database + * @return - to finish the call */ - _commitBatch(db) { + _commitBatch(db: string) { const dbState = this.dbState[db]; const bCache = dbState.batchCache; // do nothing if no batch to replicate @@ -120,11 +138,14 @@ class WriteGatheringManager { /** * Respond to all requests of a batch after it has been committed. * - * @param {object} error - error of committing the batch - * @param {object} batch - the committed batch - * @return {undefined} - nothing + * @param error - error of committing the batch + * @param batch - the committed batch + * @return - nothing */ - _batchCommitted(error, batch) { + _batchCommitted( + error: ArsenalError | null, + batch: { callback: (((error: ArsenalError | null) => void) | null)[] }, + ) { batch.callback.forEach(callback => { if (callback) { callback(error); @@ -132,5 +153,3 @@ class WriteGatheringManager { }); } } - -module.exports = WriteGatheringManager; From eae29c53dd4770c3ce990631c697dabff3d7a8e4 Mon Sep 17 00:00:00 2001 From: Guillaume Hivert Date: Thu, 12 May 2022 15:15:52 +0200 Subject: [PATCH 6/9] ARSN-201 Type check constants --- lib/versioning/constants.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/versioning/constants.ts b/lib/versioning/constants.ts index 932dcb8a6..b2a89572e 100644 --- a/lib/versioning/constants.ts +++ b/lib/versioning/constants.ts @@ -1,4 +1,4 @@ -module.exports.VersioningConstants = { +export const VersioningConstants = { VersionId: { Separator: '\0', }, From f98c65ffb475173c4c787485e8e1a0d3a03e3bfa Mon Sep 17 00:00:00 2001 From: Guillaume Hivert Date: Thu, 12 May 2022 15:16:00 +0200 Subject: [PATCH 7/9] ARSN-201 Type check VersioningRequestProcessor --- lib/versioning/VersioningRequestProcessor.ts | 196 ++++++++++++------- 1 file changed, 126 insertions(+), 70 deletions(-) diff --git a/lib/versioning/VersioningRequestProcessor.ts b/lib/versioning/VersioningRequestProcessor.ts index 3b4701a2b..e96ad2b71 100644 --- a/lib/versioning/VersioningRequestProcessor.ts +++ b/lib/versioning/VersioningRequestProcessor.ts @@ -1,18 +1,20 @@ -const errors = require('../errors').default; -const Version = require('./Version').Version; - -const genVID = require('./VersionID').generateVersionId; +import errors, { ArsenalError } from '../errors'; +import { Version } from './Version'; +import { generateVersionId as genVID } from './VersionID'; +import WriteCache from './WriteCache'; +import WriteGatheringManager from './WriteGatheringManager'; // some predefined constants -const VID_SEP = require('./constants').VersioningConstants.VersionId.Separator; +import { VersioningConstants } from './constants'; +const VID_SEP = VersioningConstants.VersionId.Separator; /** * Increment the charCode of the last character of a valid string. * - * @param {string} prefix - the input string - * @return {string} - the incremented string, or the input if it is not valid + * @param prefix - the input string + * @return - the incremented string, or the input if it is not valid */ -function getPrefixUpperBoundary(prefix) { +function getPrefixUpperBoundary(prefix: string): string { if (prefix) { return prefix.slice(0, prefix.length - 1) + String.fromCharCode(prefix.charCodeAt(prefix.length - 1) + 1); @@ -20,30 +22,41 @@ function getPrefixUpperBoundary(prefix) { return prefix; } -function formatVersionKey(key, versionId) { +function formatVersionKey(key: string, versionId: string) { return `${key}${VID_SEP}${versionId}`; } -function formatCacheKey(db, key) { +function formatCacheKey(db: string, key: string) { // using double VID_SEP to make sure the cache key is unique return `${db}${VID_SEP}${VID_SEP}${key}`; } const VID_SEPPLUS = getPrefixUpperBoundary(VID_SEP); -class VersioningRequestProcessor { +export default class VersioningRequestProcessor { + writeCache: WriteCache; + wgm: WriteGatheringManager; + replicationGroupId: string; + uidCounter: number; + queue: {}; + repairing: {}; + /** * This class takes a random string generator as additional input. - * @param {WriteCache} writeCache - the WriteCache to which this + * @param writeCache - the WriteCache to which this * will forward the cachable processed requests - * @param {writeGatheringManager} writeGatheringManager - the + * @param writeGatheringManager - the * WriteGatheringManager to which this will forward the * non-cachable processed requests - * @param {object} versioning - versioning configurations - * @param {string} versioning.replicationGroupId - replication group id + * @param versioning - versioning configurations + * @param versioning.replicationGroupId - replication group id * @constructor */ - constructor(writeCache, writeGatheringManager, versioning) { + constructor( + writeCache: WriteCache, + writeGatheringManager: WriteGatheringManager, + versioning: { replicationGroupId: string }, + ) { this.writeCache = writeCache; this.wgm = writeGatheringManager; this.replicationGroupId = versioning.replicationGroupId; @@ -54,7 +67,8 @@ class VersioningRequestProcessor { } generateVersionId() { - return genVID(this.uidCounter++, this.replicationGroupId); + const info = this.uidCounter++; + return genVID(info.toString(), this.replicationGroupId); } /** @@ -62,14 +76,18 @@ class VersioningRequestProcessor { * deletion, search by listing for the latest version then repair * it. * - * @param {object} request - the request in original + * @param request - the request in original * RepdConnection format { db, key * [, value][, type], method, options } - * @param {object} logger - logger - * @param {function} callback - callback function - * @return {any} - to finish the call + * @param logger - logger + * @param callback - callback function + * @return - to finish the call */ - get(request, logger, callback) { + get( + request: any, + logger: RequestLogger, + callback: (error: ArsenalError | null, data?: any) => void, + ) { const { db, key, options } = request; if (options && options.versionId) { const versionKey = formatVersionKey(key, options.versionId); @@ -96,14 +114,18 @@ class VersioningRequestProcessor { * single process is performed at any moment. Subsequent get-by-listing * requests are queued up and these requests will have the same response. * - * @param {object} request - the request in original + * @param request - the request in original * RepdConnection format { db, key * [, value][, type], method, options } - * @param {object} logger - logger - * @param {function} callback - callback function - * @return {any} - to finish the call + * @param logger - logger + * @param callback - callback function + * @return - to finish the call */ - getByListing(request, logger, callback) { + getByListing( + request: any, + logger: RequestLogger, + callback: (error: ArsenalError | null, data?: any) => void, + ) { // enqueue the get entry; do nothing if another is processing it // this is to manage the number of expensive listings when there // are multiple concurrent gets on the same key which is a PHD version @@ -149,14 +171,18 @@ class VersioningRequestProcessor { /** * Enqueue a get-by-listing request. * - * @param {object} request - the request in original + * @param request - the request in original * RepdConnection format { db, key * [, value][, type], method, options } - * @param {object} logger - logger - * @param {function} callback - callback function - * @return {boolean} - this request is the first in the queue or not + * @param logger - logger + * @param callback - callback function + * @return - this request is the first in the queue or not */ - enqueueGet(request, logger, callback) { + enqueueGet( + request: any, + logger: RequestLogger, + callback: (error: ArsenalError | null, data?: any) => void, + ): boolean { const cacheKey = formatCacheKey(request.db, request.key); // enqueue the get entry if another is processing it if (this.queue[cacheKey]) { @@ -172,14 +198,14 @@ class VersioningRequestProcessor { * Dequeue all pending get-by-listing requests by the result of the first * request in the queue. * - * @param {object} request - the request in original + * @param request - the request in original * RepdConnection format { db, key * [, value][, type], method, options } - * @param {object} err - resulting error of the first request - * @param {string} value - resulting value of the first request - * @return {undefined} + * @param err - resulting error of the first request + * @param value - resulting value of the first request + * @return */ - dequeueGet(request, err, value) { + dequeueGet(request: any, err: ArsenalError | null, value?: string) { const cacheKey = formatCacheKey(request.db, request.key); if (this.queue[cacheKey]) { this.queue[cacheKey].forEach(entry => { @@ -197,18 +223,22 @@ class VersioningRequestProcessor { * Search for the latest version of an object to update its master version * in an atomic manner when the master version is a PHD. * - * @param {object} request - the request in original + * @param request - the request in original * RepdConnection format { db, key * [, value][, type], method, options } - * @param {object} logger - logger - * @param {object} hints - storing reparing hints - * @param {string} hints.type - type of repair operation ('put' or 'del') - * @param {string} hints.value - existing value of the master version (PHD) - * @param {string} hints.nextValue - the suggested latest version + * @param logger - logger + * @param hints - storing reparing hints + * @param hints.type - type of repair operation ('put' or 'del') + * @param hints.value - existing value of the master version (PHD) + * @param hints.nextValue - the suggested latest version (for 'put') - * @return {any} - to finish the call + * @return - to finish the call */ - repairMaster(request, logger, hints) { + repairMaster(request: any, logger: RequestLogger, hints: { + type: 'put' | 'del'; + value: string; + nextValue?: string; + }) { const { db, key } = request; logger.info('start repair process', { request }); this.writeCache.get({ db, key }, logger, (err, value) => { @@ -241,14 +271,18 @@ class VersioningRequestProcessor { * Process the request if it is a versioning request, or send it to the * next level replicator if it is not. * - * @param {object} request - the request in original + * @param request - the request in original * RepdConnection format { db, key * [, value][, type], method, options } - * @param {object} logger - logger - * @param {function} callback - expect callback(err, data) - * @return {any} - to finish the call + * @param logger - logger + * @param callback - expect callback(err, data) + * @return - to finish the call */ - put(request, logger, callback) { + put( + request: any, + logger: RequestLogger, + callback: (error: ArsenalError | null, data?: any) => void, + ) { const { db, key, value, options } = request; // valid combinations of versioning options: // - !versioning && !versionId: normal non-versioning put @@ -287,14 +321,22 @@ class VersioningRequestProcessor { * operations for updating the master version and creating the specific * version. * - * @param {object} request - the request in original + * @param request - the request in original * RepdConnection format { db, key * [, value][, type], method, options } - * @param {object} logger - logger - * @param {function} callback - expect callback(err, batch, versionId) - * @return {any} - to finish the call + * @param logger - logger + * @param callback - expect callback(err, batch, versionId) + * @return - to finish the call */ - processNewVersionPut(request, logger, callback) { + processNewVersionPut( + request: any, + logger: RequestLogger, + callback: ( + error: null, + data: { key: string; value: string }[], + versionId: string, + ) => void, + ) { // making a new versionId and a new version key const versionId = this.generateVersionId(); const versionKey = formatVersionKey(request.key, versionId); @@ -311,14 +353,18 @@ class VersioningRequestProcessor { * of operations for updating the target version, and the master version if * the target version is the latest. * - * @param {object} request - the request in original + * @param request - the request in original * RepdConnection format { db, key * [, value][, type], method, options } - * @param {object} logger - logger - * @param {function} callback - expect callback(err, batch, versionId) - * @return {any} - to finish the call + * @param logger - logger + * @param callback - expect callback(err, batch, versionId) + * @return - to finish the call */ - processVersionSpecificPut(request, logger, callback) { + processVersionSpecificPut( + request: any, + logger: RequestLogger, + callback: (err: ArsenalError | null, data?: any, versionId?: string) => void, + ) { const { db, key } = request; // versionId is empty: update the master version if (request.options.versionId === '') { @@ -335,7 +381,7 @@ class VersioningRequestProcessor { const versionKey = formatVersionKey(request.key, versionId); const ops = [{ key: versionKey, value: request.value }]; if (data === undefined || - Version.from(data).getVersionId() >= versionId) { + (Version.from(data).getVersionId() ?? '') >= versionId) { // master does not exist or is not newer than put // version and needs to be updated as well. // Note that older versions have a greater version ID. @@ -347,7 +393,11 @@ class VersioningRequestProcessor { } - del(request, logger, callback) { + del( + request: any, + logger: RequestLogger, + callback: (err: ArsenalError | null, data?: any) => void, + ) { const { db, key, options } = request; // no versioning or versioning configuration off if (!(options && options.versionId)) { @@ -372,14 +422,22 @@ class VersioningRequestProcessor { * master version of the object as a place holder for deletion if the * specific version is also the master version. * - * @param {object} request - the request in original + * @param request - the request in original * RepdConnection format { db, key * [, value][, type], method, options } - * @param {object} logger - logger - * @param {function} callback - expect callback(err, batch, versionId) - * @return {any} - to finish the call + * @param logger - logger + * @param callback - expect callback(err, batch, versionId) + * @return - to finish the call */ - processVersionSpecificDelete(request, logger, callback) { + processVersionSpecificDelete( + request: any, + logger: RequestLogger, + callback: ( + error: ArsenalError | null, + batch?: any, + versionId?: string, + ) => void, + ) { const { db, key, options } = request; // deleting a specific version this.writeCache.get({ db, key }, logger, (err, data) => { @@ -389,7 +447,7 @@ class VersioningRequestProcessor { // delete the specific version const versionId = options.versionId; const versionKey = formatVersionKey(key, versionId); - const ops = [{ key: versionKey, type: 'del' }]; + const ops: any = [{ key: versionKey, type: 'del' }]; // update the master version as PHD if it is the deleting version if (Version.isPHD(data) || Version.from(data).getVersionId() === versionId) { @@ -405,5 +463,3 @@ class VersioningRequestProcessor { }); } } - -module.exports = VersioningRequestProcessor; From bd498d414be9ae600e5ac330b54ca89b3da4d693 Mon Sep 17 00:00:00 2001 From: Guillaume Hivert Date: Thu, 12 May 2022 15:16:19 +0200 Subject: [PATCH 8/9] ARSN-201 Export in index --- index.ts | 10 +--------- lib/versioning/index.ts | 6 ++++++ 2 files changed, 7 insertions(+), 9 deletions(-) create mode 100644 lib/versioning/index.ts diff --git a/index.ts b/index.ts index 51b289a47..6459f0d50 100644 --- a/index.ts +++ b/index.ts @@ -26,6 +26,7 @@ export * as constants from './lib/constants'; export * as https from './lib/https'; export * as metrics from './lib/metrics'; export * as network from './lib/network'; +export * as versioning from './lib/versioning'; export const db = require('./lib/db'); export const shuffle = require('./lib/shuffle'); @@ -66,15 +67,6 @@ export const testing = { matrix: require('./lib/testing/matrix.js'), }; -export const versioning = { - VersioningConstants: require('./lib/versioning/constants.js').VersioningConstants, - Version: require('./lib/versioning/Version.js').Version, - VersionID: require('./lib/versioning/VersionID.js'), - WriteGatheringManager: require('./lib/versioning/WriteGatheringManager.js'), - WriteCache: require('./lib/versioning/WriteCache.js'), - VersioningRequestProcessor: require('./lib/versioning/VersioningRequestProcessor.js'), -}; - export const s3routes = { routes: require('./lib/s3routes/routes'), routesUtils: require('./lib/s3routes/routesUtils'), diff --git a/lib/versioning/index.ts b/lib/versioning/index.ts new file mode 100644 index 000000000..ef64efcbd --- /dev/null +++ b/lib/versioning/index.ts @@ -0,0 +1,6 @@ +export { VersioningConstants } from './constants'; +export { Version } from './Version'; +export * as VersionID from './VersionID'; +export { default as WriteGatheringManager } from './WriteGatheringManager'; +export { default as WriteCache } from './WriteCache'; +export { default as VersioningRequestProcessor } from './VersioningRequestProcessor'; From 76bffb2a23614f901e1d058301a185a3eed97f97 Mon Sep 17 00:00:00 2001 From: Guillaume Hivert Date: Thu, 12 May 2022 15:16:23 +0200 Subject: [PATCH 9/9] ARSN-201 Fix tests --- tests/unit/versioning/VersionID.spec.js | 2 +- tests/unit/versioning/VersioningRequestProcessor.spec.js | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit/versioning/VersionID.spec.js b/tests/unit/versioning/VersionID.spec.js index afc831ba8..81702392e 100644 --- a/tests/unit/versioning/VersionID.spec.js +++ b/tests/unit/versioning/VersionID.spec.js @@ -1,4 +1,4 @@ -const VID = require('../../../lib/versioning/VersionID.js'); +const VID = require('../../../lib/versioning/VersionID'); const assert = require('assert'); const { env } = require('process'); diff --git a/tests/unit/versioning/VersioningRequestProcessor.spec.js b/tests/unit/versioning/VersioningRequestProcessor.spec.js index 84a613051..0946741bc 100644 --- a/tests/unit/versioning/VersioningRequestProcessor.spec.js +++ b/tests/unit/versioning/VersioningRequestProcessor.spec.js @@ -4,9 +4,9 @@ const async = require('async'); const Version = require('../../../lib/versioning/Version').Version; const errors = require('../../../lib/errors').default; -const WGM = require('../../../lib/versioning/WriteGatheringManager'); -const WriteCache = require('../../../lib/versioning/WriteCache'); -const VSP = require('../../../lib/versioning/VersioningRequestProcessor'); +const WGM = require('../../../lib/versioning/WriteGatheringManager').default; +const WriteCache = require('../../../lib/versioning/WriteCache').default; +const VSP = require('../../../lib/versioning/VersioningRequestProcessor').default; const DELAY_MIN = 1; const DELAY_MAX = 5;