diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a52493..4782bc8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,16 @@ +## 0.16.0 + +**Breaking change, databases created using earlier versions is not compatible with 0.16.0 and up.** + +- Use 32 bit MurmurHash3 (from new dependency `npm:ohash` instead of SHA-1 for transaction integrity check, + saving time and storage +- Precalculate the Uint32Array of the transaction signature, improving + performance +- Use CBOR-encoding of key instead of custom implementation, improving + performance +- Avoid copying arraybuffers in certain situations +- Prefetch data on transaction read, severely reducing the number of disk reads + ## 0.15.11 - Remove option doLock from `.sync()` diff --git a/deno.json b/deno.json index 15507e1..d0ba13a 100644 --- a/deno.json +++ b/deno.json @@ -1,6 +1,6 @@ { "name": "@cross/kv", - "version": "0.15.11", + "version": "0.16.0", "exports": { ".": "./mod.ts", "./cli": "./src/cli/mod.ts" @@ -12,7 +12,8 @@ "@cross/utils": "jsr:@cross/utils@^0.13.0", "@std/assert": "jsr:@std/assert@^0.226.0", "@std/path": "jsr:@std/path@^0.225.1", - "cbor-x": "npm:cbor-x@^1.5.9" + "cbor-x": "npm:cbor-x@^1.5.9", + "ohash": "npm:ohash@^1.1.3" }, "publish": { "exclude": [".github", "test/*"] diff --git a/src/lib/cache.ts b/src/lib/cache.ts index e9635f8..dda210a 100644 --- a/src/lib/cache.ts +++ b/src/lib/cache.ts @@ -2,7 +2,7 @@ import { LEDGER_CACHE_MEMORY_FACTOR } from "./constants.ts"; import type { KVLedgerResult } from "./ledger.ts"; /** - * An in-memory cache for `KVLedgerResult` objects, optimized for append-only ledgers. + * An in-memory cache for `KVLedgerResult` objects. * * This cache stores transaction results (`KVLedgerResult`) associated with their offsets within the ledger. * It maintains a fixed maximum size and evicts the oldest entries (Least Recently Used - LRU) diff --git a/src/lib/constants.ts b/src/lib/constants.ts index 881a23f..c5be825 100644 --- a/src/lib/constants.ts +++ b/src/lib/constants.ts @@ -2,15 +2,12 @@ export const LOCK_DEFAULT_MAX_RETRIES = 32; export const LOCK_DEFAULT_INITIAL_RETRY_INTERVAL_MS = 30; // Increased with itself on each retry, so the actual retry interval is 20, 40, 60 etc. 32 and 20 become about 10 seconds total. export const LOCK_STALE_TIMEOUT_MS = 6 * 60 * 60 * 1000; // Automatically unlock a ledger that has been locked for more than 2*60*60*1000 = 2 hours. -export const LEDGER_CURRENT_VERSION: string = "B015"; +export const LEDGER_CURRENT_VERSION: string = "B016"; export const SUPPORTED_LEDGER_VERSIONS: string[] = [ LEDGER_CURRENT_VERSION, - "B011", - "B012", - "B013", ]; +export const LEDGER_PREFETCH_BYTES = 50 * 1024; // Prefetch chunks of 50KB of data while reading the ledger export const LEDGER_MAX_READ_FAILURES = 10; -export const LEDGER_PREFETCH_BYTES = 256; export const SYNC_INTERVAL_MS = 2_500; // Overridable with instance configuration export const LEDGER_CACHE_MB = 100; // Allow 100 MBytes of the ledger to exist in RAM. Not an exact science due to LEDGER_CACHE_MEMORY_FACTOR. export const LEDGER_CACHE_MEMORY_FACTOR = 3; // Assume that ledger entries take about n times as much space when unwrapped in RAM. Used for ledger cache memory limit, does not need to be exakt. @@ -21,5 +18,7 @@ export const LOCKED_BYTES_LENGTH = 8; // Length of timestamp export const LOCK_BYTE_OFFSET = LEDGER_BASE_OFFSET - LOCKED_BYTES_LENGTH; // Last 8 bytes of the header export const KV_KEY_ALLOWED_CHARS = /^[@\p{L}\p{N}_-]+$/u; // Unicode letters and numbers, undescore, hyphen and at export const LEDGER_FILE_ID: string = "CKVD"; // Cross/KV Database -export const TRANSACTION_SIGNATURE: string = "T;"; // Cross/Kv Transaction +export const ENCODED_TRANSACTION_SIGNATURE: Uint8Array = new TextEncoder() + .encode("T;"); // Cross/Kv Transaction export const UNLOCKED_BYTES = new Uint8Array(LOCKED_BYTES_LENGTH); +export const LOCKED_BYTES = new Uint8Array(LOCKED_BYTES_LENGTH); diff --git a/src/lib/key.ts b/src/lib/key.ts index 2425925..cbddfe6 100644 --- a/src/lib/key.ts +++ b/src/lib/key.ts @@ -1,3 +1,4 @@ +import { decode, encode } from "cbor-x"; import { KV_KEY_ALLOWED_CHARS } from "./constants.ts"; // Helper function to stringify range values correctly @@ -68,11 +69,11 @@ export class KVKeyInstance { private isQuery: boolean; public byteLength?: number; constructor( - key: KVQuery | KVKey | DataView, + key: KVQuery | KVKey | Uint8Array, isQuery: boolean = false, validate: boolean = true, ) { - if (key instanceof DataView) { + if (key instanceof Uint8Array) { this.key = this.fromUint8Array(key); } else { this.key = key; @@ -87,48 +88,8 @@ export class KVKeyInstance { * Encodes the key elements into a byte array suitable for storage in a transaction header. */ public toUint8Array(): Uint8Array { - const keyBytesArray = []; - for (const element of this.key) { - if (typeof element === "string") { - keyBytesArray.push(new Uint8Array([0])); // Type: String - const strBytes = new TextEncoder().encode(element); - const strLengthBytes = new Uint8Array(4); - new DataView(strLengthBytes.buffer).setUint32( - 0, - strBytes.length, - false, - ); - keyBytesArray.push(strLengthBytes); - keyBytesArray.push(strBytes); - } else if (typeof element === "number") { - keyBytesArray.push(new Uint8Array([1])); // Type: Number - const numBytes = new Uint8Array(8); - new DataView(numBytes.buffer).setFloat64(0, element, false); - keyBytesArray.push(numBytes); - } else { - // This should never happen if validate() is working correctly - throw new TypeError("Invalid key element type"); - } - } - - // Encode the number of key elements - const numKeyElementsBytes = new Uint8Array(1); - new DataView(numKeyElementsBytes.buffer).setUint8( - 0, - this.key.length, - ); - keyBytesArray.unshift(numKeyElementsBytes); // Add to the beginning - - const keyArray = new Uint8Array( - keyBytesArray.reduce((a, b) => a + b.length, 0), - ); - let keyOffset = 0; - for (const bytes of keyBytesArray) { - keyArray.set(bytes, keyOffset); - keyOffset += bytes.length; - } - - return keyArray; + const data = encode(this.key); + return new Uint8Array(data, 0, data.byteLength); } /** @@ -137,44 +98,9 @@ export class KVKeyInstance { * @param data - The byte array containing the encoded key. * @throws {Error} If the key cannot be decoded. */ - private fromUint8Array(dataView: DataView): KVKey { - let offset = 0; - - // 1. Decode Number of Key Elements (uint32) - const numKeyElements = dataView.getUint8(offset); - offset += 1; - - const keyToBe: KVKey = []; - - for (let i = 0; i < numKeyElements; i++) { - // 2. Decode Element Type (uint8): 0 for string, 1 for number - const elementType = dataView.getUint8(offset); - offset += 1; - - if (elementType === 0) { // String - // 3a. Decode String Length (uint32) - const strLength = dataView.getUint32(offset, false); - offset += 4; - - // 3b. Decode String Bytes - const strBytes = new DataView( - dataView.buffer, - dataView.byteOffset + offset, - strLength, - ); - keyToBe.push(new TextDecoder().decode(strBytes)); - offset += strLength; - } else if (elementType === 1) { // Number - // 3c. Decode Number (float64) - const numValue = dataView.getFloat64(offset, false); - keyToBe.push(numValue); - offset += 8; - } else { - throw new Error(`Invalid key element type ${elementType}`); - } - } - this.byteLength = offset; - return keyToBe; + private fromUint8Array(data: Uint8Array): KVKey { + this.key = decode(data); + return this.key as KVKey; } get(): KVQuery | KVKey { @@ -301,7 +227,7 @@ export class KVKeyInstance { } } - const instance = new KVKeyInstance(result, isQuery); + const instance = new KVKeyInstance(result, isQuery, false); instance.validate(); return isQuery ? result : result as KVKey; @@ -380,7 +306,10 @@ export class KVKeyInstance { const subquery = query.slice(i + 1); const subkey = thisKey.slice(i + 1); if ( - !new KVKeyInstance(subkey, true).matchesQuery(subquery, recursive) + !new KVKeyInstance(subkey, true, false).matchesQuery( + subquery, + recursive, + ) ) { return false; } diff --git a/src/lib/kv.ts b/src/lib/kv.ts index dab5f66..bc37b26 100644 --- a/src/lib/kv.ts +++ b/src/lib/kv.ts @@ -159,7 +159,7 @@ export class KV extends EventEmitter { "Invalid option: syncIntervalMs must be a positive integer", ); } - this.syncIntervalMs = options.syncIntervalMs ?? SYNC_INTERVAL_MS; + this.syncIntervalMs = options.syncIntervalMs ?? this.syncIntervalMs; // - ledgerCacheSize if ( options.ledgerCacheSize !== undefined && @@ -209,9 +209,9 @@ export class KV extends EventEmitter { this.ledger = new KVLedger(filePath, this.ledgerCacheSize); this.ledgerPath = filePath; await this.ledger.open(createIfMissing); - // Do the initial synchronization // - If `this.autoSync` is enabled, additional synchronizations will be carried out every `this.syncIntervalMs` + const syncResult = await this.sync(); if (syncResult.error) { throw syncResult.error; @@ -747,8 +747,12 @@ export class KV extends EventEmitter { this.ensureOpen(); this.ensureIndex(); + const validatedQuery: KVKeyInstance | null = key === null + ? null + : new KVKeyInstance(key, true); + return this.index.getChildKeys( - key === null ? null : new KVKeyInstance(key, true), + key === null ? null : validatedQuery, ); } diff --git a/src/lib/ledger.ts b/src/lib/ledger.ts index 70c1fc8..86d8252 100644 --- a/src/lib/ledger.ts +++ b/src/lib/ledger.ts @@ -6,6 +6,7 @@ import { writeAtPosition, } from "./utils/file.ts"; import { + ENCODED_TRANSACTION_SIGNATURE, LEDGER_BASE_OFFSET, LEDGER_CURRENT_VERSION, LEDGER_FILE_ID, @@ -14,9 +15,9 @@ import { LOCK_DEFAULT_INITIAL_RETRY_INTERVAL_MS, LOCK_DEFAULT_MAX_RETRIES, LOCK_STALE_TIMEOUT_MS, + LOCKED_BYTES, LOCKED_BYTES_LENGTH, SUPPORTED_LEDGER_VERSIONS, - TRANSACTION_SIGNATURE, UNLOCKED_BYTES, } from "./constants.ts"; import { KVOperation, KVTransaction } from "./transaction.ts"; @@ -24,6 +25,7 @@ import { rename, unlink } from "@cross/fs"; import type { FileHandle } from "node:fs/promises"; import type { KVQuery } from "./key.ts"; import { KVLedgerCache } from "./cache.ts"; +import { KVPrefetcher } from "./prefetcher.ts"; /** * This file handles the ledger file, which is where all persisted data of an cross/kv instance is stored. @@ -60,9 +62,12 @@ export class KVLedger { private opened: boolean = false; private dataPath: string; - // Cache + // Cache for decoded transactions public cache: KVLedgerCache; + // Rolling pre-fetch cache for raw transaction data + public prefetch: KVPrefetcher; + public header: KVLedgerHeader = { fileId: LEDGER_FILE_ID, ledgerVersion: LEDGER_CURRENT_VERSION, @@ -73,6 +78,7 @@ export class KVLedger { constructor(filePath: string, maxCacheSizeMBytes: number) { this.dataPath = toNormalizedAbsolutePath(filePath); this.cache = new KVLedgerCache(maxCacheSizeMBytes * 1024 * 1024); + this.prefetch = new KVPrefetcher(); } /** @@ -146,6 +152,7 @@ export class KVLedger { currentOffset += result.length; // Advance the offset failures = 0; } catch (_e) { + console.error(_e); failures++; } } @@ -221,6 +228,7 @@ export class KVLedger { try { reusableFd = await rawOpen(this.dataPath, false); // Keep file open during scan + while (currentOffset < this.header.currentOffset) { // Allow getting partial results (2nd parameter false) to re-use ledger cache to the maximum const result = await this.rawGetTransaction( @@ -261,12 +269,14 @@ export class KVLedger { // Encode fileId new TextEncoder().encodeInto( this.header.fileId, + // - Creates a uint8 view into the existing ArrayBuffer new Uint8Array(headerBuffer, 0, 4), ); // Encode ledgerVersion new TextEncoder().encodeInto( this.header.ledgerVersion, + // - Creates a uint8 view into the existing ArrayBuffer new Uint8Array(headerBuffer, 4, 4), ); @@ -326,30 +336,30 @@ export class KVLedger { externalFd?: Deno.FsFile | FileHandle, ): Promise { this.ensureOpen(); - // Check cache first const cachedResult = this.cache.getTransactionData(baseOffset); if (cachedResult && (!readData || cachedResult.complete)) { return cachedResult; } - let fd = externalFd; try { if (!externalFd) fd = await rawOpen(this.dataPath, false); // Fetch 2 + 4 + 4 bytes (signature, header length, data length) - const transactionLengthData = await readAtPosition( + const baseData = await this.prefetch.read( fd!, - TRANSACTION_SIGNATURE.length + 4 + 4, + ENCODED_TRANSACTION_SIGNATURE.length + 4 + 4, baseOffset, ); const transactionLengthDataView = new DataView( - transactionLengthData.buffer, + baseData.buffer, + 0, + ENCODED_TRANSACTION_SIGNATURE.length + 4 + 4, ); let headerOffset = 0; - headerOffset += TRANSACTION_SIGNATURE.length; + headerOffset += ENCODED_TRANSACTION_SIGNATURE.length; // Read header length (offset by 4 bytes for header length uint32) const headerLength = transactionLengthDataView.getUint32( @@ -366,9 +376,8 @@ export class KVLedger { headerOffset += 4; const transaction = new KVTransaction(); - // Read transaction header - const transactionHeaderData = await readAtPosition( + const transactionHeaderData = await this.prefetch.read( fd!, headerLength, baseOffset + headerOffset, @@ -378,14 +387,13 @@ export class KVLedger { // Read transaction data (optional) const complete = readData || !(dataLength > 0); if (readData && dataLength > 0) { - const transactionData = await readAtPosition( + const transactionData = await this.prefetch.read( fd!, dataLength, baseOffset + headerOffset + headerLength, ); - await transaction.dataFromUint8Array(transactionData); + transaction.dataFromUint8Array(transactionData); } - // Get transaction result const result = { offset: baseOffset, @@ -396,7 +404,6 @@ export class KVLedger { // Cache transaction this.cache.cacheTransactionData(baseOffset, result); - return result; } finally { if (fd && !externalFd) fd.close(); @@ -465,8 +472,9 @@ export class KVLedger { } this.header.currentOffset = tempLedger.header.currentOffset; - // 6. Clear cache + // 6. Clear cache and prefetch this.cache.clear(); + this.prefetch.clear(); // 7. Replace Original File // - The lock flag is now set independently, no need to unlock from this point on @@ -521,7 +529,7 @@ export class KVLedger { } // 2. Prepare lock data - const lockBytes = new Uint8Array(LOCKED_BYTES_LENGTH); + const lockBytes = LOCKED_BYTES; const lockView = new DataView(lockBytes.buffer); lockView.setBigUint64(0, BigInt(Date.now()), false); diff --git a/src/lib/prefetcher.ts b/src/lib/prefetcher.ts new file mode 100644 index 0000000..ef6ade7 --- /dev/null +++ b/src/lib/prefetcher.ts @@ -0,0 +1,58 @@ +import { readAtPosition } from "./utils/file.ts"; +import type { FileHandle } from "node:fs/promises"; +import { LEDGER_PREFETCH_BYTES } from "./constants.ts"; + +export class KVPrefetcher { + private cache?: Uint8Array; + private currentChunkStart: number; + private currentChunkEnd: number; + + constructor() { + this.currentChunkStart = 0; + this.currentChunkEnd = 0; + } + + private async fetchChunk( + fd: Deno.FsFile | FileHandle, + startPosition: number, + length: number, + ): Promise { + const chunk = await readAtPosition( + fd, + length > LEDGER_PREFETCH_BYTES ? length : LEDGER_PREFETCH_BYTES, + startPosition, + ); + this.cache = chunk; + this.currentChunkStart = startPosition; + this.currentChunkEnd = startPosition + chunk.length; + } + + public async read( + fd: Deno.FsFile | FileHandle, + length: number, + position: number, + ): Promise { + // Ensure we have the required chunk + if ( + position < this.currentChunkStart || + position + length > this.currentChunkEnd + ) { + await this.fetchChunk(fd, position, length); + } + + if (!this.cache) { + throw new Error("Failed to fetch data"); + } + + // Use slice to always return a fresh Uint8Array without an internal offset + // to the underlying buffer + return this.cache.slice( + position - this.currentChunkStart, + position - this.currentChunkStart + length, + ); + } + + public clear() { + this.cache = undefined; + } +} diff --git a/src/lib/transaction.ts b/src/lib/transaction.ts index 9dd4c9c..675ebd1 100644 --- a/src/lib/transaction.ts +++ b/src/lib/transaction.ts @@ -1,7 +1,8 @@ -import { compareHash, sha1 } from "./utils/hash.ts"; +//import { compareHash, sha1 } from "./utils/hash.ts"; import { type KVKey, KVKeyInstance } from "./key.ts"; import { decode, encode } from "cbor-x"; -import { TRANSACTION_SIGNATURE } from "./constants.ts"; +import { ENCODED_TRANSACTION_SIGNATURE } from "./constants.ts"; +import { murmurHash } from "ohash"; /** * Data structure of a Cross/kv transaction: @@ -16,22 +17,14 @@ import { TRANSACTION_SIGNATURE } from "./constants.ts"; * * Header Bytes Structure: * - * | Key Data... | Operation (uint8) | Timestamp (uint32) | Hash Length (uint32) | Hash Bytes... | - * + * Key Length (uint32) | Key Data... | Operation (uint8) | Timestamp (uint32) | Hash Length (uint32) | Hash Bytes... | + + * - Key Length: The length of the key in bytes. * - Key Data: Data returned by the key * - Operation: The type of operation (SET or DELETE). * - Timestamp: The timestamp of the operation. - * - Hash Length: The length of the hash in bytes. - * - Hash Bytes: The hash of the data (optional). - * - * Key Element Structure (repeated for each key element): - * - * | Element Type (uint8) | Element Data... | - * - * - Element Type: 0 for string, 1 for number. - * - String Element Data: String Length (uint32) | String Bytes... - * - Number Element Data: Number Value (float64) - */ + * - Hash Length: The hash (32 bit positive integer). + **/ /** * Enumerates the possible operations that can be performed on a key-value pair in the KV store. @@ -104,7 +97,7 @@ export interface KVTransactionResult { * The hash of the raw transaction data. This can be used for * verification and integrity checks. */ - hash: Uint8Array | null; + hash: number | null; } // Concrete implementation of the KVTransaction interface @@ -113,11 +106,11 @@ export class KVTransaction { public operation?: KVOperation; public timestamp?: number; public data?: Uint8Array; - public hash?: Uint8Array; + public hash?: number; constructor() { } - public async create( + public create( key: KVKeyInstance, operation: KVOperation, timestamp: number, @@ -127,15 +120,13 @@ export class KVTransaction { if (this.operation === KVOperation.SET && value === undefined) { throw new Error("Set operation needs data"); } - - // Assign this.key = key; this.operation = operation; this.timestamp = timestamp; - if (value) { + if (operation !== KVOperation.DELETE && value) { const valueData = new Uint8Array(encode(value)); this.data = valueData; - this.hash = await sha1(valueData); + this.hash = murmurHash(valueData); } } @@ -150,11 +141,25 @@ export class KVTransaction { ); let offset = 0; + // Decode key length + const keyLength = dataView.getUint32(offset, false); + offset += 4; + // Decode key - this.key = new KVKeyInstance(dataView, false, false); - offset += this.key.byteLength!; + // - Create a view into the original buffer + const keyView = new Uint8Array( + data.buffer, + data.byteOffset + offset, + keyLength, + ); + this.key = new KVKeyInstance( + keyView, + false, + false, + ); + offset += keyLength; - // Decode operation (assuming it's encoded as uint8) + // Decode operation this.operation = dataView.getUint8(offset); offset += 1; @@ -162,18 +167,11 @@ export class KVTransaction { this.timestamp = dataView.getFloat64(offset, false); offset += 8; - // Decode hash length (assuming it's encoded as uint32) - const hashLength = dataView.getUint32(offset, false); - offset += 4; - // Decode hash bytes if (readHash) { - this.hash = data.slice( - offset, - offset + hashLength, - ); + this.hash = dataView.getUint32(offset, false); } - offset += hashLength; + offset += 4; // Do not allow extra data if (offset !== data.byteLength) { @@ -181,8 +179,8 @@ export class KVTransaction { } } - public async dataFromUint8Array(data: Uint8Array) { - if (!compareHash(await sha1(data), this.hash!)) { + public dataFromUint8Array(data: Uint8Array) { + if (murmurHash(data) !== this.hash!) { throw new Error("Invalid data: Read data not matching hash"); } this.data = data; @@ -197,11 +195,12 @@ export class KVTransaction { const pendingTransactionData = this.data; // Calculate total sizes - const headerSize = keyBytes.length + 1 + 8 + 4 + (hashBytes?.length ?? 0); + const headerSize = 4 + keyBytes.length + 1 + 8 + 4; const dataLength = pendingTransactionData ? pendingTransactionData.length : 0; - const fullDataSize = TRANSACTION_SIGNATURE.length + 4 + 4 + headerSize + + const fullDataSize = ENCODED_TRANSACTION_SIGNATURE.length + 4 + 4 + + headerSize + dataLength; const fullData = new Uint8Array(fullDataSize); @@ -210,9 +209,8 @@ export class KVTransaction { let offset = 0; // Encode transaction signature - const signature = new TextEncoder().encode(TRANSACTION_SIGNATURE); - fullData.set(signature, 0); - offset += TRANSACTION_SIGNATURE.length; + fullData.set(ENCODED_TRANSACTION_SIGNATURE, 0); + offset += ENCODED_TRANSACTION_SIGNATURE.length; // Encode header and data lengths fullDataView.setUint32(offset, headerSize, false); @@ -221,6 +219,10 @@ export class KVTransaction { fullDataView.setUint32(offset, dataLength, false); offset += 4; + // Encode key length + fullDataView.setUint32(offset, keyBytes.length, false); + offset += 4; + // Encode key bytes fullData.set(keyBytes, offset); offset += keyBytes.length; @@ -229,12 +231,8 @@ export class KVTransaction { fullDataView.setUint8(offset++, this.operation!); fullDataView.setFloat64(offset, this.timestamp!, false); offset += 8; - fullDataView.setUint32(offset, hashBytes?.length ?? 0, false); + fullDataView.setUint32(offset, hashBytes ?? 0, false); offset += 4; - if (hashBytes) { - fullData.set(hashBytes, offset); - offset += hashBytes.length; - } // Encode data (if present) if (pendingTransactionData) { @@ -245,12 +243,7 @@ export class KVTransaction { } private getData(): T | null { - // Return data, should be validated through create or fromUint8Array - if (this.data) { - return decode(this.data); - } else { - return null; - } + return this.data ? decode(this.data) : null; } /** diff --git a/src/lib/utils/file.ts b/src/lib/utils/file.ts index f27fcea..10bf18c 100644 --- a/src/lib/utils/file.ts +++ b/src/lib/utils/file.ts @@ -1,4 +1,9 @@ -import { type FileHandle, open, writeFile } from "node:fs/promises"; +import { + type FileHandle, + type FileReadResult, + open, + writeFile, +} from "node:fs/promises"; import { CurrentRuntime, Runtime } from "@cross/runtime"; import { cwd, isDir, isFile, mkdir } from "@cross/fs"; import { dirname, isAbsolute, join, resolve } from "@std/path"; @@ -57,15 +62,21 @@ export async function readAtPosition( if (CurrentRuntime === Runtime.Deno) { await (fd as Deno.FsFile).seek(position, Deno.SeekMode.Start); const buffer = new Uint8Array(length); - await fd.read(buffer); - return buffer; - + const bytesRead = await (fd as Deno.FsFile).read(buffer); + return buffer.subarray(0, bytesRead ?? 0); // Node or Bun } else { // @ts-ignore cross-runtime const buffer = Buffer.alloc(length); - await fd.read(buffer, 0, length, position); - return buffer; + const readResult = await fd.read( + buffer, + 0, + length, + position, + // deno-lint-ignore no-explicit-any + ) as FileReadResult; + const bytesRead = readResult.bytesRead as number; + return buffer.subarray(0, bytesRead); } } diff --git a/src/lib/utils/hash.ts b/src/lib/utils/hash.ts deleted file mode 100644 index 397d74d..0000000 --- a/src/lib/utils/hash.ts +++ /dev/null @@ -1,17 +0,0 @@ -/** - * node:crypto is used instead of global crypto.subtle to get Node/Bun-support without polyfilling - */ -import { subtle } from "node:crypto"; - -export async function sha1(data: Uint8Array): Promise { - return new Uint8Array(await subtle.digest("SHA-1", data)); -} - -export function compareHash(arr1: Uint8Array, arr2: Uint8Array): boolean { - if (arr1.length !== arr2.length) return false; // Length mismatch - - for (let i = 0; i < arr1.length; i++) { - if (arr1[i] !== arr2[i]) return false; // Value mismatch - } - return true; -} diff --git a/test/key.test.ts b/test/key.test.ts index fdb71fa..fa78bdb 100644 --- a/test/key.test.ts +++ b/test/key.test.ts @@ -302,8 +302,7 @@ test("KVKeyInstance: toUint8Array and fromUint8Array roundtrip", () => { const instance = new KVKeyInstance(key); const uint8Array = instance.toUint8Array(); - const dataView = new DataView(uint8Array.buffer); - const newKeyInstance = new KVKeyInstance(dataView); + const newKeyInstance = new KVKeyInstance(uint8Array); assertEquals(newKeyInstance.get(), key); }); diff --git a/test/transaction.test.ts b/test/transaction.test.ts index ef48648..26db432 100644 --- a/test/transaction.test.ts +++ b/test/transaction.test.ts @@ -2,7 +2,7 @@ import { assertEquals, assertThrows } from "@std/assert"; import { test } from "@cross/test"; import { KVKeyInstance } from "../src/lib/key.ts"; import { KVOperation, KVTransaction } from "../src/lib/transaction.ts"; -import { TRANSACTION_SIGNATURE } from "../src/lib/constants.ts"; +import { ENCODED_TRANSACTION_SIGNATURE } from "../src/lib/constants.ts"; test("KVTransaction: create and toUint8Array", async () => { const key = new KVKeyInstance(["testKey"]); @@ -15,10 +15,10 @@ test("KVTransaction: create and toUint8Array", async () => { const uint8Array = transaction.toUint8Array(); const decodedTransaction = new KVTransaction(); - const headerOffset = TRANSACTION_SIGNATURE.length + 4 + 4; // <2 bytes "T;"> + const headerOffset = ENCODED_TRANSACTION_SIGNATURE.length + 4 + 4; // <2 bytes "T;"> const headerLength = new DataView(uint8Array.buffer).getUint32( - TRANSACTION_SIGNATURE.length, + ENCODED_TRANSACTION_SIGNATURE.length, ); decodedTransaction.headerFromUint8Array(