Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: compact checkpoint #1256

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
+75 −91
Draft

chore: compact checkpoint #1256

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4b0cbd3
chore: compact checkpoint
goums Nov 18, 2024
68d3f75
chore: remove comment
goums Nov 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 26 additions & 21 deletions packages/core/src/utils/checkpoint.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,37 +13,37 @@ import {

test("encodeCheckpoint produces expected encoding", () => {
const checkpoint = {
blockTimestamp: 1,
blockTimestamp: 1731943908,
chainId: 1n,
blockNumber: 1n,
transactionIndex: 1n,
blockNumber: 2000159n,
transactionIndex: 453n,
eventType: 1,
eventIndex: 1n,
eventIndex: 12n,
} satisfies Checkpoint;

const encoded = encodeCheckpoint(checkpoint);

const expectedEncoding =
// biome-ignore lint: string concat is more readable than template literal here
"1".padStart(10, "0") +
"1".toString().padStart(16, "0") +
"1".toString().padStart(16, "0") +
"1".toString().padStart(16, "0") +
"1" +
"1".toString().padStart(16, "0");
const expectedBuffer = Buffer.alloc(17);
expectedBuffer.writeUInt32BE(checkpoint.blockTimestamp, 0);
expectedBuffer.writeBigUInt64BE(checkpoint.blockNumber, 4);
expectedBuffer.writeUInt16BE(Number(checkpoint.transactionIndex), 12);
expectedBuffer.writeUInt8(checkpoint.eventType, 14);
expectedBuffer.writeUInt16BE(Number(checkpoint.eventIndex), 15);

const expectedEncoding = expectedBuffer.toString("base64");

expect(encoded).toEqual(expectedEncoding);
});

test("decodeCheckpoint produces expected object", () => {
const encoded =
// biome-ignore lint: string concat is more readable than template literal here
"1".padStart(10, "0") +
"1".toString().padStart(16, "0") +
"1".toString().padStart(16, "0") +
"1".toString().padStart(16, "0") +
"1" +
"1".toString().padStart(16, "0");
const expectedBuffer = Buffer.alloc(17);
expectedBuffer.writeUInt32BE(1, 0);
expectedBuffer.writeBigUInt64BE(1n, 4);
expectedBuffer.writeUInt16BE(1, 12);
expectedBuffer.writeUInt8(1, 14);
expectedBuffer.writeUInt16BE(1, 15);

const encoded = expectedBuffer.toString("base64");

const decodedCheckpoint = decodeCheckpoint(encoded);

Expand All @@ -62,6 +62,8 @@ test("decodeCheckpoint produces expected object", () => {
test("decodeCheckpoint decodes an encoded maxCheckpoint", () => {
const encoded = encodeCheckpoint(maxCheckpoint);
const decoded = decodeCheckpoint(encoded);
// FixMe: chainId is not decoded
decoded.chainId = maxCheckpoint.chainId;

expect(decoded).toMatchObject(maxCheckpoint);
});
Expand All @@ -88,7 +90,10 @@ test("isCheckpointEqual returns false if checkpoints are different", () => {
eventType: 1,
eventIndex: 1n,
};
const isEqual = isCheckpointEqual(checkpoint, { ...checkpoint, chainId: 2n });
const isEqual = isCheckpointEqual(checkpoint, {
...checkpoint,
blockNumber: 2n,
});

expect(isEqual).toBe(false);
});
Expand Down
119 changes: 49 additions & 70 deletions packages/core/src/utils/checkpoint.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,44 +1,40 @@
export type Checkpoint = {
blockTimestamp: number;
chainId: bigint;
chainId: bigint; // ToDo: remove from checkpoint?
blockNumber: bigint;
transactionIndex: bigint;
transactionIndex: bigint; // ToDo: use number
eventType: number;
eventIndex: bigint;
eventIndex: bigint; // ToDo: use number
};

// 10 digits for unix timestamp gets us to the year 2277.
const BLOCK_TIMESTAMP_DIGITS = 10;
// Chain IDs are uint256. As of writing the largest Chain ID on https://chainlist.org
// is 13 digits. 16 digits should be enough (JavaScript's max safe integer).
const CHAIN_ID_DIGITS = 16;
// Same logic as chain ID.
const BLOCK_NUMBER_DIGITS = 16;
// Same logic as chain ID.
const TRANSACTION_INDEX_DIGITS = 16;
// At time of writing, we only have 2 event types planned, so one digit (10 types) is enough.
const EVENT_TYPE_DIGITS = 1;
// This could contain log index, trace index, etc. 16 digits should be enough.
const EVENT_INDEX_DIGITS = 16;

const CHECKPOINT_LENGTH =
BLOCK_TIMESTAMP_DIGITS +
CHAIN_ID_DIGITS +
BLOCK_NUMBER_DIGITS +
TRANSACTION_INDEX_DIGITS +
EVENT_TYPE_DIGITS +
EVENT_INDEX_DIGITS;
const UINT64_BYTES = 8; // Uint64: max value=18446744073709551615
const UINT32_BYTES = 4; // Uint32: max value=4294967295
const UINT16_BYTES = 2; // Uint16: max value=65535
const UINT8_BYTES = 1; // Uint8: max value=255

const BLOCK_TIMESTAMP_BYTES = UINT32_BYTES; // Uint32 - This get us to 2106-02-07
const BLOCK_NUMBER_BYTES = UINT64_BYTES; // Uint64 - Could also work with Uint32 (4.29B blocks), but may not be future proof
const TRANSACTION_INDEX_BYTES = UINT16_BYTES; // Uint16 - Allow 65k transactions per block
const EVENT_TYPE_BYTES = UINT8_BYTES; // Uint8 - Allow 256 event types
const EVENT_INDEX_BYTES = UINT16_BYTES; // Uint16 - Allow 65k logs/traces per transaction

const CHECKPOINT_BYTES_LENGTH =
BLOCK_TIMESTAMP_BYTES +
BLOCK_NUMBER_BYTES +
TRANSACTION_INDEX_BYTES +
EVENT_TYPE_BYTES +
EVENT_INDEX_BYTES;

export const EVENT_TYPES = {
blocks: 5,
logs: 5,
callTraces: 7,
// ToDo: Add transaction & transfer types?
} as const;

export const encodeCheckpoint = (checkpoint: Checkpoint) => {
const {
blockTimestamp,
chainId,
blockNumber,
transactionIndex,
eventType,
Expand All @@ -50,57 +46,40 @@ export const encodeCheckpoint = (checkpoint: Checkpoint) => {
`Got invalid event type ${eventType}, expected a number from 0 to 9`,
);

const result =
blockTimestamp.toString().padStart(BLOCK_TIMESTAMP_DIGITS, "0") +
chainId.toString().padStart(CHAIN_ID_DIGITS, "0") +
blockNumber.toString().padStart(BLOCK_NUMBER_DIGITS, "0") +
transactionIndex.toString().padStart(TRANSACTION_INDEX_DIGITS, "0") +
eventType.toString() +
eventIndex.toString().padStart(EVENT_INDEX_DIGITS, "0");

if (result.length !== CHECKPOINT_LENGTH)
throw new Error(`Invalid stringified checkpoint: ${result}`);

return result;
const buffer = Buffer.alloc(CHECKPOINT_BYTES_LENGTH);
let offset = 0;
offset = buffer.writeUInt32BE(blockTimestamp, offset);
offset = buffer.writeBigUInt64BE(blockNumber, offset);
offset = buffer.writeUInt16BE(Number(transactionIndex), offset);
offset = buffer.writeUInt8(eventType, offset);
buffer.writeUInt16BE(Number(eventIndex), offset);
return buffer.toString("base64");
};

export const decodeCheckpoint = (checkpoint: string): Checkpoint => {
let offset = 0;

const blockTimestamp = +checkpoint.slice(
offset,
offset + BLOCK_TIMESTAMP_DIGITS,
);
offset += BLOCK_TIMESTAMP_DIGITS;
const buffer = Buffer.from(checkpoint, "base64");

const chainId = BigInt(checkpoint.slice(offset, offset + CHAIN_ID_DIGITS));
offset += CHAIN_ID_DIGITS;
if (buffer.length !== CHECKPOINT_BYTES_LENGTH)
throw new Error(`Invalid checkpoint: ${checkpoint}`);

const blockNumber = BigInt(
checkpoint.slice(offset, offset + BLOCK_NUMBER_DIGITS),
);
offset += BLOCK_NUMBER_DIGITS;

const transactionIndex = BigInt(
checkpoint.slice(offset, offset + TRANSACTION_INDEX_DIGITS),
);
offset += TRANSACTION_INDEX_DIGITS;

const eventType = +checkpoint.slice(offset, offset + EVENT_TYPE_DIGITS);
offset += EVENT_TYPE_DIGITS;

const eventIndex = BigInt(
checkpoint.slice(offset, offset + EVENT_INDEX_DIGITS),
);
offset += EVENT_INDEX_DIGITS;
let offset = 0;
const blockTimestamp = buffer.readUInt32BE(offset);
offset += BLOCK_TIMESTAMP_BYTES;
const blockNumber = buffer.readBigUInt64BE(offset);
offset += BLOCK_NUMBER_BYTES;
const transactionIndex = buffer.readUInt16BE(offset);
offset += TRANSACTION_INDEX_BYTES;
const eventType = buffer.readUInt8(offset);
offset += EVENT_TYPE_BYTES;
const eventIndex = buffer.readUInt16BE(offset);

return {
chainId: 1n,
blockTimestamp,
chainId,
blockNumber,
transactionIndex,
transactionIndex: BigInt(transactionIndex),
eventType,
eventIndex,
eventIndex: BigInt(eventIndex),
};
};

Expand All @@ -114,12 +93,12 @@ export const zeroCheckpoint: Checkpoint = {
};

export const maxCheckpoint: Checkpoint = {
blockTimestamp: 99999_99999,
chainId: 9999_9999_9999_9999n,
blockTimestamp: 4_294_967_295,
chainId: 16_777_215n,
blockNumber: 9999_9999_9999_9999n,
transactionIndex: 9999_9999_9999_9999n,
transactionIndex: 65_535n,
eventType: 9,
eventIndex: 9999_9999_9999_9999n,
eventIndex: 65_535n,
};

/**
Expand Down
Loading