Skip to content

Commit

Permalink
bench: Add a script to sync benchmark results for all versions (coral…
Browse files Browse the repository at this point in the history
  • Loading branch information
acheroncrypto authored May 3, 2023
1 parent a195106 commit c166712
Show file tree
Hide file tree
Showing 8 changed files with 236 additions and 48 deletions.
7 changes: 4 additions & 3 deletions tests/bench/Anchor.toml
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,14 @@ cluster = "localnet"
wallet = "~/.config/solana/id.json"

[programs.localnet]
bench = "Fg6PaFpoGXkYsidMpWTK6W2BeZ7FEfcYkg476zPFsLnS"
bench = "Bench11111111111111111111111111111111111111"

[workspace]
members = ["programs/bench"]

[scripts]
test = "yarn run ts-mocha -t 1000000 -p ./tsconfig.json -t 1000000 tests/**/*.ts"
update-bench = "yarn run ts-node scripts/update-bench.ts"
test = "yarn run ts-mocha -t 1000000 -p ./tsconfig.json tests/**/*.ts"
sync = "yarn run ts-node scripts/sync.ts"
sync-markdown = "yarn run ts-node scripts/sync-markdown.ts"
generate-ix = "yarn run ts-node scripts/generate-ix.ts"
bump-version = "yarn run ts-node scripts/bump-version.ts"
14 changes: 9 additions & 5 deletions tests/bench/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,21 +4,25 @@ The bench program and its tests are used to measure the performance of Anchor pr

## How

Create a program -> Write tests that measure usage -> Compare the results -> Save the new result

The script will check whether there is a difference between the current result and the last saved result(in `bench.json`) at the end of the tests. If the difference between the results is greater than 1%, the new data will be saved in `bench.json` and Markdown files in [/bench](https://github.com/coral-xyz/anchor/tree/master/bench) will be updated accordingly.
We run the same tests that measure some metric for each Anchor version starting from `0.27.0`. If the difference between the results is greater than 1%, the new data will be saved in `bench.json` and Markdown files in [/bench](https://github.com/coral-xyz/anchor/tree/master/bench) will be updated accordingly.

## Scripts

| :memo: TL;DR |
| :----------------------------------------------------------------------------------------------------------------------------- |
| If you've made changes to programs or tests in this directory, run `anchor run sync`, otherwise run `anchor test --skip-lint`. |

`anchor test --skip-lint`: Run all tests and update benchmark files when necessary. This is the only command that needs to be run for most use cases.

---

The following scripts are useful when making changes to how benchmarking works.

`anchor run update-bench`: Update Markdown files in [/bench](https://github.com/coral-xyz/anchor/tree/master/bench) based on the data from `bench.json`.
`anchor run sync`: Sync all benchmark files by running tests for each version. If you've made changes to the bench program or its tests, you should run this command to sync the results.

`anchor run sync-markdown`: Sync Markdown files in [/bench](https://github.com/coral-xyz/anchor/tree/master/bench) based on the data from `bench.json`.

`anchor run generate-ix`: Generate instructions with repetitive accounts.
`anchor run generate-ix`: Generate program instructions with repetitive accounts.

---

Expand Down
5 changes: 5 additions & 0 deletions tests/bench/programs/bench/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -14,3 +14,8 @@ cpi = ["no-entrypoint"]
[dependencies]
anchor-lang = { path = "../../../../lang" }
anchor-spl = { path = "../../../../spl" }

# TODO: Remove this and store lock files for each version instead.
# Latest solana version(1.14.17) as of 2023-05-01 comes with rustc 1.62.0-dev but MSRV for latest
# version of this crate is 1.64.0. See https://github.com/solana-labs/solana/pull/31418
winnow = "=0.4.1"
4 changes: 3 additions & 1 deletion tests/bench/programs/bench/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
//! This program is used to measure the performance of Anchor programs.
//!
//! If you are making a change to this program, run `anchor run sync`.
use anchor_lang::prelude::*;
use anchor_spl::token_interface::{Mint, TokenAccount, TokenInterface};

declare_id!("Fg6PaFpoGXkYsidMpWTK6W2BeZ7FEfcYkg476zPFsLnS");
declare_id!("Bench11111111111111111111111111111111111111");

#[program]
pub mod bench {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
/** Update Markdown files in /bench */
/** Sync Markdown files in /bench based on the data from bench.json */

import { BenchData, Markdown } from "./utils";

Expand Down Expand Up @@ -33,19 +33,27 @@ import { BenchData, Markdown } from "./utils";
bench.compareComputeUnits(
newComputeUnitsResult,
oldComputeUnitsResult,
(ixName, newComputeUnits, oldComputeUnits) => {
const percentChange = (
(newComputeUnits / oldComputeUnits - 1) *
100
).toFixed(2);
({ ixName, newComputeUnits, oldComputeUnits }) => {
if (newComputeUnits === null) {
// Deleted instruction
return;
}

let changeText;
if (isNaN(oldComputeUnits)) {
if (oldComputeUnits === null) {
// New instruction
changeText = "N/A";
} else if (+percentChange > 0) {
changeText = `🔴 **+${percentChange}%**`;
} else {
changeText = `🟢 **${percentChange}%**`;
const percentChange = (
(newComputeUnits / oldComputeUnits - 1) *
100
).toFixed(2);

if (+percentChange > 0) {
changeText = `🔴 **+${percentChange}%**`;
} else {
changeText = `🟢 **${percentChange}%**`;
}
}

table.insert(ixName, newComputeUnits.toString(), changeText);
Expand Down
66 changes: 66 additions & 0 deletions tests/bench/scripts/sync.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
/**
* Sync all saved data by re-running the tests for each version.
*
* This script should be used when the bench program or its tests has changed
* and all data needs to be updated.
*/

import path from "path";
import { spawnSync } from "child_process";

import { ANCHOR_VERSION_ARG, BenchData, Toml } from "./utils";

(async () => {
const bench = await BenchData.open();

const cargoToml = await Toml.open(
path.join("..", "programs", "bench", "Cargo.toml")
);
const anchorToml = await Toml.open(path.join("..", "Anchor.toml"));

for (const version of bench.getVersions()) {
console.log(`Updating '${version}'...`);

const isUnreleased = version === "unreleased";

// Update the anchor dependency versions
for (const dependency of ["lang", "spl"]) {
cargoToml.replaceValue(`anchor-${dependency}`, () => {
return isUnreleased
? `{ path = "../../../../${dependency}" }`
: `"${version}"`;
});
}

// Save Cargo.toml
await cargoToml.save();

// Update `anchor test` command to pass version in Anchor.toml
anchorToml.replaceValue(
"test",
(cmd) => {
return cmd.includes(ANCHOR_VERSION_ARG)
? cmd.replace(
new RegExp(`\\s*${ANCHOR_VERSION_ARG}\\s+(.+)`),
(arg, ver) => (isUnreleased ? "" : arg.replace(ver, version))
)
: `${cmd} ${ANCHOR_VERSION_ARG} ${version}`;
},
{ insideQuotes: true }
);

// Save Anchor.toml
await anchorToml.save();

// Run the command to update the current version's results
const result = spawnSync("anchor", ["test", "--skip-lint"]);
console.log(result.output.toString());

// Check for failure
if (result.status !== 0) {
console.error("Please fix the error and re-run this command.");
process.exitCode = 1;
return;
}
}
})();
140 changes: 117 additions & 23 deletions tests/bench/scripts/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@ import * as fs from "fs/promises";
import path from "path";
import { spawnSync } from "child_process";

/** Version that is used in bench data file */
export type Version = "unreleased" | (`${number}.${number}.${number}` & {});

/** Persistent benchmark data(mapping of `Version -> Data`) */
type Bench = {
[key: string]: {
Expand All @@ -21,7 +24,10 @@ export type ComputeUnits = { [key: string]: number };
export const THRESHOLD_PERCENTAGE = 1;

/** Path to the benchmark Markdown files */
export const BENCH_DIR_PATH = "../../bench";
export const BENCH_DIR_PATH = path.join("..", "..", "bench");

/** Command line argument for Anchor version */
export const ANCHOR_VERSION_ARG = "--anchor-version";

/** Utility class to handle benchmark data related operations */
export class BenchData {
Expand Down Expand Up @@ -56,43 +62,74 @@ export class BenchData {
}

/** Get the stored results based on version */
get(version: string) {
get(version: Version) {
return this.#data[version];
}

/** Get unreleased version results */
getUnreleased() {
return this.get("unreleased");
}

/** Get all versions */
getVersions() {
return Object.keys(this.#data);
return Object.keys(this.#data) as Version[];
}

/** Compare and update compute units changes */
compareComputeUnits(
newComputeUnitsResult: ComputeUnits,
oldComputeUnitsResult: ComputeUnits,
changeCb: (
ixName: string,
newComputeUnits: number,
oldComputeUnits: number
) => void,
changeCb: (args: {
ixName: string;
newComputeUnits: number | null;
oldComputeUnits: number | null;
}) => void,
noChangeCb?: (ixName: string, computeUnits: number) => void
) {
let needsUpdate = false;

const checkIxs = (
comparedFrom: ComputeUnits,
comparedTo: ComputeUnits,
cb: (ixName: string, computeUnits: number) => void
) => {
for (const ixName in comparedFrom) {
if (comparedTo[ixName] === undefined) {
cb(ixName, comparedFrom[ixName]);
}
}
};

// New instruction
checkIxs(
newComputeUnitsResult,
oldComputeUnitsResult,
(ixName, computeUnits) => {
console.log(`New instruction '${ixName}'`);
changeCb({
ixName,
newComputeUnits: computeUnits,
oldComputeUnits: null,
});
needsUpdate = true;
}
);

// Deleted instruction
checkIxs(
oldComputeUnitsResult,
newComputeUnitsResult,
(ixName, computeUnits) => {
console.log(`Deleted instruction '${ixName}'`);
changeCb({
ixName,
newComputeUnits: null,
oldComputeUnits: computeUnits,
});
needsUpdate = true;
}
);

// Compare compute units changes
for (const ixName in newComputeUnitsResult) {
const oldComputeUnits = oldComputeUnitsResult[ixName];
const newComputeUnits = newComputeUnitsResult[ixName];
if (!oldComputeUnits) {
console.log(`New instruction '${ixName}'`);
needsUpdate = true;
changeCb(ixName, newComputeUnits, NaN);
continue;
}

const percentage = THRESHOLD_PERCENTAGE / 100;
const oldMaximumAllowedDelta = oldComputeUnits * percentage;
Expand All @@ -119,8 +156,12 @@ export class BenchData {
`Compute units change '${ixName}' (${oldComputeUnits} -> ${newComputeUnits})`
);

changeCb({
ixName,
newComputeUnits,
oldComputeUnits,
});
needsUpdate = true;
changeCb(ixName, newComputeUnits, oldComputeUnits);
} else {
noChangeCb?.(ixName, newComputeUnits);
}
Expand All @@ -131,14 +172,14 @@ export class BenchData {

/** Bump benchmark data version to the given version */
bumpVersion(newVersion: string) {
const versions = Object.keys(this.#data);
const unreleasedVersion = versions[versions.length - 1];

if (this.#data[newVersion]) {
console.error(`Version '${newVersion}' already exists!`);
process.exit(1);
}

const versions = this.getVersions();
const unreleasedVersion = versions[versions.length - 1];

// Add the new version
this.#data[newVersion] = this.get(unreleasedVersion);

Expand Down Expand Up @@ -296,3 +337,56 @@ class MarkdownTable {
);
}
}

/** Utility class to handle TOML related operations */
export class Toml {
/** TOML filepath */
#path: string;

/** TOML text */
#text: string;

constructor(path: string, text: string) {
this.#path = path;
this.#text = text;
}

/** Open the TOML file */
static async open(tomlPath: string) {
tomlPath = path.join(__dirname, tomlPath);
const text = await fs.readFile(tomlPath, {
encoding: "utf8",
});
return new Toml(tomlPath, text);
}

/** Save the TOML file */
async save() {
await fs.writeFile(this.#path, this.#text);
}

/** Replace the value for the given key */
replaceValue(
key: string,
cb: (previous: string) => string,
opts?: { insideQuotes: boolean }
) {
this.#text = this.#text.replace(
new RegExp(`${key}\\s*=\\s*${opts?.insideQuotes ? `"(.*)"` : "(.*)"}`),
(line, value) => line.replace(value, cb(value))
);
}
}

/**
* Get Anchor version from the passed arguments.
*
* Defaults to `unreleased`.
*/
export const getVersionFromArgs = () => {
const args = process.argv;
const anchorVersionArgIndex = args.indexOf(ANCHOR_VERSION_ARG);
return anchorVersionArgIndex === -1
? "unreleased"
: (args[anchorVersionArgIndex + 1] as Version);
};
Loading

0 comments on commit c166712

Please sign in to comment.