Skip to content

Commit

Permalink
Frankenstein the benchmarks
Browse files Browse the repository at this point in the history
  • Loading branch information
gavlyukovskiy committed Sep 24, 2024
1 parent b4d2417 commit c6daf63
Show file tree
Hide file tree
Showing 8 changed files with 2,527 additions and 44 deletions.
72 changes: 64 additions & 8 deletions .github/workflows/benchmark-new.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,22 +4,77 @@ on:
branches:
- improve-benchmark-pipeline

concurrency:
group: ${{ format('{0}-{1}', github.workflow_ref, github.head_ref) }}
cancel-in-progress: true

jobs:
define-matrix:
runs-on: ubuntu-latest

outputs:
commits: ${{ steps.commits.outputs.commits }}

steps:
- id: commits
run: |
# Notable commits
# e2b53ec87538b416db00d61da9158282125e2725 Random gen changed package
# 5b137816d80105124ba93d7a6143bc5a75c1c22b v1.0.0-rc2
# 5bad2969f1126fc499af42691bcbbfdbe4048bb8 v1.0.0-rc3
# c0e63df3c186be9a7fa4c11ed385d8cbedfd4e8f v1.0.0, ValueMaskers added
# 7bc09acfdf5f331ba6baa56bdb866987aeb6f90e v1.0.1
# ba6154a80531757249916d533ef6f5bdd6bb42c8 Streaming added
echo 'commits=["e2b53ec87538b416db00d61da9158282125e2725","84cb10506042cffb5e8aab82f405536ffea90ae7","1508a8cdcd55a2c6f15cbbcb172fd9a0c92c5c85","1430cb001fcf3d1636c1a923a99c82c35ef24bf4","3358805d0bdfa1fb4a3793eaf80af5ed6dbc5407","8f4788a20614794316d97f0ab6f1852505cd92b1","299f2a8b3f7ffc629d9887b14495e74e35c34b81","5932607c19c2dfa2fb617844c7a4c67cc825af32","9246c24ef85349fdfecc77775e59f28b962faced","6dd707f57e215d843177e58430e36f9040ea44fa","da82ad7933cfb53d012a31ea2408af3fd646e700","5b137816d80105124ba93d7a6143bc5a75c1c22b","09e7d152fd1930a3d79475ad3b6e56ca0be25fbd","564c268f2e97aa6587ee4a107dbfe1dc146bda1f","84d1e903202c32fd1d913c6562f72a5a412c4a4e","d8627a4b121f4997214c1c8aa5cf0b7e69665588","ba76534ba5d8c5f7ec6c7932b62202d670370fd3","13589a195329ce613bbcf7015e58d74b25c32d5a","775265e68b464f09d3487dbe80c27a28b901b815","49698e8f1f8219699e846f51519271ad50d40a9c","831df5a15dfa0e4df8dd80b3534450f829e61093","d7a7d5635093e35b0b6631383b48fe59bf6b9a8f","5bad2969f1126fc499af42691bcbbfdbe4048bb8","0fdaf7d9f98ca36a1f321362687b463fab007905","3f618fde3a1ab52c1b39cbaee4c3505f52ee383f","be13b37bb16c89c64a4effea7464ef4e1504b006","da987bf26e58c8f30dcd0c96c53824bca000065e","b4a0cc1838c9dc3572b8081a97b2ee79754f1aa9","518a78b113fb1eb3692c80398a5780a13818fa54","556fdf98d69938023e28eac7435cf85ec9b62178","98db254e96d194722950b4c13f8a41df653b08ac","97cae59fa76229f87a40ea72f729c3f4a0ae87b3","03d5744c3429a3822677db7e63149e64e4434f3a","82c901bdc42074c3abfa27166e6edd607c61ef59","12324ffb915b9333b8b52d10e2ee7c629040fb72","c0e63df3c186be9a7fa4c11ed385d8cbedfd4e8f","bc96c81d7d3a0a7d1e18220e4d87718317f8e2e0","a2d753db1948713da7395a1e59179cd9575eec97","c5ec892454ecc797406525600343e47e718eeecc","84cb657b6a8540f178bc0d1e8847525569c6711e","96d0afc5de28256c1b27860255ee08603bc84d1a","04140866d42b3ec1f5639225f3faa0a00897920b","63056ff8be941ef8a0cc1dec09d6f273e17a44d6","7eadb0c71ad61fdc331d3bec078e69bf03a1d31d","b5740af7656fea664647c0a42c4fef36524cc964","a918150d8f7efb5372d63c0d5d6e0a9107f7f444","0273c3c8f36d80c3c59d8d495ca6f715713da668","eb487d5db59671cf92f3d34dacbb9ae439a22490","6afab482d9a65b8d0fc53b938d86d1a5e1af0ecd","5ed1e110f48dedaaab35615f03c2f2f5a4c2271b","c0d0067a1494c29644577fc6e337bc06a65ecfe4","aab32f5c5c0770d355b6db87f04b7e3bd4702895","0fa83bf3ba2817e890db2f996a934d87a5bff034","7bc09acfdf5f331ba6baa56bdb866987aeb6f90e","ec96ef1ca1d6bb8f01250e78d3ed3797a0a7e94b","0ffdf387de52d06ff784c27886a9127e40dc2888","72cafa7b9b1e75765d16882256aa9149ce043a20","bc9002b2d0ec2b08f408cb460ad9822d8ff624e3","d3422b092acfd5d843a4dc01283ccc3ec4804de3","899bacd419db3aa138476b68b612d3c07f53057b","a7ce47364b96f350282a34613072009fc960e21a","2e81c1990dc219b07cdd7bab085621c1c055838e","76c4f517a84fec15b10649378e0470123385772c","06991511583a8b5010cfc1386037d1705a973bab","4890345f43909d4e32ad8a545ec88574357bb15b","341ff88579891e4045c31552cfeb505d162b1644","00bf0bb53a6a2ddb174146f4f4bbd72bed9b988f","50f5f1175e32e209db269f92e603639a47e1bfd8","f1cbd27c378499fcf63c267ac30505833540aa28","0d5af5437b2b64ab353f159780a88640b4cb1a5f","56d1c2f59c4f48b8a2dc1103d76dab8a12c361d6","fa66f6862fce74a4834ad6fb9014957250d92fef","f8aeda7492b95b44b4097c588c1d56c5d0739776","a55717ddea3e0b01998a7a5ddd3e84a49fe515a1","5bf2e9c3fc7a14e29c1f59a4357c8be69dbd503f","4da8dfa40e2992e542c56a46b84091e611f6b053","4a47d413817dc76942fb94b9bf9b2c12ba2dd773","af769ef4ddf015bb1ac00fc1fdb6b55dd99a1f1e","e8d990e11db351c9bc114b7e247030c3efeb27cb","4c98d6e25a642e6af68835718b5ce67aea110539","9568ca4ed5bb4eedb020b2ee9a21097f599c3aa0","2458b81a40ff94aeb469a23b8fb5f4fefbfeaf2b","ba6154a80531757249916d533ef6f5bdd6bb42c8","3a0643bc5090428b7a2a60d36effa617e9b9d1a6","070553b21d5e1354c10f67d14114a841c8d54052"]' >> "$GITHUB_OUTPUT"
benchmarks:
if: ${{ github.repository == 'Breus/json-masker' }}
needs:
- define-matrix
permissions:
contents: read
pull-requests: write # for benchmark comment
env:
GITHUB_PAT_1: github_p
GITHUB_PAT_2: at_11ADUR3ZY0rtQnrriME4t4_eKeXZAaMj7h0cdaZ6CFkARS9m3Vac85DFgXiglr2Lrx44XN7CE63jmftTra
strategy:
matrix:
commit: ${{ fromJSON(needs.define-matrix.outputs.commits) }}
max-parallel: 1

runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v4 # checkout latest in this branch

- uses: actions/checkout@v4 # checkout older commit to run benchmarks against
with:
ref: ${{ matrix.commit }}
path: json-masker-ref

- id: frankenstein-the-benchmarks
uses: actions/github-script@v7
with:
script: |
const fs = require('fs')
let currentCommit = '${{ matrix.commit }}'
let allCommits = JSON.parse('${{ needs.define-matrix.outputs.commits }}')
// checking ValueMaskers availability
if (allCommits.indexOf(currentCommit) < allCommits.indexOf('7bc09acfdf5f331ba6baa56bdb866987aeb6f90e')) {
fs.unlinkSync('src/jmh/java/dev/blaauwendraad/masker/json/ValueMaskerBenchmark.java')
}
// checking streaming availability
if (allCommits.indexOf(currentCommit) < allCommits.indexOf('ba6154a80531757249916d533ef6f5bdd6bb42c8')) {
fs.unlinkSync('src/jmh/java/dev/blaauwendraad/masker/json/StreamTypeBenchmark.java')
fs.unlinkSync('src/jmh/java/dev/blaauwendraad/masker/json/JsonMaskerBenchmark.java')
fs.renameSync('src/jmh/java/dev/blaauwendraad/masker/json/JsonMaskerBenchmark_no_streaming.java', 'src/jmh/java/dev/blaauwendraad/masker/json/JsonMaskerBenchmark.java')
} else {
fs.unlinkSync('src/jmh/java/dev/blaauwendraad/masker/json/JsonMaskerBenchmark_no_streaming.java')
}
// Remove the code from existing branch
fs.rmdirSync('src/main', { recursive: true, force: true })
fs.rmdirSync('src/test', { recursive: true, force: true })
// Copy the code from the ref branch
fs.cpSync('json-masker-ref/src/main', 'src/main', { recursive: true })
fs.cpSync('json-masker-ref/src/test', 'src/test', { recursive: true })
- name: Setup JDK 17
uses: actions/setup-java@v4
Expand All @@ -32,19 +87,20 @@ jobs:
run: ./gradlew jmhJar

- name: Run JMH benchmark
run: java -jar build/libs/json-masker-*-jmh.jar -wi 1 -i 3 -f 1 -rf json "KeyMatcherBenchmark.*"
run: java -jar build/libs/json-masker-*-jmh.jar -wi 1 -i 3 -f 1 -rf json

- name: Update benchmark results
uses: benchmark-action/github-action-benchmark@v1
uses: gavlyukovskiy/github-action-benchmark@master
with:
name: JMH Benchmark
tool: 'jmh'
output-file-path: jmh-result.json
benchmark-data-dir-path: data
benchmark-data-dir-path: .
github-token: ${{ env.GITHUB_PAT_1 }}${{ env.GITHUB_PAT_2 }}
gh-repository: github.com/gavlyukovskiy/json-masker-benchmarks
alert-threshold: '120%'
comment-on-alert: true
comment-always: true
fail-on-alert: true
auto-push: ${{ github.event_name != 'pull_request' }}
ref: ${{ matrix.commit }}
30 changes: 1 addition & 29 deletions build.gradle.kts
Original file line number Diff line number Diff line change
@@ -1,12 +1,10 @@
import net.ltgt.gradle.errorprone.errorprone
import org.sonarqube.gradle.SonarTask

plugins {
alias(libs.plugins.sonarqube)
alias(libs.plugins.test.logger)
alias(libs.plugins.nexus.publish)
alias(libs.plugins.jmh)
alias(libs.plugins.errorprone)
`maven-publish`
`java-library`
signing
Expand Down Expand Up @@ -37,6 +35,7 @@ java {

dependencies {
"nullabilityAnnotationsImplementation"(libs.jspecify)
"nullabilityAnnotationsImplementation"("com.google.code.findbugs:jsr305:3.0.2")

testImplementation(libs.assertj.core)
testImplementation(libs.jackson.databind)
Expand All @@ -46,8 +45,6 @@ dependencies {

jmh(libs.jmh.core)
jmhAnnotationProcessor(libs.jmh.generator.annproccesor)
errorprone(libs.nullaway)
errorprone(libs.errorprone.core)
}

publishing {
Expand Down Expand Up @@ -165,31 +162,6 @@ tasks {
}

withType<JavaCompile>().configureEach {
options.errorprone {
error(
"CheckedExceptionNotThrown",
"FunctionalInterfaceClash",
"NonFinalStaticField",
"NullAway",
"RedundantOverride",
"RedundantThrows",
"RemoveUnusedImports",
"UnnecessarilyFullyQualified",
"UnnecessarilyUsedValue",
"UnnecessaryBoxedAssignment",
"UnnecessaryBoxedVariable",
"UnnecessaryFinal",
"UnusedException",
"WildcardImport",
)
disable(
"StringCaseLocaleUsage",
"MissingSummary",
)
option("NullAway:JSpecifyMode")
option("NullAway:AnnotatedPackages", "dev.blaauwendraad.masker")
excludedPaths = ".*/build/generated/.*"
}
options.encoding = "UTF-8"
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ public static String randomJson(Set<String> targetKeys, String jsonSize, String
.setTargetKeys(targetKeys)
.setTargetKeyPercentage(targetKeyPercentage)
.setTargetJsonSizeBytes(BenchmarkUtils.parseSize(jsonSize))
.setRandomSeed(RandomJsonGenerator.STATIC_RANDOM_SEED)
.setRandomSeed(1285756302517652226L)
.createConfig();

return new RandomJsonGenerator(config).createRandomJsonNode().toString();
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package dev.blaauwendraad.masker.json;

import dev.blaauwendraad.masker.randomgen.RandomJsonGenerator;
import org.jspecify.annotations.NullUnmarked;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
Expand Down Expand Up @@ -40,7 +39,7 @@ public static class State {
public synchronized void setup() {
List<String> targetKeyList = BenchmarkUtils.loadSampleTargetKeys();
if (numberOfTargetKeys <= targetKeyList.size()) {
Collections.shuffle(targetKeyList, new Random(RandomJsonGenerator.STATIC_RANDOM_SEED));
Collections.shuffle(targetKeyList, new Random(1285756302517652226L));
targetKeys = targetKeyList.stream().limit(numberOfTargetKeys).collect(Collectors.toSet());
} else {
targetKeys = new HashSet<>(targetKeyList);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
package dev.blaauwendraad.masker.json;

import dev.blaauwendraad.masker.json.config.JsonMaskingConfig;
import dev.blaauwendraad.masker.json.util.JsonPathTestUtils;
import org.jspecify.annotations.NullUnmarked;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
import org.openjdk.jmh.annotations.Measurement;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.OutputTimeUnit;
import org.openjdk.jmh.annotations.Param;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.Warmup;

import java.nio.charset.StandardCharsets;
import java.util.Set;
import java.util.concurrent.TimeUnit;

@Warmup(iterations = 1, time = 3)
@Fork(value = 1)
@Measurement(iterations = 1, time = 3)
@OutputTimeUnit(TimeUnit.SECONDS)
@BenchmarkMode(Mode.Throughput)
public class JsonMaskerBenchmark {

@org.openjdk.jmh.annotations.State(Scope.Thread)
@NullUnmarked
public static class State {
@Param({ "1kb", "32kb", "1mb" })
String jsonSize;
@Param({ "unicode" })
String characters;
@Param({ "0.1" })
double maskedKeyProbability;
@Param({ "false", "true" })
boolean jsonPath;

private String jsonString;
private byte[] jsonBytes;
private JsonMasker jsonMasker;

@Setup
public synchronized void setup() {
Set<String> targetKeys = BenchmarkUtils.getTargetKeys(20);
jsonString = BenchmarkUtils.randomJson(targetKeys, jsonSize, characters, maskedKeyProbability);
jsonBytes = jsonString.getBytes(StandardCharsets.UTF_8);

JsonMaskingConfig.Builder builder = JsonMaskingConfig.builder();
if (jsonPath) {
builder.maskJsonPaths(JsonPathTestUtils.transformToJsonPathKeys(targetKeys, jsonString));
} else {
builder.maskKeys(targetKeys);
}
jsonMasker = JsonMasker.getMasker(builder.build());
}
}

@Benchmark
public String jsonMaskerString(State state) {
return state.jsonMasker.mask(state.jsonString);
}

@Benchmark
public byte[] jsonMaskerBytes(State state) {
return state.jsonMasker.mask(state.jsonBytes);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ public static class State {

@Setup
public synchronized void setup() throws IOException {
Random random = new Random(RandomJsonGenerator.STATIC_RANDOM_SEED);
Random random = new Random(1285756302517652226L);
List<Character> characters =
JsonStringCharacters.mergeCharSets(
JsonStringCharacters.getPrintableAsciiCharacters(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,20 +36,20 @@ public static class State {
double maskedKeyProbability;

private final JsonMasker nativeMasker = JsonMasker.getMasker(JsonMaskingConfig.builder()
.maskKeys("targetKey")
.maskKeys(Set.of("targetKey"))
.build()
);

private final JsonMasker rawValueMasker = JsonMasker.getMasker(JsonMaskingConfig.builder()
.maskKeys("targetKey")
.maskKeys(Set.of("targetKey"))
.maskStringsWith(ValueMaskers.withRawValueFunction(value -> "\"***\""))
.maskNumbersWith(ValueMaskers.withRawValueFunction(value -> "\"###\""))
.maskBooleansWith(ValueMaskers.withRawValueFunction(value -> "\"&&&\""))
.build()
);

private final JsonMasker textValueMasker = JsonMasker.getMasker(JsonMaskingConfig.builder()
.maskKeys("targetKey")
.maskKeys(Set.of("targetKey"))
.maskStringsWith(ValueMaskers.withTextFunction(value -> "***"))
.maskNumbersWith(ValueMaskers.withTextFunction(value -> "###"))
.maskBooleansWith(ValueMaskers.withTextFunction(value -> "&&&"))
Expand Down
Loading

0 comments on commit c6daf63

Please sign in to comment.