feat(perf): add (automation, provision, build, run) tooling (#184)

This project includes the following components: - `terraform/`: a Terraform scripts to provision infrastructure - `impl/`: implementations of the [libp2p perf protocol](https://github.com/libp2p/specs/blob/master/perf/perf.md) running on top of e.g. go-libp2p, rust-libp2p or Go's std-library https stack - `runner/`: a set of scripts building and running the above implementations on the above infrastructure, reporting the results in `benchmark-results.json` Benchmark results can be visualized with https://observablehq.com/@mxinden-workspace/libp2p-performance-dashboard. Co-authored-by: Marco Munizaga <[email protected]> Co-authored-by: Marten Seemann <[email protected]> Co-authored-by: Piotr Galar <[email protected]>
libp2p · Jun 22, 2023 · 53ff8b5 · 53ff8b5
1 parent 3c7d726
commit 53ff8b5
Show file tree

Hide file tree

Showing 48 changed files with 7,281 additions and 0 deletions.
diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml
@@ -0,0 +1,101 @@
+name: libp2p perf test
+
+# How to configure a repository for running this workflow:
+# 1. Configure auth for the AWS provider as per https://registry.terraform.io/providers/hashicorp/aws/latest/docs#authentication-and-configuration
+# 2. Run 'terraform init' and 'terraform apply' in 'perf/terraform/configs/remote' to create the resources needed for this workflow
+# 3. Go to https://console.aws.amazon.com/iamv2/home?#/users/details/perf?section=security_credentials
+# 4. Click 'Create access key' to get the access key ID and secret access key
+# 5. Go to https://github.com/libp2p/test-plans/settings/secrets/actions
+# 6. Click 'New repository secret', set the name to 'PERF_AWS_SECRET_ACCESS_KEY', and paste the secret access key from step 5
+# 7. Go to https://github.com/libp2p/test-plans/settings/variables/actions
+# 8. Click 'New repository variable', set the name to 'PERF_AWS_ACCESS_KEY_ID', and paste the access key ID from step 5
+
+on:
+  workflow_dispatch:
+    inputs:
+      push:
+        description: 'Push the benchmark results to the repository'
+        required: false
+        default: 'true'
+
+jobs:
+  perf:
+    name: Perf
+    runs-on: ubuntu-latest
+    timeout-minutes: 40
+    defaults:
+      run:
+        shell: bash
+        working-directory: perf
+    env:
+      AWS_ACCESS_KEY_ID: ${{ vars.PERF_AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.PERF_AWS_SECRET_ACCESS_KEY }}
+      TF_IN_AUTOMATION: 1
+      TF_INPUT: 0
+    steps:
+      - name: Checkout test-plans
+        uses: actions/checkout@v3
+        with:
+          repository: ${{ github.repository }}
+          ref: ${{ github.ref }}
+      - id: ssh
+        name: Generate SSH key
+        run: |
+          make ssh-keygen
+          echo "key<<EOF" >> $GITHUB_OUTPUT
+          while read -r line; do
+            echo "::add-mask::$line"
+            echo "$line" >> $GITHUB_OUTPUT
+          done < terraform/modules/short_lived/files/perf
+          echo "EOF" >> $GITHUB_OUTPUT
+      - name: Configure SSH
+        uses: webfactory/ssh-agent@d4b9b8ff72958532804b70bbe600ad43b36d5f2e # v0.8.0
+        with:
+          ssh-private-key: ${{ steps.ssh.outputs.key }}
+      - name: Configure git
+        run: |
+          git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com>"
+          git config --global user.name "${GITHUB_ACTOR}"
+      - name: Configure terraform
+        uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # v2.0.3
+      - name: Init terraform
+        id: init
+        run: terraform init
+        working-directory: perf/terraform/configs/local
+      - name: Apply terraform
+        run: terraform apply -auto-approve
+        working-directory: perf/terraform/configs/local
+      - id: server
+        name: Retrieve server's IP
+        run: terraform output -raw server_ip
+        working-directory: perf/terraform/configs/local
+      - id: client
+        name: Retrieve client's IP
+        run: terraform output -raw client_ip
+        working-directory: perf/terraform/configs/local
+      - name: Download dependencies
+        run: npm ci
+        working-directory: perf/runner
+      - name: Run tests
+        env:
+          SERVER_IP: ${{ steps.server.outputs.stdout }}
+          CLIENT_IP: ${{ steps.client.outputs.stdout }}
+        run: npm run start -- --client-public-ip $CLIENT_IP --server-public-ip $SERVER_IP
+        working-directory: perf/runner
+      - name: Push
+        if: github.event.inputs.push == 'true'
+        run: |
+          git add benchmark-results.json
+          git commit -m "perf: update benchmark results"
+          git push
+        working-directory: perf/runner
+      - name: Archive
+        if: github.event.intputs.push == 'false'
+        uses: actions/upload-artifact@v2
+        with:
+          name: benchmark-results
+          path: perf/runner/benchmark-results.json
+      - name: Destroy terraform
+        if: always() && steps.init.outputs.exitcode == 0
+        run: terraform destroy -auto-approve
+        working-directory: perf/terraform/configs/local
diff --git a/perf/Makefile b/perf/Makefile
@@ -0,0 +1,5 @@
+ssh-keygen:
+	ssh-keygen -t ed25519 -f ./terraform/modules/short_lived/files/perf -N ''
+
+ssh-add:
+	ssh-add ./terraform/modules/short_lived/files/perf
diff --git a/perf/README.md b/perf/README.md
@@ -0,0 +1,57 @@
+# libp2p performance benchmarking
+
+This project includes the following components:
+
+- `terraform/`: a Terraform scripts to provision infrastructure
+- `impl/`: implementations of the [libp2p perf protocol](https://github.com/libp2p/specs/blob/master/perf/perf.md) running on top of e.g. go-libp2p, rust-libp2p or Go's std-library https stack
+- `runner/`: a set of scripts building and running the above implementations on the above infrastructure, reporting the results in `benchmark-results.json`
+
+Benchmark results can be visualized with https://observablehq.com/@mxinden-workspace/libp2p-performance-dashboard.
+
+## Provision infrastructure
+
+### Bootstrap
+
+1. Save your public SSH key as the file `./terraform/modules/short_lived/files/perf.pub`; or generate a new key pair with `make ssh-keygen` and add it to your SSH agent with `make ssh-add`.
+2. `cd terraform/configs/local`
+3. `terraform init`
+4. `terraform apply`
+5. `CLIENT_IP=$(terraform output -raw client_ip)`
+6. `SERVER_IP=$(terraform output -raw server_ip)`
+
+## Build and run implementations
+
+_WARNING_: Running the perf tests might take a while.
+
+1. `cd runner`
+2. `npm ci`
+3. `npm run start -- --client-public-ip $CLIENT_IP --server-public-ip $SERVER_IP`
+
+## Deprovision infrastructure
+
+1. `cd terraform/configs/local`
+2. `terraform destroy`
+
+## Adding a new implementation
+
+1. Add implementation to `impl/`.
+  - Create a folder `impl/<your-implementation-name>/<your-implementation-version>`.
+  - In that folder include a `Makefile` that builds an executable and stores it next to the `Makefile` under the name `perf`.
+  - Requirements for the executable:
+    - Running as a libp2p-perf server
+      - Command line flags
+        - `--run-server`
+    - Running as a libp2p-perf client
+        - Input via command line
+          - `--server-ip-address`
+          - `--transport` (see `runner/versions.ts` for possible variants)
+          - `--upload-bytes` number of bytes to upload per stream.
+          - `--download-bytes` number of bytes to download per stream.
+        - Output
+          - Logging MUST go to stderr.
+          - Measurement output is printed to stdout as JSON in the form of:
+            ```json
+            {"connectionEstablishedSeconds":0.246442851,"uploadSeconds":0.000002077,"downloadSeconds":0.060712241}
+            ```
+2. In `impl/Makefile` include your implementation in the `all` target.
+3. Reference implementation in `runner/src/versions.ts`.
diff --git a/perf/impl/Makefile b/perf/impl/Makefile
@@ -0,0 +1,24 @@
+GO_SUBDIRS := $(wildcard go-libp2p/*/.)
+RUST_SUBDIRS := $(wildcard rust-libp2p/*/.)
+RUST_QUINN_SUBDIRS := $(wildcard rust-libp2p-quinn/*/.)
+HTTPS_SUBDIRS := $(wildcard https/*/.)
+QUIC_GO_SUBDIRS := $(wildcard quic-go/*/.)
+
+all: $(RUST_SUBDIRS) $(RUST_QUINN_SUBDIRS) $(GO_SUBDIRS) $(HTTPS_SUBDIRS) $(QUIC_GO_SUBDIRS)
+$(RUST_SUBDIRS):
+	$(MAKE) -C $@
+$(RUST_QUINN_SUBDIRS):
+	$(MAKE) -C $@
+$(GO_SUBDIRS):
+	$(MAKE) -C $@
+$(HTTPS_SUBDIRS):
+	$(MAKE) -C $@
+$(QUIC_GO_SUBDIRS):
+	$(MAKE) -C $@
+
+clean: $(RUST_SUBDIRS:%=%clean) $(RUST_QUINN_SUBDIRS:%=%clean) $(GO_SUBDIRS:%=%clean) $(HTTPS_SUBDIRS:%=%clean) $(QUIC_GO_SUBDIRS:%=%clean)
+
+%clean:
+	$(MAKE) -C $* clean
+
+.PHONY: $(RUST_SUBDIRS) $(RUST_QUINN_SUBDIRS) $(GO_SUBDIRS) $(HTTPS_SUBDIRS) $(QUIC_GO_SUBDIRS) all clean
diff --git a/perf/impl/go-libp2p/.gitignore b/perf/impl/go-libp2p/.gitignore
@@ -0,0 +1,4 @@
+go-libp2p-*.zip
+go-libp2p-*
+go-libp2p-*/*
+image.json
diff --git a/perf/impl/go-libp2p/v0.27/.gitignore b/perf/impl/go-libp2p/v0.27/.gitignore
@@ -0,0 +1,3 @@
+perf
+.cache
+v0.27
diff --git a/perf/impl/go-libp2p/v0.27/Dockerfile b/perf/impl/go-libp2p/v0.27/Dockerfile
@@ -0,0 +1,14 @@
+# Build Go Binary
+FROM golang:1.20-alpine AS builder
+
+WORKDIR /app
+COPY go.mod go.sum ./
+RUN go mod download
+COPY *.go ./
+RUN go build -o perf .
+
+FROM alpine
+
+COPY --from=builder /app/perf /app/perf
+
+ENTRYPOINT [ "/app/perf" ]
diff --git a/perf/impl/go-libp2p/v0.27/Makefile b/perf/impl/go-libp2p/v0.27/Makefile
@@ -0,0 +1,13 @@
+GO_FILES := $(wildcard *.go)
+
+all: perf
+
+perf: $(GO_FILES)
+	docker run --rm --user "$(shell id -u):$(shell id -g)" -v "$(shell pwd)":/usr/src/myapp -w /usr/src/myapp -e GOCACHE=/usr/src/myapp/.cache golang:1.20 go build -o perf .
+
+clean:
+	rm perf
+	rm .cache
+	rm v0.27
+
+.PHONY: all clean
diff --git a/perf/impl/go-libp2p/v0.27/go.mod b/perf/impl/go-libp2p/v0.27/go.mod
@@ -0,0 +1,98 @@
+module github.com/libp2p/test-plans/perf/go/v0.27/v2
+
+go 1.19
+
+require (
+	github.com/ipfs/go-log/v2 v2.5.1
+	github.com/libp2p/go-buffer-pool v0.1.0
+	github.com/libp2p/go-libp2p v0.27.0
+	github.com/multiformats/go-multiaddr v0.9.0
+	github.com/stretchr/testify v1.8.2
+)
+
+require (
+	github.com/benbjohnson/clock v1.3.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/containerd/cgroups v1.1.0 // indirect
+	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/elastic/gosigar v0.14.2 // indirect
+	github.com/flynn/noise v1.0.0 // indirect
+	github.com/francoispqt/gojay v1.2.13 // indirect
+	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
+	github.com/godbus/dbus/v5 v5.1.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/mock v1.6.0 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/gopacket v1.1.19 // indirect
+	github.com/google/pprof v0.0.0-20230405160723-4a4c7d95572b // indirect
+	github.com/huin/goupnp v1.1.0 // indirect
+	github.com/ipfs/go-cid v0.4.1 // indirect
+	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
+	github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
+	github.com/klauspost/compress v1.16.4 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
+	github.com/koron/go-ssdp v0.0.4 // indirect
+	github.com/libp2p/go-cidranger v1.1.0 // indirect
+	github.com/libp2p/go-flow-metrics v0.1.0 // indirect
+	github.com/libp2p/go-libp2p-asn-util v0.3.0 // indirect
+	github.com/libp2p/go-msgio v0.3.0 // indirect
+	github.com/libp2p/go-nat v0.1.0 // indirect
+	github.com/libp2p/go-netroute v0.2.1 // indirect
+	github.com/libp2p/go-reuseport v0.2.0 // indirect
+	github.com/libp2p/go-yamux/v4 v4.0.0 // indirect
+	github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
+	github.com/mattn/go-isatty v0.0.18 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/miekg/dns v1.1.53 // indirect
+	github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
+	github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
+	github.com/minio/sha256-simd v1.0.0 // indirect
+	github.com/mr-tron/base58 v1.2.0 // indirect
+	github.com/multiformats/go-base32 v0.1.0 // indirect
+	github.com/multiformats/go-base36 v0.2.0 // indirect
+	github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect
+	github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
+	github.com/multiformats/go-multibase v0.2.0 // indirect
+	github.com/multiformats/go-multicodec v0.8.1 // indirect
+	github.com/multiformats/go-multihash v0.2.1 // indirect
+	github.com/multiformats/go-multistream v0.4.1 // indirect
+	github.com/multiformats/go-varint v0.0.7 // indirect
+	github.com/onsi/ginkgo/v2 v2.9.2 // indirect
+	github.com/opencontainers/runtime-spec v1.0.2 // indirect
+	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_golang v1.14.0 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.42.0 // indirect
+	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/quic-go/qpack v0.4.0 // indirect
+	github.com/quic-go/qtls-go1-19 v0.3.2 // indirect
+	github.com/quic-go/qtls-go1-20 v0.2.2 // indirect
+	github.com/quic-go/quic-go v0.33.0 // indirect
+	github.com/quic-go/webtransport-go v0.5.2 // indirect
+	github.com/raulk/go-watchdog v1.3.0 // indirect
+	github.com/spaolacci/murmur3 v1.1.0 // indirect
+	go.uber.org/atomic v1.10.0 // indirect
+	go.uber.org/dig v1.16.1 // indirect
+	go.uber.org/fx v1.19.2 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.24.0 // indirect
+	golang.org/x/crypto v0.7.0 // indirect
+	golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect
+	golang.org/x/mod v0.10.0 // indirect
+	golang.org/x/net v0.8.0 // indirect
+	golang.org/x/sync v0.1.0 // indirect
+	golang.org/x/sys v0.7.0 // indirect
+	golang.org/x/text v0.8.0 // indirect
+	golang.org/x/tools v0.7.0 // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	lukechampine.com/blake3 v1.1.7 // indirect
+	nhooyr.io/websocket v1.8.7 // indirect
+)