From d3c67d8502f6d5c2f79861da76a527907bca09f5 Mon Sep 17 00:00:00 2001 From: Lauri Junkkari Date: Wed, 19 Jan 2022 14:49:17 +0200 Subject: [PATCH 1/5] Don't set ticker to nil --- chain_events/listener.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/chain_events/listener.go b/chain_events/listener.go index ca2845c9..ff7b5d5a 100644 --- a/chain_events/listener.go +++ b/chain_events/listener.go @@ -224,8 +224,6 @@ func (l *ListenerImpl) Stop() { if l.ticker != nil { l.ticker.Stop() } - - l.ticker = nil } func (l *ListenerImpl) systemHalted() (bool, error) { From 190e1b21849a996bb6c8bd04b389f8430bacda67 Mon Sep 17 00:00:00 2001 From: Lauri Junkkari Date: Wed, 19 Jan 2022 15:19:36 +0200 Subject: [PATCH 2/5] Add a fully dockerized e2e test setup --- .dockerignore | 7 ++-- Makefile | 11 ++++++ docker-compose.test-suite.yml | 69 +++++++++++++++++++++++++++++++++++ docker-compose.yml | 2 +- docker/flow-cli/Dockerfile | 13 +++++++ docker/wallet/Dockerfile | 6 ++- 6 files changed, 102 insertions(+), 6 deletions(-) create mode 100644 docker-compose.test-suite.yml create mode 100644 docker/flow-cli/Dockerfile diff --git a/.dockerignore b/.dockerignore index 80544825..f05b0c2e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -22,14 +22,15 @@ tmp/ *.pid # Docker specific -flow/ -api/ .github/ +api/ examples/ docker/ +docker-compose.yml +docker-compose.test-suite.yml +Makefile # Not required for build -flow/ api-test-scripts/ *.md *.svg diff --git a/Makefile b/Makefile index d36f6f2e..b581073a 100644 --- a/Makefile +++ b/Makefile @@ -58,3 +58,14 @@ emulator.pid: .PHONY: lint lint: @golangci-lint run + +.PHONY: run-test-suite +run-test-suite: + @docker-compose -f docker-compose.test-suite.yml build flow test-api + @docker-compose -f docker-compose.test-suite.yml up --remove-orphans -d db redis flow + @echo "\nRunning tests, hang on...\n" \ + ; docker-compose -f docker-compose.test-suite.yml run --rm test-api go test ./... -p 1 \ + ; echo "\nRunning linter, hang on...\n" \ + ; docker-compose -f docker-compose.test-suite.yml run --rm lint golangci-lint run \ + ; echo "\nStopping services...\n" \ + ; docker-compose -f docker-compose.test-suite.yml stop diff --git a/docker-compose.test-suite.yml b/docker-compose.test-suite.yml new file mode 100644 index 00000000..ad6ec050 --- /dev/null +++ b/docker-compose.test-suite.yml @@ -0,0 +1,69 @@ +version: "3.9" +services: + db: + image: postgres:13-alpine + environment: + POSTGRES_DB: wallet_test + POSTGRES_USER: wallet_test + POSTGRES_PASSWORD: wallet_test + + redis: + image: redis:6.2-alpine + command: redis-server /usr/local/etc/redis/redis.conf + volumes: + - ./redis-config/redis.conf:/usr/local/etc/redis/redis.conf + - ./redis-config/users.acl:/usr/local/etc/redis/users.acl + + flow: + build: + context: ./docker/flow-cli + args: + FLOW_CLI_VERSION: v0.28.3 + entrypoint: ["/bin/sh","-c"] + command: + - | + flow emulator -b 100ms & + sleep 1 + flow project deploy --network=emulator --update=true + tail -F anything + environment: + FLOW_SERVICEPRIVATEKEY: 91a22fbd87392b019fbe332c32695c14cf2ba5b6521476a8540228bdf1987068 + FLOW_SERVICEKEYSIGALGO: ECDSA_P256 + FLOW_SERVICEKEYHASHALGO: SHA3_256 + working_dir: /flow + volumes: + - "./flow:/flow:ro" + + test-api: + build: + context: . + dockerfile: ./docker/wallet/Dockerfile + target: dependencies + network: host # docker build sometimes has problems fetching from alpine's CDN + environment: + FLOW_WALLET_ACCESS_API_HOST: flow:3569 + FLOW_WALLET_ADMIN_ADDRESS: "0xf8d6e0586b0a20c7" + FLOW_WALLET_ADMIN_PRIVATE_KEY: 91a22fbd87392b019fbe332c32695c14cf2ba5b6521476a8540228bdf1987068 + FLOW_WALLET_ADMIN_PROPOSAL_KEY_COUNT: 100 + FLOW_WALLET_CHAIN_ID: flow-emulator + FLOW_WALLET_ENABLED_TOKENS: FUSD:0xf8d6e0586b0a20c7:fusd,FlowToken:0x0ae53cb6e3f42a79:flowToken + FLOW_WALLET_ENCRYPTION_KEY: faae4ed1c30f4e4555ee3a71f1044a8e + FLOW_WALLET_ENCRYPTION_KEY_TYPE: local + FLOW_WALLET_DATABASE_DSN: postgresql://wallet_test:wallet_test@db:5432/wallet_test + FLOW_WALLET_DATABASE_TYPE: psql + FLOW_WALLET_DEFAULT_KEY_TYPE: local + FLOW_WALLET_IDEMPOTENCY_MIDDLEWARE_DATABASE_TYPE: redis + FLOW_WALLET_IDEMPOTENCY_MIDDLEWARE_REDIS_URL: redis://walletapi:wallet-api-redis@localhost:6379/ + working_dir: /flow-wallet-api + volumes: + - ".:/flow-wallet-api:ro" + depends_on: + - db + - redis + - flow + + lint: + image: golangci/golangci-lint + working_dir: /flow-wallet-api + volumes: + - ".:/flow-wallet-api:ro" diff --git a/docker-compose.yml b/docker-compose.yml index b66c0114..4408f25e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,7 +30,7 @@ services: ports: - "5050:80" - wallet: + api: build: context: . dockerfile: ./docker/wallet/Dockerfile diff --git a/docker/flow-cli/Dockerfile b/docker/flow-cli/Dockerfile new file mode 100644 index 00000000..a17f5011 --- /dev/null +++ b/docker/flow-cli/Dockerfile @@ -0,0 +1,13 @@ +FROM ubuntu:20.04 + +ARG FLOW_CLI_VERSION + +RUN apt-get update && apt-get install -y \ + ca-certificates \ + curl \ + && rm -rf /var/lib/apt/lists/* + +RUN sh -c "$(curl -fsSL https://storage.googleapis.com/flow-cli/install.sh)" 0 $FLOW_CLI_VERSION \ + && mv /root/.local/bin/flow /usr/local/bin + +ENTRYPOINT ["/usr/local/bin/flow"] diff --git a/docker/wallet/Dockerfile b/docker/wallet/Dockerfile index 8de4bed2..41ff6a30 100644 --- a/docker/wallet/Dockerfile +++ b/docker/wallet/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:alpine AS builder +FROM golang:1.17-alpine AS dependencies RUN apk update && apk add --no-cache \ ca-certificates \ @@ -18,6 +18,8 @@ COPY go.mod . COPY go.sum . RUN go mod download +FROM dependencies AS builder + COPY . . RUN ./build.sh @@ -26,7 +28,7 @@ WORKDIR /dist RUN cp /build/main . -FROM scratch +FROM scratch as dist COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ COPY --from=builder /dist/main / From 55f1e6354ab95707ae8ad5ab76bca85e9cfd23dd Mon Sep 17 00:00:00 2001 From: Lauri Junkkari Date: Thu, 20 Jan 2022 12:40:56 +0200 Subject: [PATCH 3/5] Speed up tests and separate dev and test projects --- Makefile | 25 ++++++----- docker-compose.yml => docker-compose.dev.yml | 45 ++++++++++---------- docker-compose.test-suite.yml | 14 ++++-- docker/wallet/Dockerfile | 3 +- 4 files changed, 50 insertions(+), 37 deletions(-) rename docker-compose.yml => docker-compose.dev.yml (88%) diff --git a/Makefile b/Makefile index b581073a..9c4f277d 100644 --- a/Makefile +++ b/Makefile @@ -10,18 +10,21 @@ ifeq (, $(shell which go)) $(error "No go in PATH") endif + +dev = docker-compose -f docker-compose.dev.yml -p flow-wallet-api-dev +test = docker-compose -f docker-compose.test-suite.yml -p flow-wallet-api-test + .PHONY: dev dev: - @docker-compose up -d db pgadmin emulator redis - @docker-compose logs -f + @$(dev) up --remove-orphans -d db pgadmin emulator redis .PHONY: stop stop: - @docker-compose stop + @$(dev) stop .PHONY: down down: - @docker-compose down + @$(dev) down --remove-orphans .PHONY: reset reset: down dev @@ -61,11 +64,13 @@ lint: .PHONY: run-test-suite run-test-suite: - @docker-compose -f docker-compose.test-suite.yml build flow test-api - @docker-compose -f docker-compose.test-suite.yml up --remove-orphans -d db redis flow + @$(test) build flow api + @$(test) up --remove-orphans -d db redis flow @echo "\nRunning tests, hang on...\n" \ - ; docker-compose -f docker-compose.test-suite.yml run --rm test-api go test ./... -p 1 \ + ; $(test) run --rm api go test ./... -p 1 \ ; echo "\nRunning linter, hang on...\n" \ - ; docker-compose -f docker-compose.test-suite.yml run --rm lint golangci-lint run \ - ; echo "\nStopping services...\n" \ - ; docker-compose -f docker-compose.test-suite.yml stop + ; $(test) run --rm lint golangci-lint run + +.PHONY: stop-test-suite +stop-test-suite: + @$(test) down --remove-orphans diff --git a/docker-compose.yml b/docker-compose.dev.yml similarity index 88% rename from docker-compose.yml rename to docker-compose.dev.yml index 4408f25e..5ee781c1 100644 --- a/docker-compose.yml +++ b/docker-compose.dev.yml @@ -2,7 +2,6 @@ version: "3.9" services: db: image: postgres:13-alpine - restart: unless-stopped ports: - "5432:5432" environment: @@ -12,30 +11,42 @@ services: redis: image: redis:6.2-alpine - restart: unless-stopped command: redis-server /usr/local/etc/redis/redis.conf ports: - "6379:6379" volumes: - - redis_data:/data - ./redis-config/redis.conf:/usr/local/etc/redis/redis.conf - ./redis-config/users.acl:/usr/local/etc/redis/users.acl pgadmin: image: dpage/pgadmin4 - restart: unless-stopped environment: PGADMIN_DEFAULT_EMAIL: admin@example.com PGADMIN_DEFAULT_PASSWORD: admin ports: - "5050:80" + emulator: + image: gcr.io/flow-container-registry/emulator:0.27.3 + command: emulator -v --persist + ports: + - "3569:3569" + volumes: + - emulator_persist:/flowdb + env_file: + - ./.env + environment: + - FLOW_SERVICEPRIVATEKEY=${FLOW_WALLET_ADMIN_PRIVATE_KEY} + - FLOW_SERVICEKEYSIGALGO=ECDSA_P256 + - FLOW_SERVICEKEYHASHALGO=SHA3_256 + - FLOW_DBPATH=/flowdb + api: build: context: . dockerfile: ./docker/wallet/Dockerfile + target: dependencies network: host # docker build sometimes has problems fetching from alpine's CDN - restart: unless-stopped ports: - "3000:3000" env_file: @@ -45,27 +56,17 @@ services: FLOW_WALLET_DATABASE_TYPE: psql FLOW_WALLET_ACCESS_API_HOST: emulator:3569 FLOW_WALLET_CHAIN_ID: flow-emulator + working_dir: /flow-wallet-api + volumes: + - .:/flow-wallet-api:ro + - go-modules:/go/pkg/mod + - go-cache:/root/.cache/go-build depends_on: - db - emulator - redis - emulator: - image: gcr.io/flow-container-registry/emulator:0.27.3 - restart: unless-stopped - command: emulator -v --persist - ports: - - "3569:3569" - volumes: - - emulator_persist:/flowdb - env_file: - - ./.env - environment: - - FLOW_SERVICEPRIVATEKEY=${FLOW_WALLET_ADMIN_PRIVATE_KEY} - - FLOW_SERVICEKEYSIGALGO=ECDSA_P256 - - FLOW_SERVICEKEYHASHALGO=SHA3_256 - - FLOW_DBPATH=/flowdb - volumes: - redis_data: emulator_persist: + go-modules: + go-cache: diff --git a/docker-compose.test-suite.yml b/docker-compose.test-suite.yml index ad6ec050..6e1af174 100644 --- a/docker-compose.test-suite.yml +++ b/docker-compose.test-suite.yml @@ -34,7 +34,7 @@ services: volumes: - "./flow:/flow:ro" - test-api: + api: build: context: . dockerfile: ./docker/wallet/Dockerfile @@ -56,7 +56,9 @@ services: FLOW_WALLET_IDEMPOTENCY_MIDDLEWARE_REDIS_URL: redis://walletapi:wallet-api-redis@localhost:6379/ working_dir: /flow-wallet-api volumes: - - ".:/flow-wallet-api:ro" + - .:/flow-wallet-api:ro + - go-modules:/go/pkg/mod + - go-cache:/root/.cache/go-build depends_on: - db - redis @@ -66,4 +68,10 @@ services: image: golangci/golangci-lint working_dir: /flow-wallet-api volumes: - - ".:/flow-wallet-api:ro" + - .:/flow-wallet-api:ro + - go-modules:/go/pkg/mod + - go-cache:/root/.cache/go-build + +volumes: + go-modules: + go-cache: diff --git a/docker/wallet/Dockerfile b/docker/wallet/Dockerfile index 41ff6a30..e527d430 100644 --- a/docker/wallet/Dockerfile +++ b/docker/wallet/Dockerfile @@ -14,8 +14,7 @@ ENV GO111MODULE=on \ WORKDIR /build -COPY go.mod . -COPY go.sum . +COPY go.mod go.sum ./ RUN go mod download FROM dependencies AS builder From f04c6f2c787ec956eba85a40114cfcb8bd164cf6 Mon Sep 17 00:00:00 2001 From: Lauri Junkkari Date: Thu, 20 Jan 2022 12:44:27 +0200 Subject: [PATCH 4/5] Add note about test suite to readme --- README.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 16edb1de..bcde7f9b 100644 --- a/README.md +++ b/README.md @@ -257,7 +257,6 @@ If you want to use an AWS KMS symmetric encryption key for encrypting the stored | `EncryptionKeyType` | `FLOW_WALLET_ENCRYPTION_KEY_TYPE` | Encryption key type | `local` | `aws_kms` | | `EncryptionKey` | `FLOW_WALLET_ENCRYPTION_KEY` | KMS encryption key ARN | - | `arn:aws:kms:eu-central-1:012345678910:key/00000000-aaaa-bbbb-cccc-12345678910` | - ### Idempotency middleware Idempotency middleware ensures that `POST` requests are idempotent. When the middleware is enabled an `Idempotency-Key` HTTP header is required for `POST` requests. The header value should be a unique identifier for the request (UUID or similar is recommended). Trying to send a request with a duplicate idempotency key will result in a `409 Conflict` HTTP response. @@ -271,6 +270,7 @@ To configure the middleware set the following configuration settings; | `IdempotencyMiddlewareRedisURL` | `FLOW_WALLET_IDEMPOTENCY_MIDDLEWARE_REDIS_URL` | Redis URL for idempotency key middleware storage | - | `redis://walletapi:wallet-api-redis@localhost:6379/` | NOTE: + - The `local` option for `IdempotencyMiddlewareDatabaseType` does not support multiple instances. - The provided `docker-compose.yml` provides a basic Redis instance for local development purposes, with basic configuration files in the [`redis-config`](redis-config) directory. - There is currently no automatic cleanup of old idempotency keys when using the `shared` (sql) database. Redis is recommended for production use. @@ -305,3 +305,13 @@ The Flow Wallet API is developed and maintained by [Equilibrium](https://equilib with support from the Flow core contributors. Equilibrium + +## Testing + +You can run a fully dockerized test suite if you have _Docker_ and _docker-compose_ installed. + + # Run the test suite + make run-test-suite + + # If you don't want to leave background services running (database, redis, flow-emulator) + make stop-test-suite From be8df0bc310f323472c54b7cf90ccad262313881 Mon Sep 17 00:00:00 2001 From: Lauri Junkkari Date: Thu, 20 Jan 2022 12:52:54 +0200 Subject: [PATCH 5/5] Allow defining database in env for tests --- tests/test/config.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/test/config.go b/tests/test/config.go index 873ac177..55ab27cc 100644 --- a/tests/test/config.go +++ b/tests/test/config.go @@ -35,12 +35,13 @@ func LoadConfig(t *testing.T, cfgFile ...string) *configs.Config { t.Fatal(err) } - cfg.DatabaseDSN = path.Join(t.TempDir(), "test.db") - cfg.DatabaseType = "sqlite" - cfg.ChainID = flow.Emulator + // Check if using default + if cfg.DatabaseDSN == "wallet.db" { + cfg.DatabaseDSN = path.Join(t.TempDir(), "test.db") + cfg.DatabaseType = "sqlite" + } - cfg.WorkerQueueCapacity = 100 - cfg.WorkerCount = 1 + cfg.ChainID = flow.Emulator return cfg }