From ffe7bad5a1a68629891536db2fe734ceb195bd6f Mon Sep 17 00:00:00 2001 From: Rasyid Hakim Date: Tue, 6 Apr 2021 13:35:46 +0700 Subject: [PATCH 1/5] build: setup pipeline --- .github/workflows/build.yaml | 36 +++++++++++++++++++++++++++++++++ .github/workflows/package.yaml | 37 ++++++++++++++++++++++++++++++++++ Dockerfile | 9 +++++++++ Makefile | 4 ++-- websocket/handler_test.go | 4 +++- 5 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/build.yaml create mode 100644 .github/workflows/package.yaml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 00000000..c5c2dc27 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,36 @@ +name: Test & Build + +on: + - push + - pull_request + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - name: Setup Go + uses: actions/setup-go@v2.1.3 + with: + go-version: '1.14' + - name: Install Protoc + uses: arduino/setup-protoc@v1 + - name: Checkout repo + uses: actions/checkout@v2 + - name: Setup Project + run: make install-protoc; make setup + - name: Invoking go test + run: make test + build: + runs-on: ubuntu-latest + + steps: + - name: Setup Go + uses: actions/setup-go@v2.1.3 + with: + go-version: '1.14' + - name: Install Protoc + uses: arduino/setup-protoc@v1 + - uses: actions/checkout@v2 + - name: Build + run: make all diff --git a/.github/workflows/package.yaml b/.github/workflows/package.yaml new file mode 100644 index 00000000..16e9ce51 --- /dev/null +++ b/.github/workflows/package.yaml @@ -0,0 +1,37 @@ +name: Package +on: + push: + # Sequence of patterns matched against refs/tags + tags: + - "v*.*.*" # Push events to matching v*, i.e. v1.0, v20.15.10 + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Get release tag + id: get_version + uses: battila7/get-version-action@v2 + - name: Login to GitHub Packages Docker Registry + uses: docker/login-action@v1 + with: + registry: docker.pkg.github.com + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build and Push + uses: docker/build-push-action@v2 + with: + context: . + push: true + tags: | + docker.pkg.github.com/${{ github.repository }}/raccoon:latest + docker.pkg.github.com/${{ github.repository }}/raccoon:${{ steps.get_version.outputs.version-without-v }} + odpf/raccoon:latest + odpf/raccoon:${{ steps.get_version.outputs.version-without-v }} diff --git a/Dockerfile b/Dockerfile index 1e050ae7..1bbf4ec7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,16 @@ FROM golang:1.14 WORKDIR /app +RUN apt-get update && apt-get install unzip --no-install-recommends --assume-yes +RUN PROTOC_ZIP=protoc-3.14.0-linux-x86_64.zip && \ +curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v3.14.0/$PROTOC_ZIP && \ +unzip -o $PROTOC_ZIP -d /usr/local bin/protoc && \ +unzip -o $PROTOC_ZIP -d /usr/local 'include/*' && \ +rm -f $PROTOC_ZIP COPY . . +RUN make install-protoc +RUN make generate-proto +RUN make update-deps RUN make compile EXPOSE 8080 diff --git a/Makefile b/Makefile index 3d7d322f..83c60fed 100644 --- a/Makefile +++ b/Makefile @@ -26,8 +26,8 @@ PROTO_PACKAGE=/websocket/proto generate-proto: rm -rf .temp mkdir -p .temp - cd ./.temp; curl -o proton.tar.gz -L http://api.github.com/repos/odpf/proton/tarball/main; tar xvf proton.tar.gz -C . --strip-components 1 - protoc --proto_path=.temp/ $(wildcard .temp/odpf/raccoon/*.proto) --go_out=./ --go_opt=paths=import --go_opt=Modpf/raccoon/Event.proto=$(PROTO_PACKAGE) --go_opt=Modpf/raccoon/EventRequest.proto=$(PROTO_PACKAGE) --go_opt=Modpf/raccoon/EventResponse.proto=$(PROTO_PACKAGE) + curl -o .temp/proton.tar.gz -L http://api.github.com/repos/odpf/proton/tarball/main; tar xvf .temp/proton.tar.gz -C .temp/ --strip-components 1 + protoc --proto_path=.temp/ .temp/odpf/raccoon/*.proto --go_out=./ --go_opt=paths=import --go_opt=Modpf/raccoon/Event.proto=$(PROTO_PACKAGE) --go_opt=Modpf/raccoon/EventRequest.proto=$(PROTO_PACKAGE) --go_opt=Modpf/raccoon/EventResponse.proto=$(PROTO_PACKAGE) # Build Lifecycle compile: diff --git a/websocket/handler_test.go b/websocket/handler_test.go index 4369b531..76b82931 100644 --- a/websocket/handler_test.go +++ b/websocket/handler_test.go @@ -95,7 +95,9 @@ func TestHandler_HandlerWSEvents(t *testing.T) { }) t.Run("Should return unknown request when request fail to deserialize", func(t *testing.T) { - wss, _, err := websocket.DefaultDialer.Dial(url, header) + wss, _, err := websocket.DefaultDialer.Dial(url, http.Header{ + "x-user-id": []string{"test2-user2"}, + }) defer wss.Close() require.NoError(t, err) From f82c2f8d08802570c5a24e73365a1dac51ff742e Mon Sep 17 00:00:00 2001 From: Rasyid Hakim Date: Fri, 9 Apr 2021 10:19:53 +0700 Subject: [PATCH 2/5] fix: cyclic dependency when initializing log --- logger/logger.go | 17 +++++++++-------- main.go | 1 + 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/logger/logger.go b/logger/logger.go index 95382a8d..b22fd4f3 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -4,7 +4,6 @@ import ( "fmt" "io" "os" - "raccoon/config" log "github.com/sirupsen/logrus" ) @@ -16,19 +15,13 @@ func init() { if logger != nil { return } - logLevel, err := log.ParseLevel(config.Log.Level) - if err != nil { - fmt.Printf("[init] Fail to parse log level during init: %s\n", err) - fmt.Println("[init] Fallback to info log level") - logLevel = log.InfoLevel - } logger = &log.Logger{ Out: os.Stdout, Formatter: &log.TextFormatter{ FullTimestamp: true, }, Hooks: make(log.LevelHooks), - Level: logLevel, + Level: defaultLevel, } return @@ -73,3 +66,11 @@ func Set(log *log.Logger) { func SetOutput(out io.Writer) { logger.SetOutput(out) } + +func SetLevel(level string) { + if l, err := log.ParseLevel(level); err != nil { + logger.SetLevel(l) + } else { + fmt.Printf("[logger] NoOps, Fail to parse log level: %s\n", err) + } +} diff --git a/main.go b/main.go index 2e4b7989..373fc1ac 100644 --- a/main.go +++ b/main.go @@ -10,6 +10,7 @@ import ( func main() { config.Load() metrics.Setup() + logger.SetLevel(config.Log.Level) err := app.Run() metrics.Close() if err != nil { From 066a2891bb1408e7f92bb4e9039705ae9e095500 Mon Sep 17 00:00:00 2001 From: Rasyid Hakim Date: Fri, 9 Apr 2021 11:43:44 +0700 Subject: [PATCH 3/5] build: add integration test --- .github/workflows/integration-test.yaml | 23 ++++++++++++++++++ application.yml.integration-test | 31 +++++++++++++++++++++++++ docker-compose.yml | 2 +- 3 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/integration-test.yaml create mode 100644 application.yml.integration-test diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml new file mode 100644 index 00000000..37662ecd --- /dev/null +++ b/.github/workflows/integration-test.yaml @@ -0,0 +1,23 @@ +name: Integration Test + +on: + - push + - pull_request + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Setup Docker + uses: docker-practice/actions-setup-docker@master + - name: Install Protoc + uses: arduino/setup-protoc@v1 + - name: Checkout repo + uses: actions/checkout@v2 + - name: Copy integration config + run: cp application.yml.integration-test application.yml + - name: Run Raccoon + run: make docker-run + - run: make install-protoc && make generate-proto + - name: Invoking go test + run: INTEGTEST_BOOTSTRAP_SERVER=localhost:9094 INTEGTEST_HOST=ws://localhost:8080 INTEGTEST_TOPIC_FORMAT="clickstream-%s-log" go test ./integration -v diff --git a/application.yml.integration-test b/application.yml.integration-test new file mode 100644 index 00000000..49c9a437 --- /dev/null +++ b/application.yml.integration-test @@ -0,0 +1,31 @@ +SERVER_WEBSOCKET_PORT: "8080" +SERVER_WEBSOCKET_CHECK_ORIGIN: "true" +SERVER_WEBSOCKET_MAX_CONN: "30000" +SERVER_WEBSOCKET_READ_BUFFER_SIZE: "10240" +SERVER_WEBSOCKET_WRITE_BUFFER_SIZE: "10240" +SERVER_WEBSOCKET_CONN_UNIQ_ID_HEADER: "x-user-id" +SERVER_WEBSOCKET_PING_INTERVAL_MS: 30000 +SERVER_WEBSOCKET_PONG_WAIT_INTERVAL_MS: 60000 +SERVER_WEBSOCKET_WRITE_WAIT_INTERVAL_MS: 5000 +SERVER_WEBSOCKET_SERVER_SHUTDOWN_GRACE_PERIOD_MS: 3000 +SERVER_WEBSOCKET_PINGER_SIZE: 1 + +WORKER_BUFFER_CHANNEL_SIZE: 5 +WORKER_BUFFER_FLUSH_TIMEOUT_MS: 5000 +WORKER_POOL_SIZE: 5 +WORKER_KAFKA_DELIVERY_CHANNEL_SIZE: 100 + +EVENT_DISTRIBUTION_PUBLISHER_PATTERN: "clickstream-%s-log" + +PUBLISHER_KAFKA_CLIENT_BOOTSTRAP_SERVERS: "kafka:9092" +PUBLISHER_KAFKA_CLIENT_ACKS: "1" +PUBLISHER_KAFKA_CLIENT_RETRIES: "2" +PUBLISHER_KAFKA_CLIENT_RETRY_BACKOFF_MS: "100" +PUBLISHER_KAFKA_CLIENT_STATISTICS_INTERVAL_MS: 5000 +PUBLISHER_KAFKA_CLIENT_QUEUE_BUFFERING_MAX_MESSAGES: 100000 +PUBLISHER_KAFKA_FLUSH_INTERVAL_MS: 1000 + +METRIC_STATSD_ADDRESS: ":8125" +METRIC_STATSD_FLUSH_PERIOD_MS: 100 + +LOG_LEVEL: "info" diff --git a/docker-compose.yml b/docker-compose.yml index b5a3a327..baba8187 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3.2' +version: '3.9' networks: cs-network: From 54d568dfb8159deea2ccffc25ea66cafdcf0868e Mon Sep 17 00:00:00 2001 From: Rasyid Hakim Date: Tue, 13 Apr 2021 17:06:29 +0700 Subject: [PATCH 4/5] build: use multi stage dockerfile --- Dockerfile | 13 ++++++------- docker-compose.yml | 2 +- logger/logger.go | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1bbf4ec7..7c637090 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,12 +8,11 @@ unzip -o $PROTOC_ZIP -d /usr/local bin/protoc && \ unzip -o $PROTOC_ZIP -d /usr/local 'include/*' && \ rm -f $PROTOC_ZIP COPY . . -RUN make install-protoc -RUN make generate-proto -RUN make update-deps -RUN make compile +RUN make install-protoc && make generate-proto && make update-deps && make compile +FROM debian:buster-slim +WORKDIR /app +COPY --from=0 /app/out/raccoon ./raccoon +COPY ./application.yml . EXPOSE 8080 - -# Command to run the executable -CMD ["./out/raccoon"] +CMD ["./raccoon"] diff --git a/docker-compose.yml b/docker-compose.yml index baba8187..ca1e1174 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -41,7 +41,7 @@ services: cs: build: context: . - command: ["/bin/sh", "-c", "./out/raccoon"] + command: ["/bin/sh", "-c", "./raccoon"] hostname: cs container_name: cs stdin_open: true diff --git a/logger/logger.go b/logger/logger.go index b22fd4f3..f4f4011e 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -71,6 +71,6 @@ func SetLevel(level string) { if l, err := log.ParseLevel(level); err != nil { logger.SetLevel(l) } else { - fmt.Printf("[logger] NoOps, Fail to parse log level: %s\n", err) + fmt.Printf("[logger] NoOps, Fail to parse log level: %v\n", err) } } From e184f6ef74a2b632921434fc6afc606837e006af Mon Sep 17 00:00:00 2001 From: M Rasyid Hakim Date: Tue, 13 Apr 2021 20:07:13 +0700 Subject: [PATCH 5/5] fix: parse level on logger Co-authored-by: Riteek Srivastav --- logger/logger.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logger/logger.go b/logger/logger.go index f4f4011e..8633c9bb 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -68,7 +68,7 @@ func SetOutput(out io.Writer) { } func SetLevel(level string) { - if l, err := log.ParseLevel(level); err != nil { + if l, err := log.ParseLevel(level); err == nil { logger.SetLevel(l) } else { fmt.Printf("[logger] NoOps, Fail to parse log level: %v\n", err)