diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 00000000..c5c2dc27 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,36 @@ +name: Test & Build + +on: + - push + - pull_request + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - name: Setup Go + uses: actions/setup-go@v2.1.3 + with: + go-version: '1.14' + - name: Install Protoc + uses: arduino/setup-protoc@v1 + - name: Checkout repo + uses: actions/checkout@v2 + - name: Setup Project + run: make install-protoc; make setup + - name: Invoking go test + run: make test + build: + runs-on: ubuntu-latest + + steps: + - name: Setup Go + uses: actions/setup-go@v2.1.3 + with: + go-version: '1.14' + - name: Install Protoc + uses: arduino/setup-protoc@v1 + - uses: actions/checkout@v2 + - name: Build + run: make all diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml new file mode 100644 index 00000000..37662ecd --- /dev/null +++ b/.github/workflows/integration-test.yaml @@ -0,0 +1,23 @@ +name: Integration Test + +on: + - push + - pull_request + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Setup Docker + uses: docker-practice/actions-setup-docker@master + - name: Install Protoc + uses: arduino/setup-protoc@v1 + - name: Checkout repo + uses: actions/checkout@v2 + - name: Copy integration config + run: cp application.yml.integration-test application.yml + - name: Run Raccoon + run: make docker-run + - run: make install-protoc && make generate-proto + - name: Invoking go test + run: INTEGTEST_BOOTSTRAP_SERVER=localhost:9094 INTEGTEST_HOST=ws://localhost:8080 INTEGTEST_TOPIC_FORMAT="clickstream-%s-log" go test ./integration -v diff --git a/.github/workflows/package.yaml b/.github/workflows/package.yaml new file mode 100644 index 00000000..16e9ce51 --- /dev/null +++ b/.github/workflows/package.yaml @@ -0,0 +1,37 @@ +name: Package +on: + push: + # Sequence of patterns matched against refs/tags + tags: + - "v*.*.*" # Push events to matching v*, i.e. v1.0, v20.15.10 + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Get release tag + id: get_version + uses: battila7/get-version-action@v2 + - name: Login to GitHub Packages Docker Registry + uses: docker/login-action@v1 + with: + registry: docker.pkg.github.com + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build and Push + uses: docker/build-push-action@v2 + with: + context: . + push: true + tags: | + docker.pkg.github.com/${{ github.repository }}/raccoon:latest + docker.pkg.github.com/${{ github.repository }}/raccoon:${{ steps.get_version.outputs.version-without-v }} + odpf/raccoon:latest + odpf/raccoon:${{ steps.get_version.outputs.version-without-v }} diff --git a/Dockerfile b/Dockerfile index 1e050ae7..7c637090 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,18 @@ FROM golang:1.14 WORKDIR /app +RUN apt-get update && apt-get install unzip --no-install-recommends --assume-yes +RUN PROTOC_ZIP=protoc-3.14.0-linux-x86_64.zip && \ +curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v3.14.0/$PROTOC_ZIP && \ +unzip -o $PROTOC_ZIP -d /usr/local bin/protoc && \ +unzip -o $PROTOC_ZIP -d /usr/local 'include/*' && \ +rm -f $PROTOC_ZIP COPY . . -RUN make compile +RUN make install-protoc && make generate-proto && make update-deps && make compile +FROM debian:buster-slim +WORKDIR /app +COPY --from=0 /app/out/raccoon ./raccoon +COPY ./application.yml . EXPOSE 8080 - -# Command to run the executable -CMD ["./out/raccoon"] +CMD ["./raccoon"] diff --git a/Makefile b/Makefile index 3d7d322f..83c60fed 100644 --- a/Makefile +++ b/Makefile @@ -26,8 +26,8 @@ PROTO_PACKAGE=/websocket/proto generate-proto: rm -rf .temp mkdir -p .temp - cd ./.temp; curl -o proton.tar.gz -L http://api.github.com/repos/odpf/proton/tarball/main; tar xvf proton.tar.gz -C . --strip-components 1 - protoc --proto_path=.temp/ $(wildcard .temp/odpf/raccoon/*.proto) --go_out=./ --go_opt=paths=import --go_opt=Modpf/raccoon/Event.proto=$(PROTO_PACKAGE) --go_opt=Modpf/raccoon/EventRequest.proto=$(PROTO_PACKAGE) --go_opt=Modpf/raccoon/EventResponse.proto=$(PROTO_PACKAGE) + curl -o .temp/proton.tar.gz -L http://api.github.com/repos/odpf/proton/tarball/main; tar xvf .temp/proton.tar.gz -C .temp/ --strip-components 1 + protoc --proto_path=.temp/ .temp/odpf/raccoon/*.proto --go_out=./ --go_opt=paths=import --go_opt=Modpf/raccoon/Event.proto=$(PROTO_PACKAGE) --go_opt=Modpf/raccoon/EventRequest.proto=$(PROTO_PACKAGE) --go_opt=Modpf/raccoon/EventResponse.proto=$(PROTO_PACKAGE) # Build Lifecycle compile: diff --git a/application.yml.integration-test b/application.yml.integration-test new file mode 100644 index 00000000..49c9a437 --- /dev/null +++ b/application.yml.integration-test @@ -0,0 +1,31 @@ +SERVER_WEBSOCKET_PORT: "8080" +SERVER_WEBSOCKET_CHECK_ORIGIN: "true" +SERVER_WEBSOCKET_MAX_CONN: "30000" +SERVER_WEBSOCKET_READ_BUFFER_SIZE: "10240" +SERVER_WEBSOCKET_WRITE_BUFFER_SIZE: "10240" +SERVER_WEBSOCKET_CONN_UNIQ_ID_HEADER: "x-user-id" +SERVER_WEBSOCKET_PING_INTERVAL_MS: 30000 +SERVER_WEBSOCKET_PONG_WAIT_INTERVAL_MS: 60000 +SERVER_WEBSOCKET_WRITE_WAIT_INTERVAL_MS: 5000 +SERVER_WEBSOCKET_SERVER_SHUTDOWN_GRACE_PERIOD_MS: 3000 +SERVER_WEBSOCKET_PINGER_SIZE: 1 + +WORKER_BUFFER_CHANNEL_SIZE: 5 +WORKER_BUFFER_FLUSH_TIMEOUT_MS: 5000 +WORKER_POOL_SIZE: 5 +WORKER_KAFKA_DELIVERY_CHANNEL_SIZE: 100 + +EVENT_DISTRIBUTION_PUBLISHER_PATTERN: "clickstream-%s-log" + +PUBLISHER_KAFKA_CLIENT_BOOTSTRAP_SERVERS: "kafka:9092" +PUBLISHER_KAFKA_CLIENT_ACKS: "1" +PUBLISHER_KAFKA_CLIENT_RETRIES: "2" +PUBLISHER_KAFKA_CLIENT_RETRY_BACKOFF_MS: "100" +PUBLISHER_KAFKA_CLIENT_STATISTICS_INTERVAL_MS: 5000 +PUBLISHER_KAFKA_CLIENT_QUEUE_BUFFERING_MAX_MESSAGES: 100000 +PUBLISHER_KAFKA_FLUSH_INTERVAL_MS: 1000 + +METRIC_STATSD_ADDRESS: ":8125" +METRIC_STATSD_FLUSH_PERIOD_MS: 100 + +LOG_LEVEL: "info" diff --git a/docker-compose.yml b/docker-compose.yml index b5a3a327..ca1e1174 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3.2' +version: '3.9' networks: cs-network: @@ -41,7 +41,7 @@ services: cs: build: context: . - command: ["/bin/sh", "-c", "./out/raccoon"] + command: ["/bin/sh", "-c", "./raccoon"] hostname: cs container_name: cs stdin_open: true diff --git a/logger/logger.go b/logger/logger.go index 95382a8d..8633c9bb 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -4,7 +4,6 @@ import ( "fmt" "io" "os" - "raccoon/config" log "github.com/sirupsen/logrus" ) @@ -16,19 +15,13 @@ func init() { if logger != nil { return } - logLevel, err := log.ParseLevel(config.Log.Level) - if err != nil { - fmt.Printf("[init] Fail to parse log level during init: %s\n", err) - fmt.Println("[init] Fallback to info log level") - logLevel = log.InfoLevel - } logger = &log.Logger{ Out: os.Stdout, Formatter: &log.TextFormatter{ FullTimestamp: true, }, Hooks: make(log.LevelHooks), - Level: logLevel, + Level: defaultLevel, } return @@ -73,3 +66,11 @@ func Set(log *log.Logger) { func SetOutput(out io.Writer) { logger.SetOutput(out) } + +func SetLevel(level string) { + if l, err := log.ParseLevel(level); err == nil { + logger.SetLevel(l) + } else { + fmt.Printf("[logger] NoOps, Fail to parse log level: %v\n", err) + } +} diff --git a/main.go b/main.go index 2e4b7989..373fc1ac 100644 --- a/main.go +++ b/main.go @@ -10,6 +10,7 @@ import ( func main() { config.Load() metrics.Setup() + logger.SetLevel(config.Log.Level) err := app.Run() metrics.Close() if err != nil { diff --git a/websocket/handler_test.go b/websocket/handler_test.go index 4369b531..76b82931 100644 --- a/websocket/handler_test.go +++ b/websocket/handler_test.go @@ -95,7 +95,9 @@ func TestHandler_HandlerWSEvents(t *testing.T) { }) t.Run("Should return unknown request when request fail to deserialize", func(t *testing.T) { - wss, _, err := websocket.DefaultDialer.Dial(url, header) + wss, _, err := websocket.DefaultDialer.Dial(url, http.Header{ + "x-user-id": []string{"test2-user2"}, + }) defer wss.Close() require.NoError(t, err)