diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f243565d..dbb5aa3d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,3 +34,22 @@ jobs: load: true cache-from: type=gha cache-to: type=gha,mode=max + + - name: setup@test + uses: gacts/install-hurl@v1 + + - name: e2e@test + working-directory: ./tests/e2e + run: sh ./run.sh + + - name: report@test + uses: mikepenz/action-junit-report@v5.0.0-a02 + if: ${{ github.event_name == 'pull_request' && (success() || failure()) }} + with: + report_paths: "./tests/e2e/reports/junit.xml" + include_passed: true + check_name: "End-To-End Test Report" + job_summary: true + comment: true + updateComment: true + fail_on_failure: true diff --git a/README.md b/README.md index 36cc2356..fad74502 100644 --- a/README.md +++ b/README.md @@ -63,15 +63,6 @@ Result: - **Total time:** 481.65s - **Average:** 2076.18 request/s -## :warning: Disclaimer - -This project is in a very early stage of development, as such: - - - it has not been tested in production - - not even an automated test suite - -Those points will be improved on given enough time. - ## :construction: Build **Requirements:** diff --git a/Taskfile.yml b/Taskfile.yml index 590b9f33..ad72e048 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -8,12 +8,23 @@ tasks: - task: "gen:js" - go build -o bin/ ./... - test: - desc: "Run tests" + run: + desc: "Run the project" + cmds: + - ./bin/flowg serve {{.CLI_ARGS}} + + "test:unit": + desc: "Run unit tests" cmds: - task: "test:go" - task: "test:rust" + "test:e2e": + desc: "Run end-to-end tests" + dir: ./tests/e2e + cmds: + - sh run.sh + doc: desc: "Generate documentation" cmds: diff --git a/api/create_token.go b/api/create_token.go index 32e8d9b1..ef976d24 100644 --- a/api/create_token.go +++ b/api/create_token.go @@ -13,8 +13,9 @@ import ( type CreateTokenRequest struct{} type CreateTokenResponse struct { - Success bool `json:"success"` - Token string `json:"token"` + Success bool `json:"success"` + Token string `json:"token"` + TokenUUID string `json:"token-uuid"` } func CreateTokenUsecase(authDb *auth.Database) usecase.Interactor { @@ -28,7 +29,7 @@ func CreateTokenUsecase(authDb *auth.Database) usecase.Interactor { ) error { user := auth.GetContextUser(ctx) - token, err := tokenSys.CreateToken(user.Name) + token, tokenUuid, err := tokenSys.CreateToken(user.Name) if err != nil { slog.ErrorContext( ctx, @@ -44,6 +45,7 @@ func CreateTokenUsecase(authDb *auth.Database) usecase.Interactor { resp.Success = true resp.Token = token + resp.TokenUUID = tokenUuid return nil }, diff --git a/api/list_users.go b/api/list_users.go index 3b4de59d..79f7eca1 100644 --- a/api/list_users.go +++ b/api/list_users.go @@ -12,7 +12,7 @@ import ( type ListUsersRequest struct{} type ListUsersResponse struct { Success bool `json:"success"` - Users []auth.User `json:"Users"` + Users []auth.User `json:"users"` } func ListUsersUsecase(authDb *auth.Database) usecase.Interactor { diff --git a/api/main.go b/api/main.go index 04465ca4..ea14448a 100644 --- a/api/main.go +++ b/api/main.go @@ -94,6 +94,8 @@ func NewHandler( r.Put("/api/v1/users/{user}", SaveUserUsecase(authDb)) r.Delete("/api/v1/users/{user}", DeleteUserUsecase(authDb)) + r.Get("/api/v1/whoami", WhoamiUsecase(authDb)) + r.Get("/api/v1/tokens", ListTokensUsecase(authDb)) r.Post("/api/v1/token", CreateTokenUsecase(authDb)) r.Delete("/api/v1/tokens/{token-uuid}", DeleteTokenUsecase(authDb)) diff --git a/api/save_role.go b/api/save_role.go index 6a2eefce..fd420cee 100644 --- a/api/save_role.go +++ b/api/save_role.go @@ -31,7 +31,7 @@ func SaveRoleUsecase(authDb *auth.Database) usecase.Interactor { req SaveRoleRequest, resp *SaveRoleResponse, ) error { - scopes := make([]auth.Scope, 0, len(req.Scopes)) + scopes := make([]auth.Scope, len(req.Scopes)) for i, scopeName := range req.Scopes { scope, err := auth.ParseScope(scopeName) diff --git a/api/whoami.go b/api/whoami.go new file mode 100644 index 00000000..b2377b81 --- /dev/null +++ b/api/whoami.go @@ -0,0 +1,39 @@ +package api + +import ( + "context" + + "github.com/swaggest/usecase" + "github.com/swaggest/usecase/status" + + "link-society.com/flowg/internal/data/auth" +) + +type WhoamiRequest struct{} +type WhoamiResponse struct { + Success bool `json:"success"` + User *auth.User `json:"user"` +} + +func WhoamiUsecase(authDb *auth.Database) usecase.Interactor { + u := usecase.NewInteractor( + func( + ctx context.Context, + req WhoamiRequest, + resp *WhoamiResponse, + ) error { + resp.Success = true + resp.User = auth.GetContextUser(ctx) + return nil + }, + ) + + u.SetName("whoami") + u.SetTitle("Fetch current profile") + u.SetDescription("Fetch the profile of the currently authenticated user") + u.SetTags("acls") + + u.SetExpectedErrors(status.PermissionDenied) + + return u +} diff --git a/cmd/flowg/admin_token_create.go b/cmd/flowg/admin_token_create.go index 4420c0a3..3ef8cacf 100644 --- a/cmd/flowg/admin_token_create.go +++ b/cmd/flowg/admin_token_create.go @@ -54,7 +54,7 @@ func NewAdminTokenCreateCommand() *cobra.Command { return } - token, err := tokenSys.CreateToken(user.Name) + token, _, err := tokenSys.CreateToken(user.Name) if err != nil { fmt.Fprintln(os.Stderr, "ERROR: Failed to generate token:", err) exitCode = 1 diff --git a/docker/flowg.dockerfile b/docker/flowg.dockerfile index 3f1ea526..4b4a023d 100644 --- a/docker/flowg.dockerfile +++ b/docker/flowg.dockerfile @@ -89,7 +89,7 @@ RUN npm run build FROM golang:1.23-alpine3.20 AS builder-go RUN apk add --no-cache gcc musl-dev -RUN go install github.com/a-h/templ/cmd/templ@v0.2.771 +RUN go install github.com/a-h/templ/cmd/templ@v0.2.778 COPY --from=sources-go /src /workspace COPY --from=builder-rust-filterdsl /workspace/internal/ffi/filterdsl/rust-crate/target/release/libflowg_filterdsl.a /workspace/internal/ffi/filterdsl/rust-crate/target/release/libflowg_filterdsl.a diff --git a/internal/data/auth/system_token.go b/internal/data/auth/system_token.go index 55faad52..802b513c 100644 --- a/internal/data/auth/system_token.go +++ b/internal/data/auth/system_token.go @@ -21,17 +21,19 @@ func NewTokenSystem(backend *Database) *TokenSystem { return &TokenSystem{backend: backend} } -func (sys *TokenSystem) CreateToken(username string) (string, error) { +func (sys *TokenSystem) CreateToken(username string) (string, string, error) { token, err := newToken(32) if err != nil { - return "", err + return "", "", err } tokenHash, err := hash.HashPassword(token) if err != nil { - return "", fmt.Errorf("failed to hash token: %w", err) + return "", "", fmt.Errorf("failed to hash token: %w", err) } + tokenUuid := uuid.New().String() + err = sys.backend.db.Update(func(txn *badger.Txn) error { userKey := []byte(fmt.Sprintf("index:user:%s", username)) _, err := txn.Get(userKey) @@ -43,7 +45,7 @@ func (sys *TokenSystem) CreateToken(username string) (string, error) { return fmt.Errorf("failed to check if user '%s' exists: %w", username, err) } - tokenKey := []byte(fmt.Sprintf("pat:%s:%s", username, uuid.New().String())) + tokenKey := []byte(fmt.Sprintf("pat:%s:%s", username, tokenUuid)) err = txn.Set(tokenKey, []byte(tokenHash)) if err != nil { return fmt.Errorf("failed to add token to user '%s': %w", username, err) @@ -53,10 +55,10 @@ func (sys *TokenSystem) CreateToken(username string) (string, error) { }) if err != nil { - return "", err + return "", "", err } - return token, nil + return token, tokenUuid, nil } func (sys *TokenSystem) VerifyToken(token string) (*User, error) { diff --git a/internal/data/lognotify/notify_test.go b/internal/data/lognotify/notify_test.go index 05c0cc18..59dbe123 100644 --- a/internal/data/lognotify/notify_test.go +++ b/internal/data/lognotify/notify_test.go @@ -2,9 +2,11 @@ package lognotify_test import ( "reflect" - "sync" "testing" + "sync" + "time" + "link-society.com/flowg/internal/data/logstorage" "link-society.com/flowg/internal/data/lognotify" @@ -15,11 +17,12 @@ func TestLogNotifier(t *testing.T) { notifier.Start() defer notifier.Stop() - doneC := make(chan struct{}) - logC := notifier.Subscribe("test", doneC) + logDoneC := make(chan struct{}) + logC := notifier.Subscribe("test", logDoneC) logEntry := logstorage.NewLogEntry(map[string]string{}) + wgDoneC := make(chan struct{}) wg := sync.WaitGroup{} wg.Add(2) @@ -34,7 +37,17 @@ func TestLogNotifier(t *testing.T) { result = <-logC }() - wg.Wait() + go func() { + wg.Wait() + wgDoneC <- struct{}{} + close(wgDoneC) + }() + + select { + case <-wgDoneC: + case <-time.After(5 * time.Second): + t.Fatalf("timed out waiting for log") + } if result.Stream != "test" { t.Fatalf("unexpected stream: %s", result.Stream) @@ -48,6 +61,6 @@ func TestLogNotifier(t *testing.T) { t.Fatalf("unexpected log entry: %v", result.LogEntry) } - doneC <- struct{}{} - close(doneC) + logDoneC <- struct{}{} + close(logDoneC) } diff --git a/internal/data/logstorage/system_collector.go b/internal/data/logstorage/system_collector.go index d0d8cdad..b7aaf115 100644 --- a/internal/data/logstorage/system_collector.go +++ b/internal/data/logstorage/system_collector.go @@ -24,87 +24,101 @@ func (sys *CollectorSystem) Ingest( stream string, logEntry *LogEntry, ) ([]byte, error) { - key := logEntry.NewDbKey(stream) - val, err := json.Marshal(logEntry) - if err != nil { - slog.DebugContext( - ctx, - "Could not marshal log entry", - "channel", "storage", - "stream", stream, - "error", err.Error(), - ) - return nil, fmt.Errorf("could not marshal log entry: %w", err) - } - - err = sys.storage.db.Update(func(txn *badger.Txn) error { - slog.DebugContext( - ctx, - "Fetch stream configuration", - "channel", "storage", - "stream", stream, - ) - - streamConfig, err := getOrCreateStreamConfig(txn, stream) + for { + key := logEntry.NewDbKey(stream) + val, err := json.Marshal(logEntry) if err != nil { - return err + slog.DebugContext( + ctx, + "Could not marshal log entry", + "channel", "storage", + "stream", stream, + "error", err.Error(), + ) + return nil, fmt.Errorf("could not marshal log entry: %w", err) } - slog.DebugContext( - ctx, - "Save log entry in BadgerDB", - "channel", "storage", - "stream", stream, - "key", key, - ) - - entry := badger.NewEntry(key, val) - if streamConfig.RetentionTime > 0 { - entry = entry.WithTTL(time.Duration(streamConfig.RetentionTime) * time.Second) - } + err = sys.storage.db.Update(func(txn *badger.Txn) error { + slog.DebugContext( + ctx, + "Fetch stream configuration", + "channel", "storage", + "stream", stream, + ) - if err := txn.SetEntry(entry); err != nil { - return fmt.Errorf( - "could not add log entry '%s' to stream '%s': %w", - key, stream, err, + streamConfig, err := getOrCreateStreamConfig(txn, stream) + if err != nil { + return err + } + + slog.DebugContext( + ctx, + "Save log entry in BadgerDB", + "channel", "storage", + "stream", stream, + "key", key, ) - } - for field, value := range logEntry.Fields { - fieldKey := []byte(fmt.Sprintf("stream:field:%s:%s", stream, field)) - if err := txn.Set(fieldKey, []byte{}); err != nil { - return fmt.Errorf( - "could not save field '%s' of log entry '%s' to stream '%s': %w", - field, key, stream, err, - ) + entry := badger.NewEntry(key, val) + if streamConfig.RetentionTime > 0 { + entry = entry.WithTTL(time.Duration(streamConfig.RetentionTime) * time.Second) } - if streamConfig.IsFieldIndexed(field) { - slog.DebugContext( - ctx, - "Save field index in BadgerDB", - "channel", "storage", - "stream", stream, - "key", key, - "field", field, + if err := txn.SetEntry(entry); err != nil { + return fmt.Errorf( + "could not add log entry '%s' to stream '%s': %w", + key, stream, err, ) + } - fieldIndex := newFieldIndex(txn, stream, field, value) - if err := fieldIndex.AddKey(key, streamConfig.RetentionTime); err != nil { + for field, value := range logEntry.Fields { + fieldKey := []byte(fmt.Sprintf("stream:field:%s:%s", stream, field)) + if err := txn.Set(fieldKey, []byte{}); err != nil { return fmt.Errorf( - "could not add field index '%s' of log entry '%s' to stream '%s': %w", + "could not save field '%s' of log entry '%s' to stream '%s': %w", field, key, stream, err, ) } + + if streamConfig.IsFieldIndexed(field) { + slog.DebugContext( + ctx, + "Save field index in BadgerDB", + "channel", "storage", + "stream", stream, + "key", key, + "field", field, + ) + + fieldIndex := newFieldIndex(txn, stream, field, value) + if err := fieldIndex.AddKey(key, streamConfig.RetentionTime); err != nil { + return fmt.Errorf( + "could not add field index '%s' of log entry '%s' to stream '%s': %w", + field, key, stream, err, + ) + } + } } - } - return nil - }) + return nil + }) - if err != nil { - return nil, err - } + switch err { + case nil: + return key, nil + + case badger.ErrConflict: + slog.DebugContext( + ctx, + "Retry log entry ingestion", + "channel", "storage", + "stream", stream, + "key", key, + ) + continue - return key, nil + default: + return nil, err + } + } } diff --git a/internal/data/logstorage/system_meta.go b/internal/data/logstorage/system_meta.go index de9aa9f8..8fb1a9d7 100644 --- a/internal/data/logstorage/system_meta.go +++ b/internal/data/logstorage/system_meta.go @@ -200,6 +200,10 @@ func getOrCreateStreamConfig(txn *badger.Txn, stream string) (StreamConfig, erro } } + if streamConfig.IndexedFields == nil { + streamConfig.IndexedFields = []string{} + } + return streamConfig, nil } diff --git a/tests/benchmark/.gitignore b/tests/benchmark/.gitignore index ffb295ee..07e08dcd 100644 --- a/tests/benchmark/.gitignore +++ b/tests/benchmark/.gitignore @@ -1,3 +1,2 @@ /data/logs/ /data/auth/ -/logs.txt diff --git a/tests/benchmark/run.sh b/tests/benchmark/run.sh index 17b90764..5c494546 100644 --- a/tests/benchmark/run.sh +++ b/tests/benchmark/run.sh @@ -1,37 +1,7 @@ #!/bin/sh -rm -rf logs.txt data/logs data/auth +set -e -../../bin/flowg admin role create \ - --auth-dir ./data/auth \ - --name admin \ - write_streams \ - write_transformers \ - write_pipelines \ - write_acls \ - send_logs +. ../flowg.sh -../../bin/flowg admin user create \ - --auth-dir ./data/auth \ - --name root \ - --password root \ - admin - -token=$( - ../../bin/flowg admin token create \ - --auth-dir ./data/auth \ - --user root -) - -../../bin/flowg serve \ - --log-dir ./data/logs \ - --config-dir ./data/config \ - --bind 127.0.0.1:5080 \ - --verbose \ - > logs.txt & -pid=$! -trap "kill $pid" EXIT - -sleep 0.1 - -python generate-logs.py --token $token +python generate-logs.py --token $FLOWG_ADMIN_TOKEN diff --git a/tests/e2e/.gitignore b/tests/e2e/.gitignore new file mode 100644 index 00000000..05a17d31 --- /dev/null +++ b/tests/e2e/.gitignore @@ -0,0 +1,3 @@ +/data/logs/ +/data/auth/ +/reports diff --git a/tests/e2e/benchmark/stress.hurl b/tests/e2e/benchmark/stress.hurl new file mode 100644 index 00000000..c121b4eb --- /dev/null +++ b/tests/e2e/benchmark/stress.hurl @@ -0,0 +1,10 @@ +POST http://localhost:5080/api/v1/pipelines/default/logs +Authorization: Bearer {{admin_token}} +Content-Type: application/json +{ + "record": { + "level": "info", + "message": "Hello, World!" + } +} +HTTP 200 diff --git a/tests/e2e/integration/acl_roles.hurl b/tests/e2e/integration/acl_roles.hurl new file mode 100644 index 00000000..ff5146a5 --- /dev/null +++ b/tests/e2e/integration/acl_roles.hurl @@ -0,0 +1,75 @@ +GET http://localhost:5080/api/v1/roles +Authorization: Bearer {{admin_token}} +HTTP 200 +[Asserts] +jsonpath "$.success" == true +jsonpath "$.roles" count == 1 +jsonpath "$.roles[*].name" includes "admin" + +PUT http://localhost:5080/api/v1/roles/test +Authorization: Bearer {{admin_token}} +Content-Type: application/json +{ + "scopes": [ + "read_acls" + ] +} +HTTP 200 +[Asserts] +jsonpath "$.success" == true + +GET http://localhost:5080/api/v1/roles +Authorization: Bearer {{admin_token}} +HTTP 200 +[Asserts] +jsonpath "$.success" == true +jsonpath "$.roles" count == 2 +jsonpath "$.roles[*].name" includes "test" + +GET http://localhost:5080/api/v1/users +Authorization: Bearer {{guest_token}} +HTTP 403 + +PUT http://localhost:5080/api/v1/users/guest +Authorization: Bearer {{admin_token}} +Content-Type: application/json +{ + "password": "guest", + "roles": ["test"] +} +HTTP 200 +[Asserts] +jsonpath "$.success" == true + +GET http://localhost:5080/api/v1/users +Authorization: Bearer {{guest_token}} +HTTP 200 + +PUT http://localhost:5080/api/v1/users/guest +Authorization: Bearer {{admin_token}} +Content-Type: application/json +{ + "password": "guest", + "roles": [] +} +HTTP 200 +[Asserts] +jsonpath "$.success" == true + +GET http://localhost:5080/api/v1/users +Authorization: Bearer {{guest_token}} +HTTP 403 + +DELETE http://localhost:5080/api/v1/roles/test +Authorization: Bearer {{admin_token}} +HTTP 200 +[Asserts] +jsonpath "$.success" == true + +GET http://localhost:5080/api/v1/roles +Authorization: Bearer {{admin_token}} +HTTP 200 +[Asserts] +jsonpath "$.success" == true +jsonpath "$.roles" count == 1 +jsonpath "$.roles[*].name" not includes "test" diff --git a/tests/e2e/integration/acl_tokens.hurl b/tests/e2e/integration/acl_tokens.hurl new file mode 100644 index 00000000..00e69ddc --- /dev/null +++ b/tests/e2e/integration/acl_tokens.hurl @@ -0,0 +1,40 @@ +GET http://localhost:5080/api/v1/tokens +Authorization: Bearer {{admin_token}} +HTTP 200 +[Asserts] +jsonpath "$.success" == true +jsonpath "$.token-uuids" count == 1 + +POST http://localhost:5080/api/v1/token +Authorization: Bearer {{admin_token}} +HTTP 200 +[Captures] +new_admin_token: jsonpath "$.token" +new_admin_token_uuid: jsonpath "$.token-uuid" +[Asserts] +jsonpath "$.success" == true + +GET http://localhost:5080/api/v1/tokens +Authorization: Bearer {{admin_token}} +HTTP 200 +[Asserts] +jsonpath "$.success" == true +jsonpath "$.token-uuids" count == 2 +jsonpath "$.token-uuids" includes "{{new_admin_token_uuid}}" + +GET http://localhost:5080/api/v1/whoami +Authorization: Bearer {{new_admin_token}} +HTTP 200 +[Asserts] +jsonpath "$.success" == true +jsonpath "$.user.name" == "root" + +DELETE http://localhost:5080/api/v1/tokens/{{new_admin_token_uuid}} +Authorization: Bearer {{admin_token}} +HTTP 200 +[Asserts] +jsonpath "$.success" == true + +GET http://localhost:5080/api/v1/whoami +Authorization: Bearer {{new_admin_token}} +HTTP 401 diff --git a/tests/e2e/integration/acl_users.hurl b/tests/e2e/integration/acl_users.hurl new file mode 100644 index 00000000..5a0b1ab0 --- /dev/null +++ b/tests/e2e/integration/acl_users.hurl @@ -0,0 +1,42 @@ +GET http://localhost:5080/api/v1/users +Authorization: Bearer {{admin_token}} +HTTP 200 +[Asserts] +jsonpath "$.success" == true +jsonpath "$.users" count == 2 +jsonpath "$.users[*].name" includes "root" +jsonpath "$.users[*].name" includes "guest" +jsonpath "$.users[?(@.name == 'root')].roles[*]" count == 1 +jsonpath "$..users[?(@.name == 'root')].roles[*]" includes "admin" +jsonpath "$.users[?(@.name == 'guest')].roles[*]" count == 0 + +PUT http://localhost:5080/api/v1/users/test +Authorization: Bearer {{admin_token}} +Content-Type: application/json +{ + "password": "test", + "roles": [] +} +HTTP 200 +[Asserts] +jsonpath "$.success" == true + +GET http://localhost:5080/api/v1/users +Authorization: Bearer {{admin_token}} +HTTP 200 +[Asserts] +jsonpath "$.success" == true +jsonpath "$.users" count == 3 +jsonpath "$.users[*].name" includes "test" + +DELETE http://localhost:5080/api/v1/users/test +Authorization: Bearer {{admin_token}} +HTTP 200 + +GET http://localhost:5080/api/v1/users +Authorization: Bearer {{admin_token}} +HTTP 200 +[Asserts] +jsonpath "$.success" == true +jsonpath "$.users" count == 2 +jsonpath "$.users[*].name" not includes "test" diff --git a/tests/e2e/integration/alerts.hurl b/tests/e2e/integration/alerts.hurl new file mode 100644 index 00000000..64723de5 --- /dev/null +++ b/tests/e2e/integration/alerts.hurl @@ -0,0 +1,92 @@ +GET http://localhost:5080/api/v1/alerts +Authorization: Bearer {{admin_token}} +HTTP 200 +[Asserts] +jsonpath "$.success" == true +jsonpath "$.alerts" count == 0 + +GET http://localhost:5080/api/v1/alerts/httpbin +Authorization: Bearer {{admin_token}} +HTTP 404 + +PUT http://localhost:5080/api/v1/alerts/httpbin +Authorization: Bearer {{admin_token}} +Content-Type: application/json +{ + "webhook": { + "url": "http://httpbin.org/anything", + "headers": { + "Foo": "Bar" + } + } +} +HTTP 200 +[Asserts] +jsonpath "$.success" == true + +GET http://localhost:5080/api/v1/alerts +Authorization: Bearer {{admin_token}} +HTTP 200 +[Asserts] +jsonpath "$.success" == true +jsonpath "$.alerts" count == 1 +jsonpath "$.alerts[0]" == "httpbin" + +GET http://localhost:5080/api/v1/alerts/httpbin +Authorization: Bearer {{admin_token}} +HTTP 200 +[Asserts] +jsonpath "$.success" == true + +POST http://localhost:5080/api/v1/alerts/httpbin/test +Authorization: Bearer {{admin_token}} +Content-Type: application/json +{ + "record": { + "message": "Hello, World!" + } +} +HTTP 200 +[Asserts] +jsonpath "$.success" == true + +DELETE http://localhost:5080/api/v1/alerts/httpbin +Authorization: Bearer {{admin_token}} +HTTP 200 +[Asserts] +jsonpath "$.success" == true + +GET http://localhost:5080/api/v1/alerts/httpbin +Authorization: Bearer {{admin_token}} +HTTP 404 + +PUT http://localhost:5080/api/v1/alerts/fail +Authorization: Bearer {{admin_token}} +Content-Type: application/json +{ + "webhook": { + "url": "http://httpbin.org/status/500", + "headers": { + "Foo": "Bar" + } + } +} +HTTP 200 +[Asserts] +jsonpath "$.success" == true + +POST http://localhost:5080/api/v1/alerts/fail/test +Authorization: Bearer {{admin_token}} +Content-Type: application/json +{ + "record": { + "message": "Hello, World!" + } +} +HTTP 500 + +DELETE http://localhost:5080/api/v1/alerts/fail +Authorization: Bearer {{admin_token}} +HTTP 200 +[Asserts] +jsonpath "$.success" == true diff --git a/tests/e2e/integration/auth.hurl b/tests/e2e/integration/auth.hurl new file mode 100644 index 00000000..46303fd4 --- /dev/null +++ b/tests/e2e/integration/auth.hurl @@ -0,0 +1,10 @@ +GET http://localhost:5080/api/v1/whoami +HTTP 401 + +GET http://localhost:5080/api/v1/whoami +Authorization: Bearer {{admin_token}} +HTTP 200 + +GET http://localhost:5080/api/v1/whoami +Authorization: Bearer {{guest_token}} +HTTP 200 diff --git a/tests/e2e/integration/monitoring.hurl b/tests/e2e/integration/monitoring.hurl new file mode 100644 index 00000000..754b3130 --- /dev/null +++ b/tests/e2e/integration/monitoring.hurl @@ -0,0 +1,5 @@ +GET http://localhost:5080/health +HTTP 200 + +GET http://localhost:5080/metrics +HTTP 200 diff --git a/tests/e2e/integration/permissions.hurl b/tests/e2e/integration/permissions.hurl new file mode 100644 index 00000000..8b47ab08 --- /dev/null +++ b/tests/e2e/integration/permissions.hurl @@ -0,0 +1,65 @@ +GET http://localhost:5080/api/v1/streams +HTTP 401 + +GET http://localhost:5080/api/v1/streams +Authorization: Bearer {{admin_token}} +HTTP 200 + +GET http://localhost:5080/api/v1/streams +Authorization: Bearer {{guest_token}} +HTTP 403 + +GET http://localhost:5080/api/v1/transformers +HTTP 401 + +GET http://localhost:5080/api/v1/transformers +Authorization: Bearer {{admin_token}} +HTTP 200 + +GET http://localhost:5080/api/v1/transformers +Authorization: Bearer {{guest_token}} +HTTP 403 + +GET http://localhost:5080/api/v1/pipelines +HTTP 401 + +GET http://localhost:5080/api/v1/pipelines +Authorization: Bearer {{admin_token}} +HTTP 200 + +GET http://localhost:5080/api/v1/pipelines +Authorization: Bearer {{guest_token}} +HTTP 403 + +GET http://localhost:5080/api/v1/alerts +HTTP 401 + +GET http://localhost:5080/api/v1/alerts +Authorization: Bearer {{admin_token}} +HTTP 200 + +GET http://localhost:5080/api/v1/alerts +Authorization: Bearer {{guest_token}} +HTTP 403 + +GET http://localhost:5080/api/v1/users +HTTP 401 + +GET http://localhost:5080/api/v1/users +Authorization: Bearer {{admin_token}} +HTTP 200 + +GET http://localhost:5080/api/v1/users +Authorization: Bearer {{guest_token}} +HTTP 403 + +GET http://localhost:5080/api/v1/roles +HTTP 401 + +GET http://localhost:5080/api/v1/roles +Authorization: Bearer {{admin_token}} +HTTP 200 + +GET http://localhost:5080/api/v1/roles +Authorization: Bearer {{guest_token}} +HTTP 403 diff --git a/tests/e2e/integration/pipelines.hurl b/tests/e2e/integration/pipelines.hurl new file mode 100644 index 00000000..5e7d00d0 --- /dev/null +++ b/tests/e2e/integration/pipelines.hurl @@ -0,0 +1,132 @@ +GET http://localhost:5080/api/v1/pipelines +Authorization: Bearer {{admin_token}} +HTTP 200 +[Asserts] +jsonpath "$.success" == true +jsonpath "$.pipelines" count == 1 +jsonpath "$.pipelines[0]" == "default" + +GET http://localhost:5080/api/v1/pipelines/default +Authorization: Bearer {{admin_token}} +HTTP 200 +[Asserts] +jsonpath "$.success" == true +jsonpath "$.flow" exists + +GET http://localhost:5080/api/v1/pipelines/test +Authorization: Bearer {{admin_token}} +HTTP 404 + +PUT http://localhost:5080/api/v1/pipelines/test +Authorization: Bearer {{admin_token}} +Content-Type: application/json +{ + "flow": { + "nodes": [ + { + "id": "__builtin__source_direct", + "type": "source", + "position": {"x": 210, "y": 195}, + "deletable": false, + "data": {"type": "direct"}, + "measured": {"width": 136, "height": 38}, + "selected": true, + "dragging": false + }, + { + "id": "__builtin__source_syslog", + "type": "source", + "position": {"x": 210, "y": 250}, + "deletable": false, + "data": {"type": "syslog"}, + "measured": {"width": 136, "height": 38}, + "selected": true, + "dragging": false + }, + { + "id": "node-1", + "type": "router", + "position": {"x": 405, "y": 195}, + "data": {"stream": "test"}, + "measured": {"width": 241,"height": 91}, + "selected": false, + "dragging": false + } + ], + "edges": [ + { + "id": "xy-edge____builtin__source_direct-node-1", + "type": "smoothstep", + "source": "__builtin__source_direct", + "target": "node-1", + "animated": true + }, + { + "id": "xy-edge____builtin__source_syslog-node-1", + "type": "smoothstep", + "source": "__builtin__source_syslog", + "target": "node-1", + "animated": true + } + ] + } +} +HTTP 200 +[Asserts] +jsonpath "$.success" == true + +GET http://localhost:5080/api/v1/pipelines +Authorization: Bearer {{admin_token}} +HTTP 200 +[Asserts] +jsonpath "$.success" == true +jsonpath "$.pipelines" count == 2 +jsonpath "$.pipelines" includes "default" +jsonpath "$.pipelines" includes "test" + +POST http://localhost:5080/api/v1/pipelines/test/logs +Authorization: Bearer {{admin_token}} +Content-Type: application/json +{ + "record": { + "level": "info", + "message": "test" + } +} +HTTP 200 + +GET http://localhost:5080/api/v1/streams/test/logs +Authorization: Bearer {{admin_token}} +[QueryStringParams] +from: {{timewindow_begin}} +to: {{timewindow_end}} + +HTTP 200 +[Asserts] +jsonpath "$.success" == true +jsonpath "$.records" count == 1 +jsonpath "$.records[0].fields.level" == "info" +jsonpath "$.records[0].fields.message" == "test" + +DELETE http://localhost:5080/api/v1/streams/test +Authorization: Bearer {{admin_token}} +HTTP 200 +[Asserts] +jsonpath "$.success" == true + +DELETE http://localhost:5080/api/v1/pipelines/test +Authorization: Bearer {{admin_token}} +HTTP 200 +[Asserts] +jsonpath "$.success" == true + +GET http://localhost:5080/api/v1/pipelines/test +Authorization: Bearer {{admin_token}} +HTTP 404 + +GET http://localhost:5080/api/v1/pipelines +Authorization: Bearer {{admin_token}} +HTTP 200 +[Asserts] +jsonpath "$.success" == true +jsonpath "$.pipelines" not includes "test" diff --git a/tests/e2e/integration/retention.hurl b/tests/e2e/integration/retention.hurl new file mode 100644 index 00000000..0bcf1324 --- /dev/null +++ b/tests/e2e/integration/retention.hurl @@ -0,0 +1,55 @@ +PUT http://localhost:5080/api/v1/streams/default +Authorization: Bearer {{admin_token}} +Content-Type: application/json +{ + "config": { + "indexed_fields": [], + "ttl": 2, + "size": 0 + } +} +HTTP 200 +[Asserts] +jsonpath "$.success" == true + + +POST http://localhost:5080/api/v1/pipelines/default/logs +Authorization: Bearer {{admin_token}} +Content-Type: application/json +{ + "record": { + "level": "info", + "message": "Hello, World!" + } +} +HTTP 200 + +GET http://localhost:5080/api/v1/streams/default/logs +Authorization: Bearer {{admin_token}} +[QueryStringParams] +from: {{timewindow_begin}} +to: {{timewindow_end}} + +HTTP 200 +[Asserts] +jsonpath "$.success" == true +jsonpath "$.records" count == 1 +jsonpath "$.records[0].fields.level" == "info" +jsonpath "$.records[0].fields.message" == "Hello, World!" + +GET http://localhost:5080/api/v1/streams/default/logs +Authorization: Bearer {{admin_token}} +[QueryStringParams] +from: {{timewindow_begin}} +to: {{timewindow_end}} +[Options] +delay: 2s + +HTTP 200 +[Asserts] +jsonpath "$.success" == true +jsonpath "$.records" count == 0 + +DELETE http://localhost:5080/api/v1/streams/default +Authorization: Bearer {{admin_token}} +HTTP 200 diff --git a/tests/e2e/integration/streams.hurl b/tests/e2e/integration/streams.hurl new file mode 100644 index 00000000..bfbcad16 --- /dev/null +++ b/tests/e2e/integration/streams.hurl @@ -0,0 +1,85 @@ +GET http://localhost:5080/api/v1/streams +Authorization: Bearer {{admin_token}} +HTTP 200 +[Asserts] +jsonpath "$.success" == true +jsonpath "$.streams" isEmpty + +PUT http://localhost:5080/api/v1/streams/test +Authorization: Bearer {{admin_token}} +Content-Type: application/json +{ + "config": { + "indexed_fields": ["level"], + "ttl": 3600, + "size": 1024 + } +} +HTTP 200 +[Asserts] +jsonpath "$.success" == true + +GET http://localhost:5080/api/v1/streams +Authorization: Bearer {{admin_token}} +HTTP 200 +[Asserts] +jsonpath "$.success" == true +jsonpath "$.streams.test" exists +jsonpath "$.streams.test.indexed_fields" count == 1 +jsonpath "$.streams.test.indexed_fields[0]" == "level" +jsonpath "$.streams.test.ttl" == 3600 +jsonpath "$.streams.test.size" == 1024 + +GET http://localhost:5080/api/v1/streams/test +Authorization: Bearer {{admin_token}} +HTTP 200 +[Asserts] +jsonpath "$.success" == true +jsonpath "$.config.indexed_fields" count == 1 +jsonpath "$.config.indexed_fields[0]" == "level" +jsonpath "$.config.ttl" == 3600 +jsonpath "$.config.size" == 1024 + +GET http://localhost:5080/api/v1/streams/test_getorcreate +Authorization: Bearer {{admin_token}} +HTTP 200 +[Asserts] +jsonpath "$.success" == true +jsonpath "$.config.indexed_fields" count == 0 +jsonpath "$.config.ttl" == 0 +jsonpath "$.config.size" == 0 + +GET http://localhost:5080/api/v1/streams +Authorization: Bearer {{admin_token}} +HTTP 200 +[Asserts] +jsonpath "$.success" == true +jsonpath "$.streams.test" exists +jsonpath "$.streams.test_getorcreate" exists + +DELETE http://localhost:5080/api/v1/streams/test_getorcreate +Authorization: Bearer {{admin_token}} +HTTP 200 +[Asserts] +jsonpath "$.success" == true + +GET http://localhost:5080/api/v1/streams +Authorization: Bearer {{admin_token}} +HTTP 200 +[Asserts] +jsonpath "$.success" == true +jsonpath "$.streams.test" exists +jsonpath "$.streams.test_getorcreate" not exists + +DELETE http://localhost:5080/api/v1/streams/test +Authorization: Bearer {{admin_token}} +HTTP 200 +[Asserts] +jsonpath "$.success" == true + +GET http://localhost:5080/api/v1/streams +Authorization: Bearer {{admin_token}} +HTTP 200 +[Asserts] +jsonpath "$.success" == true +jsonpath "$.streams" isEmpty diff --git a/tests/e2e/integration/transformers.hurl b/tests/e2e/integration/transformers.hurl new file mode 100644 index 00000000..1aa020c8 --- /dev/null +++ b/tests/e2e/integration/transformers.hurl @@ -0,0 +1,58 @@ +GET http://localhost:5080/api/v1/transformers +Authorization: Bearer {{admin_token}} +HTTP 200 +[Asserts] +jsonpath "$.success" == true +jsonpath "$.transformers" count == 0 + +GET http://localhost:5080/api/v1/transformers/logfmt +Authorization: Bearer {{admin_token}} +HTTP 404 + +PUT http://localhost:5080/api/v1/transformers/logfmt +Authorization: Bearer {{admin_token}} +Content-Type: application/json +{ + "script": ". = parse_logfmt!(.message)" +} +HTTP 200 +[Asserts] +jsonpath "$.success" == true + +GET http://localhost:5080/api/v1/transformers +Authorization: Bearer {{admin_token}} +HTTP 200 +[Asserts] +jsonpath "$.success" == true +jsonpath "$.transformers" count == 1 +jsonpath "$.transformers[0]" == "logfmt" + +GET http://localhost:5080/api/v1/transformers/logfmt +Authorization: Bearer {{admin_token}} +HTTP 200 +[Asserts] +jsonpath "$.success" == true + +POST http://localhost:5080/api/v1/transformers/logfmt/test +Authorization: Bearer {{admin_token}} +Content-Type: application/json +{ + "record": { + "message": "level=info message=\"foo bar\"" + } +} +HTTP 200 +[Asserts] +jsonpath "$.success" == true +jsonpath "$.record.level" == "info" +jsonpath "$.record.message" == "foo bar" + +DELETE http://localhost:5080/api/v1/transformers/logfmt +Authorization: Bearer {{admin_token}} +HTTP 200 +[Asserts] +jsonpath "$.success" == true + +GET http://localhost:5080/api/v1/transformers/logfmt +Authorization: Bearer {{admin_token}} +HTTP 404 diff --git a/tests/e2e/run.sh b/tests/e2e/run.sh new file mode 100644 index 00000000..c912da2a --- /dev/null +++ b/tests/e2e/run.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +set -e + +. ../flowg.sh + +hurl \ + --variable admin_token=${FLOWG_ADMIN_TOKEN} \ + --variable guest_token=${FLOWG_GUEST_TOKEN} \ + --variable timewindow_begin=$(date -d "5 minutes ago" -u +"%Y-%m-%dT%H:%M:%SZ") \ + --variable timewindow_end=$(date -d "+5 minutes" -u +"%Y-%m-%dT%H:%M:%SZ") \ + --error-format long \ + --report-html reports/html \ + --report-junit reports/junit.xml \ + --jobs 1 \ + --test integration/ + +if [ -z "$NOBENCHMARK" ] +then + hurl \ + --variable admin_token=${FLOWG_ADMIN_TOKEN} \ + --variable guest_token=${FLOWG_GUEST_TOKEN} \ + --repeat 1000 \ + --test benchmark/ +fi diff --git a/tests/flowg.sh b/tests/flowg.sh new file mode 100644 index 00000000..076172b7 --- /dev/null +++ b/tests/flowg.sh @@ -0,0 +1,63 @@ +#!/bin/sh + +FLOWG_CMD_FG="docker run --rm -v ./data:/data -p 5080:5080/tcp -p 5514:5514/udp linksociety/flowg:latest" +FLOWG_CMD_BG="docker run -d --rm -v ./data:/data -p 5080:5080/tcp -p 5514:5514/udp linksociety/flowg:latest" + +flowg_cleanup_data() { + echo -n "Cleaning up data..." + sudo rm -rf ./data/ + echo " ok" +} + +flowg_acl_admin() { + ${FLOWG_CMD_FG} admin role create --name admin \ + write_streams \ + write_transformers \ + write_pipelines \ + write_acls \ + write_alerts \ + send_logs \ + > /dev/null + + ${FLOWG_CMD_FG} admin user create --name root --password root admin \ + > /dev/null + + ${FLOWG_CMD_FG} admin token create --user root +} + +flowg_acl_guest() { + ${FLOWG_CMD_FG} admin user create --name guest --password guest \ + > /dev/null + + ${FLOWG_CMD_FG} admin token create --user guest +} + +flowg_start() { + echo -n "Starting FlowG..." + + DOCKER_CONTAINER_ID=$(${FLOWG_CMD_BG} serve) + trap "docker kill ${DOCKER_CONTAINER_ID} >/dev/null && flowg_cleanup_data" EXIT + + for _ in $(seq 1 10) + do + echo -n "." + sleep 0.25 + nc -z localhost 5080 2>/dev/null && echo " ok" && break + done + + nc -z localhost 5080 2>/dev/null || (echo " timeout" && exit 1) +} + + +echo "--------------------------------------------------------------------------------" + +flowg_cleanup_data + +echo -n "Setup ACLs..." +export FLOWG_ADMIN_TOKEN=$(flowg_acl_admin) +export FLOWG_GUEST_TOKEN=$(flowg_acl_guest) +echo " ok" + +flowg_start + +echo "--------------------------------------------------------------------------------" diff --git a/web/apps/account/controllers/create_token.go b/web/apps/account/controllers/create_token.go index 7db7407d..19efa228 100644 --- a/web/apps/account/controllers/create_token.go +++ b/web/apps/account/controllers/create_token.go @@ -21,7 +21,7 @@ func CreateToken( r = r.WithContext(webutils.WithNotificationSystem(r.Context())) user := auth.GetContextUser(r.Context()) - token, err := tokenSys.CreateToken(user.Name) + token, _, err := tokenSys.CreateToken(user.Name) if err != nil { webutils.LogError(r.Context(), "Failed to create token", err) webutils.NotifyError(r.Context(), "Could not create token")