Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(task): add Kafka consumer to create tasks from Kafka topic #378

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

feat(task): add Kafka consumer to create tasks from Kafka topic #378

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions engine/engine.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
"github.com/ovh/utask/models/task"
"github.com/ovh/utask/models/tasktemplate"
"github.com/ovh/utask/pkg/jsonschema"
"github.com/ovh/utask/pkg/kafkaconsumer"
"github.com/ovh/utask/pkg/metadata"
"github.com/ovh/utask/pkg/now"
"github.com/ovh/utask/pkg/taskutils"
Expand Down Expand Up @@ -147,6 +148,31 @@ func Init(ctx context.Context, wg *sync.WaitGroup, store *configstore.Store) err
if err := RetryCollector(ctx); err != nil {
return err
}

// init KafkaConsumer
kcItems, err := configstore.Filter().Store(store).Slice(kafkaconsumer.ConfigKey).Unmarshal(func() interface{} { return &kafkaconsumer.KafkaConfig{} }).Squash().GetItemList()
if err != nil {
return err
}

for _, cfgItem := range kcItems.Items {
i, err := cfgItem.Unmarshaled()
if err != nil {
return err
}

cfg := i.(*kafkaconsumer.KafkaConfig)
consumer, err := kafkaconsumer.StartNewTaskConsumer(ctx, *cfg)
if err != nil {
return err
}

if err := consumer.SetDefaultConsumer(ctx); err != nil {
return err
}

consumer.StartConsumer(ctx)
}
}
return nil
}
Expand Down
33 changes: 22 additions & 11 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,14 @@ go 1.19
require (
github.com/Masterminds/sprig/v3 v3.1.0
github.com/Masterminds/squirrel v1.4.0
github.com/Shopify/sarama v1.37.2
github.com/cenkalti/backoff v2.2.1+incompatible
github.com/fabienm/go-logrus-formatters v1.0.0
github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32
github.com/gin-gonic/gin v1.7.7
github.com/go-gorp/gorp v2.2.0+incompatible
github.com/go-ping/ping v0.0.0-20210506233800-ff8be3320020
github.com/go-playground/validator/v10 v10.11.1
github.com/gofrs/uuid v3.3.0+incompatible
github.com/jpillora/backoff v1.0.0
github.com/juju/errors v0.0.0-20200330140219-3fe23663418f
Expand All @@ -29,13 +31,13 @@ require (
github.com/sirupsen/logrus v1.6.0
github.com/spf13/cobra v0.0.6
github.com/spf13/viper v1.4.0
github.com/stretchr/testify v1.5.1
github.com/stretchr/testify v1.8.0
github.com/tidwall/gjson v1.9.3
github.com/wI2L/fizz v0.17.0
github.com/ybriffa/go-http-digest-auth-client v0.6.3
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa
golang.org/x/net v0.7.0
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f
golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7
gopkg.in/mail.v2 v2.3.1
)

Expand All @@ -47,38 +49,44 @@ require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/eapache/go-resiliency v1.3.0 // indirect
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect
github.com/eapache/queue v1.1.0 // indirect
github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 // indirect
github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9 // indirect
github.com/facebookgo/httpcontrol v0.0.0-20150708234001-ccde4420e1fe // indirect
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 // indirect
github.com/fsnotify/fsnotify v1.4.7 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-playground/locales v0.13.0 // indirect
github.com/go-playground/universal-translator v0.17.0 // indirect
github.com/go-playground/validator/v10 v10.4.1 // indirect
github.com/go-playground/locales v0.14.0 // indirect
github.com/go-playground/universal-translator v0.18.0 // indirect
github.com/go-sql-driver/mysql v1.4.1 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/uuid v1.1.1 // indirect
github.com/hashicorp/errwrap v1.0.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-retryablehttp v0.5.1 // indirect
github.com/hashicorp/go-uuid v1.0.2 // indirect
github.com/hashicorp/go-uuid v1.0.3 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/huandu/xstrings v1.3.1 // indirect
github.com/imdario/mergo v0.3.9 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/jcmturner/aescts/v2 v2.0.0 // indirect
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
github.com/jcmturner/gofork v1.0.0 // indirect
github.com/jcmturner/gofork v1.7.6 // indirect
github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
github.com/jcmturner/gokrb5/v8 v8.3.0 // indirect
github.com/jcmturner/rpc/v2 v2.0.2 // indirect
github.com/jcmturner/gokrb5/v8 v8.4.3 // indirect
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
github.com/jinzhu/now v1.0.1 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.15.11 // indirect
github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
github.com/leodido/go-urn v1.2.0 // indirect
github.com/leodido/go-urn v1.2.1 // indirect
github.com/magiconair/properties v1.8.1 // indirect
github.com/mattn/go-isatty v0.0.12 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
Expand All @@ -89,12 +97,14 @@ require (
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pelletier/go-toml v1.4.0 // indirect
github.com/pierrec/lz4/v4 v4.1.17 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/poy/onpar v1.0.0 // indirect
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/procfs v0.8.0 // indirect
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 // indirect
github.com/spf13/afero v1.2.2 // indirect
github.com/spf13/cast v1.3.1 // indirect
Expand All @@ -112,4 +122,5 @@ require (
gopkg.in/ini.v1 v1.46.0 // indirect
gopkg.in/sourcemap.v1 v1.0.5 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
Loading