diff --git a/go.mod b/go.mod index bec9fe6..3d96310 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,6 @@ go 1.18 require google.golang.org/grpc v1.47.0 -require ( - github.com/fluent/fluent-logger-golang v1.9.0 // indirect - github.com/philhofer/fwd v1.1.1 // indirect - github.com/tinylib/msgp v1.1.6 // indirect -) - require ( github.com/confluentinc/confluent-kafka-go v1.8.2 github.com/golang/protobuf v1.5.2 // indirect @@ -17,6 +11,6 @@ require ( golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 // indirect golang.org/x/text v0.3.3 // indirect google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 // indirect - google.golang.org/protobuf v1.27.1 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + google.golang.org/protobuf v1.27.1 + gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index f2edeca..8d24a25 100644 --- a/go.sum +++ b/go.sum @@ -20,8 +20,6 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/fluent/fluent-logger-golang v1.9.0 h1:zUdY44CHX2oIUc7VTNZc+4m+ORuO/mldQDA7czhWXEg= -github.com/fluent/fluent-logger-golang v1.9.0/go.mod h1:2/HCT/jTy78yGyeNGQLGQsjF3zzzAuy6Xlk6FCMV5eU= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -49,33 +47,25 @@ github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ= -github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/tinylib/msgp v1.1.6 h1:i+SbKraHhnrf9M5MYmvQhFnbLhAXSDWF8WWsuyRdocw= -github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -85,7 +75,6 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -101,10 +90,6 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -136,6 +121,7 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/src/conf/datasage.yaml b/src/conf/datasage.yaml index ee882a7..3d55d31 100644 --- a/src/conf/datasage.yaml +++ b/src/conf/datasage.yaml @@ -28,13 +28,13 @@ adaptors: integrations: kafka: - - topic: datalog - broker: broker.kafka.com - port: 11211 + - topic: dblogs + broker: 127.0.0.1 + port: 9092 rpc: - - host: grpc.com - port: 1989 + - host: 127.0.0.1 + port: 2222 http: - endpoint: https://accuknox.com/service/log method: post - \ No newline at end of file + diff --git a/src/integrations/config.go b/src/integrations/config.go index 98a55d4..fc80c88 100644 --- a/src/integrations/config.go +++ b/src/integrations/config.go @@ -8,7 +8,7 @@ import ( ) type Config struct { - Integrations Integrations + Integrations } type Integrations struct { @@ -27,14 +27,16 @@ type KafkaLogConfig struct { Port string } -func ReadLogConfig(path string) Config { +func ReadLogConfig(path string) (Config, error) { config := Config{} data, err := os.ReadFile(path) if err != nil { log.Printf("err: %s", err) + return config, err } if yaml.Unmarshal(data, &config) != nil { log.Printf("err: %s", err) + return config, err } - return config + return config, nil } diff --git a/src/integrations/fluentd.go b/src/integrations/fluentd.go deleted file mode 100644 index 33ed1cc..0000000 --- a/src/integrations/fluentd.go +++ /dev/null @@ -1,20 +0,0 @@ -package integrations - -import ( - "log" - - "github.com/fluent/fluent-logger-golang/fluent" -) - -func SendLogToFluentd(Log string, config fluent.Config) { - logger, err := fluent.New(config) - if err != nil { - log.Println(err) - } - defer logger.Close() - tag := "datasage.access" - err = logger.Post(tag, Log) - if err != nil { - log.Println("err: ", err) - } -} diff --git a/src/integrations/grpc.go b/src/integrations/grpc.go index f7261e0..0419371 100644 --- a/src/integrations/grpc.go +++ b/src/integrations/grpc.go @@ -13,14 +13,15 @@ import ( "google.golang.org/grpc/credentials/insecure" ) -func StreamLogToGRPC(Log string, grpcConfigs []GRPCLogConfig) { +func StreamLogToGRPC(Log string, grpcConfigs []GRPCLogConfig) error { for _, config := range grpcConfigs { var conn *grpc.ClientConn log.Printf("[GRPC] Dialing %s:%s \n", config.Host, config.Port) - conn, err := grpc.Dial(config.Host, + conn, err := grpc.Dial(config.Host+":"+config.Port, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { log.Fatalf("did not connect: %s", err) + return err } defer conn.Close() c := grpc_config.NewDataSageServerClient(conn) @@ -28,6 +29,9 @@ func StreamLogToGRPC(Log string, grpcConfigs []GRPCLogConfig) { defer cancel() if _, err = c.LogSend(ctx, &grpc_config.Log{Body: Log}); err != nil { log.Printf("could not send the data: %v", err) + return err } + log.Printf("[GRPC] Log send to %s:%s\n", config.Host, config.Port) } + return nil } diff --git a/src/integrations/http_server.go b/src/integrations/http_server.go new file mode 100644 index 0000000..9858bbd --- /dev/null +++ b/src/integrations/http_server.go @@ -0,0 +1,44 @@ +package integrations + +import ( + "fmt" + "io/ioutil" + "log" + "net/http" +) + +func loghandler(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/log" { + http.Error(w, "404 not found.", http.StatusNotFound) + return + } + if r.Method != "POST" { + fmt.Fprintf(w, "Only POST method is supported") + return + } + body, err := ioutil.ReadAll(r.Body) + if err != nil { + fmt.Fprintf(w, "err : %v", err) + } + w.Header().Set("Content-Type", "application/json") + config, err := ReadLogConfig("src/conf/datasage.yaml") + if err != nil { + fmt.Fprintf(w, `{status: "not ok", "error": %v}`, err) + return + } + config.StreamLogToAll(string(body)) + if err != nil { + fmt.Fprintf(w, `{status: "not ok", "error": %v}`, err) + return + } + fmt.Fprintf(w, `{status : "ok"}`) +} +func RunServer() { + + http.HandleFunc("/log", loghandler) + fmt.Printf("HTTP Server listening on :8080\n") + fmt.Printf("Send logs to 127.0.0.1:8080/log endpoint\n") + if err := http.ListenAndServe(":8080", nil); err != nil { + log.Fatal(err) + } +} diff --git a/src/integrations/integration_handler.go b/src/integrations/integration_handler.go index f8711c9..3653ad6 100644 --- a/src/integrations/integration_handler.go +++ b/src/integrations/integration_handler.go @@ -1,6 +1,15 @@ package integrations -func (intg Integrations) StreamLogToAll(Log string) { - StreamLogToGRPC(Log, intg.Rpc) - StreamLogToKafka(Log, intg.Kafka) +import "log" + +func (intg Integrations) StreamLogToAll(Log string) error { + err := StreamLogToGRPC(Log, intg.Rpc) + if err != nil { + log.Printf("err: %s\n", err) + } + err = StreamLogToKafka(Log, intg.Kafka) + if err != nil { + log.Printf("err: %s\n", err) + } + return nil } diff --git a/src/integrations/kafka.go b/src/integrations/kafka.go index cafa0dc..04e180c 100644 --- a/src/integrations/kafka.go +++ b/src/integrations/kafka.go @@ -6,7 +6,7 @@ import ( "github.com/confluentinc/confluent-kafka-go/kafka" ) -func StreamLogToKafka(Log string, kafkaConfigs []KafkaLogConfig) { +func StreamLogToKafka(Log string, kafkaConfigs []KafkaLogConfig) error { for _, config := range kafkaConfigs { log.Printf("[KAFKA] Dialing %v:%v Topic: %s", config.Broker, config.Port, config.Topic) producer, err := kafka.NewProducer(&kafka.ConfigMap{ @@ -14,6 +14,7 @@ func StreamLogToKafka(Log string, kafkaConfigs []KafkaLogConfig) { }) if err != nil { log.Printf("err: %s", err) + return err } defer producer.Close() if producer.Produce(&kafka.Message{ @@ -21,7 +22,10 @@ func StreamLogToKafka(Log string, kafkaConfigs []KafkaLogConfig) { Value: []byte(Log), }, nil) != nil { log.Printf("err: %s", err) + return err } - producer.Flush(100 * 2) + producer.Flush(1000 * 2) + log.Printf("[KAFKA] Log send to %v:%v", config.Broker, config.Port) } + return nil }