Skip to content

Commit

Permalink
Added fluentd config
Browse files Browse the repository at this point in the history
  • Loading branch information
stefink committed Jun 27, 2022
1 parent be7c65e commit 5653e3d
Show file tree
Hide file tree
Showing 9 changed files with 82 additions and 59 deletions.
10 changes: 2 additions & 8 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,13 @@ go 1.18

require google.golang.org/grpc v1.47.0

require (
github.com/fluent/fluent-logger-golang v1.9.0 // indirect
github.com/philhofer/fwd v1.1.1 // indirect
github.com/tinylib/msgp v1.1.6 // indirect
)

require (
github.com/confluentinc/confluent-kafka-go v1.8.2
github.com/golang/protobuf v1.5.2 // indirect
golang.org/x/net v0.0.0-20201021035429-f5854403a974 // indirect
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 // indirect
golang.org/x/text v0.3.3 // indirect
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 // indirect
google.golang.org/protobuf v1.27.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
google.golang.org/protobuf v1.27.1
gopkg.in/yaml.v2 v2.4.0
)
16 changes: 1 addition & 15 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,6 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fluent/fluent-logger-golang v1.9.0 h1:zUdY44CHX2oIUc7VTNZc+4m+ORuO/mldQDA7czhWXEg=
github.com/fluent/fluent-logger-golang v1.9.0/go.mod h1:2/HCT/jTy78yGyeNGQLGQsjF3zzzAuy6Xlk6FCMV5eU=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
Expand Down Expand Up @@ -49,33 +47,25 @@ github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ=
github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/tinylib/msgp v1.1.6 h1:i+SbKraHhnrf9M5MYmvQhFnbLhAXSDWF8WWsuyRdocw=
github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
Expand All @@ -85,7 +75,6 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
Expand All @@ -101,10 +90,6 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
Expand Down Expand Up @@ -136,6 +121,7 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
Expand Down
12 changes: 6 additions & 6 deletions src/conf/datasage.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,13 @@ adaptors:

integrations:
kafka:
- topic: datalog
broker: broker.kafka.com
port: 11211
- topic: dblogs
broker: 127.0.0.1
port: 9092
rpc:
- host: grpc.com
port: 1989
- host: 127.0.0.1
port: 2222
http:
- endpoint: https://accuknox.com/service/log
method: post


8 changes: 5 additions & 3 deletions src/integrations/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import (
)

type Config struct {
Integrations Integrations
Integrations
}

type Integrations struct {
Expand All @@ -27,14 +27,16 @@ type KafkaLogConfig struct {
Port string
}

func ReadLogConfig(path string) Config {
func ReadLogConfig(path string) (Config, error) {
config := Config{}
data, err := os.ReadFile(path)
if err != nil {
log.Printf("err: %s", err)
return config, err
}
if yaml.Unmarshal(data, &config) != nil {
log.Printf("err: %s", err)
return config, err
}
return config
return config, nil
}
20 changes: 0 additions & 20 deletions src/integrations/fluentd.go

This file was deleted.

8 changes: 6 additions & 2 deletions src/integrations/grpc.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,21 +13,25 @@ import (
"google.golang.org/grpc/credentials/insecure"
)

func StreamLogToGRPC(Log string, grpcConfigs []GRPCLogConfig) {
func StreamLogToGRPC(Log string, grpcConfigs []GRPCLogConfig) error {
for _, config := range grpcConfigs {
var conn *grpc.ClientConn
log.Printf("[GRPC] Dialing %s:%s \n", config.Host, config.Port)
conn, err := grpc.Dial(config.Host,
conn, err := grpc.Dial(config.Host+":"+config.Port,
grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
log.Fatalf("did not connect: %s", err)
return err
}
defer conn.Close()
c := grpc_config.NewDataSageServerClient(conn)
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
if _, err = c.LogSend(ctx, &grpc_config.Log{Body: Log}); err != nil {
log.Printf("could not send the data: %v", err)
return err
}
log.Printf("[GRPC] Log send to %s:%s\n", config.Host, config.Port)
}
return nil
}
44 changes: 44 additions & 0 deletions src/integrations/http_server.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
package integrations

import (
"fmt"
"io/ioutil"
"log"
"net/http"
)

func loghandler(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/log" {
http.Error(w, "404 not found.", http.StatusNotFound)
return
}
if r.Method != "POST" {
fmt.Fprintf(w, "Only POST method is supported")
return
}
body, err := ioutil.ReadAll(r.Body)
if err != nil {
fmt.Fprintf(w, "err : %v", err)
}
w.Header().Set("Content-Type", "application/json")
config, err := ReadLogConfig("src/conf/datasage.yaml")
if err != nil {
fmt.Fprintf(w, `{status: "not ok", "error": %v}`, err)
return
}
config.StreamLogToAll(string(body))
if err != nil {
fmt.Fprintf(w, `{status: "not ok", "error": %v}`, err)
return
}
fmt.Fprintf(w, `{status : "ok"}`)
}
func RunServer() {

http.HandleFunc("/log", loghandler)
fmt.Printf("HTTP Server listening on :8080\n")
fmt.Printf("Send logs to 127.0.0.1:8080/log endpoint\n")
if err := http.ListenAndServe(":8080", nil); err != nil {
log.Fatal(err)
}
}
15 changes: 12 additions & 3 deletions src/integrations/integration_handler.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,15 @@
package integrations

func (intg Integrations) StreamLogToAll(Log string) {
StreamLogToGRPC(Log, intg.Rpc)
StreamLogToKafka(Log, intg.Kafka)
import "log"

func (intg Integrations) StreamLogToAll(Log string) error {
err := StreamLogToGRPC(Log, intg.Rpc)
if err != nil {
log.Printf("err: %s\n", err)
}
err = StreamLogToKafka(Log, intg.Kafka)
if err != nil {
log.Printf("err: %s\n", err)
}
return nil
}
8 changes: 6 additions & 2 deletions src/integrations/kafka.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,22 +6,26 @@ import (
"github.com/confluentinc/confluent-kafka-go/kafka"
)

func StreamLogToKafka(Log string, kafkaConfigs []KafkaLogConfig) {
func StreamLogToKafka(Log string, kafkaConfigs []KafkaLogConfig) error {
for _, config := range kafkaConfigs {
log.Printf("[KAFKA] Dialing %v:%v Topic: %s", config.Broker, config.Port, config.Topic)
producer, err := kafka.NewProducer(&kafka.ConfigMap{
"bootstrap.servers": config.Broker + ":" + config.Port,
})
if err != nil {
log.Printf("err: %s", err)
return err
}
defer producer.Close()
if producer.Produce(&kafka.Message{
TopicPartition: kafka.TopicPartition{Topic: &config.Topic, Partition: kafka.PartitionAny},
Value: []byte(Log),
}, nil) != nil {
log.Printf("err: %s", err)
return err
}
producer.Flush(100 * 2)
producer.Flush(1000 * 2)
log.Printf("[KAFKA] Log send to %v:%v", config.Broker, config.Port)
}
return nil
}

0 comments on commit 5653e3d

Please sign in to comment.