From b74cb7078c0d3b7bc278d7941aa41ea1c41c8f61 Mon Sep 17 00:00:00 2001 From: stefink Date: Tue, 28 Jun 2022 15:58:02 +0530 Subject: [PATCH 1/4] Added test code --- test/Makefile | 18 +++++++++++++++++ test/docker-compose.yaml | 27 +++++++++++++++++++++++++ test/main.go | 9 +++++++++ test/server/grpc_server.go | 37 ++++++++++++++++++++++++++++++++++ test/src/conf/datasage.yaml | 40 +++++++++++++++++++++++++++++++++++++ 5 files changed, 131 insertions(+) create mode 100644 test/Makefile create mode 100644 test/docker-compose.yaml create mode 100644 test/main.go create mode 100644 test/server/grpc_server.go create mode 100644 test/src/conf/datasage.yaml diff --git a/test/Makefile b/test/Makefile new file mode 100644 index 0000000..32ac9b9 --- /dev/null +++ b/test/Makefile @@ -0,0 +1,18 @@ +all: run-kafka-server run-grpc-server + +run-kafka-server: + docker-compose -f ./resources/docker-compose.yaml up -d + docker exec broker kafka-topics --bootstrap-server broker:9092 --create --topic dblogs + +run-grpc-server: + go run server/grpc_server.go + +send-log-to-endpoint: + curl --header "Content-Type: application/json" --request POST --data '{"DataDomainID":"5", "Database":"accuknox","Operation":"select", "OperationDetails":"none", "Timestamp":"2022-06-17T09:35:38Z","User":"ada_demo_user"}' http://127.0.0.1:8080/log + +demo: + go run main.go + +clean: + docker-compose -f ./resources/docker-compose.yaml down + diff --git a/test/docker-compose.yaml b/test/docker-compose.yaml new file mode 100644 index 0000000..33a45a4 --- /dev/null +++ b/test/docker-compose.yaml @@ -0,0 +1,27 @@ +--- +version: '3' +services: + zookeeper: + image: confluentinc/cp-zookeeper:7.0.1 + container_name: zookeeper + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_TICK_TIME: 2000 + + broker: + image: confluentinc/cp-kafka:7.0.1 + container_name: broker + ports: + # To learn about configuring Kafka for access across networks see + # https://www.confluent.io/blog/kafka-client-cannot-connect-to-broker-on-aws-on-docker-etc/ + - "9092:9092" + depends_on: + - zookeeper + environment: + KAFKA_BROKER_ID: 1 + KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_INTERNAL:PLAINTEXT + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092,PLAINTEXT_INTERNAL://broker:29092 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 diff --git a/test/main.go b/test/main.go new file mode 100644 index 0000000..c6be9cb --- /dev/null +++ b/test/main.go @@ -0,0 +1,9 @@ +package main + +import ( + "github.com/datasage-io/datasage/src/integrations" +) + +func main() { + integrations.RunServer() +} diff --git a/test/server/grpc_server.go b/test/server/grpc_server.go new file mode 100644 index 0000000..b7b55f0 --- /dev/null +++ b/test/server/grpc_server.go @@ -0,0 +1,37 @@ +package main + +import ( + "context" + "fmt" + "log" + "net" + + config "github.com/datasage-io/datasage/src/integrations/grpc_config" + "google.golang.org/grpc" +) + +type server struct { + config.UnimplementedDataSageServerServer +} + +func (s *server) LogSend(ctx context.Context, in *config.Log) (*config.Null, error) { + log.Printf("GRPC: Receive message body from client: %s", in.Body) + return &config.Null{}, nil +} + +func RunGRPCServer() { + listen, err := net.Listen("tcp", ":2222") + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + grpcServer := grpc.NewServer() + fmt.Println("gRPC Server listening on :2222") + config.RegisterDataSageServerServer(grpcServer, &server{}) + if err := grpcServer.Serve(listen); err != nil { + log.Fatalf("failed to serve: %s", err) + } +} + +func main() { + RunGRPCServer() +} diff --git a/test/src/conf/datasage.yaml b/test/src/conf/datasage.yaml new file mode 100644 index 0000000..3d55d31 --- /dev/null +++ b/test/src/conf/datasage.yaml @@ -0,0 +1,40 @@ +application: + name: Datasage + +logging: + level: INFO + +server: + port: 1989 + database: sqlite + +database: + external: + name: datasage + host: localhost + user: datasage + pass: datasage + port: 1234 + +adaptors: + name: mysql + version: 5.7 + env: k8s + is_operator: yes + percona: yes + host: 122.122.122.1 + username: root + password: abcdef + +integrations: + kafka: + - topic: dblogs + broker: 127.0.0.1 + port: 9092 + rpc: + - host: 127.0.0.1 + port: 2222 + http: + - endpoint: https://accuknox.com/service/log + method: post + From 99756face82fcb66084bf0605a362099eae3a20a Mon Sep 17 00:00:00 2001 From: stefink Date: Fri, 8 Jul 2022 12:42:49 +0530 Subject: [PATCH 2/4] Change dblogs --- src/conf/datasage.yaml | 4 ++-- test/Makefile | 2 +- test/src/conf/datasage.yaml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/conf/datasage.yaml b/src/conf/datasage.yaml index 3d55d31..c77c568 100644 --- a/src/conf/datasage.yaml +++ b/src/conf/datasage.yaml @@ -28,13 +28,13 @@ adaptors: integrations: kafka: - - topic: dblogs + - topic: datasage-logs broker: 127.0.0.1 port: 9092 rpc: - host: 127.0.0.1 port: 2222 http: - - endpoint: https://accuknox.com/service/log + - endpoint: https://datasage.com/service/log method: post diff --git a/test/Makefile b/test/Makefile index 32ac9b9..ed68861 100644 --- a/test/Makefile +++ b/test/Makefile @@ -2,7 +2,7 @@ all: run-kafka-server run-grpc-server run-kafka-server: docker-compose -f ./resources/docker-compose.yaml up -d - docker exec broker kafka-topics --bootstrap-server broker:9092 --create --topic dblogs + docker exec broker kafka-topics --bootstrap-server broker:9092 --create --topic datasage-logs run-grpc-server: go run server/grpc_server.go diff --git a/test/src/conf/datasage.yaml b/test/src/conf/datasage.yaml index 3d55d31..c31b603 100644 --- a/test/src/conf/datasage.yaml +++ b/test/src/conf/datasage.yaml @@ -28,13 +28,13 @@ adaptors: integrations: kafka: - - topic: dblogs + - topic: datasage-logs broker: 127.0.0.1 port: 9092 rpc: - host: 127.0.0.1 port: 2222 http: - - endpoint: https://accuknox.com/service/log + - endpoint: https://datasage.com/service/log method: post From 6a1851ea31a1e2e68c931b810a36dbffb21c0e73 Mon Sep 17 00:00:00 2001 From: stefink Date: Thu, 14 Jul 2022 16:39:49 +0530 Subject: [PATCH 3/4] renamed main.go --- test/{main.go => test_integrations.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/{main.go => test_integrations.go} (100%) diff --git a/test/main.go b/test/test_integrations.go similarity index 100% rename from test/main.go rename to test/test_integrations.go From 5775eb99194d7d366539cbdc0be659dd5b4daf51 Mon Sep 17 00:00:00 2001 From: stefink Date: Thu, 14 Jul 2022 16:53:25 +0530 Subject: [PATCH 4/4] removed datasage.yaml --- test/src/conf/datasage.yaml | 40 ------------------------------------- 1 file changed, 40 deletions(-) delete mode 100644 test/src/conf/datasage.yaml diff --git a/test/src/conf/datasage.yaml b/test/src/conf/datasage.yaml deleted file mode 100644 index c31b603..0000000 --- a/test/src/conf/datasage.yaml +++ /dev/null @@ -1,40 +0,0 @@ -application: - name: Datasage - -logging: - level: INFO - -server: - port: 1989 - database: sqlite - -database: - external: - name: datasage - host: localhost - user: datasage - pass: datasage - port: 1234 - -adaptors: - name: mysql - version: 5.7 - env: k8s - is_operator: yes - percona: yes - host: 122.122.122.1 - username: root - password: abcdef - -integrations: - kafka: - - topic: datasage-logs - broker: 127.0.0.1 - port: 9092 - rpc: - - host: 127.0.0.1 - port: 2222 - http: - - endpoint: https://datasage.com/service/log - method: post -