From b5a0475c24738e33007d1f725b7d232d772fd8d2 Mon Sep 17 00:00:00 2001 From: Sourr_cream Date: Thu, 25 Apr 2024 18:37:55 +0300 Subject: [PATCH] docker-compose --- .dockerignore | 3 + .gitignore | 2 + backend/cmd/agent/main.go | 7 +- backend/cmd/auth/main.go | 2 +- backend/cmd/orchestrator/main.go | 7 +- backend/config/local.yaml | 12 ++- backend/go.mod | 2 +- backend/internal/config/config.go | 19 +---- backend/internal/lib/jwt/jwt.go | 12 --- .../lib/logger/logcleaner/logcleaner.go | 61 -------------- backend/internal/lib/logger/setup/logger.go | 17 +--- .../lib/logger/setup/pretty_logger.go | 3 +- backend/sql/daec.sql | 60 ++++++++++++++ docker-compose.yml | 79 +++++++++++++++++++ docker/backend/agent.Dockerfile | 22 ++++++ docker/backend/auth.Dockerfile | 22 ++++++ docker/backend/orchestrator.Dockerfile | 22 ++++++ docker/database/postgres.Dockerfile | 7 ++ docker/frontend/frontend.Dockerfile | 15 ++++ frontend/src/pages/Login/LoginPage.tsx | 4 +- local.env.example | 2 - 21 files changed, 250 insertions(+), 130 deletions(-) create mode 100644 .dockerignore delete mode 100644 backend/internal/lib/logger/logcleaner/logcleaner.go create mode 100644 backend/sql/daec.sql create mode 100644 docker-compose.yml create mode 100644 docker/backend/agent.Dockerfile create mode 100644 docker/backend/auth.Dockerfile create mode 100644 docker/backend/orchestrator.Dockerfile create mode 100644 docker/database/postgres.Dockerfile create mode 100644 docker/frontend/frontend.Dockerfile delete mode 100644 local.env.example diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..28ec8da --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +frontend/node_modules/ +backend/vendor +sqlc.yaml \ No newline at end of file diff --git a/.gitignore b/.gitignore index 20a3e89..231884d 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,8 @@ vendor/ # Go workspace file go.work .env +local.env +local.yaml # Logs logs diff --git a/backend/cmd/agent/main.go b/backend/cmd/agent/main.go index b883b92..8bdc31f 100644 --- a/backend/cmd/agent/main.go +++ b/backend/cmd/agent/main.go @@ -6,11 +6,9 @@ import ( "os" "os/signal" "syscall" - "time" agentapp "github.com/Prrromanssss/DAEC-fullstack/internal/app/agent" "github.com/Prrromanssss/DAEC-fullstack/internal/config" - "github.com/Prrromanssss/DAEC-fullstack/internal/lib/logger/logcleaner" "github.com/Prrromanssss/DAEC-fullstack/internal/lib/logger/setup" "github.com/Prrromanssss/DAEC-fullstack/internal/storage" ) @@ -23,15 +21,12 @@ func main() { cfg := config.MustLoad() // Configuration Logger - log := setup.SetupLogger(cfg.Env, cfg.LogPathAgent) + log := setup.SetupLogger(cfg.Env) log.Info( "start agent", slog.String("env", cfg.Env), slog.String("version", "2"), ) - log.Debug("debug messages are enabled") - - go logcleaner.CleanLog(10*time.Minute, cfg.LogPathAgent, 100) // Configuration Storage dbCfg := storage.NewStorage(log, cfg.StorageURL) diff --git a/backend/cmd/auth/main.go b/backend/cmd/auth/main.go index a87ea18..9245a7e 100644 --- a/backend/cmd/auth/main.go +++ b/backend/cmd/auth/main.go @@ -18,7 +18,7 @@ func main() { cfg := config.MustLoad() // Configuration Logger - log := setup.SetupLogger(cfg.Env, cfg.LogPathAuth) + log := setup.SetupLogger(cfg.Env) log.Info( "start grpc server", slog.String("env", cfg.Env), diff --git a/backend/cmd/orchestrator/main.go b/backend/cmd/orchestrator/main.go index 7811b90..c72cccf 100644 --- a/backend/cmd/orchestrator/main.go +++ b/backend/cmd/orchestrator/main.go @@ -7,7 +7,6 @@ import ( "os" "os/signal" "syscall" - "time" orchestratorapp "github.com/Prrromanssss/DAEC-fullstack/internal/app/orchestrator" daecv1 "github.com/Prrromanssss/DAEC-fullstack/internal/protos/gen/go/daec" @@ -17,7 +16,6 @@ import ( "github.com/Prrromanssss/DAEC-fullstack/internal/config" "github.com/Prrromanssss/DAEC-fullstack/internal/http-server/handlers" mwlogger "github.com/Prrromanssss/DAEC-fullstack/internal/http-server/middleware/logger" - "github.com/Prrromanssss/DAEC-fullstack/internal/lib/logger/logcleaner" "github.com/Prrromanssss/DAEC-fullstack/internal/lib/logger/setup" "github.com/Prrromanssss/DAEC-fullstack/internal/lib/logger/sl" "github.com/Prrromanssss/DAEC-fullstack/internal/storage" @@ -35,15 +33,12 @@ func main() { cfg := config.MustLoad() // Configuration Logger - log := setup.SetupLogger(cfg.Env, cfg.LogPathOrchestrator) + log := setup.SetupLogger(cfg.Env) log.Info( "start orchestrator", slog.String("env", cfg.Env), slog.String("version", "2"), ) - log.Debug("debug messages are enabled") - - go logcleaner.CleanLog(10*time.Minute, cfg.LogPathOrchestrator, 100) // Configuration Storage dbCfg := storage.NewStorage(log, cfg.StorageURL) diff --git a/backend/config/local.yaml b/backend/config/local.yaml index 6085eae..1105c64 100644 --- a/backend/config/local.yaml +++ b/backend/config/local.yaml @@ -1,20 +1,18 @@ env: "local" -log_path_agent: "../../log/agent.log" -log_path_orchestrator: "../../log/orchestrator.log" -log_path_auth: "../../log/auth.log" inactive_time_for_agent: 20 time_for_ping: 10 tokenTTL: 1h grpc_server: - address: "localhost:44044" + address: ":44044" + grpc_client_connection_string: "auth:44044" rabbit_queue: - rabbitmq_url: "amqp://guest:guest@localhost:5672/" + rabbitmq_url: "amqp://guest:guest@rabbitmq:5672/" queue_for_expressions_to_agents: "Expressions to agents" queue_for_results_from_agents: "Results from agents" http_server: - address: "localhost:3000" + address: ":3000" timeout: 4s idle_timeout: 60s database_instance: goose_migration_dir: "./backend/sql/schema" - storage_url: "postgres://postgres:6,62607004@localhost:5434/daec?sslmode=disable" \ No newline at end of file + storage_url: "postgres://postgres:postgres@db:5432/daec?sslmode=disable" \ No newline at end of file diff --git a/backend/go.mod b/backend/go.mod index 069058a..600e882 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -5,7 +5,7 @@ go 1.21.1 require ( github.com/go-chi/chi v1.5.5 github.com/go-chi/cors v1.2.1 - github.com/joho/godotenv v1.5.1 + github.com/joho/godotenv v1.5.1 // indirect ) require github.com/lib/pq v1.10.9 diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go index c696f03..ae09147 100644 --- a/backend/internal/config/config.go +++ b/backend/internal/config/config.go @@ -1,21 +1,15 @@ package config import ( - "fmt" "log" "os" - "path/filepath" "time" "github.com/ilyakaznacheev/cleanenv" - "github.com/joho/godotenv" ) type Config struct { Env string `yaml:"env" env:"ENV" env-default:"local"` - LogPathAgent string `yaml:"log_path_agent" env-required:"true"` - LogPathOrchestrator string `yaml:"log_path_orchestrator" env-required:"true"` - LogPathAuth string `yaml:"log_path_auth" env-required:"true"` InactiveTimeForAgent int32 `yaml:"inactive_time_for_agent" env-default:"200"` TimeForPing int32 `yaml:"time_for_ping" end-default:"100"` TokenTTL time.Duration `yaml:"tokenTTL" env-default:"1h"` @@ -44,20 +38,11 @@ type DatabaseInstance struct { } type GRPCServer struct { - Address string `yaml:"address" env-default:"localhost:44044"` + Address string `yaml:"address" env-default:"localhost:44044"` + GRPCClientConnectionString string `yaml:"grpc_client_connection_string" env-default:"auth:44044"` } func MustLoad() *Config { - path, err := os.Getwd() - if err != nil { - log.Fatalf("can't get pwd: %v", err) - } - - err = godotenv.Load(fmt.Sprintf("%s/local.env", filepath.Dir(filepath.Dir(filepath.Dir(path))))) - if err != nil { - log.Fatalf("can't parse env file: %v", err) - } - configPath := os.Getenv("CONFIG_PATH") if configPath == "" { log.Fatal("CONFIG_PATH is not set") diff --git a/backend/internal/lib/jwt/jwt.go b/backend/internal/lib/jwt/jwt.go index 6a9e9f1..2a0d0c6 100644 --- a/backend/internal/lib/jwt/jwt.go +++ b/backend/internal/lib/jwt/jwt.go @@ -6,27 +6,15 @@ import ( "log" "net/http" "os" - "path/filepath" "strings" "time" "github.com/Prrromanssss/DAEC-fullstack/internal/storage/postgres" "github.com/golang-jwt/jwt/v5" - "github.com/joho/godotenv" ) // NewToken creates new JWT token. func NewToken(user postgres.User, duration time.Duration) (string, error) { - path, err := os.Getwd() - if err != nil { - log.Fatalf("can't get pwd: %v", err) - } - - err = godotenv.Load(fmt.Sprintf("%s/local.env", filepath.Dir(filepath.Dir(filepath.Dir(path))))) - if err != nil { - log.Fatalf("can't parse env file: %v", err) - } - jwtSecret := os.Getenv("JWT_SECRET") if jwtSecret == "" { log.Fatal("JWT_SECRET is not set") diff --git a/backend/internal/lib/logger/logcleaner/logcleaner.go b/backend/internal/lib/logger/logcleaner/logcleaner.go deleted file mode 100644 index 0095ae0..0000000 --- a/backend/internal/lib/logger/logcleaner/logcleaner.go +++ /dev/null @@ -1,61 +0,0 @@ -package logcleaner - -import ( - "bufio" - "log" - "os" - "time" -) - -// CleanLog cleans log file every timeBetweenCleanning -func CleanLog( - timeBetweenCleanning time.Duration, - filePath string, - linesToRemove int, -) { - timer := time.NewTimer(timeBetweenCleanning) - defer timer.Stop() - <-timer.C - - ticker := time.NewTicker(timeBetweenCleanning) - for ; ; <-ticker.C { - file, err := os.Open(filePath) - if err != nil { - log.Fatalf("Log file is not found in environment: %v", err) - } - defer file.Close() - - tmpFilename := filePath + ".tmp" - tmpFile, err := os.Create(tmpFilename) - if err != nil { - log.Fatalf("Can't create temporary file: %v", err) - return - } - defer os.Remove(tmpFilename) - defer tmpFile.Close() - - scanner := bufio.NewScanner(file) - for i := 0; i < linesToRemove; i++ { - if !scanner.Scan() { - break - } - } - for scanner.Scan() { - if _, err := tmpFile.WriteString(scanner.Text() + "\n"); err != nil { - log.Fatalf("Can't write to temporary file: %v", err) - return - } - } - - if err := scanner.Err(); err != nil { - log.Fatalf("Can't read source file: %v", err) - return - } - if err := os.Rename(tmpFilename, filePath); err != nil { - log.Fatalf("Can't rename temporary file: %v", err) - return - } - - log.Println("First", linesToRemove, "lines was successfully removed from: ", filePath) - } -} diff --git a/backend/internal/lib/logger/setup/logger.go b/backend/internal/lib/logger/setup/logger.go index 9bcda12..285128e 100644 --- a/backend/internal/lib/logger/setup/logger.go +++ b/backend/internal/lib/logger/setup/logger.go @@ -1,7 +1,6 @@ package setup import ( - "io" "log/slog" "os" ) @@ -12,29 +11,21 @@ const ( envProd = "prod" ) -func SetupLogger(env, logPath string) *slog.Logger { +func SetupLogger(env string) *slog.Logger { var log *slog.Logger - logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) - if err != nil { - panic("failed to open log file: " + err.Error()) - } - defer logFile.Close() - - writer := io.Writer(logFile) - switch env { case envLocal: - log = SetupPrettySlog(writer) + log = SetupPrettySlog() case envDev: log = slog.New( - slog.NewJSONHandler(writer, &slog.HandlerOptions{ + slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ Level: slog.LevelDebug, }), ) case envProd: log = slog.New( - slog.NewJSONHandler(writer, &slog.HandlerOptions{ + slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ Level: slog.LevelInfo, }), ) diff --git a/backend/internal/lib/logger/setup/pretty_logger.go b/backend/internal/lib/logger/setup/pretty_logger.go index 58e4b8c..56e1ee3 100644 --- a/backend/internal/lib/logger/setup/pretty_logger.go +++ b/backend/internal/lib/logger/setup/pretty_logger.go @@ -1,14 +1,13 @@ package setup import ( - "io" "log/slog" "os" "github.com/Prrromanssss/DAEC-fullstack/internal/lib/logger/handlers/slogpretty" ) -func SetupPrettySlog(logFile io.Writer) *slog.Logger { +func SetupPrettySlog() *slog.Logger { opts := slogpretty.PrettyHandlerOptions{ SlogOpts: &slog.HandlerOptions{ Level: slog.LevelDebug, diff --git a/backend/sql/daec.sql b/backend/sql/daec.sql new file mode 100644 index 0000000..ff2a00e --- /dev/null +++ b/backend/sql/daec.sql @@ -0,0 +1,60 @@ +DROP TYPE IF EXISTS agent_status; +CREATE TYPE agent_status AS ENUM ('running', 'waiting', 'sleeping', 'terminated'); + +CREATE TABLE IF NOT EXISTS agents ( + agent_id int GENERATED ALWAYS AS IDENTITY, + number_of_parallel_calculations int NOT NULL DEFAULT 5, + last_ping timestamp NOT NULL, + status agent_status NOT NULL, + + PRIMARY KEY(agent_id) +); + +ALTER TABLE agents ADD COLUMN created_at timestamp NOT NULL; + +CREATE TABLE IF NOT EXISTS users ( + user_id int GENERATED ALWAYS AS IDENTITY, + email text UNIQUE NOT NULL, + password_hash bytea NOT NULL, + + PRIMARY KEY(user_id) +); + +DROP TYPE IF EXISTS expression_status; +CREATE TYPE expression_status AS ENUM ('ready_for_computation', 'computing', 'result', 'terminated'); + +CREATE TABLE IF NOT EXISTS expressions ( + expression_id int GENERATED ALWAYS AS IDENTITY, + user_id int NOT NULL, + agent_id int, + created_at timestamp NOT NULL, + updated_at timestamp NOT NULL, + data text NOT NULL, + parse_data text NOT NULL, + status expression_status NOT NULL, + result int NOT NULL DEFAULT 0, + is_ready boolean NOT NULL DEFAULT false, + + PRIMARY KEY(expression_id), + FOREIGN KEY(agent_id) + REFERENCES agents(agent_id) + ON DELETE SET NULL, + FOREIGN KEY(user_id) + REFERENCES users(user_id) + ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS operations ( + operation_id int GENERATED ALWAYS AS IDENTITY, + operation_type varchar(1) NOT NULL, + execution_time int NOT NULL DEFAULT 100, + user_id int NOT NULL, + + PRIMARY KEY(operation_id), + CONSTRAINT operation_type_user_id UNIQUE(operation_type, user_id), + FOREIGN KEY(user_id) + REFERENCES users(user_id) + ON DELETE CASCADE +); + +ALTER TABLE agents ADD COLUMN number_of_active_calculations int NOT NULL DEFAULT 0; \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..b70ef64 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,79 @@ +version: "3.9" +services: + db: + build: + context: . + dockerfile: ./docker/database/postgres.Dockerfile + container_name: daec-db + restart: always + environment: + POSTGRES_USER: "postgres" + POSTGRES_PASSWORD: "postgres" + POSTGRES_DB: "daec" + ports: + - "5432:5432" + rabbitmq: + image: rabbitmq:3-management + hostname: rabbitmq + container_name: daec-rabbitmq + restart: unless-stopped + ports: + - "5672:5672" + - "15672:15672" + expose: + - "5672" + - "15672" + environment: + RABBITMQ_DEFAULT_USER: "guest" + RABBITMQ_DEFAULT_PASS: "guest" + orchestrator: + build: + context: . + dockerfile: ./docker/backend/orchestrator.Dockerfile + container_name: daec-orchestrator + restart: always + ports: + - "3000:3000" + depends_on: + - db + - rabbitmq + agent1: + build: + context: . + dockerfile: ./docker/backend/agent.Dockerfile + container_name: daec-agent-1 + restart: unless-stopped + depends_on: + - rabbitmq + agent2: + build: + context: . + dockerfile: ./docker/backend/agent.Dockerfile + container_name: daec-agent-2 + restart: unless-stopped + depends_on: + - rabbitmq + agent3: + build: + context: . + dockerfile: ./docker/backend/agent.Dockerfile + container_name: daec-agent-3 + restart: unless-stopped + depends_on: + - rabbitmq + auth: + build: + context: . + dockerfile: ./docker/backend/auth.Dockerfile + container_name: daec-auth + restart: always + ports: + - "44044:44044" + frontend: + build: + context: . + dockerfile: ./docker/frontend/frontend.Dockerfile + container_name: daec-frontend + restart: unless-stopped + ports: + - "5173:5173" \ No newline at end of file diff --git a/docker/backend/agent.Dockerfile b/docker/backend/agent.Dockerfile new file mode 100644 index 0000000..1840fcb --- /dev/null +++ b/docker/backend/agent.Dockerfile @@ -0,0 +1,22 @@ +FROM golang:alpine as builder + +WORKDIR /build + +ADD backend/go.mod . + +COPY backend . + +RUN go build -o agent cmd/agent/main.go + +FROM alpine + +WORKDIR /build + +COPY --from=builder /build/agent /build/agent +COPY backend/config/local.yaml /app/backend/config/local.yaml + +ENV JWT_SECRET "super-super-secret" + +ENV CONFIG_PATH /app/backend/config/local.yaml + +CMD ["./agent"] \ No newline at end of file diff --git a/docker/backend/auth.Dockerfile b/docker/backend/auth.Dockerfile new file mode 100644 index 0000000..5c9aff3 --- /dev/null +++ b/docker/backend/auth.Dockerfile @@ -0,0 +1,22 @@ +FROM golang:alpine as builder + +WORKDIR /build + +ADD backend/go.mod . + +COPY backend . + +RUN go build -o auth cmd/auth/main.go + +FROM alpine + +WORKDIR /build + +COPY --from=builder /build/auth /build/auth +COPY backend/config/local.yaml /app/backend/config/local.yaml + +ENV JWT_SECRET "super-super-secret" + +ENV CONFIG_PATH /app/backend/config/local.yaml + +CMD ["./auth"] \ No newline at end of file diff --git a/docker/backend/orchestrator.Dockerfile b/docker/backend/orchestrator.Dockerfile new file mode 100644 index 0000000..cac3c46 --- /dev/null +++ b/docker/backend/orchestrator.Dockerfile @@ -0,0 +1,22 @@ +FROM golang:alpine as builder + +WORKDIR /build + +ADD backend/go.mod . + +COPY backend . + +RUN go build -o orchestrator cmd/orchestrator/main.go + +FROM alpine + +WORKDIR /build + +COPY --from=builder /build/orchestrator /build/orchestrator +COPY backend/config/local.yaml /app/backend/config/local.yaml + +ENV JWT_SECRET "super-super-secret" + +ENV CONFIG_PATH /app/backend/config/local.yaml + +CMD ["./orchestrator"] \ No newline at end of file diff --git a/docker/database/postgres.Dockerfile b/docker/database/postgres.Dockerfile new file mode 100644 index 0000000..f1cf8a7 --- /dev/null +++ b/docker/database/postgres.Dockerfile @@ -0,0 +1,7 @@ +FROM postgres:16-alpine3.19 + +ENV POSTGRES_PASSWORD=postgres + +ENV POSTGRES_DB=daec + +COPY ./backend/sql/daec.sql /docker-entrypoint-initdb.d/ \ No newline at end of file diff --git a/docker/frontend/frontend.Dockerfile b/docker/frontend/frontend.Dockerfile new file mode 100644 index 0000000..da540f6 --- /dev/null +++ b/docker/frontend/frontend.Dockerfile @@ -0,0 +1,15 @@ +FROM node:18-alpine + +WORKDIR /app + +COPY frontend/package.json . + +RUN npm install + +COPY frontend/ . + +RUN npm run build + +EXPOSE 5173 + +CMD [ "npm", "run", "dev", "--", "--host"] \ No newline at end of file diff --git a/frontend/src/pages/Login/LoginPage.tsx b/frontend/src/pages/Login/LoginPage.tsx index 5640aea..1769617 100644 --- a/frontend/src/pages/Login/LoginPage.tsx +++ b/frontend/src/pages/Login/LoginPage.tsx @@ -16,11 +16,11 @@ export const LoginPage = () => { if (variant === "login") { login(data).then(() => { setData({ email: "", password: "" }) - toast.success("Успешно!"); + toast.success("Success!"); }); } else { registration(data).then(() => { - toast.success("Успешно!"); + toast.success("Success!"); setVariant("login"); }) .catch((err) => { diff --git a/local.env.example b/local.env.example deleted file mode 100644 index 13fca64..0000000 --- a/local.env.example +++ /dev/null @@ -1,2 +0,0 @@ -CONFIG_PATH="./backend/config/local.yaml" -JWT_SECRET="super-super-secret" \ No newline at end of file