-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
WTFFFFFF??????? WHY DELETED FROM STAGE?????
- Loading branch information
1 parent
1403f6f
commit b0ceffb
Showing
107 changed files
with
8,848 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,33 @@ | ||
name: golangci-lint | ||
on: | ||
push: | ||
branches: | ||
- master | ||
- main | ||
pull_request: | ||
branches: | ||
- master | ||
- main | ||
|
||
permissions: | ||
contents: read | ||
|
||
jobs: | ||
golangci: | ||
strategy: | ||
matrix: | ||
go: ['1.21'] | ||
os: [macos-latest] | ||
name: lint | ||
runs-on: ${{ matrix.os }} | ||
steps: | ||
- uses: actions/checkout@v4 | ||
- uses: actions/setup-go@v5 | ||
with: | ||
go-version: ${{ matrix.go }} | ||
cache: false | ||
- name: golangci-lint | ||
uses: golangci/golangci-lint-action@v4 | ||
with: | ||
version: v1.54 | ||
working-directory: backend |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,47 @@ | ||
# If you prefer the allow list template instead of the deny list, see community template: | ||
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore | ||
# | ||
# Binaries for programs and plugins | ||
*.exe | ||
*.exe~ | ||
*.dll | ||
*.so | ||
*.dylib | ||
|
||
# Test binary, built with `go test -c` | ||
*.test | ||
|
||
# Output of the go coverage tool, specifically when used with LiteIDE | ||
*.out | ||
|
||
# Dependency directories (remove the comment below to include it) | ||
vendor/ | ||
|
||
# Go workspace file | ||
go.work | ||
.env | ||
|
||
# Logs | ||
logs | ||
*.log | ||
npm-debug.log* | ||
yarn-debug.log* | ||
yarn-error.log* | ||
pnpm-debug.log* | ||
lerna-debug.log* | ||
|
||
node_modules | ||
dist | ||
dist-ssr | ||
*.local | ||
|
||
# Editor directories and files | ||
.vscode/* | ||
!.vscode/extensions.json | ||
.idea | ||
.DS_Store | ||
*.suo | ||
*.ntvs* | ||
*.njsproj | ||
*.sln | ||
*.sw? |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,10 @@ | ||
run-rabbitmq: | ||
docker run -d --name my-rabbitmq -p 5672:5672 -p 15672:15672 rabbitmq:3-management | ||
|
||
run-postgres: | ||
docker run --name habr-pg-13.3 -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=daee -d postgres:13.3 | ||
|
||
generate: | ||
cd ./backend/internal/protos && protoc -I proto proto/daee/daee.proto \ | ||
--go_out=./gen/go --go_opt=paths=source_relative \ | ||
--go-grpc_out=./gen/go --go-grpc_opt=paths=source_relative |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,168 @@ | ||
# DAEE-fullstack | ||
|
||
![Main page](https://github.com/Prrromanssss/DAEE-fullstack/raw/main/images/expressions.png) | ||
![Agents](https://github.com/Prrromanssss/DAEE-fullstack/raw/main/images/agents.png) | ||
|
||
|
||
## Deployment instructions | ||
|
||
### 1. Cloning project from GitHub | ||
|
||
Run this command | ||
```commandline | ||
git clone https://github.com/Prrromanssss/DAEE-fullstack | ||
``` | ||
|
||
### 2. Installing PostgreSQL | ||
If you have your database - skip this step(just write to .env DB_URL) | ||
From root directory run this commands(firstly download docker) | ||
```commandline | ||
make run-postgres | ||
``` | ||
|
||
### 3. Installing RabbitMQ | ||
If you have your rabbitMQ - skip this step(just write to .env RABBIT_MQ_URL) | ||
From root directory run this commands(firstly download docker) | ||
```commandline | ||
make pull-rabbitmq | ||
make run-rabbitmq | ||
``` | ||
|
||
### 4. Generate file with virtual environment variables (.env) in root directory | ||
|
||
Generate file '.env' in root directory with the structure presented in the .env.example file | ||
|
||
If you didn't skip 2 and 3 just write | ||
```text | ||
DB_URL=postgres://postgres:postgres@localhost:5432/daee?sslmode=disable | ||
RABBIT_MQ_URL=amqp://guest:guest@localhost:5672/ | ||
``` | ||
|
||
### 5. Installing dependencies for backend | ||
|
||
From root directory run this command | ||
```commandline | ||
cd backend | ||
go mod download | ||
``` | ||
|
||
### 6. Installing goose for migrations | ||
|
||
From backend directory run this command | ||
```commandline | ||
go install github.com/pressly/goose/cmd/goose@latest | ||
``` | ||
|
||
### 7. Make migrations | ||
From backend directory run this command | ||
```commandline | ||
cd sql/schema | ||
goose postgres postgres://postgres:postgres@localhost:5432/daee up | ||
``` | ||
|
||
### 8. Running backend | ||
|
||
Run this command from backend directory | ||
```commandline | ||
cd cmd/daee | ||
./main | ||
``` | ||
|
||
### 9. Installing dependencies for frontend | ||
|
||
From root directory run this command | ||
```commandline | ||
cd frontend | ||
npm i | ||
``` | ||
|
||
### 10. Running Frontend | ||
Run this command from frontend directory | ||
```commandline | ||
npm run dev | ||
``` | ||
|
||
### 11. Follow link | ||
```commandline | ||
http://127.0.0.1:5173/ | ||
``` | ||
|
||
## About | ||
|
||
This project is transitional to the next sprint on the Yandex Lyceum course. | ||
This is distributed arithmetic expression evaluator. | ||
|
||
**Some description** | ||
|
||
The user wants to calculate arithmetic expressions. He enters the code 2 + 2 * 2 and wants to get the answer 6. But our addition and multiplication (also subtraction) operations take a “very, very” long time. Therefore, the option in which the user makes an http request and receives the result as a response is impossible. | ||
Moreover: the calculation of each such operation in our “alternative reality” takes “giant” computing power. Accordingly, we must be able to perform each action separately and we can scale this system by adding computing power to our system in the form of new “machines”. | ||
Therefore, when a user sends an expression, he receives an expression identifier in response and can, at some periodicity, check with the server whether the expression has been counted? If the expression is finally evaluated, he will get the result. Remember that some parts of an arithmetic expression can be evaluated in parallel. | ||
|
||
|
||
**How to use it?** | ||
|
||
/expressions - You can write some expressions to calculate | ||
|
||
/operations - You can change the execution time of each operation | ||
|
||
/agents - You can see how many servers can currently process expressions | ||
|
||
**How does it work?** | ||
|
||
*Orchestrator*: | ||
1. HTTP-server | ||
2. Accepts client requests | ||
3. Parses its expression and sends it to Agent Agregator | ||
4. Writing the data to the database | ||
|
||
*Agent Agregator*: | ||
1. Consumes expressions from the orchestrator, | ||
breaks it into tokens and writes it to the RabbitMQ queue for processing by agents | ||
2. Consumes results from agents, insert them to the expressions and sends new tokens to agents to calculate | ||
3. Consumes pings from agents, **BUT** I didn’t have time to create any mechanism that would check the pings of each server and if there had been no ping for a long time, kill it. (Ping every 200 seconds) | ||
|
||
*Agent*: | ||
1. Consumes expressions from the Agent Agregator and gives it to its goroutines for calculations. | ||
2. Consumes results from each goroutine and sends it to Agent Agregator | ||
3. Sends pings to Agent Agregator | ||
4. Every agent have 5 goroutines | ||
5. There are 3 agents | ||
|
||
**What about parallelism?** | ||
|
||
Some example: | ||
|
||
I uses reverse Polish notation | ||
|
||
2 + 2 --parse--> 2 2 + | ||
|
||
And we can give 2 2 + to some goroutine to calculate. | ||
|
||
But what about this example? | ||
|
||
2 + 2 + 2 + 2 --parse--> 2 2 + 2 + 2 + | ||
|
||
I think it's slow, because we need to solve 2 2 +, then 4 2 +, then 6 2 + | ||
|
||
SO, I parses it to RPN differently. | ||
|
||
I just add some brackets to expression. | ||
|
||
2 + 2 + 2 + 2 --add-brackets--> (2 + 2) + (2 + 2) --parse--> 2 2 + 2 2 + + | ||
|
||
And now we can run parallel 2 2 + and 2 2 + and then just add up their results. | ||
|
||
We have N expressions, every expression is processed by some agent. | ||
But that's not all, inside each expression we process subexpressions with different agents. | ||
|
||
If the HTTP-server crashed and we have expressions that did not have time to be calculated, by rebooting the server we will return to their calculations. | ||
|
||
## Some expressions to testing site | ||
1. 4 + -2 + 5 * 6 | ||
2. 2 + 2 + 2 + 2 | ||
3. 2 + 2 * 4 + 3 - 4 + 5 | ||
|
||
## Schema | ||
![Schema of the project](https://github.com/Prrromanssss/DAEE-fullstack/raw/main/images/schema.png) | ||
|
||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,43 @@ | ||
package main | ||
|
||
import ( | ||
"context" | ||
"log/slog" | ||
"time" | ||
|
||
agentapp "github.com/Prrromanssss/DAEE-fullstack/internal/app/agent" | ||
"github.com/Prrromanssss/DAEE-fullstack/internal/config" | ||
"github.com/Prrromanssss/DAEE-fullstack/internal/lib/logger/logcleaner" | ||
"github.com/Prrromanssss/DAEE-fullstack/internal/lib/logger/setup" | ||
"github.com/Prrromanssss/DAEE-fullstack/internal/storage" | ||
) | ||
|
||
func main() { | ||
ctxWithCancel, cancel := context.WithCancel(context.Background()) | ||
defer cancel() | ||
|
||
// Load Config | ||
cfg := config.MustLoad() | ||
|
||
// Configuration Logger | ||
log := setup.SetupLogger(cfg.Env, cfg.LogPathAgent) | ||
log.Info( | ||
"start agent", | ||
slog.String("env", cfg.Env), | ||
slog.String("version", "2"), | ||
) | ||
log.Debug("debug messages are enabled") | ||
|
||
go logcleaner.CleanLog(10*time.Minute, cfg.LogPathAgent, 100) | ||
|
||
// Configuration Storage | ||
dbCfg := storage.NewStorage(cfg.StorageURL) | ||
|
||
// Configuration Agent | ||
application, err := agentapp.New(log, cfg, dbCfg, cancel) | ||
if err != nil { | ||
panic(err) | ||
} | ||
|
||
go application.MustRun(ctxWithCancel) | ||
} |
Oops, something went wrong.