-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
755a0b7
commit 211b5b1
Showing
114 changed files
with
10,047 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,42 @@ | ||
name: golangci-lint | ||
on: | ||
push: | ||
branches: | ||
- master | ||
- main | ||
pull_request: | ||
branches: | ||
- master | ||
- main | ||
|
||
permissions: | ||
contents: read | ||
|
||
jobs: | ||
golangci: | ||
strategy: | ||
matrix: | ||
go: ['1.21'] | ||
os: [macos-latest] | ||
name: lint | ||
runs-on: ${{ matrix.os }} | ||
steps: | ||
- uses: actions/checkout@v4 | ||
- uses: actions/setup-go@v5 | ||
with: | ||
go-version: ${{ matrix.go }} | ||
cache: false | ||
- name: golangci-lint | ||
uses: golangci/golangci-lint-action@v4 | ||
with: | ||
version: v1.54 | ||
working-directory: backend | ||
- name: Run tests | ||
run: | | ||
if grep -q "ok" <<< "$(cd backend && go test ./...)"; then | ||
echo "All tests passed" | ||
exit 0 | ||
else | ||
echo "Some tests failed" | ||
exit 1 | ||
fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,47 @@ | ||
# If you prefer the allow list template instead of the deny list, see community template: | ||
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore | ||
# | ||
# Binaries for programs and plugins | ||
*.exe | ||
*.exe~ | ||
*.dll | ||
*.so | ||
*.dylib | ||
|
||
# Test binary, built with `go test -c` | ||
*.test | ||
|
||
# Output of the go coverage tool, specifically when used with LiteIDE | ||
*.out | ||
|
||
# Dependency directories (remove the comment below to include it) | ||
vendor/ | ||
|
||
# Go workspace file | ||
go.work | ||
.env | ||
|
||
# Logs | ||
logs | ||
*.log | ||
npm-debug.log* | ||
yarn-debug.log* | ||
yarn-error.log* | ||
pnpm-debug.log* | ||
lerna-debug.log* | ||
|
||
node_modules | ||
dist | ||
dist-ssr | ||
*.local | ||
|
||
# Editor directories and files | ||
.vscode/* | ||
!.vscode/extensions.json | ||
.idea | ||
.DS_Store | ||
*.suo | ||
*.ntvs* | ||
*.njsproj | ||
*.sln | ||
*.sw? |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,10 @@ | ||
run-rabbitmq: | ||
docker run -d --name my-rabbitmq -p 5672:5672 -p 15672:15672 rabbitmq:3-management | ||
|
||
run-postgres: | ||
docker run --name habr-pg-13.3 -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=daec -d postgres:13.3 | ||
|
||
generate: | ||
cd ./backend/internal/protos && protoc -I proto proto/daec/daec.proto \ | ||
--go_out=./gen/go --go_opt=paths=source_relative \ | ||
--go-grpc_out=./gen/go --go-grpc_opt=paths=source_relative |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,126 @@ | ||
# DAEC-fullstack | ||
|
||
![Main page](https://github.com/Prrromanssss/DAEC-fullstack/raw/main/images/expressions.png) | ||
![Agents](https://github.com/Prrromanssss/DAEC-fullstack/raw/main/images/agents.png) | ||
In the photo, we can see two agents working (one gorutine from each) and one agent died. | ||
|
||
## How to contact me: | ||
#### NOTE!!! If there aren't docker-compose.yaml file you have to contact me!!! | ||
|
||
``` | ||
https://t.me/sourr_cream | ||
``` | ||
|
||
## Deployment instructions | ||
|
||
### 1. Cloning project from GitHub | ||
|
||
Run this command | ||
```commandline | ||
git clone https://github.com/Prrromanssss/DAEC-fullstack | ||
``` | ||
|
||
### 2. Build application | ||
Run this command | ||
```comandline | ||
docker-compose up -d | ||
``` | ||
|
||
### 3. Follow link | ||
```commandline | ||
http://127.0.0.1:5173/ | ||
``` | ||
|
||
## About | ||
|
||
This is distributed arithmetic expression calculator. | ||
|
||
**Some description** | ||
|
||
The user wants to calculate arithmetic expressions. He enters the code 2 + 2 * 2 and wants to get the answer 6. But our addition and multiplication (also subtraction) operations take a “very, very” long time. Therefore, the option in which the user makes an http request and receives the result as a response is impossible. | ||
Moreover: the calculation of each such operation in our “alternative reality” takes “giant” computing power. Accordingly, we must be able to perform each action separately and we can scale this system by adding computing power to our system in the form of new “machines”. | ||
Therefore, when a user sends an expression, he receives an expression identifier in response and can, at some periodicity, check with the server whether the expression has been counted? If the expression is finally evaluated, he will get the result. Remember that some parts of an arithmetic expression can be evaluated in parallel. | ||
|
||
|
||
**How to use it?** | ||
|
||
Expressions - You can write some expressions to calculate (registered users only). | ||
|
||
Operations - You can change the execution time of each operation (registered users only). | ||
|
||
Agents - You can see how many servers can currently process expressions. | ||
|
||
Login - You can register or log in to your account. | ||
|
||
**How does it work?** | ||
|
||
*Orchestrator*: | ||
1. HTTP-server | ||
2. Parses expression from users and sends to Agents through RabbitMQ. | ||
3. Consumes results from Agents, inserts them to the expressions and sends new tokens to Agents to calculate. | ||
4. Consumes pings from Agents and kills those who didn't send anything. | ||
5. Writing the data to the database. | ||
|
||
*Agent*: | ||
1. Consumes expressions from the Orchestartor and gives it to its goroutines for calculations. | ||
2. Consumes results from each goroutine and sends it to Orchestartor through RabbitMQ. | ||
3. Sends pings to Orchestartor. | ||
4. Every agent have 5 goroutines. | ||
5. There are 3 agents. | ||
|
||
*Auth*: | ||
1. Log in to user's account. | ||
2. Register new users. | ||
|
||
**What about parallelism?** | ||
|
||
Some example: | ||
|
||
I uses reverse Polish notation | ||
|
||
2 + 2 --parse--> 2 2 + | ||
|
||
And we can give 2 2 + to some goroutine to calculate. | ||
|
||
But what about this example? | ||
|
||
2 + 2 + 2 + 2 --parse--> 2 2 + 2 + 2 + | ||
|
||
I think it's slow, because we need to solve 2 2 +, then 4 2 +, then 6 2 + | ||
|
||
SO, I parses it to RPN differently. | ||
|
||
I just add some brackets to expression. | ||
|
||
2 + 2 + 2 + 2 --add-brackets--> (2 + 2) + (2 + 2) --parse--> 2 2 + 2 2 + + | ||
|
||
And now we can run parallel 2 2 + and 2 2 + and then just add up their results. | ||
|
||
We have N expressions, every expression is processed by some agent. | ||
But that's not all, inside each expression we process subexpressions with different agents. | ||
|
||
If the HTTP-server crashed and we have expressions that did not have time to be calculated, by rebooting the server we will return to their calculations. | ||
|
||
## Some expressions to testing site | ||
- Valid cases | ||
1. 4 + -2 + 5 * 6 | ||
2. 2 + 2 + 2 + 2 | ||
3. 2 + 2 * 4 + 3 - 4 + 5 | ||
4. (23 + 125) - 567 * 23 | ||
5. -3 +6 | ||
- Invalid cases | ||
1. 4 / 0 | ||
2. 45 + x - 5 | ||
3. 45 + 4* | ||
4. ---4 + 5 | ||
5. 52 * 3 / | ||
|
||
## Testing | ||
I have unit-tests to test the work of my parser. | ||
You can see that all tests have passed in github actions. | ||
|
||
## Schema | ||
![Schema of the project](https://github.com/Prrromanssss/DAEC-fullstack/raw/main/images/schema.png) | ||
|
||
## ER-diagram | ||
![ER-diagram of the project](https://github.com/Prrromanssss/DAEC-fullstack/raw/main/images/ERD.png) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,58 @@ | ||
package main | ||
|
||
import ( | ||
"context" | ||
"log/slog" | ||
"os" | ||
"os/signal" | ||
"syscall" | ||
"time" | ||
|
||
agentapp "github.com/Prrromanssss/DAEC-fullstack/internal/app/agent" | ||
"github.com/Prrromanssss/DAEC-fullstack/internal/config" | ||
"github.com/Prrromanssss/DAEC-fullstack/internal/lib/logger/logcleaner" | ||
"github.com/Prrromanssss/DAEC-fullstack/internal/lib/logger/setup" | ||
"github.com/Prrromanssss/DAEC-fullstack/internal/storage" | ||
) | ||
|
||
func main() { | ||
ctxWithCancel, cancel := context.WithCancel(context.Background()) | ||
defer cancel() | ||
|
||
// Load Config | ||
cfg := config.MustLoad() | ||
|
||
// Configuration Logger | ||
log := setup.SetupLogger(cfg.Env, cfg.LogPathAgent) | ||
log.Info( | ||
"start agent", | ||
slog.String("env", cfg.Env), | ||
slog.String("version", "2"), | ||
) | ||
log.Debug("debug messages are enabled") | ||
|
||
go logcleaner.CleanLog(10*time.Minute, cfg.LogPathAgent, 100) | ||
|
||
// Configuration Storage | ||
dbCfg := storage.NewStorage(log, cfg.StorageURL) | ||
|
||
// Configuration Agent | ||
application, err := agentapp.New(log, cfg, dbCfg, cancel) | ||
if err != nil { | ||
panic(err) | ||
} | ||
|
||
go application.MustRun(ctxWithCancel) | ||
|
||
// Graceful shotdown | ||
stop := make(chan os.Signal, 1) | ||
signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT) | ||
|
||
sign := <-stop | ||
|
||
log.Info("stopping agent", slog.String("signal", sign.String())) | ||
|
||
application.Stop(ctxWithCancel) | ||
|
||
log.Info("agent stopped") | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,49 @@ | ||
package main | ||
|
||
import ( | ||
"log/slog" | ||
"os" | ||
"os/signal" | ||
"syscall" | ||
|
||
grpcapp "github.com/Prrromanssss/DAEC-fullstack/internal/app/grpc" | ||
"github.com/Prrromanssss/DAEC-fullstack/internal/config" | ||
"github.com/Prrromanssss/DAEC-fullstack/internal/lib/logger/setup" | ||
"github.com/Prrromanssss/DAEC-fullstack/internal/services/auth" | ||
"github.com/Prrromanssss/DAEC-fullstack/internal/storage" | ||
) | ||
|
||
func main() { | ||
// Load Config | ||
cfg := config.MustLoad() | ||
|
||
// Configuration Logger | ||
log := setup.SetupLogger(cfg.Env, cfg.LogPathAuth) | ||
log.Info( | ||
"start grpc server", | ||
slog.String("env", cfg.Env), | ||
slog.String("version", "2"), | ||
) | ||
log.Debug("debug messages are enabled") | ||
|
||
// Configuration Storage | ||
dbCfg := storage.NewStorage(log, cfg.StorageURL) | ||
|
||
authService := auth.New(log, dbCfg, dbCfg, cfg.TokenTTL) | ||
|
||
grpcApp := grpcapp.New(log, authService, cfg.GRPCServer.Address) | ||
|
||
go grpcApp.MustRun() | ||
|
||
// Graceful shotdown | ||
stop := make(chan os.Signal, 1) | ||
signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT) | ||
|
||
sign := <-stop | ||
|
||
log.Info("stopping application", slog.String("signal", sign.String())) | ||
|
||
grpcApp.Stop() | ||
|
||
log.Info("grpc server stopped") | ||
} |
Oops, something went wrong.