diff --git a/.env.example b/.env.example index a1fb16dc..ab0c8055 100644 --- a/.env.example +++ b/.env.example @@ -5,8 +5,10 @@ ENV=development #* Database DB_NAME=urbantree -DB_USER=root +# If user is root, both PASS and ROOT_PASS should be the same +DB_USER=urbantree DB_PASS=J0HMXAJ6XE +DB_ROOT_PASS="J0¿H.MXA;'J6!XE" # If you are using docker-compose, you can ignore the following variables DB_HOST=database DB_PORT=3306 diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 58f07d08..e792d7bc 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -12,6 +12,13 @@ updates: labels: - "kind/dependencies" - "needs-triage" + - package-ecosystem: "pip" + directory: "/api" + schedule: + interval: "weekly" + labels: + - "kind/dependencies" + - "needs-triage" - package-ecosystem: "docker" directory: "/" schedule: @@ -25,4 +32,4 @@ updates: interval: "weekly" labels: - "kind/dependencies" - - "needs-triage" \ No newline at end of file + - "needs-triage" diff --git a/.github/file-filters.yml b/.github/file-filters.yml new file mode 100644 index 00000000..caed5ba4 --- /dev/null +++ b/.github/file-filters.yml @@ -0,0 +1,4 @@ +api: &api + - "api/**" +urbantree: + - "!(*api)" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0c4e6d14..39647a92 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -14,23 +14,44 @@ env: REGISTRY: ghcr.io jobs: + changes: + name: 🔄 Detect changes + runs-on: ubuntu-latest + permissions: + pull-requests: read + outputs: + images: ${{ steps.filter.outputs.changes }} + steps: + # https://github.com/actions/checkout/tree/11bd71901bbe5b1630ceea73d27597364c9af683 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + # https://github.com/dorny/paths-filter/tree/de90cc6fb38fc0963ad72b210f1f284cd68cea36 + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 + id: filter + with: + filters: .github/file-filters.yml + tests: name: 🧪 Tests + needs: changes uses: ./.github/workflows/tests.yml + with: + images: ${{ needs.changes.outputs.images }} secrets: inherit build: name: 🐳 Docker runs-on: ubuntu-latest - needs: tests + needs: [changes, tests] strategy: fail-fast: false # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/running-variations-of-jobs-in-a-workflow#example-adding-configurations matrix: - image: [urbantree] + image: ${{ fromJSON(needs.changes.outputs.images) }} include: - image: urbantree context: . + - image: api + context: ./api permissions: contents: read packages: write @@ -80,6 +101,7 @@ jobs: with: path: | composer-cache + python-cache key: ${{ runner.os }}-docker-${{ hashFiles('**/Dockerfile') }} # https://github.com/reproducible-containers/buildkit-cache-dance/tree/5b6db76d1da5c8b307d5d2e0706d266521b710de @@ -88,7 +110,8 @@ jobs: with: cache-map: | { - "composer-cache": "/tmp/cache" + "composer-cache": "/tmp/cache", + "python-cache": "/root/.cache/pip" } skip-extraction: ${{ steps.cache.outputs.cache-hit }} @@ -110,7 +133,7 @@ jobs: # This will only write to the public Rekor transparency log when the Docker # repository is public to avoid leaking data. If you would like to publish # transparency data even for private images, pass --force to cosign below. - # https://github.com/sigstore/cosign + # https://github.com/sigstore/cosign/ - name: 🖋️ Sign the published Docker image if: ${{ github.event_name != 'pull_request' }} env: @@ -141,12 +164,14 @@ jobs: - name: 🚚 SSH into production server uses: appleboy/ssh-action@7eaf76671a0d7eec5d98ee897acda4f968735a17 with: - host: ${{ secrets.PRODUCTION_SERVER }} - username: ${{ secrets.PRODUCTION_SERVER_USERNAME }} - password: ${{ secrets.PRODUCTION_SERVER_KEY }} - port: ${{ secrets.PRODUCTION_SERVER_PORT }} + host: ${{ secrets.SSH_MANAGER_HOST }} + username: ${{ secrets.SSH_MANAGER_USER }} + password: ${{ secrets.SSH_MANAGER_PASS }} + port: ${{ secrets.SSH_MANAGER_PORT }} script: | - docker compose -f docker-compose.prod.yml up -d --no-deps + docker stack deploy --with-registry-auth -c compose.prod.yml urbantree - name: 🕵️ Check the deployment - run: curl -sSf http://${{ secrets.PRODUCTION_SERVER }}/ + run: | + # curl -sSf http://${{ env.WEB_URL }}/ + curl -sSf http://${{ env.API_URL }}/ diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3033a731..46be349e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,11 +2,16 @@ name: 🧪 Tests on: workflow_call: + inputs: + images: + required: true + type: string jobs: urbantree: name: 🧪 UrbanTree runs-on: ubuntu-latest + if: ${{ contains(fromJSON(inputs.images), 'urbantree') }} steps: # https://github.com/actions/checkout/tree/11bd71901bbe5b1630ceea73d27597364c9af683 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 @@ -49,3 +54,48 @@ jobs: verbose: true env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + api: + name: 🧪 API + runs-on: ubuntu-latest + if: ${{ contains(fromJSON(inputs.images), 'api') }} + steps: + # https://github.com/actions/checkout/tree/11bd71901bbe5b1630ceea73d27597364c9af683 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + # https://github.com/actions/cache/tree/6849a6489940f00c2f30c0fb92c6274307ccb58a + - name: 📦 Cache Python dependencies + uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a + with: + path: /root/.cache/pip + key: ${{ runner.os }}-${{ hashFiles('**/requirements.txt', '**/requirements-dev.txt') }} + + # https://github.com/actions/setup-python/tree/0b93645e9fea7318ecaed2b359559ac225c90a2b + - name: 🐍 Setup Python + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b + with: + python-version: "3.13.0" + + - name: 📦 Install Python dependencies + shell: bash + working-directory: api + run: pip install -r requirements-dev.txt + + - name: 🧪 Run Python tests with coverage + shell: bash + working-directory: api + run: pytest tests --cov=./src --cov-report=xml --junitxml=junit.xml + + # https://github.com/codecov/test-results-action/tree/9739113ad922ea0a9abb4b2c0f8bf6a4aa8ef820 + - name: 📊 Upload test results to Codecov + if: ${{ !cancelled() }} + uses: codecov/test-results-action@9739113ad922ea0a9abb4b2c0f8bf6a4aa8ef820 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + # https://github.com/codecov/codecov-action/tree/015f24e6818733317a2da2edd6290ab26238649a + - name: Upload coverage to Codecov + uses: codecov/codecov-action@015f24e6818733317a2da2edd6290ab26238649a + with: + verbose: true + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.gitignore b/.gitignore index 32e8f148..ec8531bd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,8 @@ +#! Custom + +junit.xml +/coverage/ + # Created by https://www.toptal.com/developers/gitignore/api/git,dotenv,phpunit,composer,visualstudiocode # Edit at https://www.toptal.com/developers/gitignore?templates=git,dotenv,phpunit,composer,visualstudiocode diff --git a/.vscode/settings.json b/.vscode/settings.json index ae0a0259..c8530f87 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,6 @@ { "yaml.schemas": { - "https://json.schemastore.org/pull-request-labeler-5.json": ".github/labeler.yml" + "https://json.schemastore.org/pull-request-labeler-5.json": ".github/labeler.yml", + "https://raw.githubusercontent.com/compose-spec/compose-spec/refs/heads/main/schema/compose-spec.json": "compose.test.yml" } } diff --git a/Dockerfile b/Dockerfile index 94894dd0..7b5f9c00 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,8 +30,9 @@ COPY ./src /var/www/html FROM base AS development # Add PECL extensions, and enable Xdebug. # See https://github.com/docker-library/docs/tree/master/php#pecl-extensions -# RUN pecl install xdebug-3.2.1 \ +# RUN pecl install xdebug \ # && docker-php-ext-enable xdebug +# COPY ./.docker/xdebug.ini /usr/local/etc/php/conf.d/xdebug.ini RUN mv "$PHP_INI_DIR/php.ini-development" "$PHP_INI_DIR/php.ini" COPY --from=dev-deps app/vendor/ /var/www/html/vendor COPY ./tests /var/www/html/tests @@ -40,7 +41,9 @@ COPY ./phpunit.xml** /var/www/html #* Run tests when building FROM development AS test WORKDIR /var/www/html -RUN ./vendor/bin/phpunit +# ENV XDEBUG_MODE coverage +# CMD ["./vendor/bin/phpunit", "--log-junit", "junit.xml", "--coverage-clover=coverage.xml"] +CMD ["./vendor/bin/phpunit"] #* Create a production stage. FROM base AS final diff --git a/README.md b/README.md index b5adc1cf..9d9a4d51 100644 --- a/README.md +++ b/README.md @@ -4,17 +4,23 @@ Aplicatiu web per a la gestió del manteniment d'arbrat urbà i periurbà. ## Instruccions d'instal·lació -**Arrencar l'entorn amb Docker:** +**1. Configurar les variables d'entorn** + +En base al fitxer `.env.example`, crea un fitxer similar a la mateixa carpeta amb el nom `.env`, modificant les variables d'entorn per tal que s'adaptin a les teves necessitats. + +**2. Arrencar l'entorn amb Docker** Executa el següent comandament: + ```bash docker-compose up --build --watch --remove-orphans ``` -Executar-ho crearà els contenidors necessaris per al projecte i posarà en marxa l'aplicació. +Executar-ho crearà els contenidors necessaris per al projecte i posarà en marxa el projecte! --- +Aquestes són les URL's d'accés a les diferents parts de l'aplicació en entorn de desenvolupament: -Accedir a l'aplicació en entorn de desenvolupament: [http://localhost:8000](http://localhost:8000) - -Accedir al phpMyAdmin:: [http://localhost:8080](http://localhost:8080) +- L'aplicatiu: [http://localhost:8000](http://localhost:8000) +- L'API: [http://localhost:8001](http://localhost:8001) +- El phpMyAdmin: [http://localhost:8080](http://localhost:8080) diff --git a/api/.dockerignore b/api/.dockerignore new file mode 100644 index 00000000..03a268b8 --- /dev/null +++ b/api/.dockerignore @@ -0,0 +1,34 @@ +# Include any files or directories that you don't want to be copied to your +# container here (e.g., local build artifacts, temporary files, etc.). +# +# For more help, visit the .dockerignore file reference guide at +# https://docs.docker.com/go/build-context-dockerignore/ + +**/.DS_Store +**/__pycache__ +**/.venv +**/.classpath +**/.dockerignore +**/.env +**/.git +**/.gitignore +**/.project +**/.settings +**/.toolstarget +**/.vs +**/.vscode +**/*.*proj.user +**/*.dbmdl +**/*.jfm +**/bin +**/charts +**/docker-compose* +**/compose.y*ml +**/Dockerfile* +**/node_modules +**/npm-debug.log +**/obj +**/secrets.dev.yaml +**/values.dev.yaml +LICENSE +README.md diff --git a/api/.gitignore b/api/.gitignore new file mode 100644 index 00000000..90f8fbb3 --- /dev/null +++ b/api/.gitignore @@ -0,0 +1,176 @@ +# Created by https://www.toptal.com/developers/gitignore/api/python +# Edit at https://www.toptal.com/developers/gitignore?templates=python + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +### Python Patch ### +# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration +poetry.toml + +# ruff +.ruff_cache/ + +# LSP config files +pyrightconfig.json + +# End of https://www.toptal.com/developers/gitignore/api/python \ No newline at end of file diff --git a/api/Dockerfile b/api/Dockerfile new file mode 100644 index 00000000..5fa35c9e --- /dev/null +++ b/api/Dockerfile @@ -0,0 +1,61 @@ +# syntax=docker/dockerfile:1 + +ARG PYTHON_VERSION=3.13.0 + +# Download dependencies as a separate step to take advantage of Docker's caching. +# Leverage a cache mount to /root/.cache/pip to speed up subsequent builds. +# Leverage a bind mount to requirements.txt to avoid having to copy them into +# into this layer. + +FROM python:${PYTHON_VERSION}-slim AS prod-deps +WORKDIR /app +RUN --mount=type=cache,target=/root/.cache/pip \ + --mount=type=bind,source=requirements.txt,target=requirements.txt \ + python -m pip install -r requirements.txt + +FROM python:${PYTHON_VERSION}-slim AS dev-deps +WORKDIR /app +RUN --mount=type=cache,target=/root/.cache/pip \ + --mount=type=bind,source=requirements.txt,target=requirements.txt \ + --mount=type=bind,source=requirements-dev.txt,target=requirements-dev.txt \ + python -m pip install -r requirements-dev.txt + +FROM dev-deps AS development +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 +WORKDIR /app +COPY . . +EXPOSE 8000 +CMD ["python3", "-m", "uvicorn", "src.app:app", "--host=0.0.0.0", "--port=8000"] + +FROM development AS test +WORKDIR /app +CMD ["pytest", "tests", "--cov=./src", "--cov-report=xml", "--junitxml=./coverage/api.xml"] + +FROM prod-deps AS final +# Prevents Python from writing pyc files. +ENV PYTHONDONTWRITEBYTECODE=1 +# Keeps Python from buffering stdout and stderr to avoid situations where +# the application crashes without emitting any logs due to buffering. +ENV PYTHONUNBUFFERED=1 +WORKDIR /app +# Create a non-privileged user that the app will run under. +# See https://docs.docker.com/go/dockerfile-user-best-practices/ +ARG UID=10001 +RUN adduser \ + --disabled-password \ + --gecos "" \ + --home "/nonexistent" \ + --shell "/sbin/nologin" \ + --no-create-home \ + --uid "${UID}" \ + appuser +# Switch to the non-privileged user to run the application. +USER appuser +# Copy the source code into the container. +COPY . . +# Expose the port that the application listens on. +EXPOSE 8000 +# Run the application. +CMD ["python3", "-m", "uvicorn", "src.app:app", "--host=0.0.0.0", "--port=8000"] +# CMD ["python", "src/main.py"] diff --git a/api/requirements-dev.txt b/api/requirements-dev.txt new file mode 100644 index 00000000..8087a6ed --- /dev/null +++ b/api/requirements-dev.txt @@ -0,0 +1,5 @@ +-r requirements.txt +pytest +pytest-cov +pytest-asyncio +httpx diff --git a/api/requirements.txt b/api/requirements.txt new file mode 100644 index 00000000..66d8512b --- /dev/null +++ b/api/requirements.txt @@ -0,0 +1,14 @@ +fastapi==0.111.0 +pydantic==2.8.2 +pydantic_core==2.20.1 +pydantic-settings==2.3.4 +uvicorn==0.30.1 +# SQLAlchemy +sqlmodel==0.0.19 +# mysqlclient +databases[mysql] +# alembic +# python-dotenv +# python-jose==3.3.0 +# python-multipart==0.0.5 +# passlib==1.7.4 diff --git a/api/sensors.json b/api/sensors.json new file mode 100644 index 00000000..19a2c6c2 --- /dev/null +++ b/api/sensors.json @@ -0,0 +1,50 @@ +[ + { + "id": 1, + "sensor_id": 1, + "temperature": 22.5, + "humidity": 55.3, + "inclination": 0.12, + "created_at": "2024-11-19T12:00:00" + }, + { + "id": 2, + "sensor_id": 1, + "temperature": 23.0, + "humidity": 50.1, + "inclination": 0.15, + "created_at": "2024-11-19T12:30:00" + }, + { + "id": 3, + "sensor_id": 2, + "temperature": 21.8, + "humidity": 60.4, + "inclination": 0.10, + "created_at": "2024-11-19T13:00:00" + }, + { + "id": 4, + "sensor_id": 2, + "temperature": 22.2, + "humidity": 57.9, + "inclination": 0.20, + "created_at": "2024-11-19T13:30:00" + }, + { + "id": 5, + "sensor_id": 3, + "temperature": 24.0, + "humidity": 49.0, + "inclination": 0.18, + "created_at": "2024-11-19T14:00:00" + }, + { + "id": 6, + "sensor_id": 3, + "temperature": 23.5, + "humidity": 51.2, + "inclination": 0.14, + "created_at": "2024-11-19T14:30:00" + } +] diff --git a/api/src/__init__.py b/api/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/api/src/app.py b/api/src/app.py new file mode 100644 index 00000000..66a97f09 --- /dev/null +++ b/api/src/app.py @@ -0,0 +1,41 @@ +from fastapi import FastAPI +from sqlmodel import Session, SQLModel, create_engine +from src.config import settings +from src.services.sensor_service import insert_data +from src.utils.file_loader import load_sensor_data + +engine = create_engine(str(settings.SQLALCHEMY_DATABASE_URI)) + +def create_db_and_tables(): + SQLModel.metadata.create_all(engine) + + +app = FastAPI() + +@app.on_event("startup") +def on_startup(): + create_db_and_tables() + sensor_data = load_sensor_data("./sensors.json") + with Session(engine) as session: + if sensor_data: + insert_data(sensor_data, session) + else: + print("No data to insert") + +@app.get("/") +def hello(): + return "Hello, Docker!" + +# @app.post("/heroes/") +# def create_hero(hero: Hero): +# with Session(engine) as session: +# session.add(hero) +# session.commit() +# session.refresh(hero) +# return hero + +# @app.get("/heroes/") +# def read_heroes(): +# with Session(engine) as session: +# heroes = session.exec(select(Hero)).all() +# return heroes \ No newline at end of file diff --git a/api/src/config.py b/api/src/config.py new file mode 100644 index 00000000..9ffb4b28 --- /dev/null +++ b/api/src/config.py @@ -0,0 +1,57 @@ +import os + +from typing import Any + +from pydantic import ( + MariaDBDsn, + computed_field, + field_validator, + model_validator, +) +from pydantic_core import MultiHostUrl +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + if os.path.exists('/run/secrets'): + model_config = SettingsConfigDict(secrets_dir='/run/secrets') + + MARIADB_SERVER: str + MARIADB_PORT: int = 3306 + MARIADB_USER: str + MARIADB_PASSWORD: str | None = None + MARIADB_PASSWORD_FILE: str | None = None + MARIADB_DB: str + + @model_validator(mode="before") + @classmethod + def check_mariadb_password(cls, data: Any) -> Any: + if isinstance(data, dict): + if data.get("MARIADB_PASSWORD_FILE") is None and data.get("MARIADB_PASSWORD") is None: + raise ValueError("At least one of MARIADB_PASSWORD_FILE and MARIADB_PASSWORD must be set.") + return data + + # @validator('MARIADB_PASSWORD_FILE', pre=True, always=True) + @field_validator("MARIADB_PASSWORD_FILE") + def read_password_from_file(cls, v): + if v is not None: + file_path = v + if os.path.exists(file_path): + with open(file_path, 'r') as file: + return file.read().strip() + raise ValueError(f"Password file {file_path} does not exist.") + return v + + @computed_field + @property + def SQLALCHEMY_DATABASE_URI(self) -> MariaDBDsn: + return MultiHostUrl.build( + scheme="mysql+pymysql", + username=self.MARIADB_USER, + password=self.MARIADB_PASSWORD if self.MARIADB_PASSWORD else self.MARIADB_PASSWORD_FILE, + host=self.MARIADB_SERVER, + port=self.MARIADB_PORT, + path=self.MARIADB_DB, + ) + +settings = Settings() diff --git a/api/src/main.py b/api/src/main.py new file mode 100644 index 00000000..43c28893 --- /dev/null +++ b/api/src/main.py @@ -0,0 +1,19 @@ +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from src.config import settings +from src.services.sensor_service import insert_data +from src.utils.file_loader import load_sensor_data + +engine = create_engine(str(settings.SQLALCHEMY_DATABASE_URI)) +Session = sessionmaker(bind=engine) +session = Session() +def main(): + sensor_data = load_sensor_data("./sensors.json") + + if sensor_data: + insert_data(sensor_data, session) + else: + print("No data to insert") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/api/src/models/__init__.py b/api/src/models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/api/src/models/sensor_model.py b/api/src/models/sensor_model.py new file mode 100644 index 00000000..358bc7a3 --- /dev/null +++ b/api/src/models/sensor_model.py @@ -0,0 +1,36 @@ +from sqlalchemy import Column, Integer, String, Float, Boolean, ForeignKey, TIMESTAMP, func +from sqlalchemy.orm import relationship, declarative_base + +Base = declarative_base() + +class Sensor(Base): + __tablename__ = 'sensors' + + id = Column(Integer, primary_key=True, autoincrement=True) + model = Column(String(255), nullable=True) + is_active = Column(Boolean, nullable=True) + created_at = Column(TIMESTAMP, default=func.current_timestamp()) + + histories = relationship("SensorHistory", back_populates="sensor") + + def __repr__(self): + return (f"") + + +class SensorHistory(Base): + __tablename__ = 'sensor_history' + + id = Column(Integer, primary_key=True, autoincrement=True) + sensor_id = Column(Integer, ForeignKey('sensors.id'), nullable=False) + temperature = Column(Float, nullable=True) + humidity = Column(Float, nullable=True) + inclination = Column(Float, nullable=True) + created_at = Column(TIMESTAMP, default=func.current_timestamp()) + + sensor = relationship("Sensor", back_populates="histories") + + def __repr__(self): + return (f"") diff --git a/api/src/schemas/__init__.py b/api/src/schemas/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/api/src/schemas/sensor.py b/api/src/schemas/sensor.py new file mode 100644 index 00000000..b0e204da --- /dev/null +++ b/api/src/schemas/sensor.py @@ -0,0 +1,15 @@ +from pydantic import BaseModel, RootModel +from datetime import datetime +from typing import List, Optional + +class ModelItem(BaseModel): + id: Optional[int] + sensor_id: int + temperature: float + humidity: float + inclination: float + created_at: datetime + + +class Model(RootModel): + root: List[ModelItem] diff --git a/api/src/services/__init__.py b/api/src/services/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/api/src/services/sensor_service.py b/api/src/services/sensor_service.py new file mode 100644 index 00000000..510112c8 --- /dev/null +++ b/api/src/services/sensor_service.py @@ -0,0 +1,28 @@ +from typing import List +from src.schemas.sensor import ModelItem +from src.models.sensor_model import Sensor, SensorHistory +from sqlalchemy.orm import Session +from datetime import datetime + + +def insert_data(sensor_data: List[ModelItem], session: Session): + try: + for sensor in sensor_data: + # Search existent sensor + db_sensor = session.query(Sensor).filter_by(id=sensor.sensor_id).first() + + if db_sensor: + db_history = SensorHistory( + sensor_id=sensor.sensor_id, + temperature=sensor.temperature, + humidity=sensor.humidity, + inclination=sensor.inclination, + created_at=sensor.created_at + ) + session.add(db_history) + + session.commit() + print("Datos insertados correctamente 👍🏼") + except Exception as e: + session.rollback() + print(f"Error al insertar los datos: {e}") diff --git a/api/src/utils/__init__.py b/api/src/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/api/src/utils/file_loader.py b/api/src/utils/file_loader.py new file mode 100644 index 00000000..638a02a4 --- /dev/null +++ b/api/src/utils/file_loader.py @@ -0,0 +1,14 @@ +import json +from typing import List +from src.schemas.sensor import ModelItem + +def load_sensor_data(file_path: str) -> List[ModelItem]: + try: + with open(file_path, "r") as f: + raw_data = json.load(f) + # raw_data: List[dict] + + return [ModelItem(**sensor) for sensor in raw_data] + except Exception as e: + print('Error', e) + return [] diff --git a/api/tests/__init__.py b/api/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/api/tests/resources/test_file_loader.json b/api/tests/resources/test_file_loader.json new file mode 100644 index 00000000..998d4aa4 --- /dev/null +++ b/api/tests/resources/test_file_loader.json @@ -0,0 +1,18 @@ +[ + { + "id": 1, + "sensor_id": 101, + "temperature": 23.5, + "humidity": 45.2, + "inclination": 0.15, + "created_at": "2024-11-17T10:30:00" + }, + { + "id": 2, + "sensor_id": 102, + "temperature": 25.3, + "humidity": 40.8, + "inclination": 0.30, + "created_at": "2024-11-17T11:00:00" + } +] diff --git a/api/tests/test_file_loader.py b/api/tests/test_file_loader.py new file mode 100644 index 00000000..9eb83b31 --- /dev/null +++ b/api/tests/test_file_loader.py @@ -0,0 +1,15 @@ +import pytest +from src.utils.file_loader import load_sensor_data + +@pytest.fixture +def mock_json_data(): + json_file_path = 'tests/resources/test_file_loader.json' + return json_file_path + +# verify that the 'load_sensor_data' function loads the file as expected +def test_load_sensor_data(mock_json_data): + result = load_sensor_data(mock_json_data) + + assert len(result) == 2 + assert result[0].id == 1 + assert result[1].sensor_id == 102 diff --git a/api/tests/test_integration.py b/api/tests/test_integration.py new file mode 100644 index 00000000..f50d671b --- /dev/null +++ b/api/tests/test_integration.py @@ -0,0 +1,73 @@ +import os +import json +import pytest +from src.models.sensor_model import Sensor, SensorHistory +from src.services.sensor_service import insert_data +from src.schemas.sensor import ModelItem + + +@pytest.fixture(scope="module") +def db_session(): + from sqlalchemy import create_engine + from sqlalchemy.orm import sessionmaker + from src.models.sensor_model import Base + + # create a persistent sqlite database for integration tests + engine = create_engine('sqlite:///test_integration.db') + Base.metadata.create_all(engine) + + # create a database session + Session = sessionmaker(bind=engine) + session = Session() + + yield session + + session.close() + # comment this line to retain tables in the integration database + Base.metadata.drop_all(engine) + + +@pytest.fixture +def load_test_json(): + + with open(os.path.join(os.path.dirname(__file__), 'resources', 'test_file_loader.json'), 'r') as file: + return json.load(file) + + +def test_integration(db_session, load_test_json): + # create and commit sensor entries + sensor_1 = Sensor(id=101, model="Model1", is_active=True) + sensor_2 = Sensor(id=102, model="Model2", is_active=True) + + db_session.add(sensor_1) + db_session.add(sensor_2) + db_session.commit() + + # convert the loaded JSON data to ModelItems + mock_data = [ModelItem(**sensor) for sensor in load_test_json] + + # insert the data into the database + insert_data(mock_data, db_session) + + # call the main function to process the data + # main() + + + history_records = db_session.query(SensorHistory).all() + assert len(history_records) == 2 + + + assert history_records[0].sensor_id == 101 + assert history_records[1].sensor_id == 102 + assert history_records[0].temperature == 23.5 + assert history_records[1].temperature == 25.3 + + + sensor_records = db_session.query(Sensor).all() + assert len(sensor_records) == 2 + assert sensor_records[0].id == 101 + assert sensor_records[1].id == 102 + + + assert sensor_records[0].id == history_records[0].sensor_id + assert sensor_records[1].id == history_records[1].sensor_id diff --git a/api/tests/test_sensor_service.py b/api/tests/test_sensor_service.py new file mode 100644 index 00000000..8473b2c0 --- /dev/null +++ b/api/tests/test_sensor_service.py @@ -0,0 +1,68 @@ +import pytest +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from src.schemas.sensor import ModelItem +from src.services.sensor_service import insert_data +from src.models.sensor_model import Base, Sensor, SensorHistory + + +# fixture for the in-memory database +@pytest.fixture +def db_session(): + # create in-memory database + engine = create_engine('sqlite:///:memory:') + # create all tables (sensor, sensorhistory) + Base.metadata.create_all(engine) + + # create session + Session = sessionmaker(bind=engine) + session = Session() + + # return the session to be used in tests + yield session + + # close the session and drop tables after the test + session.close() + Base.metadata.drop_all(engine) + + +# test to insert data +def test_insert_data(db_session): + # insert sensors into the database before inserting history + sensor_1 = Sensor(id=101, model="Model1", is_active=True) + sensor_2 = Sensor(id=102, model="Model2", is_active=True) + + db_session.add(sensor_1) + db_session.add(sensor_2) + db_session.commit() + + # verify that the sensors were inserted correctly into the database + sensor_records = db_session.query(Sensor).all() + assert len(sensor_records) == 2 # verify that 2 sensors were inserted + + # mock data including 'sensor_id' and other fields + mock_data = [ + ModelItem(id=1, sensor_id=101, temperature=23.5, humidity=45.2, inclination=0.15, created_at="2024-11-17T10:30:00"), + ModelItem(id=2, sensor_id=102, temperature=25.3, humidity=40.8, inclination=0.30, created_at="2024-11-17T11:00:00") + ] + + # call the data insertion function passing the database session + insert_data(mock_data, db_session) + + # verify that records have been inserted into SensorHistory + history_records = db_session.query(SensorHistory).all() + + assert len(history_records) == 2 # verify that 2 records were inserted into SensorHistory + + # verify that history records are correctly associated + assert history_records[0].sensor_id == 101 + assert history_records[1].sensor_id == 102 + assert history_records[0].temperature == 23.5 + assert history_records[1].temperature == 25.3 + + # verify that records have been inserted into Sensor + sensor_records = db_session.query(Sensor).all() + + assert len(sensor_records) == 2 + assert sensor_records[0].id == 101 + assert sensor_records[1].id == 102 diff --git a/compose.prod.yml b/compose.prod.yml new file mode 100644 index 00000000..6feefeb0 --- /dev/null +++ b/compose.prod.yml @@ -0,0 +1,106 @@ +version: "3.8" +services: + api: + depends_on: + - database + deploy: + restart_policy: + condition: on-failure + delay: 5s + max_attempts: 3 + window: 120s + placement: + constraints: + - node.labels.type==api + environment: + APP_NAME: ${APP_NAME} + APP_ENV: ${APP_ENV} + MARIADB_SERVER: database + MARIADB_PORT: "3306" + MARIADB_DB: ${DB_NAME} + MARIADB_USER: ${DB_USER} + MARIADB_PASSWORD_FILE: /run/secrets/db_pass + image: ghcr.io/projecte-urbantree/api:main + ports: + - mode: host + target: 8000 + published: 8000 + secrets: + - source: db_pass + database: + deploy: + restart_policy: + condition: any + delay: 5s + max_attempts: 3 + window: 120s + placement: + constraints: + - node.labels.type==database + environment: + MYSQL_ROOT_PASSWORD_FILE: /run/secrets/db_root_pass + MYSQL_DATABASE: ${DB_NAME} + MYSQL_USER: ${DB_USER} + MYSQL_PASSWORD_FILE: /run/secrets/db_pass + healthcheck: + test: + [ + "CMD", + "/usr/local/bin/healthcheck.sh", + "--su-mysql", + "--connect", + "--innodb_initialized", + ] + timeout: 5s + interval: 10s + retries: 5 + image: mariadb:lts + secrets: + - db_pass + - db_root_pass + user: root + volumes: + - type: volume + source: database-data + target: /var/lib/mysql + web: + depends_on: + - database + deploy: + restart_policy: + condition: on-failure + delay: 5s + max_attempts: 3 + window: 120s + placement: + constraints: + - node.labels.type==web + environment: + APP_NAME: ${APP_NAME} + DB_HOST: database + DB_PORT: "3306" + DB_NAME: ${DB_NAME} + DB_USER: ${DB_USER} + DB_PASS_FILE_PATH: /run/secrets/db_pass + LOG_FILE_PATH: /var/www/html/storage/logs/app.log + image: ghcr.io/projecte-urbantree/urbantree:main + ports: + - mode: host + target: 8000 + published: 8000 + secrets: + - source: db_pass + volumes: + - type: volume + source: storage-data + target: /var/www/html/storage +volumes: + database-data: {} + storage-data: {} +secrets: + db_pass: + name: db_pass + external: true + db_root_pass: + name: db_root_pass + external: true diff --git a/compose.test.yml b/compose.test.yml new file mode 100644 index 00000000..4a7e33e1 --- /dev/null +++ b/compose.test.yml @@ -0,0 +1,27 @@ +services: + web: + build: + context: . + target: test + ports: + - 8000:80 + volumes: + - ./coverage:/var/www/html/coverage + develop: + watch: + - action: sync + path: ./src + target: /var/www/html + api: + build: + context: ./api + target: test + ports: + - 8001:8000 + volumes: + - ./coverage:/app/coverage + develop: + watch: + - action: rebuild + path: ./api + diff --git a/docker-compose.yml b/compose.yml similarity index 65% rename from docker-compose.yml rename to compose.yml index 732001b5..54686090 100644 --- a/docker-compose.yml +++ b/compose.yml @@ -6,52 +6,23 @@ # services: - server: - build: - context: . - target: development - ports: - - 8000:80 - networks: - - app-network - depends_on: - database: - condition: service_healthy - secrets: - - database-password - volumes: - - storage-data:/var/www/html/storage - environment: - - APP_NAME=${APP_NAME} - - DB_HOST=${DB_HOST:-database} - - DB_NAME=${DB_NAME} - - DB_USER=${DB_USER:-root} - - DB_PASS_FILE_PATH=/run/secrets/database-password - - LOG_FILE_PATH=/var/www/html/storage/logs/app.log - develop: - watch: - - action: sync - path: ./src - target: /var/www/html - database: build: context: . target: database restart: always user: root - secrets: - - database-password - ports: - - 1000:3306 - networks: - - app-network # This will store the database in memory, so it will be lost when the container is stopped tmpfs: - /var/lib/mysql + secrets: + - db_pass + - db_root_pass environment: + MYSQL_ROOT_PASSWORD_FILE: /run/secrets/db_root_pass MYSQL_DATABASE: ${DB_NAME} - MYSQL_ROOT_PASSWORD_FILE: /run/secrets/database-password + MYSQL_USER: ${DB_USER} + MYSQL_PASSWORD_FILE: /run/secrets/db_pass healthcheck: test: [ @@ -69,26 +40,73 @@ services: - action: rebuild path: ./database + web: + build: + context: . + target: development + ports: + - 8000:80 + depends_on: + database: + condition: service_healthy + secrets: + - db_pass + volumes: + - storage-data:/var/www/html/storage + environment: + - APP_NAME=${APP_NAME} + - DB_HOST=database + - DB_PORT=3306 + - DB_NAME=${DB_NAME} + - DB_USER=${DB_USER} + - DB_PASS_FILE_PATH=/run/secrets/db_pass + - LOG_FILE_PATH=/var/www/html/storage/logs/app.log + develop: + watch: + - action: sync + path: ./src + target: /var/www/html + + api: + build: + context: ./api + target: development + ports: + - 8001:8000 + depends_on: + database: + condition: service_healthy + secrets: + - db_pass + environment: + - APP_NAME=${APP_NAME} + - APP_ENV=${APP_ENV} + - MARIADB_SERVER=database + - MARIADB_PORT=3306 + - MARIADB_DB=${DB_NAME} + - MARIADB_USER=${DB_USER} + - MARIADB_PASSWORD_FILE=/run/secrets/db_pass + develop: + watch: + - action: rebuild + path: ./api + phpmyadmin: image: phpmyadmin:latest ports: - 8080:80 - networks: - - app-network depends_on: database: condition: service_healthy environment: PMA_HOST: database -networks: - app-network: - driver: bridge - volumes: database-data: storage-data: secrets: - database-password: + db_pass: environment: DB_PASS + db_root_pass: + environment: DB_ROOT_PASS