diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a486a4..f50f86f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -138,61 +138,26 @@ jobs: - name: Run build run: make build - integration-api: - name: API Integration Tests + integration-tests: + name: Integration Tests (${{ matrix.component }}) needs: [lint, test, build] runs-on: ubuntu-latest + strategy: + matrix: + component: [api, dashboard] steps: - name: Checkout code uses: actions/checkout@v4 - - name: Set up Python 3.12 - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - - name: Cache Poetry dependencies - uses: actions/cache@v4 - with: - path: ~/.cache/poetry - key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} - restore-keys: | - ${{ runner.os }}-poetry- - - - name: Install dependencies - run: | - curl -sSL https://install.python-poetry.org | python3 - - poetry config virtualenvs.create false - poetry install - - name: Set up Docker uses: docker/setup-buildx-action@v3 - - name: Start API service and run integration tests - run: | - docker compose up -d --pull never - pytest tests/integration -v -m api_integration - docker compose down - - integration-dashboard: - name: Dashboard Integration Tests - needs: [lint, test, build] - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Docker - uses: docker/setup-buildx-action@v3 - - - name: Start Dashboard service and run integration tests - run: | - docker build -t dash-app-tests . -f tests/integration/test_dashboard.Dockerfile - docker run --rm dash-app-tests + - name: Run integration tests + run: make test-${{ matrix.component }} release: name: Release - needs: [integration-api, integration-dashboard] + needs: [integration-tests] if: github.event_name == 'push' && github.ref == 'refs/heads/master' runs-on: ubuntu-latest permissions: @@ -226,7 +191,7 @@ jobs: publish: name: Publish Docker Images - needs: release + needs: [release] runs-on: ubuntu-latest steps: - name: Checkout code diff --git a/Makefile b/Makefile index 815d376..7bb2b70 100644 --- a/Makefile +++ b/Makefile @@ -60,10 +60,15 @@ build: up down # Integration Testing test-dashboard: + make up + $(DOCKER_COMPOSE) logs api docker build -t dash-app-tests . -f tests/integration/test_dashboard.Dockerfile - docker run --rm dash-app-tests + docker run --network host --rm dash-app-tests + make down test-api: make up - pytest tests/integration -v -m api_integration + $(DOCKER_COMPOSE) logs api + docker build -t api-tests . -f tests/integration/test_api.Dockerfile + docker run --network host --rm api-tests make down diff --git a/api.Dockerfile b/api.Dockerfile index 7cceade..347409c 100644 --- a/api.Dockerfile +++ b/api.Dockerfile @@ -12,14 +12,14 @@ COPY portfolio_analytics/common portfolio_analytics/common COPY portfolio_analytics/api portfolio_analytics/api # Set ownership and switch to non-root user -RUN chown -R appuser:appuser /app -USER appuser +RUN chown -R 1000:1000 /app +USER 1000:1000 # Expose port EXPOSE 8000 # Health check -HEALTHCHECK --interval=30s --timeout=3s \ +HEALTHCHECK --interval=10s --timeout=3s \ CMD curl -f http://localhost:8000/ || exit 1 # Command for production server diff --git a/base.Dockerfile b/base.Dockerfile index 419b1cf..2bb7dc7 100644 --- a/base.Dockerfile +++ b/base.Dockerfile @@ -18,8 +18,9 @@ RUN apt-get update && apt-get install -y \ procps \ && rm -rf /var/lib/apt/lists/* -# Create non-root user -RUN groupadd -r appuser && useradd -r -g appuser appuser +# Create user with explicit UID/GID +RUN groupadd -g 1000 appgroup && \ + useradd -u 1000 -g appgroup -s /bin/bash -m appuser WORKDIR /app diff --git a/dashboard.Dockerfile b/dashboard.Dockerfile index 02cd1c9..e660e7c 100644 --- a/dashboard.Dockerfile +++ b/dashboard.Dockerfile @@ -12,14 +12,14 @@ COPY portfolio_analytics/common portfolio_analytics/common COPY portfolio_analytics/dashboard portfolio_analytics/dashboard # Set ownership and switch to non-root user -RUN chown -R appuser:appuser /app -USER appuser +RUN chown -R 1000:1000 /app +USER 1000:1000 # Expose port EXPOSE 8050 # Health check -HEALTHCHECK --interval=30s --timeout=3s \ +HEALTHCHECK --interval=10s --timeout=3s \ CMD curl -f http://localhost:8050/ || exit 1 # Command for production server diff --git a/data.Dockerfile b/data.Dockerfile index 3d8001d..211cc65 100644 --- a/data.Dockerfile +++ b/data.Dockerfile @@ -2,17 +2,24 @@ FROM alpine:3.18 WORKDIR /data -# Create appuser with same UID/GID as in python-base -RUN addgroup -g 1000 appuser && \ - adduser -D -u 1000 -G appuser appuser +# Add user with same UID/GID as other containers +RUN addgroup -g 1000 appgroup && \ + adduser -u 1000 -G appgroup -s /bin/sh -D appuser -# Copy your data directory -COPY data/ . +# Create directories first +RUN mkdir -p /data/market_data /data/cache /data/portfolios -# Make sure the files are owned by appuser -RUN chown -R appuser:appuser /data +# Copy data files +COPY --chown=1000:1000 data/ . -# Use appuser -USER appuser +# Set permissions for shared access and verify +RUN chown -R 1000:1000 /data && \ + chmod -R 775 /data && \ + chmod g+s /data /data/market_data /data/cache /data/portfolios && \ + echo "Verifying permissions:" && \ + ls -ln /data -CMD ["tail", "-f", "/dev/null"] +USER 1000:1000 + +# Add permission check to startup +CMD ls -ln /data && echo "Running as: $(id)" && tail -f /dev/null diff --git a/deploy.sh b/deploy.sh index 6d643c0..72e0527 100755 --- a/deploy.sh +++ b/deploy.sh @@ -8,14 +8,14 @@ echo "Deploying application version ${GIT_TAG:-latest}" # Data service docker run -d \ --name data \ - --user appuser:appuser \ + --user 1000:1000 \ -v shared-data:/data \ ghcr.io/gbourniq/portfolio_analytics/data:${GIT_TAG:-latest} # Dashboard service docker run -d \ --name api \ - --user appuser:appuser \ + --user 1000:1000 \ -v shared-data:/app/data \ -p 8000:8000 \ ghcr.io/gbourniq/portfolio_analytics/api:${GIT_TAG:-latest} @@ -23,7 +23,7 @@ docker run -d \ # API service docker run -d \ --name dashboard \ - --user appuser:appuser \ + --user 1000:1000 \ -v shared-data:/app/data \ -p 8050:8050 \ ghcr.io/gbourniq/portfolio_analytics/dashboard:${GIT_TAG:-latest} diff --git a/docker-compose.yml b/docker-compose.yml index 6a7b3f3..073dc3f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,11 @@ volumes: shared-data: + driver: local + # driver_opts: + # type: none + # o: bind + # device: ${PWD}/data + # name: portfolio-analytics-data services: data: @@ -8,8 +14,8 @@ services: context: . dockerfile: data.Dockerfile volumes: - - shared-data:/data - user: "appuser:appuser" + - shared-data:/data:rw + user: "1000:1000" api: image: ghcr.io/gbourniq/portfolio_analytics/api:${GIT_TAG:-latest} @@ -19,8 +25,10 @@ services: ports: - "8000:8000" volumes: - - shared-data:/app/data - user: "appuser:appuser" + - shared-data:/app/data:rw + group_add: + - "1000" + user: "1000:1000" deploy: resources: limits: @@ -28,6 +36,9 @@ services: memory: 384M reservations: memory: 192M + depends_on: + data: + condition: service_started dashboard: image: ghcr.io/gbourniq/portfolio_analytics/dashboard:${GIT_TAG:-latest} @@ -37,8 +48,10 @@ services: ports: - "8050:8050" volumes: - - shared-data:/app/data - user: "appuser:appuser" + - shared-data:/app/data:rw + group_add: + - "1000" + user: "1000:1000" deploy: resources: limits: @@ -46,3 +59,6 @@ services: memory: 1024M reservations: memory: 512M + depends_on: + data: + condition: service_started diff --git a/portfolio_analytics/common/filesystem.py b/portfolio_analytics/common/filesystem.py index 4ce6131..e347818 100644 --- a/portfolio_analytics/common/filesystem.py +++ b/portfolio_analytics/common/filesystem.py @@ -37,7 +37,7 @@ def get_version(): str: The version string from pyproject.toml, or '0.0.0' if reading fails """ try: - pyproject_path = Path(__file__).parents[3] / "pyproject.toml" + pyproject_path = Path(__file__).parents[2] / "pyproject.toml" with open(pyproject_path, "rb") as f: pyproject_data = tomli.load(f) return pyproject_data["tool"]["poetry"]["version"] diff --git a/tests/integration/test_api.Dockerfile b/tests/integration/test_api.Dockerfile new file mode 100644 index 0000000..3b92141 --- /dev/null +++ b/tests/integration/test_api.Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.12-slim + +WORKDIR /app + +# Install Python packages +RUN pip install --no-cache-dir pytest requests + +# Copy test files +COPY tests/integration/test_api.py ./test_api.py + +# Set Python path +ENV PYTHONPATH=/app + +# Run tests +CMD ["pytest", "test_api.py", "-v", "-m", "api_integration"] diff --git a/tests/integration/test_api_workflows.py b/tests/integration/test_api.py similarity index 79% rename from tests/integration/test_api_workflows.py rename to tests/integration/test_api.py index fe68d14..3e33d22 100644 --- a/tests/integration/test_api_workflows.py +++ b/tests/integration/test_api.py @@ -53,18 +53,15 @@ def test_fx_data_pipeline(self, api_client: requests.Session) -> None: response = api_client.post(f"{BASE_URL}/market_data/fx", json=payload) # Then - assert response.status_code == HTTPStatus.OK - data = response.json() - - # Verify response structure - assert "output_path" in data - assert "file_stats" in data - - # Verify data coverage - stats = data["file_stats"] - assert stats["row_count"] > 0 - assert any("USD" in ticker for ticker in stats["currencies_covered"]) - assert any("EUR" in ticker for ticker in stats["currencies_covered"]) + try: + assert response.status_code == HTTPStatus.OK, \ + f"API returned {response.status_code}: {response.text}" + except AssertionError as e: + print(f"\nRequest URL: {response.request.url}") + print(f"Request Headers: {response.request.headers}") + print(f"Request Body: {response.request.body}") + print(f"Response Headers: {response.headers}") + raise e @pytest.mark.integration @@ -86,9 +83,15 @@ def test_portfolio_workflow( files = {"file": ("test_portfolio.csv", sample_portfolio, "text/csv")} response = api_client.post(f"{BASE_URL}/portfolio", files=files) - assert response.status_code == HTTPStatus.CREATED - upload_data = response.json() - assert upload_data["filename"] == "test_portfolio.csv" + try: + assert response.status_code == HTTPStatus.CREATED, \ + f"API returned {response.status_code}: {response.text}" + except AssertionError as e: + print(f"\nRequest URL: {response.request.url}") + print(f"Request Headers: {response.request.headers}") + print(f"Request Body: {response.request.body}") + print(f"Response Headers: {response.headers}") + raise e # 2. Download all portfolios (including our upload) response = api_client.get( diff --git a/tests/integration/test_dashboard.Dockerfile b/tests/integration/test_dashboard.Dockerfile index 56d84ad..a0fef7a 100644 --- a/tests/integration/test_dashboard.Dockerfile +++ b/tests/integration/test_dashboard.Dockerfile @@ -20,9 +20,6 @@ RUN pip install --no-cache-dir \ pandas # Copy files to container -COPY portfolio_analytics/dashboard portfolio_analytics/dashboard -COPY portfolio_analytics/common portfolio_analytics/common -COPY data data COPY pyproject.toml ./pyproject.toml COPY tests/integration/test_dashboard.py ./test_dashboard.py diff --git a/tests/integration/test_dashboard.py b/tests/integration/test_dashboard.py index 774fa14..2eb36d6 100644 --- a/tests/integration/test_dashboard.py +++ b/tests/integration/test_dashboard.py @@ -4,16 +4,15 @@ """ import pytest -from dash.testing.application_runners import import_app FILESYSTEM_MODULE = "portfolio_analytics.common.filesystem" @pytest.fixture def initialized_dash(dash_duo): - """Initialize the dash application for testing.""" - app = import_app("portfolio_analytics.dashboard.dashboard_main") - dash_duo.start_server(app) + """Initialize connection to the deployed dash application for testing.""" + # Use localhost since we're using host network + dash_duo.server_url = "http://localhost:8050" yield dash_duo @@ -24,7 +23,7 @@ def test_dash_app_basic_elements(initialized_dash): dash_duo = initialized_dash # Test title is present - dash_duo.wait_for_text_to_equal("h1", "Portfolio Analytics Dashboard", timeout=4) + dash_duo.wait_for_text_to_equal("h1", "Portfolio Analytics Dashboard", timeout=10) # Test that dropdowns are present assert dash_duo.find_element("#portfolio-selector").is_displayed() @@ -34,14 +33,14 @@ def test_dash_app_basic_elements(initialized_dash): max_button = dash_duo.find_element("#max-button") assert max_button.is_displayed() - # Test that graph container is present - assert dash_duo.find_element("#pnl-graph").is_displayed() + # Test that graph container is present and wait for it to become visible + dash_duo.wait_for_element("#pnl-graph", timeout=10) @pytest.mark.integration @pytest.mark.dashboard_integration def test_no_file_error_message(initialized_dash): - """Test that 'No such file or directory' error is not shown on the page.""" + """Test that error messages are not shown on the page.""" dash_duo = initialized_dash # Wait for the page to load @@ -50,17 +49,18 @@ def test_no_file_error_message(initialized_dash): # Get the page content using a valid CSS selector page_content = dash_duo.find_element("body").text - # Assert that the error message is not present - assert ( - "No such file or directory" not in page_content - ), f"Found 'No such file or directory' error in page content: {page_content}" + # Assert that error messages are not present + error_messages = ["No such file or directory", "No data available"] + for error in error_messages: + assert ( + error not in page_content + ), f"Found '{error}' error in page content: {page_content}" - # Additionally check that the graph container is not showing an error + # Additionally check that the graph container is not showing errors graph = dash_duo.find_element("#pnl-graph") graph_text = graph.text - assert ( - "No such file or directory" not in graph_text - ), f"Found 'No such file or directory' error in graph: {graph_text}" + for error in error_messages: + assert error not in graph_text, f"Found '{error}' error in graph: {graph_text}" @pytest.mark.integration diff --git a/tests/test_common/test_utils/test_filesystem.py b/tests/test_common/test_utils/test_filesystem.py index 11c3974..f887400 100644 --- a/tests/test_common/test_utils/test_filesystem.py +++ b/tests/test_common/test_utils/test_filesystem.py @@ -58,9 +58,7 @@ def test_get_version_success(self, tmp_path): with open(mock_path, "wb") as f: tomli_w.dump(mock_toml, f) - with patch.object( - Path, "parents", property(lambda x: [None, None, None, tmp_path]) - ): + with patch.object(Path, "parents", property(lambda x: [None, None, tmp_path])): # When version = get_version() @@ -70,7 +68,7 @@ def test_get_version_success(self, tmp_path): def test_get_version_file_not_found(self): """Tests fallback version when pyproject.toml is not found.""" with patch.object( - Path, "parents", property(lambda x: [None, None, None, Path("/nonexistent")]) + Path, "parents", property(lambda x: [None, None, Path("/nonexistent")]) ): # When version = get_version()