diff --git a/.devcontainer/check_test_count.sh b/.devcontainer/check_test_count.sh index 76d2cc3d..577263d2 100644 --- a/.devcontainer/check_test_count.sh +++ b/.devcontainer/check_test_count.sh @@ -6,12 +6,14 @@ # The script must be invoked with a filter matching the paths NOT included in the matrix # $@: (Optional) Can be set to specify a filter for running python tests at the specified path. -echo "Filter (paths): '$@'" +echo "Parameters: '$@'" # Exit immediately with failure status if any command fails set -e -cd source/electrical_heating/tests/ +test_path=$1 +filter=$2 +cd $test_path # Enable extended globbing. E.g. see https://stackoverflow.com/questions/8525437/list-files-not-matching-a-pattern shopt -s extglob @@ -20,7 +22,7 @@ shopt -s extglob # 'awk' is used to get the second column of the output which contains the number of tests. # 'head' is used to get the first line of the output which contains the number of tests. # Example output line returned by the grep filter: 'collected 10 items' -executed_test_count=$(coverage run --branch -m pytest $@ --collect-only | grep collected | awk '{print $2}' | head -n 1) +executed_test_count=$(coverage run --branch -m pytest $filter --collect-only | grep collected | awk '{print $2}' | head -n 1) total_test_count=$(coverage run --branch -m pytest --collect-only | grep collected | awk '{print $2}' | head -n 1) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index d2c686b5..18a1c3f3 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -45,23 +45,32 @@ jobs: # CD Databricks # - electrical_heating: + electrical_heating_promote_prerelease: needs: changes if: ${{ needs.changes.outputs.electrical_heating == 'true' }} uses: Energinet-DataHub/.github/.github/workflows/promote-prerelease.yml@v14 with: release_name_prefix: electrical_heating + + capacity_settlement_promote_prerelease: + needs: changes + if: ${{ needs.changes.outputs.capacity_settlement == 'true' }} + uses: Energinet-DataHub/.github/.github/workflows/promote-prerelease.yml@v14 + with: + release_name_prefix: capacity_settlement + # # Dispatch deployment request # dispatch_deployment_event: - if: ${{ always() && !cancelled() && !failure() && needs.changes.outputs.electrical_heating == 'true' }} + if: ${{ always() && !cancelled() && !failure() && needs.changes.outputs.electrical_heating == 'true' || needs.changes.outputs.capacity_settlement == 'true' }} runs-on: ubuntu-latest needs: [ changes, - electrical_heating + electrical_heating_promote_prerelease, + capacity_settlement_promote_prerelease, ] steps: - run: echo "${{ toJSON(needs) }}" @@ -84,7 +93,7 @@ jobs: repository: ${{ vars.environment_repository_path }} event-type: measurements-deployment-request-domain # yamllint disable-line rule:quoted-strings - client-payload: '{"pr_number": "${{ steps.find_pull_request.outputs.pull_request_number }}", "electrical_heating": "${{ needs.changes.outputs.electrical_heating }}"}' + client-payload: '{"pr_number": "${{ steps.find_pull_request.outputs.pull_request_number }}", "electrical_heating": "${{ needs.changes.outputs.electrical_heating }}", "capacity_settlement": "${{ needs.changes.outputs.capacity_settlement }}"}' # # Send notification to teams channel if deployment dispatch failed @@ -93,7 +102,8 @@ jobs: dispatch_failed: needs: [ - electrical_heating, + electrical_heating_promote_prerelease, + capacity_settlement_promote_prerelease, dispatch_deployment_event, cd_docker ] diff --git a/.github/workflows/ci-capacity-settlement.yml b/.github/workflows/ci-capacity-settlement.yml new file mode 100644 index 00000000..c65476c7 --- /dev/null +++ b/.github/workflows/ci-capacity-settlement.yml @@ -0,0 +1,92 @@ +name: CI Capacity Settlement + +on: + workflow_call: + inputs: + image_tag: + type: string + default: latest + +jobs: + databricks_ci_build: + uses: Energinet-DataHub/.github/.github/workflows/databricks-build-prerelease.yml@v14 + with: + python_version: 3.11.7 + architecture: x64 + wheel_working_directory: ./source/capacity_settlement + prerelease_prefix: capacity_settlement + + unit_tests: + strategy: + fail-fast: false + matrix: + # IMPORTANT: When adding a new folder here it should also be added in the `unit_test_check` job! + tests_filter_expression: + - name: Capacity Settlement + paths: capacity_settlement_tests/ + uses: Energinet-DataHub/.github/.github/workflows/python-ci.yml@v14 + with: + job_name: ${{ matrix.tests_filter_expression.name }} + operating_system: dh3-ubuntu-20.04-4core + path_static_checks: ./source/capacity_settlement + # documented here: https://github.com/Energinet-DataHub/opengeh-wholesale/tree/main/source/databricks#styling-and-formatting + ignore_errors_and_warning_flake8: E501,F401,E402,E203,W503 + tests_folder_path: ./source/capacity_settlement/tests + test_report_path: ./source/capacity_settlement/tests + # See .docker/entrypoint.py on how to use the filter expression + tests_filter_expression: ${{ matrix.tests_filter_expression.paths }} + image_tag: ${{ inputs.image_tag }} + + # Check executed unit tests + capacity_settlement_unit_test_check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Execute python tests + shell: bash + id: test_count + run: | + # Small hack to get the repository name + repository=${{ github.repository }} + repository_owner=${{ github.repository_owner }} + repository_name=${repository/$repository_owner\//} + + # IMPORTANT: When adding a new folder here, one must also add the folder + # to one of the test jobs above! This is because this filter contains the sum of all folders + # from test jobs. + test_path="source/capacity_settlement/tests/" + filter="capacity_settlement_tests/" + + chmod +x ./.devcontainer/check_test_count.sh + IMAGE_TAG=${{ inputs.image_tag }} docker compose -f .devcontainer/docker-compose.yml run --rm -u root python-unit-test ./.devcontainer/check_test_count.sh $test_path $filter + + + mypy_check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.x + - name: Run pip install and mypy check of files in package + shell: bash + run: | + pip install --upgrade pip + pip install mypy types-python-dateutil + mypy ./source/capacity_settlement --disallow-untyped-defs --ignore-missing-imports + + black_check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: psf/black@stable + with: + options: --check --diff + src: ./source/capacity_settlement diff --git a/.github/workflows/ci-electrical-heating.yml b/.github/workflows/ci-electrical-heating.yml index d14ba6f3..f26f6ca4 100644 --- a/.github/workflows/ci-electrical-heating.yml +++ b/.github/workflows/ci-electrical-heating.yml @@ -3,10 +3,6 @@ on: workflow_call: inputs: - has_electrical_heating_changes: - description: Whether there are changes in the electrical heating job folder - required: true - type: boolean image_tag: type: string default: latest @@ -21,14 +17,13 @@ jobs: prerelease_prefix: electrical_heating unit_tests: - if: ${{ inputs.has_electrical_heating_changes }} strategy: fail-fast: false matrix: # IMPORTANT: When adding a new folder here it should also be added in the `unit_test_check` job! tests_filter_expression: - name: Electrical Heating - paths: entry_points/ + paths: electrical_heating_tests/ uses: Energinet-DataHub/.github/.github/workflows/python-ci.yml@v14 with: job_name: ${{ matrix.tests_filter_expression.name }} @@ -44,7 +39,6 @@ jobs: # Check executed unit tests electrical_heating_unit_test_check: - if: ${{ inputs.has_electrical_heating_changes }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -62,14 +56,14 @@ jobs: # IMPORTANT: When adding a new folder here, one must also add the folder # to one of the test jobs above! This is because this filter contains the sum of all folders # from test jobs. - filter="entry_points/" + test_path="source/electrical_heating/tests/" + filter="electrical_heating_tests/" chmod +x ./.devcontainer/check_test_count.sh - IMAGE_TAG=${{ inputs.image_tag }} docker compose -f .devcontainer/docker-compose.yml run --rm -u root python-unit-test ./.devcontainer/check_test_count.sh $filter + IMAGE_TAG=${{ inputs.image_tag }} docker compose -f .devcontainer/docker-compose.yml run --rm -u root python-unit-test ./.devcontainer/check_test_count.sh $test_path $filter mypy_check: - if: ${{ inputs.has_electrical_heating_changes }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -84,7 +78,6 @@ jobs: mypy ./source/electrical_heating --disallow-untyped-defs --ignore-missing-imports black_check: - if: ${{ inputs.has_electrical_heating_changes }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/ci-orchestrator.yml b/.github/workflows/ci-orchestrator.yml index d018367e..813d2de2 100644 --- a/.github/workflows/ci-orchestrator.yml +++ b/.github/workflows/ci-orchestrator.yml @@ -12,7 +12,7 @@ concurrency: jobs: # - # License and Markdown Check. + # License and Markdown Check # ci_base: uses: Energinet-DataHub/.github/.github/workflows/ci-base.yml@v14 @@ -36,7 +36,13 @@ jobs: if: ${{ needs.changes.outputs.electrical_heating == 'true' }} uses: ./.github/workflows/ci-electrical-heating.yml with: - has_electrical_heating_changes: ${{ needs.changes.outputs.electrical_heating == 'true' }} + image_tag: ${{ needs.ci_docker.outputs.image_tag }} + + ci_capacity_settlement: + needs: [changes, ci_docker] + if: ${{ needs.changes.outputs.capacity_settlement == 'true' }} + uses: ./.github/workflows/ci-capacity-settlement.yml + with: image_tag: ${{ needs.ci_docker.outputs.image_tag }} # @@ -49,6 +55,7 @@ jobs: changes, ci_base, ci_electrical_heating, + ci_capacity_settlement ] if: | always() diff --git a/.github/workflows/detect-changes.yml b/.github/workflows/detect-changes.yml index 1c90b801..e77bfcb0 100644 --- a/.github/workflows/detect-changes.yml +++ b/.github/workflows/detect-changes.yml @@ -11,6 +11,8 @@ on: outputs: electrical_heating: value: ${{ jobs.changes.outputs.electrical_heating }} + capacity_settlement: + value: ${{ jobs.changes.outputs.capacity_settlement }} docker: value: ${{ jobs.changes.outputs.docker }} docker_in_commit: @@ -23,10 +25,11 @@ jobs: # Set job outputs to values from filter step outputs: electrical_heating: ${{ steps.filter.outputs.electrical_heating }} + capacity_settlement: ${{ steps.filter.outputs.capacity_settlement }} docker: ${{ steps.filter.outputs.docker }} docker_in_commit: ${{ steps.docker_changed.outputs.any_changed }} steps: - # For pull requests it's not necessary to checkout the code because GitHub REST API is used to determine changes + # For pull requests it's not necessary to check out the code because GitHub REST API is used to determine changes - name: Checkout repository uses: actions/checkout@v4 with: @@ -41,10 +44,15 @@ jobs: - 'source/electrical_heating/**' - '.github/workflows/ci-electrical-heating.yml' - '.github/workflows/cd.yml' - docker: - - .docker/** - '.devcontainer/**' + capacity_settlement: + - 'source/capacity_settlement/**' + - '.github/workflows/ci-capacity-settlement.yml' - '.github/workflows/cd.yml' + - '.devcontainer/**' + docker: + - .docker/** + - name: Package content or build has changed id: docker_changed diff --git a/source/capacity_settlement/capacity_settlement/__init__.py b/source/capacity_settlement/capacity_settlement/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/source/capacity_settlement/capacity_settlement/entry_points/__init__.py b/source/capacity_settlement/capacity_settlement/entry_points/__init__.py new file mode 100644 index 00000000..fc697160 --- /dev/null +++ b/source/capacity_settlement/capacity_settlement/entry_points/__init__.py @@ -0,0 +1 @@ +from .entry_point import execute diff --git a/source/capacity_settlement/capacity_settlement/entry_points/entry_point.py b/source/capacity_settlement/capacity_settlement/entry_points/entry_point.py new file mode 100644 index 00000000..3c7536f0 --- /dev/null +++ b/source/capacity_settlement/capacity_settlement/entry_points/entry_point.py @@ -0,0 +1,72 @@ +import os +import sys +from argparse import Namespace +from collections.abc import Callable + +import telemetry_logging.logging_configuration as config +from opentelemetry.trace import SpanKind +from telemetry_logging.span_recording import span_record_exception + +from capacity_settlement.entry_points.job_args.capacity_settlement_args import ( + CapacitySettlementArgs, +) +from capacity_settlement.entry_points.job_args.capacity_settlement_job_args import ( + parse_command_line_arguments, + parse_job_arguments, +) + + +def execute() -> None: + applicationinsights_connection_string = os.getenv( + "APPLICATIONINSIGHTS_CONNECTION_STRING" + ) + + start_with_deps( + applicationinsights_connection_string=applicationinsights_connection_string, + ) + + +def start_with_deps( + *, + cloud_role_name: str = "dbr-capacity-settlement", + applicationinsights_connection_string: str | None = None, + parse_command_line_args: Callable[..., Namespace] = parse_command_line_arguments, + parse_job_args: Callable[..., CapacitySettlementArgs] = parse_job_arguments, +) -> None: + """Start overload with explicit dependencies for easier testing.""" + config.configure_logging( + cloud_role_name=cloud_role_name, + tracer_name="capacity-settlement-job", + applicationinsights_connection_string=applicationinsights_connection_string, + extras={"Subsystem": "measurements"}, + ) + + with config.get_tracer().start_as_current_span( + __name__, kind=SpanKind.SERVER + ) as span: + # Try/except added to enable adding custom fields to the exception as + # the span attributes do not appear to be included in the exception. + try: + + # The command line arguments are parsed to have necessary information for + # coming log messages + command_line_args = parse_command_line_args() + + # Add extra to structured logging data to be included in every log message. + config.add_extras( + { + "orchestration-instance-id": command_line_args.orchestration_instance_id, + } + ) + span.set_attributes(config.get_extras()) + parse_job_args(command_line_args) + + # Added as ConfigArgParse uses sys.exit() rather than raising exceptions + except SystemExit as e: + if e.code != 0: + span_record_exception(e, span) + sys.exit(e.code) + + except Exception as e: + span_record_exception(e, span) + sys.exit(4) diff --git a/source/capacity_settlement/capacity_settlement/entry_points/job_args/__init__.py b/source/capacity_settlement/capacity_settlement/entry_points/job_args/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/source/capacity_settlement/capacity_settlement/entry_points/job_args/capacity_settlement_args.py b/source/capacity_settlement/capacity_settlement/entry_points/job_args/capacity_settlement_args.py new file mode 100644 index 00000000..27bcd050 --- /dev/null +++ b/source/capacity_settlement/capacity_settlement/entry_points/job_args/capacity_settlement_args.py @@ -0,0 +1,7 @@ +from dataclasses import dataclass +from uuid import UUID + + +@dataclass +class CapacitySettlementArgs: + orchestration_instance_id: UUID diff --git a/source/capacity_settlement/capacity_settlement/entry_points/job_args/capacity_settlement_job_args.py b/source/capacity_settlement/capacity_settlement/entry_points/job_args/capacity_settlement_job_args.py new file mode 100644 index 00000000..eb3a029f --- /dev/null +++ b/source/capacity_settlement/capacity_settlement/entry_points/job_args/capacity_settlement_job_args.py @@ -0,0 +1,44 @@ +import argparse +import sys +from argparse import Namespace + +import configargparse +from telemetry_logging import Logger, logging_configuration + +from .capacity_settlement_args import CapacitySettlementArgs + + +def parse_command_line_arguments() -> Namespace: + return _parse_args_or_throw(sys.argv[1:]) + + +def parse_job_arguments( + job_args: Namespace, +) -> CapacitySettlementArgs: + logger = Logger(__name__) + logger.info(f"Command line arguments: {repr(job_args)}") + + with logging_configuration.start_span("capacity_settlement.parse_job_arguments"): + + capacity_settlement_args = CapacitySettlementArgs( + orchestration_instance_id=job_args.orchestration_instance_id, + ) + + return capacity_settlement_args + + +def _parse_args_or_throw(command_line_args: list[str]) -> argparse.Namespace: + p = configargparse.ArgParser( + description="Execute capacity settlement calculation", + formatter_class=configargparse.ArgumentDefaultsHelpFormatter, + ) + + # Run parameters + p.add_argument("--orchestration-instance-id", type=str, required=True) + + args, unknown_args = p.parse_known_args(args=command_line_args) + if len(unknown_args): + unknown_args_text = ", ".join(unknown_args) + raise Exception(f"Unknown args: {unknown_args_text}") + + return args diff --git a/source/capacity_settlement/capacity_settlement/entry_points/job_args/environment_variables.py b/source/capacity_settlement/capacity_settlement/entry_points/job_args/environment_variables.py new file mode 100644 index 00000000..5c136203 --- /dev/null +++ b/source/capacity_settlement/capacity_settlement/entry_points/job_args/environment_variables.py @@ -0,0 +1,20 @@ +import os +from enum import Enum +from typing import Any + + +# TODO: Move to shared library +class EnvironmentVariable(Enum): + CATALOG_NAME = "CATALOG_NAME" + + +def get_catalog_name() -> str: + return get_env_variable_or_throw(EnvironmentVariable.CATALOG_NAME) + + +def get_env_variable_or_throw(variable: EnvironmentVariable) -> Any: + env_variable = os.getenv(variable.name) + if env_variable is None: + raise ValueError(f"Environment variable not found: {variable.name}") + + return env_variable diff --git a/source/capacity_settlement/setup.py b/source/capacity_settlement/setup.py new file mode 100644 index 00000000..4c3db136 --- /dev/null +++ b/source/capacity_settlement/setup.py @@ -0,0 +1,26 @@ +from setuptools import setup, find_packages + +setup( + name="opengeh-capacity-settlement", + version=1.0, + description="Tools for capacity settlement", + long_description="", + long_description_content_type="text/markdown", + license="MIT", + packages=find_packages(), + # Make sure these packages are added to the docker container and pinned to the same versions + install_requires=[ + "ConfigArgParse==1.7.0", + "pyspark==3.5.1", + "delta-spark==3.2.0", + "python-dateutil==2.8.2", + "azure-monitor-opentelemetry==1.6.4", + "azure-core==1.32.0", + "opengeh-telemetry @ git+https://git@github.com/Energinet-DataHub/opengeh-python-packages@2.4.1#subdirectory=source/telemetry", + ], + entry_points={ + "console_scripts": [ + "execute = capacity_settlement.entry_points.entry_point:execute", + ] + }, +) diff --git a/source/capacity_settlement/test_common/__init__.py b/source/capacity_settlement/test_common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/source/capacity_settlement/test_common/entry_points/entry_point_test_util.py b/source/capacity_settlement/test_common/entry_points/entry_point_test_util.py new file mode 100644 index 00000000..4df00831 --- /dev/null +++ b/source/capacity_settlement/test_common/entry_points/entry_point_test_util.py @@ -0,0 +1,30 @@ +import importlib.metadata +from typing import Any + + +def assert_entry_point_exists(entry_point_name: str, module: Any) -> None: + try: + # Arrange + entry_point = importlib.metadata.entry_points( + group="console_scripts", name=entry_point_name + ) + + # Check if the entry point exists + if not entry_point: + assert False, f"The {entry_point_name} entry point was not found." + + # Check if the module exists + module_name = entry_point[entry_point_name].module + function_name = entry_point[entry_point_name].value.split(":")[1] + + if not hasattr( + module, + function_name, + ): + assert ( + False + ), f"The entry point module function {function_name} does not exist in the entry points file." + + importlib.import_module(module_name) + except importlib.metadata.PackageNotFoundError: + assert False, f"The {entry_point_name} entry point was not found." diff --git a/source/capacity_settlement/tests/__init__.py b/source/capacity_settlement/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/source/capacity_settlement/tests/capacity_settlement_tests/entry_points/job_args/test_capacity_settlement_job_args.py b/source/capacity_settlement/tests/capacity_settlement_tests/entry_points/job_args/test_capacity_settlement_job_args.py new file mode 100644 index 00000000..2d3d336e --- /dev/null +++ b/source/capacity_settlement/tests/capacity_settlement_tests/entry_points/job_args/test_capacity_settlement_job_args.py @@ -0,0 +1,67 @@ +from unittest.mock import patch + +import pytest + +from capacity_settlement.entry_points.job_args.capacity_settlement_job_args import ( + parse_command_line_arguments, + parse_job_arguments, +) +from capacity_settlement.entry_points.job_args.environment_variables import ( + EnvironmentVariable, +) + +DEFAULT_ORCHESTRATION_INSTANCE_ID = "12345678-9fc8-409a-a169-fbd49479d711" + + +def _get_contract_parameters(filename: str) -> list[str]: + with open(filename) as file: + text = file.read() + text = text.replace( + "{orchestration-instance-id}", DEFAULT_ORCHESTRATION_INSTANCE_ID + ) + lines = text.splitlines() + return list( + filter(lambda line: not line.startswith("#") and len(line) > 0, lines) + ) + + +@pytest.fixture(scope="session") +def contract_parameters(contracts_path: str) -> list[str]: + job_parameters = _get_contract_parameters( + f"{contracts_path}/parameters-reference.txt" + ) + + return job_parameters + + +@pytest.fixture(scope="session") +def sys_argv_from_contract( + contract_parameters: list[str], +) -> list[str]: + return ["dummy_script_name"] + contract_parameters + + +@pytest.fixture(scope="session") +def job_environment_variables() -> dict: + return { + EnvironmentVariable.CATALOG_NAME.name: "some_catalog", + } + + +def test_when_parameters__parses_parameters_from_contract( + job_environment_variables: dict, + sys_argv_from_contract: list[str], +) -> None: + """ + This test ensures that the job accepts + the arguments that are provided by the client. + """ + # Arrange + with patch("sys.argv", sys_argv_from_contract): + with patch.dict("os.environ", job_environment_variables): + command_line_args = parse_command_line_arguments() + # Act + actual_args = parse_job_arguments(command_line_args) + + # Assert + assert actual_args.orchestration_instance_id == DEFAULT_ORCHESTRATION_INSTANCE_ID diff --git a/source/capacity_settlement/tests/capacity_settlement_tests/entry_points/test_entry_points.py b/source/capacity_settlement/tests/capacity_settlement_tests/entry_points/test_entry_points.py new file mode 100644 index 00000000..b06d5abc --- /dev/null +++ b/source/capacity_settlement/tests/capacity_settlement_tests/entry_points/test_entry_points.py @@ -0,0 +1,17 @@ +import pytest + +from capacity_settlement import entry_points as module +from test_common.entry_points.entry_point_test_util import assert_entry_point_exists + + +@pytest.mark.parametrize( + "entry_point_name", + [ + "execute", + ], +) +def test__entry_point_exists( + installed_package: None, + entry_point_name: str, +) -> None: + assert_entry_point_exists(entry_point_name, module) diff --git a/source/capacity_settlement/tests/conftest.py b/source/capacity_settlement/tests/conftest.py new file mode 100644 index 00000000..822ce0cb --- /dev/null +++ b/source/capacity_settlement/tests/conftest.py @@ -0,0 +1,118 @@ +# Copyright 2020 Energinet DataHub A/S +# +# Licensed under the Apache License, Version 2.0 (the "License2"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import subprocess +from typing import Callable, Generator + +import pytest + + +@pytest.fixture(autouse=True) +def configure_dummy_logging() -> None: + """Ensure that logging hooks don't fail due to _TRACER_NAME not being set.""" + + from telemetry_logging.logging_configuration import configure_logging + + configure_logging( + cloud_role_name="any-cloud-role-name", tracer_name="any-tracer-name" + ) + + +@pytest.fixture(scope="session") +def file_path_finder() -> Callable[[str], str]: + """ + Returns the path of the file. + Please note that this only works if current folder haven't been changed prior using + `os.chdir()`. The correctness also relies on the prerequisite that this function is + actually located in a file located directly in the tests folder. + """ + + def finder(file: str) -> str: + return os.path.dirname(os.path.normpath(file)) + + return finder + + +@pytest.fixture(scope="session") +def source_path(file_path_finder: Callable[[str], str]) -> str: + """ + Returns the /source folder path. + Please note that this only works if current folder haven't been changed prior using + `os.chdir()`. The correctness also relies on the prerequisite that this function is + actually located in a file located directly in the tests folder. + """ + return file_path_finder(f"{__file__}/../..") + + +@pytest.fixture(scope="session") +def capacity_settlement_path(source_path: str) -> str: + """ + Returns the source/capacity_settlement/ folder path. + Please note that this only works if current folder haven't been changed prior using + `os.chdir()`. The correctness also relies on the prerequisite that this function is + actually located in a file located directly in the tests folder. + """ + return f"{source_path}/capacity_settlement" + + +@pytest.fixture(scope="session") +def contracts_path(capacity_settlement_path: str) -> str: + """ + Returns the source/contract folder path. + Please note that this only works if current folder haven't been changed prior using + `os.chdir()`. The correctness also relies on the prerequisite that this function is + actually located in a file located directly in the tests folder. + """ + return f"{capacity_settlement_path}/contracts" + + +@pytest.fixture(scope="session") +def virtual_environment() -> Generator: + """Fixture ensuring execution in a virtual environment. + Uses `virtualenv` instead of conda environments due to problems + activating the virtual environment from pytest.""" + + # Create and activate the virtual environment + subprocess.call(["virtualenv", ".test-pytest"]) + subprocess.call( + "source .test-pytest/bin/activate", shell=True, executable="/bin/bash" + ) + + yield None + + # Deactivate virtual environment upon test suite tear down + subprocess.call("deactivate", shell=True, executable="/bin/bash") + + +@pytest.fixture(scope="session") +def installed_package( + virtual_environment: Generator, capacity_settlement_path: str +) -> None: + # Build the package wheel + os.chdir(capacity_settlement_path) + subprocess.call("python -m build --wheel", shell=True, executable="/bin/bash") + + # Uninstall the package in case it was left by a cancelled test suite + subprocess.call( + "pip uninstall -y package", + shell=True, + executable="/bin/bash", + ) + + # Install wheel, which will also create console scripts for invoking the entry points of the package + subprocess.call( + f"pip install {capacity_settlement_path}/dist/opengeh_capacity_settlement-1.0-py3-none-any.whl", + shell=True, + executable="/bin/bash", + ) diff --git a/source/electrical_heating/electrical_heating/entry_points/job_args/environment_variables.py b/source/electrical_heating/electrical_heating/entry_points/job_args/environment_variables.py index 46616f46..70bb07d4 100644 --- a/source/electrical_heating/electrical_heating/entry_points/job_args/environment_variables.py +++ b/source/electrical_heating/electrical_heating/entry_points/job_args/environment_variables.py @@ -1,22 +1,9 @@ -# Copyright 2020 Energinet DataHub A/S -# -# Licensed under the Apache License, Version 2.0 (the "License2"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os +import os from enum import Enum from typing import Any +# TODO: Move to shared library class EnvironmentVariable(Enum): CATALOG_NAME = "CATALOG_NAME" TIME_ZONE = "TIME_ZONE" diff --git a/source/electrical_heating/tests/calculation_scenarios/given_a_smoke_test_scenario/then/electrical_heating_internal/calculations.csv b/source/electrical_heating/tests/calculation_scenarios/given_a_smoke_test_scenario/then/electrical_heating_internal/calculations.csv new file mode 100644 index 00000000..7b81f835 --- /dev/null +++ b/source/electrical_heating/tests/calculation_scenarios/given_a_smoke_test_scenario/then/electrical_heating_internal/calculations.csv @@ -0,0 +1,2 @@ +calculation_id;orchestration_instance_id;execution_start_datetime;execution_stop_datetime +IGNORED;00000000-orch-id00-0000-000000000001;IGNORED;IGNORED diff --git a/source/electrical_heating/tests/calculation_scenarios/given_a_smoke_test_scenario/then/measurements_bronze/measurements_bronze_v1.csv b/source/electrical_heating/tests/calculation_scenarios/given_a_smoke_test_scenario/then/measurements_bronze/measurements_bronze_v1.csv new file mode 100644 index 00000000..5e5300ce --- /dev/null +++ b/source/electrical_heating/tests/calculation_scenarios/given_a_smoke_test_scenario/then/measurements_bronze/measurements_bronze_v1.csv @@ -0,0 +1,2 @@ +orchestration_type;orchestration_instance_id;metering_point_id;transaction_id;transaction_creation_datetime;start_datetime;end_datetime;metering_point_type;product;unit;resolution;points +electrical_heating;00000000-orch-id00-0000-000000000001;140000000000170201;IGNORED;IGNORED;2024-02-01 23:00:00;2024-02-02 23:00:00;electrical_heating;8716867000030;kWh;PT1H;[{'position':1,'quantity':999.123,'quality':'calculated'},{'position':2,'quantity':0.000,'quality':'calculated'},{'position':3,'quantity':0.000,'quality':'calculated'},{'position':4,'quantity':0.000,'quality':'calculated'},{'position':5,'quantity':0.000,'quality':'calculated'},{'position':6,'quantity':0.000,'quality':'calculated'},{'position':7,'quantity':0.000,'quality':'calculated'},{'position':8,'quantity':0.000,'quality':'calculated'},{'position':9,'quantity':0.000,'quality':'calculated'},{'position':10,'quantity':0.000,'quality':'calculated'},{'position':11,'quantity':0.000,'quality':'calculated'},{'position':12,'quantity':0.000,'quality':'calculated'},{'position':13,'quantity':0.000,'quality':'calculated'},{'position':14,'quantity':0.000,'quality':'calculated'},{'position':15,'quantity':0.000,'quality':'calculated'},{'position':16,'quantity':0.000,'quality':'calculated'},{'position':17,'quantity':0.000,'quality':'calculated'},{'position':18,'quantity':0.000,'quality':'calculated'},{'position':19,'quantity':0.000,'quality':'calculated'},{'position':20,'quantity':0.000,'quality':'calculated'},{'position':21,'quantity':0.000,'quality':'calculated'},{'position':22,'quantity':0.000,'quality':'calculated'},{'position':23,'quantity':0.000,'quality':'calculated'},{'position':24,'quantity':0.000,'quality':'calculated'}] diff --git a/source/electrical_heating/tests/calculation_scenarios/given_a_smoke_test_scenario/when/electricity_market__electrical_heating/child_metering_point_periods_v1.csv b/source/electrical_heating/tests/calculation_scenarios/given_a_smoke_test_scenario/when/electricity_market__electrical_heating/child_metering_point_periods_v1.csv new file mode 100644 index 00000000..3df74efd --- /dev/null +++ b/source/electrical_heating/tests/calculation_scenarios/given_a_smoke_test_scenario/when/electricity_market__electrical_heating/child_metering_point_periods_v1.csv @@ -0,0 +1,2 @@ +metering_point_id;metering_point_type;metering_point_sub_type;resolution;parent_metering_point_id;period_from_date;period_to_date +140000000000170201;electrical_heating;calculated;PT1H;170000000000000201;2024-01-01 23:00:00; diff --git a/source/electrical_heating/tests/calculation_scenarios/given_a_smoke_test_scenario/when/electricity_market__electrical_heating/consumption_metering_point_periods_v1.csv b/source/electrical_heating/tests/calculation_scenarios/given_a_smoke_test_scenario/when/electricity_market__electrical_heating/consumption_metering_point_periods_v1.csv new file mode 100644 index 00000000..baadb27b --- /dev/null +++ b/source/electrical_heating/tests/calculation_scenarios/given_a_smoke_test_scenario/when/electricity_market__electrical_heating/consumption_metering_point_periods_v1.csv @@ -0,0 +1,2 @@ +metering_point_id;has_electrical_heating;net_settlement_group;settlement_month;period_from_date;period_to_date +170000000000000201;true;;1;2024-02-01 23:00:00;2024-03-01 23:00:00 diff --git a/source/electrical_heating/tests/calculation_scenarios/given_a_smoke_test_scenario/when/job_parameters.yml b/source/electrical_heating/tests/calculation_scenarios/given_a_smoke_test_scenario/when/job_parameters.yml new file mode 100644 index 00000000..d5209ef2 --- /dev/null +++ b/source/electrical_heating/tests/calculation_scenarios/given_a_smoke_test_scenario/when/job_parameters.yml @@ -0,0 +1 @@ +- orchestration_instance_id: 00000000-orch-id00-0000-000000000001 diff --git a/source/electrical_heating/tests/calculation_scenarios/given_a_smoke_test_scenario/when/measurements_gold/time_series_points_v1.csv b/source/electrical_heating/tests/calculation_scenarios/given_a_smoke_test_scenario/when/measurements_gold/time_series_points_v1.csv new file mode 100644 index 00000000..d4e7fe60 --- /dev/null +++ b/source/electrical_heating/tests/calculation_scenarios/given_a_smoke_test_scenario/when/measurements_gold/time_series_points_v1.csv @@ -0,0 +1,3 @@ +metering_point_id;quantity;observation_time;metering_point_type +170000000000000201;0.123;2024-02-01 23:00:00;consumption +170000000000000201;999.000;2024-02-01 23:15:00;consumption diff --git a/source/electrical_heating/tests/entry_points/job_args/test_electrical_heating_job_args.py b/source/electrical_heating/tests/electrical_heating_tests/entry_points/job_args/test_electrical_heating_job_args.py similarity index 100% rename from source/electrical_heating/tests/entry_points/job_args/test_electrical_heating_job_args.py rename to source/electrical_heating/tests/electrical_heating_tests/entry_points/job_args/test_electrical_heating_job_args.py diff --git a/source/electrical_heating/tests/entry_points/test_entry_points.py b/source/electrical_heating/tests/electrical_heating_tests/entry_points/test_entry_points.py similarity index 100% rename from source/electrical_heating/tests/entry_points/test_entry_points.py rename to source/electrical_heating/tests/electrical_heating_tests/entry_points/test_entry_points.py