Skip to content

Commit

Permalink
Merge branch 'main' into gomes/cache-monthly-amounts
Browse files Browse the repository at this point in the history
  • Loading branch information
JesperGomes committed Nov 28, 2024
2 parents 22f3fa0 + 3ca1d8a commit 9c605a0
Show file tree
Hide file tree
Showing 137 changed files with 148 additions and 158 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,6 @@ class TableColumnNames:
"aggregation_level" # TODO JVM: Remove when only using Unity Catalog
)
amount = "amount"
balance_responsible_id = (
"balance_responsible_id" # TODO JVM: Remove when only using Unity Catalog
)
balance_responsible_party_id = "balance_responsible_party_id"
charge_code = "charge_code"
charge_key = "charge_key"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ def get_metering_point_periods_basis_data(
),
f.col(Colname.energy_supplier_id).alias(TableColumnNames.energy_supplier_id),
f.col(Colname.balance_responsible_party_id).alias(
TableColumnNames.balance_responsible_id
TableColumnNames.balance_responsible_party_id
),
f.col(Colname.from_date).alias(TableColumnNames.from_date),
f.col(Colname.to_date).alias(TableColumnNames.to_date),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@
StructField(TableColumnNames.to_grid_area_code, StringType(), True),
StructField(TableColumnNames.parent_metering_point_id, StringType(), True),
StructField(TableColumnNames.energy_supplier_id, StringType(), True),
StructField(TableColumnNames.balance_responsible_id, StringType(), True),
StructField(TableColumnNames.balance_responsible_party_id, StringType(), True),
StructField(TableColumnNames.from_date, TimestampType(), False),
StructField(TableColumnNames.to_date, TimestampType(), True),
]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,13 @@
from dependency_injector.wiring import inject, Provide
from pyspark.sql import DataFrame
from pyspark.sql.types import StructType
from telemetry_logging import use_span, logging_configuration

import package.databases.wholesale_results_internal.schemas as schemas
from package.calculation.calculation_output import EnergyResultsOutput
from package.codelists import MeteringPointType
from package.container import Container
from package.databases.table_column_names import TableColumnNames
from telemetry_logging import use_span, logging_configuration
from package.infrastructure.infrastructure_settings import InfrastructureSettings
from package.infrastructure.paths import (
WholesaleResultsInternalDatabase,
Expand Down Expand Up @@ -142,17 +142,10 @@ def _write(
return None

# Adjust to match the schema
df = (
df.withColumnRenamed(
TableColumnNames.balance_responsible_id,
TableColumnNames.balance_responsible_party_id,
)
.withColumnRenamed(
TableColumnNames.calculation_result_id,
TableColumnNames.result_id,
)
.select(schema.fieldNames())
)
df = df.withColumnRenamed(
TableColumnNames.calculation_result_id,
TableColumnNames.result_id,
).select(schema.fieldNames())

df.write.format("delta").mode("append").option(
"mergeSchema", "false"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ def _map_to_storage_dataframe(results: DataFrame) -> DataFrame:
f.col(Colname.grid_area_code).alias(TableColumnNames.grid_area_code),
f.col(Colname.energy_supplier_id).alias(TableColumnNames.energy_supplier_id),
f.col(Colname.balance_responsible_party_id).alias(
TableColumnNames.balance_responsible_id
TableColumnNames.balance_responsible_party_id
),
f.col(Colname.quantity)
.alias(TableColumnNames.quantity)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,15 @@

from pyspark.sql import DataFrame, Row, SparkSession

from package.databases.migrations_wholesale.schemas import (
metering_point_periods_schema,
)
from package.codelists import (
MeteringPointType,
SettlementMethod,
MeteringPointResolution,
)
from package.constants import Colname
from package.databases.migrations_wholesale.schemas import (
metering_point_periods_schema,
)

DEFAULT_METERING_POINT_ID = "123456789012345678901234567"
DEFAULT_METERING_POINT_TYPE = MeteringPointType.PRODUCTION
Expand All @@ -36,7 +36,7 @@
DEFAULT_TO_GRID_AREA = None
DEFAULT_PARENT_METERING_POINT_ID = None
DEFAULT_ENERGY_SUPPLIER_ID = "9999999999999"
DEFAULT_BALANCE_RESPONSIBLE_ID = "1234567890123"
DEFAULT_BALANCE_RESPONSIBLE_PARTY_ID = "1234567890123"
DEFAULT_FROM_DATE = datetime(2020, 1, 1, 0, 0)
DEFAULT_TO_DATE = datetime(2020, 2, 1, 0, 0)

Expand All @@ -51,7 +51,7 @@ def create_row(
to_grid_area: str | None = DEFAULT_TO_GRID_AREA,
parent_metering_point_id: str | None = DEFAULT_PARENT_METERING_POINT_ID,
energy_supplier_id: str | None = DEFAULT_ENERGY_SUPPLIER_ID,
balance_responsible_id: str | None = DEFAULT_BALANCE_RESPONSIBLE_ID,
balance_responsible_id: str | None = DEFAULT_BALANCE_RESPONSIBLE_PARTY_ID,
from_date: datetime = DEFAULT_FROM_DATE,
to_date: datetime | None = DEFAULT_TO_DATE,
) -> Row:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ def test__columns_have_expected_values(spark: SparkSession) -> None:
TableColumnNames.to_grid_area_code: metering_point_periods_factory.DEFAULT_TO_GRID_AREA,
TableColumnNames.parent_metering_point_id: metering_point_periods_factory.DEFAULT_PARENT_METERING_POINT_ID,
TableColumnNames.energy_supplier_id: metering_point_periods_factory.DEFAULT_ENERGY_SUPPLIER_ID,
TableColumnNames.balance_responsible_id: metering_point_periods_factory.DEFAULT_BALANCE_RESPONSIBLE_ID,
TableColumnNames.balance_responsible_party_id: metering_point_periods_factory.DEFAULT_BALANCE_RESPONSIBLE_PARTY_ID,
TableColumnNames.from_date: metering_point_periods_factory.DEFAULT_FROM_DATE,
TableColumnNames.to_date: metering_point_periods_factory.DEFAULT_TO_DATE,
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
DEFAULT_FROM_GRID_AREA_CODE = "106"
DEFAULT_TO_GRID_AREA_CODE = "107"
DEFAULT_ENERGY_SUPPLIER_ID = "9876543210123"
DEFAULT_BALANCE_RESPONSIBLE_ID = "1234567890123"
DEFAULT_BALANCE_RESPONSIBLE_PARTY_ID = "1234567890123"
DEFAULT_CALCULATION_TYPE = e.CalculationType.BALANCE_FIXING
DEFAULT_CALCULATION_EXECUTION_START = datetime(2022, 6, 10, 13, 15)
DEFAULT_QUANTITY = "1.1"
Expand Down Expand Up @@ -83,7 +83,7 @@ def _create_result_row(
to_grid_area_code: str = DEFAULT_TO_GRID_AREA_CODE,
from_grid_area_code: str = DEFAULT_FROM_GRID_AREA_CODE,
energy_supplier_id: str = DEFAULT_ENERGY_SUPPLIER_ID,
balance_responsible_id: str = DEFAULT_BALANCE_RESPONSIBLE_ID,
balance_responsible_id: str = DEFAULT_BALANCE_RESPONSIBLE_PARTY_ID,
quantity: str = DEFAULT_QUANTITY,
quality: e.QuantityQuality = DEFAULT_QUALITY,
observation_time: datetime = DEFAULT_OBSERVATION_TIME,
Expand Down Expand Up @@ -183,8 +183,8 @@ def test__create__with_correct_aggregation_level(
(TableColumnNames.grid_area_code, DEFAULT_GRID_AREA_CODE),
(TableColumnNames.neighbor_grid_area_code, DEFAULT_FROM_GRID_AREA_CODE),
(
TableColumnNames.balance_responsible_id,
DEFAULT_BALANCE_RESPONSIBLE_ID,
TableColumnNames.balance_responsible_party_id,
DEFAULT_BALANCE_RESPONSIBLE_PARTY_ID,
),
(TableColumnNames.energy_supplier_id, DEFAULT_ENERGY_SUPPLIER_ID),
(TableColumnNames.time, datetime(2020, 1, 1, 0, 0)),
Expand Down Expand Up @@ -253,7 +253,7 @@ def test__create__with_correct_number_of_calculation_result_ids(
),
(
Colname.balance_responsible_party_id,
DEFAULT_BALANCE_RESPONSIBLE_ID,
DEFAULT_BALANCE_RESPONSIBLE_PARTY_ID,
OTHER_BALANCE_RESPONSIBLE_ID,
),
(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
calculation_id;metering_point_id;metering_point_type;settlement_method;grid_area_code;resolution;from_grid_area_code;to_grid_area_code;parent_metering_point_id;energy_supplier_id;balance_responsible_id;from_date;to_date
calculation_id;metering_point_id;metering_point_type;settlement_method;grid_area_code;resolution;from_grid_area_code;to_grid_area_code;parent_metering_point_id;energy_supplier_id;balance_responsible_party_id;from_date;to_date
0b15a420-9fc8-409a-a169-fbd49479d718;170000001500001000;consumption;flex;800;PT15M;;;;1000000000000;1100000000000;2023-07-02 02:00;2023-07-02 03:00
0b15a420-9fc8-409a-a169-fbd49479d718;180000001500001000;production;;800;PT15M;;;;3000000000000;2200000000000;2023-07-02 02:00;2023-07-02 03:00
0b15a420-9fc8-409a-a169-fbd49479d718;200000001500000001;exchange;;801;PT15M;801;800;;;;2023-07-02 02:00;2023-07-02 03:00
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
calculation_id;metering_point_id;metering_point_type;settlement_method;grid_area_code;resolution;from_grid_area_code;to_grid_area_code;parent_metering_point_id;energy_supplier_id;balance_responsible_id;from_date;to_date
calculation_id;metering_point_id;metering_point_type;settlement_method;grid_area_code;resolution;from_grid_area_code;to_grid_area_code;parent_metering_point_id;energy_supplier_id;balance_responsible_party_id;from_date;to_date
0b15a420-9fc8-409a-a169-fbd49479d718;180000000150001000;production;;800;PT1H;;;;2000000000000;2200000000000;2023-02-01 00:00:00;2023-02-01 01:00:00
0b15a420-9fc8-409a-a169-fbd49479d718;170000000150001000;consumption;flex;800;PT1H;;;;1000000000000;1100000000000;2023-02-01 00:00:00;2023-02-01 01:00:00
0b15a420-9fc8-409a-a169-fbd49479d718;170000000150000101;consumption;non_profiled;800;PT15M;;;;1000000000000;1100000000000;2023-02-01 00:00:00;2023-02-01 01:00:00
Expand Down
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
grid_area_code;energy_supplier_id;balance_responsible_id;quantity;quantity_qualities;time;aggregation_level;time_series_type;calculation_id;calculation_type;calculation_execution_time_start;neighbor_grid_area_code;calculation_result_id;metering_point_id;resolution
grid_area_code;energy_supplier_id;balance_responsible_party_id;quantity;quantity_qualities;time;aggregation_level;time_series_type;calculation_id;calculation_type;calculation_execution_time_start;neighbor_grid_area_code;calculation_result_id;metering_point_id;resolution
800;;;-20.000;['measured'];2023-02-01 00:00:00;total_ga;net_exchange_per_ga;0b15a420-9fc8-409a-a169-fbd49479d718;balance_fixing;2024-01-01 23:00:00;;7794ac83-9d06-5066-aff5-b6cd097b8708;;PT1H
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
grid_area_code;energy_supplier_id;balance_responsible_id;quantity;quantity_qualities;time;aggregation_level;time_series_type;calculation_id;calculation_type;calculation_execution_time_start;neighbor_grid_area_code;calculation_result_id;metering_point_id;resolution
grid_area_code;energy_supplier_id;balance_responsible_party_id;quantity;quantity_qualities;time;aggregation_level;time_series_type;calculation_id;calculation_type;calculation_execution_time_start;neighbor_grid_area_code;calculation_result_id;metering_point_id;resolution
800;;;-100.000;['measured'];2023-02-01 00:00:00;total_ga;net_exchange_per_neighboring_ga;0b15a420-9fc8-409a-a169-fbd49479d718;balance_fixing;2024-01-01 23:00:00;801;b908c3f2-265f-5693-9155-ddeb52535da0;;PT1H
800;;;80.000;['measured'];2023-02-01 00:00:00;total_ga;net_exchange_per_neighboring_ga;0b15a420-9fc8-409a-a169-fbd49479d718;balance_fixing;2024-01-01 23:00:00;802;6b403366-edf3-5923-80d8-20cc1a3ee491;;PT1H
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
grid_area_code;energy_supplier_id;balance_responsible_id;quantity;quantity_qualities;time;aggregation_level;time_series_type;calculation_id;calculation_type;calculation_execution_time_start;neighbor_grid_area_code;calculation_result_id;metering_point_id;resolution
grid_area_code;energy_supplier_id;balance_responsible_party_id;quantity;quantity_qualities;time;aggregation_level;time_series_type;calculation_id;calculation_type;calculation_execution_time_start;neighbor_grid_area_code;calculation_result_id;metering_point_id;resolution
800;1000000000000;1100000000000;21.000;['measured', 'calculated'];2023-02-01 00:00:00;es_brp_ga;flex_consumption;0b15a420-9fc8-409a-a169-fbd49479d718;balance_fixing;2024-01-01 23:00:00;;9f454f37-de20-560e-a9a9-8d1b2c50b89a;;PT1H
800;3000000000000;3300000000000;30.000;['measured'];2023-02-01 00:00:00;es_brp_ga;flex_consumption;0b15a420-9fc8-409a-a169-fbd49479d718;balance_fixing;2024-01-01 23:00:00;;d92187a4-46ca-5355-8a1d-36f667b8d754;;PT1H
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
grid_area_code;energy_supplier_id;balance_responsible_id;quantity;quantity_qualities;time;aggregation_level;time_series_type;calculation_id;calculation_type;calculation_execution_time_start;neighbor_grid_area_code;calculation_result_id;metering_point_id;resolution
grid_area_code;energy_supplier_id;balance_responsible_party_id;quantity;quantity_qualities;time;aggregation_level;time_series_type;calculation_id;calculation_type;calculation_execution_time_start;neighbor_grid_area_code;calculation_result_id;metering_point_id;resolution
800;;;1.000;['calculated'];2023-02-01 00:00:00;total_ga;grid_loss;0b15a420-9fc8-409a-a169-fbd49479d718;balance_fixing;2024-01-01 23:00:00;;7233966c-f4ab-5207-8bdc-79fe6a82ef4f;;PT1H
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
grid_area_code;energy_supplier_id;balance_responsible_id;quantity;quantity_qualities;time;aggregation_level;time_series_type;calculation_id;calculation_type;calculation_execution_time_start;neighbor_grid_area_code;calculation_result_id;metering_point_id;resolution
grid_area_code;energy_supplier_id;balance_responsible_party_id;quantity;quantity_qualities;time;aggregation_level;time_series_type;calculation_id;calculation_type;calculation_execution_time_start;neighbor_grid_area_code;calculation_result_id;metering_point_id;resolution
800;2000000000000;2200000000000;0.000;['calculated'];2023-02-01 00:00:00;total_ga;negative_grid_loss;0b15a420-9fc8-409a-a169-fbd49479d718;balance_fixing;2024-01-01 23:00:00;;71eb3492-f6fb-57f0-a2c2-5fd509e3acad;180000000150001000;PT1H
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
grid_area_code;energy_supplier_id;balance_responsible_id;quantity;quantity_qualities;time;aggregation_level;time_series_type;calculation_id;calculation_type;calculation_execution_time_start;neighbor_grid_area_code;calculation_result_id;metering_point_id;resolution
grid_area_code;energy_supplier_id;balance_responsible_party_id;quantity;quantity_qualities;time;aggregation_level;time_series_type;calculation_id;calculation_type;calculation_execution_time_start;neighbor_grid_area_code;calculation_result_id;metering_point_id;resolution
800;1000000000000;1100000000000;12.000;['measured'];2023-02-01 00:00:00;es_brp_ga;non_profiled_consumption;0b15a420-9fc8-409a-a169-fbd49479d718;balance_fixing;2024-01-01 23:00:00;;149f7825-ae6f-51c5-86af-1cf9a4ba9e68;;PT1H
800;3000000000000;3300000000000;10.000;['measured'];2023-02-01 00:00:00;es_brp_ga;non_profiled_consumption;0b15a420-9fc8-409a-a169-fbd49479d718;balance_fixing;2024-01-01 23:00:00;;ac358874-06e0-5a13-89d3-0b16695b09a8;;PT1H
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
grid_area_code;energy_supplier_id;balance_responsible_id;quantity;quantity_qualities;time;aggregation_level;time_series_type;calculation_id;calculation_type;calculation_execution_time_start;neighbor_grid_area_code;calculation_result_id;metering_point_id;resolution
grid_area_code;energy_supplier_id;balance_responsible_party_id;quantity;quantity_qualities;time;aggregation_level;time_series_type;calculation_id;calculation_type;calculation_execution_time_start;neighbor_grid_area_code;calculation_result_id;metering_point_id;resolution
800;1000000000000;1100000000000;1.000;['calculated'];2023-02-01 00:00:00;total_ga;positive_grid_loss;0b15a420-9fc8-409a-a169-fbd49479d718;balance_fixing;2024-01-01 23:00:00;;a27b5f41-7715-5e20-a403-93e90e4bf638;170000000150001000;PT1H
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
grid_area_code;energy_supplier_id;balance_responsible_id;quantity;quantity_qualities;time;aggregation_level;time_series_type;calculation_id;calculation_type;calculation_execution_time_start;neighbor_grid_area_code;calculation_result_id;metering_point_id;resolution
grid_area_code;energy_supplier_id;balance_responsible_party_id;quantity;quantity_qualities;time;aggregation_level;time_series_type;calculation_id;calculation_type;calculation_execution_time_start;neighbor_grid_area_code;calculation_result_id;metering_point_id;resolution
800;2000000000000;2200000000000;60.000;['measured', 'calculated'];2023-02-01 00:00:00;es_brp_ga;production;0b15a420-9fc8-409a-a169-fbd49479d718;balance_fixing;2024-01-01 23:00:00;;54b49055-8da5-526f-bd70-421844ab4ec7;;PT1H
800;3000000000000;3300000000000;33.000;['measured'];2023-02-01 00:00:00;es_brp_ga;production;0b15a420-9fc8-409a-a169-fbd49479d718;balance_fixing;2024-01-01 23:00:00;;ec0f7aa5-4857-506b-8c75-914ad90b24b7;;PT1H
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
grid_area_code;energy_supplier_id;balance_responsible_id;quantity;quantity_qualities;time;aggregation_level;time_series_type;calculation_id;calculation_type;calculation_execution_time_start;neighbor_grid_area_code;calculation_result_id;metering_point_id;resolution
grid_area_code;energy_supplier_id;balance_responsible_party_id;quantity;quantity_qualities;time;aggregation_level;time_series_type;calculation_id;calculation_type;calculation_execution_time_start;neighbor_grid_area_code;calculation_result_id;metering_point_id;resolution
800;;;50.000;['measured'];2023-02-01 00:00:00;total_ga;temp_flex_consumption;0b15a420-9fc8-409a-a169-fbd49479d718;balance_fixing;2024-01-01 23:00:00;;c86a1372-b525-5bde-9b00-ed85c133ebee;;PT1H
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
grid_area_code;energy_supplier_id;balance_responsible_id;quantity;quantity_qualities;time;aggregation_level;time_series_type;calculation_id;calculation_type;calculation_execution_time_start;neighbor_grid_area_code;calculation_result_id;metering_point_id;resolution
grid_area_code;energy_supplier_id;balance_responsible_party_id;quantity;quantity_qualities;time;aggregation_level;time_series_type;calculation_id;calculation_type;calculation_execution_time_start;neighbor_grid_area_code;calculation_result_id;metering_point_id;resolution
800;;;93.000;['measured'];2023-02-01 00:00:00;total_ga;temp_production;0b15a420-9fc8-409a-a169-fbd49479d718;balance_fixing;2024-01-01 23:00:00;;1ca3981e-a922-507a-8be9-8c285f947f49;;PT1H
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
grid_area_code;energy_supplier_id;balance_responsible_id;quantity;quantity_qualities;time;aggregation_level;time_series_type;calculation_id;calculation_type;calculation_execution_time_start;neighbor_grid_area_code;calculation_result_id;metering_point_id;resolution
grid_area_code;energy_supplier_id;balance_responsible_party_id;quantity;quantity_qualities;time;aggregation_level;time_series_type;calculation_id;calculation_type;calculation_execution_time_start;neighbor_grid_area_code;calculation_result_id;metering_point_id;resolution
800;;;73.000;['measured', 'calculated'];2023-02-01 00:00:00;total_ga;total_consumption;0b15a420-9fc8-409a-a169-fbd49479d718;balance_fixing;2024-01-01 23:00:00;;8df96ccd-f4be-52c9-87b6-e0f09dab1501;;PT1H
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
calculation_id;metering_point_id;metering_point_type;settlement_method;grid_area_code;resolution;from_grid_area_code;to_grid_area_code;parent_metering_point_id;energy_supplier_id;balance_responsible_id;from_date;to_date
calculation_id;metering_point_id;metering_point_type;settlement_method;grid_area_code;resolution;from_grid_area_code;to_grid_area_code;parent_metering_point_id;energy_supplier_id;balance_responsible_party_id;from_date;to_date
0b15a420-9fc8-409a-a169-fbd49479d718;170000001500001000;consumption;flex;800;PT15M;;;;1000000000000;1100000000000;2023-07-02 02:00:00;2023-07-02 03:00:00
0b15a420-9fc8-409a-a169-fbd49479d718;180000001500001000;production;;800;PT15M;;;;3000000000000;2200000000000;2023-07-02 02:00:00;2023-07-02 03:00:00
0b15a420-9fc8-409a-a169-fbd49479d718;200000001500000001;exchange;;801;PT15M;801;800;;;;2023-07-02 02:00:00;2023-07-02 03:00:00
Expand Down
Loading

0 comments on commit 9c605a0

Please sign in to comment.