Merge branch 'main' into slb/MigrationScriptChanges

.devcontainer/docker-compose-windows.yml

@@ -5,9 +5,10 @@ services:
       # Forwards the local Docker socket to the container.
       - /var/run/docker.sock:/var/run/docker-host.sock
       # Update this to wherever you want VS Code to mount the folder of your project
-      - ..:/workspaces/opengeh-wholesale:cached
+      - ..:${HOME}/work/opengeh-wholesale:cached
       # Map to Azure CLI token cache location (on Windows)
-      - "${USERPROFILE}/.azure:/home/joyvan/.azure"
+      - "${USERPROFILE}/.azure:/home/jovyan/.azure"
+    working_dir: ${HOME}/work/opengeh-wholesale
     environment:
       # Pass the environment variables from your shell straight through to your containers.
       # No warning is issued if the variable in the shell environment is not set.

.devcontainer/docker-compose.yml

@@ -5,9 +5,10 @@ services:
       # Forwards the local Docker socket to the container.
       - /var/run/docker.sock:/var/run/docker-host.sock
       # Update this to wherever you want VS Code to mount the folder of your project
-      - ..:/workspaces/opengeh-wholesale:cached
+      - ..:${HOME}/work/opengeh-wholesale:cached
       # Map to Azure CLI token cache location (on Linux)
-      - "${HOME}/.azure:/home/joyvan/.azure"
+      - "${HOME}/.azure:/home/jovyan/.azure"
+    working_dir: ${HOME}/work/opengeh-wholesale
     environment:
       # Pass the environment variables from your shell straight through to your containers.
       # No warning is issued if the variable in the shell environment is not set.

.docker/Dockerfile

@@ -51,6 +51,9 @@ ENV PATH=$SPARK_HOME/bin:$HADOOP_HOME/bin:$PATH \
 # Dynamically downloading spark dependencies from conf/spark-defaults.conf. This is done to save time in the build pipeline so that we don't need to download on every build.
 RUN spark-shell
 
-# Make $HOME owned by the root, which is the user used in the container
-# This is needed for e.g. commands that create files or folders in $HOME
-RUN sudo chown -R root:users $HOME
+# Change the username from jovyan to nayvoj and update the user id to 1001, as that is the user id used in the GitHub Actions runner
+RUN usermod -l nayvoj jovyan && \
+    usermod -u 1001 nayvoj && \
+    usermod -l jovyan nayvoj
+
+RUN usermod -u 1001 jovyan

.docker/entrypoint.sh

@@ -1,4 +1,4 @@
-#!/bin/bash -l
+#!/bin/bash
 
 # Copyright 2020 Energinet DataHub A/S
 #
@@ -21,7 +21,7 @@ echo "Tests folder path: '$1'"
 echo "Filter (paths): '$2'"
 
 # Configure Azure CLI to use token cache which must be mapped as volume from host machine
-export AZURE_CONFIG_DIR=/home/joyvan/.azure
+export AZURE_CONFIG_DIR=/home/jovyan/.azure
 
 # There env vars are important to ensure that the driver and worker nodes in spark are alligned
 export PYSPARK_PYTHON=/opt/conda/bin/python

.github/workflows/ci-databricks.yml

@@ -229,11 +229,6 @@ jobs:
         shell: bash
         id: test_count
         run: |
-          # Small hack to get the repository name
-          repository=${{ github.repository }}
-          repository_owner=${{ github.repository_owner }}
-          repository_name=${repository/$repository_owner\//}
-
           # IMPORTANT: When adding a new folder here, one must also add the folder
           # to one of the test jobs above! This is because this filter contains the sum of all folders
           # from test jobs.
@@ -255,7 +250,7 @@ jobs:
           optimize_job/
           datamigration/unity_catalog"
 
-          IMAGE_TAG=${{ inputs.image_tag }} docker compose -f .devcontainer/docker-compose.yml run --rm -u root -w //workspaces/${repository_name} python-unit-test ./.devcontainer/check_test_count.sh $filter
+          IMAGE_TAG=${{ inputs.image_tag }} docker compose -f .devcontainer/docker-compose.yml run --rm -u root python-unit-test ./.devcontainer/check_test_count.sh $filter
 
   calculator_job_mypy_check:
     if: ${{ inputs.has_calculator_job_changes }}

source/databricks/calculation_engine/contracts/data_products/wholesale_basis_data/metering_point_periods_v1.py

@@ -21,7 +21,7 @@
         # 've_production' | 'net_production' | 'supply_to_grid' |
         # 'consumption_from_grid' | 'wholesale_services_information' |
         # 'own_production' | 'net_from_grid' 'net_to_grid' | 'total_consumption' |
-        # 'electrical_heating' | 'net_consumption' | 'effect_settlement'
+        # 'electrical_heating' | 'net_consumption' | 'capacity_settlement'
         t.StructField("metering_point_type", t.StringType(), not nullable),
         #
         # 'non_profiled' | 'flex'

source/databricks/calculation_engine/contracts/data_products/wholesale_basis_data/time_series_points_v1.py

@@ -32,7 +32,7 @@
         # When wholesale calculations types also:
         # 've_production' | 'net_production' | 'supply_to_grid' 'consumption_from_grid' |
         # 'wholesale_services_information' | 'own_production' | 'net_from_grid' 'net_to_grid' |
-        # 'total_consumption' | 'electrical_heating' | 'net_consumption' | 'effect_settlement'
+        # 'total_consumption' | 'electrical_heating' | 'net_consumption' | 'capacity_settlement'
         t.StructField("metering_point_type", t.StringType(), not nullable),
         #
         # 'PT15M' | 'PT1H'

source/databricks/calculation_engine/contracts/data_products/wholesale_results/amounts_per_charge_v1.py

@@ -38,7 +38,7 @@
         # 'production' | 'consumption' | 've_production' | 'net_production' |
         # 'supply_to_grid' | 'consumption_from_grid' | 'wholesale_services_information' |
         # 'own_production' | 'net_from_grid' 'net_to_grid' | 'total_consumption' |
-        # 'electrical_heating' | 'net_consumption' | 'effect_settlement'
+        # 'electrical_heating' | 'net_consumption' | 'capacity_settlement'
         t.StructField("metering_point_type", t.StringType(), not nullable),
         #
         # 'flex' | 'non_profiled'

source/databricks/calculation_engine/contracts/data_products/wholesale_sap/amounts_per_charge_v1.py

@@ -38,7 +38,7 @@
         # 'production' | 'consumption' | 've_production' | 'net_production' |
         # 'supply_to_grid' | 'consumption_from_grid' | 'wholesale_services_information' |
         # 'own_production' | 'net_from_grid' 'net_to_grid' | 'total_consumption' |
-        # 'electrical_heating' | 'net_consumption' | 'effect_settlement'
+        # 'electrical_heating' | 'net_consumption' | 'capacity_settlement'
         t.StructField("metering_point_type", t.StringType(), not nullable),
         #
         # 'flex' | 'non_profiled'

source/databricks/calculation_engine/contracts/data_products/wholesale_settlement_reports/amounts_per_charge_v1.py

@@ -32,7 +32,7 @@
         # 'net_production' | 'supply_to_grid' 'consumption_from_grid' |
         # 'wholesale_services_information' | 'own_production' | 'net_from_grid' |
         # 'net_to_grid' | 'total_consumption' | 'electrical_heating' |
-        # 'net_consumption' | 'effect_settlement'
+        # 'net_consumption' | 'capacity_settlement'
         t.StructField("metering_point_type", t.StringType(), not nullable),
         #
         # 'flex' | 'non_profiled'

source/databricks/calculation_engine/contracts/data_products/wholesale_settlement_reports/charge_link_periods_v1.py

@@ -20,7 +20,7 @@
         # 'net_production' | 'supply_to_grid' 'consumption_from_grid' |
         # 'wholesale_services_information' | 'own_production' | 'net_from_grid' |
         # 'net_to_grid' | 'total_consumption' | 'electrical_heating' |
-        # 'net_consumption' | 'effect_settlement'
+        # 'net_consumption' | 'capacity_settlement'
         t.StructField("metering_point_type", t.StringType(), not nullable),
         #
         # 'subscription' | 'fee' | 'tariff'

source/databricks/calculation_engine/contracts/data_products/wholesale_settlement_reports/metering_point_periods_v1.py

@@ -33,7 +33,7 @@
         # 've_production' | 'net_production' | 'supply_to_grid' |
         # 'consumption_from_grid' | 'wholesale_services_information' |
         # 'own_production' | 'net_from_grid' 'net_to_grid' | 'total_consumption' |
-        # 'electrical_heating' | 'net_consumption' | 'effect_settlement'
+        # 'electrical_heating' | 'net_consumption' | 'capacity_settlement'
         t.StructField("metering_point_type", t.StringType(), not nullable),
         #
         # 'non_profiled' | 'flex'

source/databricks/calculation_engine/contracts/data_products/wholesale_settlement_reports/metering_point_time_series_v1.py

@@ -21,7 +21,7 @@
         # When wholesale calculations types also:
         # 've_production' | 'net_production' | 'supply_to_grid' 'consumption_from_grid' |
         # 'wholesale_services_information' | 'own_production' | 'net_from_grid' 'net_to_grid' |
-        # 'total_consumption' | 'electrical_heating' | 'net_consumption' | 'effect_settlement'
+        # 'total_consumption' | 'electrical_heating' | 'net_consumption' | 'capacity_settlement'
         t.StructField("metering_point_type", t.StringType(), not nullable),
         #
         # 'PT15M' | 'PT1H'

source/databricks/calculation_engine/package/calculation/preparation/transformations/metering_point_periods.py

@@ -151,8 +151,8 @@ def _fix_metering_point_type(df: DataFrame) -> DataFrame:
         )
         .when(
             col(Colname.metering_point_type)
-            == InputMeteringPointType.EFFECT_SETTLEMENT.value,
-            lit(MeteringPointType.EFFECT_SETTLEMENT.value),
+            == InputMeteringPointType.CAPACITY_SETTLEMENT.value,
+            lit(MeteringPointType.CAPACITY_SETTLEMENT.value),
         )
         .otherwise(lit("Unknown type")),
         # The otherwise is to avoid changing the nullability of the column.

source/databricks/calculation_engine/package/codelists/input_metering_point_type.py

@@ -32,4 +32,4 @@ class InputMeteringPointType(Enum):
     TOTAL_CONSUMPTION = "D12"
     ELECTRICAL_HEATING = "D14"
     NET_CONSUMPTION = "D15"
-    EFFECT_SETTLEMENT = "D19"
+    CAPACITY_SETTLEMENT = "D19"

source/databricks/calculation_engine/package/codelists/metering_point_type.py

@@ -31,4 +31,4 @@ class MeteringPointType(Enum):
     TOTAL_CONSUMPTION = "total_consumption"
     ELECTRICAL_HEATING = "electrical_heating"
     NET_CONSUMPTION = "net_consumption"
-    EFFECT_SETTLEMENT = "effect_settlement"
+    CAPACITY_SETTLEMENT = "capacity_settlement"

source/databricks/calculation_engine/package/datamigration/migration_scripts/202412161500_alter_tables__metering_point_type_constraints.sql

@@ -0,0 +1,26 @@
+ALTER TABLE {CATALOG_NAME}.{WHOLESALE_BASIS_DATA_INTERNAL_DATABASE_NAME}.time_series_points
+DROP CONSTRAINT IF EXISTS metering_point_type_chk
+GO
+
+ALTER TABLE {CATALOG_NAME}.{WHOLESALE_BASIS_DATA_INTERNAL_DATABASE_NAME}.time_series_points
+ADD CONSTRAINT metering_point_type_chk
+    CHECK (metering_point_type IN ( 'production' , 'consumption' , 'exchange' , 've_production' , 'net_production' , 'supply_to_grid' , 'consumption_from_grid' , 'wholesale_services_information' , 'own_production' , 'net_from_grid' , 'net_to_grid' , 'total_consumption' , 'electrical_heating' , 'net_consumption' , 'effect_settlement', 'capacity_settlement' ))
+GO
+
+ALTER TABLE {CATALOG_NAME}.{WHOLESALE_RESULTS_INTERNAL_DATABASE_NAME}.amounts_per_charge
+DROP CONSTRAINT IF EXISTS metering_point_type_chk
+GO
+
+ALTER TABLE {CATALOG_NAME}.{WHOLESALE_RESULTS_INTERNAL_DATABASE_NAME}.amounts_per_charge
+ADD CONSTRAINT metering_point_type_chk
+    CHECK (metering_point_type IN ( 'production' , 'consumption' , 'exchange' , 've_production' , 'net_production' , 'supply_to_grid' , 'consumption_from_grid' , 'wholesale_services_information' , 'own_production' , 'net_from_grid' , 'net_to_grid' , 'total_consumption' , 'electrical_heating' , 'net_consumption' , 'effect_settlement', 'capacity_settlement' ))
+GO
+
+ALTER TABLE {CATALOG_NAME}.{WHOLESALE_BASIS_DATA_INTERNAL_DATABASE_NAME}.metering_point_periods
+DROP CONSTRAINT IF EXISTS metering_point_type_chk
+GO
+
+ALTER TABLE {CATALOG_NAME}.{WHOLESALE_BASIS_DATA_INTERNAL_DATABASE_NAME}.metering_point_periods
+ADD CONSTRAINT metering_point_type_chk
+    CHECK (metering_point_type IN ( 'production' , 'consumption' , 'exchange' , 've_production' , 'net_production' , 'supply_to_grid' , 'consumption_from_grid' , 'wholesale_services_information' , 'own_production' , 'net_from_grid' , 'net_to_grid' , 'total_consumption' , 'electrical_heating' , 'net_consumption' , 'effect_settlement', 'capacity_settlement' ))
+GO

source/databricks/calculation_engine/package/datamigration/migration_scripts/202412170800_update_tables__metering_point_type_values.sql

@@ -0,0 +1,14 @@
+UPDATE {CATALOG_NAME}.{WHOLESALE_BASIS_DATA_INTERNAL_DATABASE_NAME}.time_series_points
+SET metering_point_type = 'capacity_settlement'
+WHERE metering_point_type = 'effect_settlement';
+GO
+
+UPDATE {CATALOG_NAME}.{WHOLESALE_RESULTS_INTERNAL_DATABASE_NAME}.amounts_per_charge
+SET metering_point_type = 'capacity_settlement'
+WHERE metering_point_type = 'effect_settlement';
+GO
+
+UPDATE {CATALOG_NAME}.{WHOLESALE_BASIS_DATA_INTERNAL_DATABASE_NAME}.metering_point_periods
+SET metering_point_type = 'capacity_settlement'
+WHERE metering_point_type = 'effect_settlement';
+GO

source/databricks/calculation_engine/package/datamigration/migration_scripts/202412170820_alter_tables__metering_point_type_constraints.sql

@@ -0,0 +1,26 @@
+ALTER TABLE {CATALOG_NAME}.{WHOLESALE_BASIS_DATA_INTERNAL_DATABASE_NAME}.time_series_points
+DROP CONSTRAINT IF EXISTS metering_point_type_chk
+GO
+
+ALTER TABLE {CATALOG_NAME}.{WHOLESALE_BASIS_DATA_INTERNAL_DATABASE_NAME}.time_series_points
+ADD CONSTRAINT metering_point_type_chk
+    CHECK (metering_point_type IN ( 'production' , 'consumption' , 'exchange' , 've_production' , 'net_production' , 'supply_to_grid' , 'consumption_from_grid' , 'wholesale_services_information' , 'own_production' , 'net_from_grid' , 'net_to_grid' , 'total_consumption' , 'electrical_heating' , 'net_consumption' , 'capacity_settlement' ))
+GO
+
+ALTER TABLE {CATALOG_NAME}.{WHOLESALE_RESULTS_INTERNAL_DATABASE_NAME}.amounts_per_charge
+DROP CONSTRAINT IF EXISTS metering_point_type_chk
+GO
+
+ALTER TABLE {CATALOG_NAME}.{WHOLESALE_RESULTS_INTERNAL_DATABASE_NAME}.amounts_per_charge
+ADD CONSTRAINT metering_point_type_chk
+    CHECK (metering_point_type IN ( 'production' , 'consumption' , 'exchange' , 've_production' , 'net_production' , 'supply_to_grid' , 'consumption_from_grid' , 'wholesale_services_information' , 'own_production' , 'net_from_grid' , 'net_to_grid' , 'total_consumption' , 'electrical_heating' , 'net_consumption' , 'capacity_settlement' ))
+GO
+
+ALTER TABLE {CATALOG_NAME}.{WHOLESALE_BASIS_DATA_INTERNAL_DATABASE_NAME}.metering_point_periods
+DROP CONSTRAINT IF EXISTS metering_point_type_chk
+GO
+
+ALTER TABLE {CATALOG_NAME}.{WHOLESALE_BASIS_DATA_INTERNAL_DATABASE_NAME}.metering_point_periods
+ADD CONSTRAINT metering_point_type_chk
+    CHECK (metering_point_type IN ( 'production' , 'consumption' , 'exchange' , 've_production' , 'net_production' , 'supply_to_grid' , 'consumption_from_grid' , 'wholesale_services_information' , 'own_production' , 'net_from_grid' , 'net_to_grid' , 'total_consumption' , 'electrical_heating' , 'net_consumption' , 'capacity_settlement' ))
+GO

source/databricks/calculation_engine/tests/calculation/preparation/transformations/test_get_metering_point_periods_df.py

@@ -92,8 +92,8 @@ class TestWhenValidInput:
             ],
             [InputMeteringPointType.NET_CONSUMPTION, MeteringPointType.NET_CONSUMPTION],
             [
-                InputMeteringPointType.EFFECT_SETTLEMENT,
-                MeteringPointType.EFFECT_SETTLEMENT,
+                InputMeteringPointType.CAPACITY_SETTLEMENT,
+                MeteringPointType.CAPACITY_SETTLEMENT,
             ],
         ],
     )

source/databricks/settlement_report/settlement_report_job/domain/utils/map_to_csv_naming.py

@@ -33,7 +33,7 @@
     MeteringPointTypeDataProductValue.TOTAL_CONSUMPTION.value: "D12",
     MeteringPointTypeDataProductValue.ELECTRICAL_HEATING.value: "D14",
     MeteringPointTypeDataProductValue.NET_CONSUMPTION.value: "D15",
-    MeteringPointTypeDataProductValue.EFFECT_SETTLEMENT.value: "D19",
+    MeteringPointTypeDataProductValue.CAPACITY_SETTLEMENT.value: "D19",
     MeteringPointTypeDataProductValue.CONSUMPTION.value: "E17",
     MeteringPointTypeDataProductValue.PRODUCTION.value: "E18",
     MeteringPointTypeDataProductValue.EXCHANGE.value: "E20",

source/databricks/settlement_report/settlement_report_job/infrastructure/wholesale/data_values/metering_point_type.py

@@ -30,4 +30,4 @@ class MeteringPointTypeDataProductValue(Enum):
     TOTAL_CONSUMPTION = "total_consumption"
     ELECTRICAL_HEATING = "electrical_heating"
     NET_CONSUMPTION = "net_consumption"
-    EFFECT_SETTLEMENT = "effect_settlement"
+    CAPACITY_SETTLEMENT = "capacity_settlement"

source/databricks/settlement_report/settlement_report_job/infrastructure/wholesale/schemas/metering_point_periods_v1.py

@@ -21,7 +21,7 @@
         # 've_production' | 'net_production' | 'supply_to_grid' |
         # 'consumption_from_grid' | 'wholesale_services_information' |
         # 'own_production' | 'net_from_grid' 'net_to_grid' | 'total_consumption' |
-        # 'electrical_heating' | 'net_consumption' | 'effect_settlement'
+        # 'electrical_heating' | 'net_consumption' | 'capacity_settlement'
         t.StructField("metering_point_type", t.StringType(), not nullable),
         #
         # 'non_profiled' | 'flex'

source/databricks/settlement_report/settlement_report_job/infrastructure/wholesale/schemas/metering_point_time_series_v1.py

@@ -21,7 +21,7 @@
         # When wholesale calculations types also:
         # 've_production' | 'net_production' | 'supply_to_grid' 'consumption_from_grid' |
         # 'wholesale_services_information' | 'own_production' | 'net_from_grid' 'net_to_grid' |
-        # 'total_consumption' | 'electrical_heating' | 'net_consumption' | 'effect_settlement'
+        # 'total_consumption' | 'electrical_heating' | 'net_consumption' | 'capacity_settlement'
         t.StructField("metering_point_type", t.StringType(), not nullable),
         #
         # 'PT15M' | 'PT1H'

source/databricks/settlement_report/tests/domain/utils/test_map_to_csv_naming.py

@@ -223,9 +223,9 @@ def test_mapping_of_energy_business_process(
             id="when metering point type is net_consumption, then type of mp is D15",
         ),
         pytest.param(
-            MeteringPointTypeDataProductValue.EFFECT_SETTLEMENT,
+            MeteringPointTypeDataProductValue.CAPACITY_SETTLEMENT,
             "D19",
-            id="when metering point type is effect_settlement, then type of mp is D19",
+            id="when metering point type is capacity_settlement, then type of mp is D19",
         ),
     ],
 )

source/dotnet/wholesale-api/CalculationResults/CalculationResults.Interfaces/CalculationResults/Model/MeteringPointType.cs

@@ -33,5 +33,5 @@ public enum MeteringPointType
     TotalConsumption = 11,
     ElectricalHeating = 12,
     NetConsumption = 13,
-    EffectSettlement = 14,
+    CapacitySettlement = 14,
 }

source/dotnet/wholesale-api/Edi.UnitTests/Mappers/MeteringPointTypeMapperTests.cs

@@ -45,7 +45,7 @@ public void Map_WhenValidMeteringPointType_ReturnsExpectedString()
             (MeteringPointType.TotalConsumption, WholesaleServicesRequestSeries.Types.MeteringPointType.TotalConsumption),
             (MeteringPointType.ElectricalHeating, WholesaleServicesRequestSeries.Types.MeteringPointType.ElectricalHeating),
             (MeteringPointType.NetConsumption, WholesaleServicesRequestSeries.Types.MeteringPointType.NetConsumption),
-            (MeteringPointType.EffectSettlement, WholesaleServicesRequestSeries.Types.MeteringPointType.EffectSettlement),
+            (MeteringPointType.CapacitySettlement, WholesaleServicesRequestSeries.Types.MeteringPointType.EffectSettlement),
         };
 
         // Act

source/dotnet/wholesale-api/Edi/Mappers/MeteringPointTypeMapper.cs

@@ -36,7 +36,7 @@ public static WholesaleServicesRequestSeries.Types.MeteringPointType Map(Meterin
             MeteringPointType.TotalConsumption => WholesaleServicesRequestSeries.Types.MeteringPointType.TotalConsumption,
             MeteringPointType.ElectricalHeating => WholesaleServicesRequestSeries.Types.MeteringPointType.ElectricalHeating,
             MeteringPointType.NetConsumption => WholesaleServicesRequestSeries.Types.MeteringPointType.NetConsumption,
-            MeteringPointType.EffectSettlement => WholesaleServicesRequestSeries.Types.MeteringPointType.EffectSettlement,
+            MeteringPointType.CapacitySettlement => WholesaleServicesRequestSeries.Types.MeteringPointType.EffectSettlement,
             _ => throw new ArgumentOutOfRangeException(
                 nameof(seriesMeteringPointType),
                 actualValue: seriesMeteringPointType,