From 0b2ee7c544ab4382d89529f8dc39598b3791bf59 Mon Sep 17 00:00:00 2001 From: EtienneLt <32468651+EtienneLt@users.noreply.github.com> Date: Fri, 26 Jul 2024 13:34:19 +0200 Subject: [PATCH 1/2] fix hvdc modification and creation logs (#504) Signed-off-by: Etienne LESOT --- .../modifications/AbstractModification.java | 4 + .../modifications/ModificationUtils.java | 7 + .../server/modifications/PropertiesUtils.java | 7 +- .../server/modifications/VscCreation.java | 18 +-- .../server/modifications/VscModification.java | 138 ++++++++++++------ 5 files changed, 115 insertions(+), 59 deletions(-) diff --git a/src/main/java/org/gridsuite/modification/server/modifications/AbstractModification.java b/src/main/java/org/gridsuite/modification/server/modifications/AbstractModification.java index 8ff5318cb..4e63f65d0 100644 --- a/src/main/java/org/gridsuite/modification/server/modifications/AbstractModification.java +++ b/src/main/java/org/gridsuite/modification/server/modifications/AbstractModification.java @@ -17,6 +17,10 @@ * @author Slimane Amar */ public abstract class AbstractModification extends AbstractNetworkModification { + + public static final String CHARACTERISTICS = "Characteristics"; + public static final String SETPOINTS = "Setpoints"; + @Override public void apply(Network network, NamingStrategy namingStrategy, boolean throwException, ComputationManager computationManager, ReportNode reportNode) { apply(network, reportNode); diff --git a/src/main/java/org/gridsuite/modification/server/modifications/ModificationUtils.java b/src/main/java/org/gridsuite/modification/server/modifications/ModificationUtils.java index cb2faeb53..78fdc5e31 100644 --- a/src/main/java/org/gridsuite/modification/server/modifications/ModificationUtils.java +++ b/src/main/java/org/gridsuite/modification/server/modifications/ModificationUtils.java @@ -519,6 +519,13 @@ public void applyElementaryModifications(Consumer setter, Supplier get } } + public ReportNode applyAndBuildModificationReport(Consumer setter, Supplier getter, AttributeModification modification, String fieldName) { + T oldValue = getter.get(); + T newValue = modification.applyModification(oldValue); + setter.accept(newValue); + return buildModificationReport(oldValue, newValue, fieldName, 1, TypedValue.INFO_SEVERITY); + } + public ReportNode buildModificationReport(T oldValue, T newValue, String fieldName) { return buildModificationReport(oldValue, newValue, fieldName, 1, TypedValue.INFO_SEVERITY); } diff --git a/src/main/java/org/gridsuite/modification/server/modifications/PropertiesUtils.java b/src/main/java/org/gridsuite/modification/server/modifications/PropertiesUtils.java index d37c1a2e4..18117cab1 100644 --- a/src/main/java/org/gridsuite/modification/server/modifications/PropertiesUtils.java +++ b/src/main/java/org/gridsuite/modification/server/modifications/PropertiesUtils.java @@ -16,15 +16,16 @@ private PropertiesUtils() { public static void applyProperties(Identifiable identifiable, ReportNode subReportNode, @Nullable List properties, String propertiesLabelKey) { List reportNodes = new ArrayList<>(); - ReportNode propertiesReporter = subReportNode.newReportNode().withMessageTemplate(PROPERTIES, PROPERTIES).add(); Optional.ofNullable(properties).ifPresent(props -> props.forEach(prop -> Optional.ofNullable(PropertiesUtils.applyProperty(identifiable, prop)) .ifPresent(reportNodes::add) ) ); - ModificationUtils.getInstance().reportModifications(propertiesReporter, reportNodes, - propertiesLabelKey, PROPERTIES, Map.of()); + if (!reportNodes.isEmpty()) { + ModificationUtils.getInstance().reportModifications(subReportNode, reportNodes, + propertiesLabelKey, PROPERTIES, Map.of()); + } } private static ReportNode applyProperty(Identifiable identifiable, FreePropertyInfos prop) { diff --git a/src/main/java/org/gridsuite/modification/server/modifications/VscCreation.java b/src/main/java/org/gridsuite/modification/server/modifications/VscCreation.java index ca5db7af8..2cf7cbb52 100644 --- a/src/main/java/org/gridsuite/modification/server/modifications/VscCreation.java +++ b/src/main/java/org/gridsuite/modification/server/modifications/VscCreation.java @@ -36,8 +36,10 @@ */ public class VscCreation extends AbstractModification { - public static final String CHARACTERISTICS = "Characteristics"; - public static final String SETPOINTS = "Setpoints"; + + public static final String VSC_SETPOINTS = "vscSetPoints"; + public static final String VSC_CHARACTERISTICS = "vscCharacteristics"; + private final VscCreationInfos modificationInfos; public VscCreation(VscCreationInfos modificationInfos) { @@ -130,24 +132,21 @@ public void apply(Network network, ReportNode subReportNode) { private void reportHvdcLineInfos(ReportNode subReportNode) { List characteristicsReports = new ArrayList<>(); - ReportNode characteristicReport = subReportNode.newReportNode().withMessageTemplate("vscCharacteristics", CHARACTERISTICS).add(); characteristicsReports.add(ModificationUtils.getInstance().buildCreationReport(modificationInfos.getNominalV(), "DC nominal voltage")); characteristicsReports.add(ModificationUtils.getInstance().buildCreationReport(modificationInfos.getR(), "DC resistance")); characteristicsReports.add(ModificationUtils.getInstance().buildCreationReport(modificationInfos.getMaxP(), "Pmax")); - ModificationUtils.getInstance().reportModifications(characteristicReport, characteristicsReports, "vscCharacteristics", CHARACTERISTICS, Map.of()); + ModificationUtils.getInstance().reportModifications(subReportNode, characteristicsReports, VSC_CHARACTERISTICS, CHARACTERISTICS, Map.of()); List limitsReports = new ArrayList<>(); - ReportNode limitsReport = subReportNode.newReportNode().withMessageTemplate("vscLimits", "Limits").add(); limitsReports.add(ModificationUtils.getInstance().buildCreationReport(modificationInfos.getOperatorActivePowerLimitFromSide1ToSide2(), "Operator active power limit (Side1 -> Side 2)")); limitsReports.add(ModificationUtils.getInstance().buildCreationReport(modificationInfos.getOperatorActivePowerLimitFromSide2ToSide1(), "Operator active power limit (Side2 -> Side 1)")); - ModificationUtils.getInstance().reportModifications(limitsReport, limitsReports, "vscLimits", "Limits", Map.of()); + ModificationUtils.getInstance().reportModifications(subReportNode, limitsReports, "vscLimits", "Limits", Map.of()); List setPointsReports = new ArrayList<>(); - ReportNode setPointsReporter = subReportNode.newReportNode().withMessageTemplate("vscSetPoints", SETPOINTS).add(); + ReportNode setPointsReporter = subReportNode.newReportNode().withMessageTemplate(VSC_SETPOINTS, SETPOINTS).add(); setPointsReports.add(ModificationUtils.getInstance().buildCreationReport(modificationInfos.getConvertersMode(), "Converters mode")); setPointsReports.add(ModificationUtils.getInstance().buildCreationReport(modificationInfos.getActivePowerSetpoint(), "Active power")); - setPointsReports.add(ModificationUtils.getInstance().buildCreationReport(modificationInfos.getMaxP(), "Pmax")); - ModificationUtils.getInstance().reportModifications(setPointsReporter, setPointsReports, "vscSetPoints", SETPOINTS, Map.of()); + ModificationUtils.getInstance().reportModifications(setPointsReporter, setPointsReports, VSC_SETPOINTS, SETPOINTS, Map.of()); List angleDroopActivePowerControlReports = new ArrayList<>(); angleDroopActivePowerControlReports.add(ModificationUtils.getInstance() @@ -173,7 +172,6 @@ private VscConverterStation createConverterStation(Network network, VscConverterStation vscConverterStation = voltageLevel.getTopologyKind() == TopologyKind.NODE_BREAKER ? createConverterStationInNodeBreaker(network, voltageLevel, converterStationCreationInfos, converterStationReporter) : createConverterStationInBusBreaker(voltageLevel, converterStationCreationInfos, converterStationReporter); - converterStationReporter.newReportNode() .withMessageTemplate("converterStationCreated" + logFieldName, "New converter station with id=${id} created") .withUntypedValue("id", converterStationCreationInfos.getEquipmentId()) diff --git a/src/main/java/org/gridsuite/modification/server/modifications/VscModification.java b/src/main/java/org/gridsuite/modification/server/modifications/VscModification.java index 45c8742a4..784c1f888 100644 --- a/src/main/java/org/gridsuite/modification/server/modifications/VscModification.java +++ b/src/main/java/org/gridsuite/modification/server/modifications/VscModification.java @@ -19,14 +19,12 @@ import org.gridsuite.modification.server.dto.ReactiveCapabilityCurveModificationInfos; import org.gridsuite.modification.server.dto.VscModificationInfos; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Optional; +import java.util.*; import static org.gridsuite.modification.server.NetworkModificationException.Type.MODIFY_BATTERY_ERROR; import static org.gridsuite.modification.server.NetworkModificationException.Type.MODIFY_VSC_ERROR; -import static org.gridsuite.modification.server.modifications.ModificationUtils.insertReportNode; +import static org.gridsuite.modification.server.modifications.VscCreation.VSC_CHARACTERISTICS; +import static org.gridsuite.modification.server.modifications.VscCreation.VSC_SETPOINTS; /** * @author jamal kheyyad @@ -34,6 +32,7 @@ public class VscModification extends AbstractModification { private final VscModificationInfos modificationInfos; + private static final String NO_VALUE = "No value"; public VscModification(VscModificationInfos vscModificationInfos) { this.modificationInfos = vscModificationInfos; @@ -70,34 +69,67 @@ private void modifyVsc(@NonNull Network network, @NonNull HvdcLine hvdcLine, Vsc .withUntypedValue("id", modificationInfos.getEquipmentId()) .withSeverity(TypedValue.INFO_SEVERITY) .add(); + // Characteristics + characteristics(hvdcLine, modificationInfos, subReportNode); + + // Set Points + // Set Points + List setPointsReports = setPoints(hvdcLine); + // hvdc droop + List droopReports = hvdcAngleDroopActivePowerControlAdder(hvdcLine); + + if (!setPointsReports.isEmpty() || !droopReports.isEmpty()) { + ReportNode setPointsReport = subReportNode.newReportNode().withMessageTemplate(VSC_SETPOINTS, SETPOINTS).add(); + if (!setPointsReports.isEmpty()) { + ModificationUtils.getInstance().reportModifications(setPointsReport, setPointsReports, VSC_SETPOINTS, SETPOINTS, Map.of()); + } + if (!droopReports.isEmpty()) { + ModificationUtils.getInstance().reportModifications(setPointsReport, droopReports, "vscAngleDroop", "Angle droop active power control", Map.of()); + } + } + + // limits + operatorActivePowerLimit(hvdcLine, modificationInfos, subReportNode); + + // stations + modifyConverterStation(network, modificationInfos.getConverterStation1(), subReportNode); + modifyConverterStation(network, modificationInfos.getConverterStation2(), subReportNode); + + PropertiesUtils.applyProperties(hvdcLine, subReportNode, modificationInfos.getProperties(), "VscProperties"); + } + private static void characteristics(HvdcLine hvdcLine, VscModificationInfos modificationInfos, ReportNode subReportNode) { + List characteristicsReportsContainer = new ArrayList<>(); if (modificationInfos.getEquipmentName() != null && modificationInfos.getEquipmentName().getValue() != null) { - ModificationUtils.getInstance().applyElementaryModifications(hvdcLine::setName, () -> hvdcLine.getOptionalName().orElse("No value"), modificationInfos.getEquipmentName(), subReportNode, "Name"); + characteristicsReportsContainer.add(ModificationUtils.getInstance().applyAndBuildModificationReport(hvdcLine::setName, + () -> hvdcLine.getOptionalName().orElse(NO_VALUE), + modificationInfos.getEquipmentName(), "Name")); } if (modificationInfos.getNominalV() != null) { - ModificationUtils.getInstance().applyElementaryModifications(hvdcLine::setNominalV, hvdcLine::getNominalV, modificationInfos.getNominalV(), subReportNode, "dcNominalVoltage"); + characteristicsReportsContainer.add(ModificationUtils.getInstance().applyAndBuildModificationReport(hvdcLine::setNominalV, hvdcLine::getNominalV, modificationInfos.getNominalV(), "DC nominal voltage")); } if (modificationInfos.getR() != null) { - ModificationUtils.getInstance().applyElementaryModifications(hvdcLine::setR, hvdcLine::getR, modificationInfos.getR(), subReportNode, "R"); - } - if (modificationInfos.getActivePowerSetpoint() != null) { - ModificationUtils.getInstance().applyElementaryModifications(hvdcLine::setActivePowerSetpoint, hvdcLine::getActivePowerSetpoint, modificationInfos.getActivePowerSetpoint(), subReportNode, "ActivePowerSetpoint"); + characteristicsReportsContainer.add(ModificationUtils.getInstance().applyAndBuildModificationReport(hvdcLine::setR, hvdcLine::getR, modificationInfos.getR(), "DC resistance")); } if (modificationInfos.getMaxP() != null) { - ModificationUtils.getInstance().applyElementaryModifications(hvdcLine::setMaxP, hvdcLine::getMaxP, modificationInfos.getMaxP(), subReportNode, "MaxP"); + characteristicsReportsContainer.add(ModificationUtils.getInstance().applyAndBuildModificationReport(hvdcLine::setMaxP, hvdcLine::getMaxP, modificationInfos.getMaxP(), "Power max")); } - if (modificationInfos.getConvertersMode() != null) { - ModificationUtils.getInstance().applyElementaryModifications(hvdcLine::setConvertersMode, hvdcLine::getConvertersMode, modificationInfos.getConvertersMode(), subReportNode, "ConvertersMode"); + if (!characteristicsReportsContainer.isEmpty()) { + ModificationUtils.getInstance().reportModifications(subReportNode, characteristicsReportsContainer, VSC_CHARACTERISTICS, CHARACTERISTICS, Map.of()); } + } - operatorActivePowerLimit(hvdcLine, modificationInfos, subReportNode); - - hvdcAngleDroopActivePowerControlAdder(hvdcLine, subReportNode); + private List setPoints(HvdcLine hvdcLine) { - modifyConverterStation(network, modificationInfos.getConverterStation1(), subReportNode); - modifyConverterStation(network, modificationInfos.getConverterStation2(), subReportNode); + List setPointsReports = new ArrayList<>(); + if (modificationInfos.getActivePowerSetpoint() != null) { + setPointsReports.add(ModificationUtils.getInstance().applyAndBuildModificationReport(hvdcLine::setActivePowerSetpoint, hvdcLine::getActivePowerSetpoint, modificationInfos.getActivePowerSetpoint(), "ActivePowerSetpoint")); + } - PropertiesUtils.applyProperties(hvdcLine, subReportNode, modificationInfos.getProperties(), "VscProperties"); + if (modificationInfos.getConvertersMode() != null) { + setPointsReports.add(ModificationUtils.getInstance().applyAndBuildModificationReport(hvdcLine::setConvertersMode, hvdcLine::getConvertersMode, modificationInfos.getConvertersMode(), "Converters mode")); + } + return setPointsReports; } private static void operatorActivePowerLimit(HvdcLine hvdcLine, VscModificationInfos modificationInfos, ReportNode subReportNode) { @@ -112,7 +144,9 @@ private static void operatorActivePowerLimit(HvdcLine hvdcLine, VscModificationI createOperatorActiveRangeExt(hvdcLine, modificationInfos, reports); } } - reports.forEach(report -> insertReportNode(subReportNode, report)); + if (!reports.isEmpty()) { + ModificationUtils.getInstance().reportModifications(subReportNode, reports, "vscLimits", "Limits", Map.of()); + } } private static void modifyOperatorActiveRange(VscModificationInfos modificationInfos, HvdcOperatorActivePowerRange operatorActivePowerRange, List reports) { @@ -121,13 +155,13 @@ private static void modifyOperatorActiveRange(VscModificationInfos modificationI Optional.ofNullable(modificationInfos.getOperatorActivePowerLimitFromSide1ToSide2()).ifPresent(info -> { if (info.getValue() != null) { operatorActivePowerRange.setOprFromCS1toCS2(info.getValue()); - reports.add(ModificationUtils.getInstance().buildModificationReport(oldCs1ToCs2, info.getValue(), "OprFromCS1toCS2")); + reports.add(ModificationUtils.getInstance().buildModificationReport(oldCs1ToCs2, info.getValue(), "Operator active power limit (Side1 -> Side 2)")); } }); Optional.ofNullable(modificationInfos.getOperatorActivePowerLimitFromSide2ToSide1()).ifPresent(info -> { if (info.getValue() != null) { operatorActivePowerRange.setOprFromCS2toCS1(info.getValue()); - reports.add(ModificationUtils.getInstance().buildModificationReport(oldCs2ToCs1, info.getValue(), "OprFromCS2toCS1")); + reports.add(ModificationUtils.getInstance().buildModificationReport(oldCs2ToCs1, info.getValue(), "Operator active power limit (Side2 -> Side 1)")); } }); } @@ -137,13 +171,13 @@ private static void createOperatorActiveRangeExt(HvdcLine hvdcLine, VscModificat Optional.ofNullable(modificationInfos.getOperatorActivePowerLimitFromSide1ToSide2()).ifPresent(info -> { if (info.getValue() != null) { hvdcOperatorActivePowerRangeAddr.withOprFromCS1toCS2(modificationInfos.getOperatorActivePowerLimitFromSide1ToSide2().getValue()); - reports.add(ModificationUtils.getInstance().buildModificationReport(null, info.getValue(), "OprFromCS1toCS2")); + reports.add(ModificationUtils.getInstance().buildModificationReport(null, info.getValue(), "Operator active power limit (Side1 -> Side 2)")); } }); Optional.ofNullable(modificationInfos.getOperatorActivePowerLimitFromSide2ToSide1()).ifPresent(info -> { if (info.getValue() != null) { hvdcOperatorActivePowerRangeAddr.withOprFromCS2toCS1(modificationInfos.getOperatorActivePowerLimitFromSide2ToSide1().getValue()); - reports.add(ModificationUtils.getInstance().buildModificationReport(null, info.getValue(), "OprFromCS2toCS1")); + reports.add(ModificationUtils.getInstance().buildModificationReport(null, info.getValue(), "Operator active power limit (Side2 -> Side 1)")); } }); hvdcOperatorActivePowerRangeAddr.add(); @@ -178,9 +212,8 @@ protected boolean checkIfChangeRequestedOnDropActiveControl() { && modificationInfos.getP0() == null; } - private void hvdcAngleDroopActivePowerControlAdder(HvdcLine hvdcLine, ReportNode subReportNode) { + private List hvdcAngleDroopActivePowerControlAdder(HvdcLine hvdcLine) { List reports = new ArrayList<>(); - var hvdcAngleDroopActivePowerControl = hvdcLine.getExtension(HvdcAngleDroopActivePowerControl.class); if (hvdcAngleDroopActivePowerControl != null) { modifyExistingHvdcAngleDroopActivePowerControl(hvdcAngleDroopActivePowerControl, reports); @@ -188,7 +221,7 @@ private void hvdcAngleDroopActivePowerControlAdder(HvdcLine hvdcLine, ReportNode var activePowerControlExtension = hvdcLine.newExtension(HvdcAngleDroopActivePowerControlAdder.class); if (checkIfChangeRequestedOnDropActiveControl()) { - return; + return Collections.emptyList(); } boolean isEnabled = modificationInfos.getAngleDroopActivePowerControl() != null && modificationInfos.getAngleDroopActivePowerControl().getValue(); if (modificationInfos.getAngleDroopActivePowerControl() != null) { @@ -209,46 +242,59 @@ private void hvdcAngleDroopActivePowerControlAdder(HvdcLine hvdcLine, ReportNode activePowerControlExtension.add(); } - reports.forEach(report -> insertReportNode(subReportNode, report)); + return reports; } private void modifyConverterStation(Network network, ConverterStationModificationInfos converterStationModificationInfos, ReportNode subReportNode) { - if (converterStationModificationInfos == null) { + if (converterStationModificationInfos == null || !isConverterStationModified(converterStationModificationInfos)) { return; } VscConverterStation converterStation = ModificationUtils.getInstance().getVscConverterStation(network, converterStationModificationInfos.getEquipmentId()); - if (!isConverterStationModified(converterStationModificationInfos)) { - return; - } ReportNode converterStationReportNode = subReportNode.newReportNode() - .withMessageTemplate("Converter Station", "Converter station ${id} modified") - .withUntypedValue("id", converterStation.getId()) - .add(); - converterStationReportNode.newReportNode() - .withMessageTemplate("converter station modification", "Converter Station with id=${id} modified :") - .withUntypedValue("id", converterStation.getId()) - .withSeverity(TypedValue.INFO_SEVERITY) - .add(); + .withMessageTemplate("Converter Station", "Converter station ${id} modified") + .withUntypedValue("id", converterStation.getId()) + .withSeverity(TypedValue.INFO_SEVERITY) + .add(); + + // characteristic + List characteristicReports = new ArrayList<>(); if (converterStationModificationInfos.getEquipmentName() != null && converterStationModificationInfos.getEquipmentName().getValue() != null) { - ModificationUtils.getInstance().applyElementaryModifications(converterStation::setName, () -> converterStation.getOptionalName().orElse("No value"), converterStationModificationInfos.getEquipmentName(), converterStationReportNode, "Name"); + characteristicReports.add(ModificationUtils.getInstance().applyAndBuildModificationReport(converterStation::setName, + () -> converterStation.getOptionalName().orElse(NO_VALUE), converterStationModificationInfos.getEquipmentName(), "Name")); } if (converterStationModificationInfos.getLossFactor() != null) { - ModificationUtils.getInstance().applyElementaryModifications(converterStation::setLossFactor, converterStation::getLossFactor, converterStationModificationInfos.getLossFactor(), converterStationReportNode, "LossFactor"); + characteristicReports.add(ModificationUtils.getInstance().applyAndBuildModificationReport(converterStation::setLossFactor, + converterStation::getLossFactor, converterStationModificationInfos.getLossFactor(), "LossFactor")); + } + + if (!characteristicReports.isEmpty()) { + ModificationUtils.getInstance().reportModifications(converterStationReportNode, + characteristicReports, "Characteristics", "Characteristics", Map.of()); } + // set points + List setPointsReports = new ArrayList<>(); if (converterStationModificationInfos.getReactivePowerSetpoint() != null) { - ModificationUtils.getInstance().applyElementaryModifications(converterStation::setReactivePowerSetpoint, converterStation::getReactivePowerSetpoint, converterStationModificationInfos.getReactivePowerSetpoint(), converterStationReportNode, "ReactivePower"); + setPointsReports.add(ModificationUtils.getInstance().applyAndBuildModificationReport(converterStation::setReactivePowerSetpoint, + converterStation::getReactivePowerSetpoint, converterStationModificationInfos.getReactivePowerSetpoint(), "Reactive Power")); } if (converterStationModificationInfos.getVoltageRegulationOn() != null) { - ModificationUtils.getInstance().applyElementaryModifications(converterStation::setVoltageRegulatorOn, converterStation::isVoltageRegulatorOn, converterStationModificationInfos.getVoltageRegulationOn(), converterStationReportNode, "VoltageRegulationOn"); + setPointsReports.add(ModificationUtils.getInstance().applyAndBuildModificationReport(converterStation::setVoltageRegulatorOn, + converterStation::isVoltageRegulatorOn, converterStationModificationInfos.getVoltageRegulationOn(), "VoltageRegulationOn")); } if (converterStationModificationInfos.getVoltageSetpoint() != null) { - ModificationUtils.getInstance().applyElementaryModifications(converterStation::setVoltageSetpoint, converterStation::getVoltageSetpoint, converterStationModificationInfos.getVoltageSetpoint(), converterStationReportNode, "Voltage"); + setPointsReports.add(ModificationUtils.getInstance().applyAndBuildModificationReport(converterStation::setVoltageSetpoint, + converterStation::getVoltageSetpoint, converterStationModificationInfos.getVoltageSetpoint(), "Voltage")); + } + if (!setPointsReports.isEmpty()) { + ModificationUtils.getInstance().reportModifications(converterStationReportNode, + setPointsReports, SETPOINTS, SETPOINTS, Map.of()); } + // limits modifyVscReactiveLimitsAttributes(converterStationModificationInfos, converterStation, converterStationReportNode, converterStationReportNode); } From 1df2e9371c3eb56a99d856927ccf0597ed42b4e8 Mon Sep 17 00:00:00 2001 From: Ayoub LABIDI <117761394+ayolab@users.noreply.github.com> Date: Fri, 26 Jul 2024 15:38:21 +0200 Subject: [PATCH 2/2] Limit concurrent large modifications applications (#501) Signed-off-by: Ayoub LABIDI --- .../NetworkModificationApplicator.java | 73 ++++++++++++++++++- src/main/resources/config/application.yaml | 4 + .../server/VoltageInitReportTest.java | 2 +- 3 files changed, 75 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/gridsuite/modification/server/modifications/NetworkModificationApplicator.java b/src/main/java/org/gridsuite/modification/server/modifications/NetworkModificationApplicator.java index 0f4d5845c..5ccbf3d68 100644 --- a/src/main/java/org/gridsuite/modification/server/modifications/NetworkModificationApplicator.java +++ b/src/main/java/org/gridsuite/modification/server/modifications/NetworkModificationApplicator.java @@ -13,9 +13,13 @@ import com.powsybl.commons.report.TypedValue; import com.powsybl.iidm.network.Network; import com.powsybl.network.store.client.NetworkStoreService; +import com.powsybl.network.store.client.PreloadingStrategy; + +import jakarta.annotation.PreDestroy; import lombok.Getter; import lombok.Setter; import org.apache.commons.lang3.tuple.Pair; +import org.gridsuite.modification.server.ModificationType; import org.gridsuite.modification.server.NetworkModificationException; import org.gridsuite.modification.server.dto.ModificationInfos; import org.gridsuite.modification.server.dto.NetworkInfos; @@ -33,6 +37,9 @@ import java.util.List; import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; /** * @author Slimane Amar @@ -51,31 +58,86 @@ public class NetworkModificationApplicator { @Getter private final FilterService filterService; + private final ExecutorService applicationExecutor; + @Value("${impacts.collection-threshold:50}") @Setter // TODO REMOVE when VoltageInitReportTest will no longer use NetworkModificationApplicator private Integer collectionThreshold; public NetworkModificationApplicator(NetworkStoreService networkStoreService, EquipmentInfosService equipmentInfosService, - ReportService reportService, FilterService filterService) { + ReportService reportService, FilterService filterService, + @Value("${max-large-concurrent-applications}") int maxConcurrentApplications) { this.networkStoreService = networkStoreService; this.equipmentInfosService = equipmentInfosService; this.reportService = reportService; this.filterService = filterService; + this.applicationExecutor = Executors.newFixedThreadPool(maxConcurrentApplications); } + /* This method is used when creating, inserting, moving or duplicating modifications + * Since there is no queue for these operations and they can be memory consuming when the preloading strategy is large + * (for example for VOLTAGE_INIT_MODIFICATION), + * we limit the number of concurrent applications of these modifications to avoid out of memory issues. + * We keep the possibility to apply small or medium modifications immediately in parallel without limits. + * And if in the future we also need to limit the memory consumption of medium modifications we can add more code here. + * Note : we currently have 3 sizes of modifications : + * small : preloadingStrategy = NONE + * medium : preloadingStrategy = COLLECTION + * large : preloadingStrategy = ALL_COLLECTIONS_NEEDED_FOR_BUS_VIEW + */ public NetworkModificationResult applyModifications(List modificationInfosList, NetworkInfos networkInfos, ReportInfos reportInfos) { + PreloadingStrategy preloadingStrategy = modificationInfosList.stream() + .map(ModificationInfos::getType) + .reduce(ModificationType::maxStrategy) + .map(ModificationType::getStrategy) + .orElse(PreloadingStrategy.NONE); + if (preloadingStrategy == PreloadingStrategy.ALL_COLLECTIONS_NEEDED_FOR_BUS_VIEW) { + CompletableFuture future = CompletableFuture.supplyAsync(() -> processApplication(modificationInfosList, networkInfos, reportInfos), applicationExecutor); + return future.join(); + } else { + return processApplication(modificationInfosList, networkInfos, reportInfos); + } + } + + // used for creating, inserting, moving or duplicating modifications + private NetworkModificationResult processApplication(List modificationInfosList, NetworkInfos networkInfos, ReportInfos reportInfos) { NetworkStoreListener listener = NetworkStoreListener.create(networkInfos.getNetwork(), networkInfos.getNetworkUuuid(), networkStoreService, equipmentInfosService, collectionThreshold); ApplicationStatus groupApplicationStatus = apply(modificationInfosList, listener.getNetwork(), reportInfos); List networkImpacts = listener.flushNetworkModifications(); - return - NetworkModificationResult.builder() + return NetworkModificationResult.builder() .applicationStatus(groupApplicationStatus) .lastGroupApplicationStatus(groupApplicationStatus) .networkImpacts(networkImpacts) .build(); } + /* This method is used when building a variant + * building a variant is limited to ${consumer.concurrency} (typically 2) concurrent builds thanks to rabbitmq queue + * but since the other operations (create, insert, move, duplicate) are not inserted in the same rabbitmq queue + * we use the same ExecutorService to globally limit the number of concurrent large modifications in order to avoid out of memory issues + * We keep the possibility to apply small or medium modifications immediately. + * And if in the future we also need to limit the memory consumption of medium modifications we can add more code here. + * Note : it is possible that the rabbitmq consumer threads here will be blocked by modifications applied directly in the other applyModifications method + * and no more builds can go through. If this causes problems we should put them in separate rabbitmq queues. + */ public NetworkModificationResult applyModifications(List>> modificationInfosGroups, NetworkInfos networkInfos, UUID reportUuid) { + PreloadingStrategy preloadingStrategy = modificationInfosGroups.stream() + .map(Pair::getRight) + .flatMap(List::stream) + .map(ModificationInfos::getType) + .reduce(ModificationType::maxStrategy) + .map(ModificationType::getStrategy) + .orElse(PreloadingStrategy.NONE); + if (preloadingStrategy == PreloadingStrategy.ALL_COLLECTIONS_NEEDED_FOR_BUS_VIEW) { + CompletableFuture future = CompletableFuture.supplyAsync(() -> processApplication(modificationInfosGroups, networkInfos, reportUuid), applicationExecutor); + return future.join(); + } else { + return processApplication(modificationInfosGroups, networkInfos, reportUuid); + } + } + + // used for building a variant + private NetworkModificationResult processApplication(List>> modificationInfosGroups, NetworkInfos networkInfos, UUID reportUuid) { NetworkStoreListener listener = NetworkStoreListener.create(networkInfos.getNetwork(), networkInfos.getNetworkUuuid(), networkStoreService, equipmentInfosService, collectionThreshold); List groupsApplicationStatuses = modificationInfosGroups.stream() @@ -158,4 +220,9 @@ public static ApplicationStatus getApplicationStatus(ReportNode reportNode) { throw new IllegalArgumentException(String.format("Report severity '%s' unknown !", severity.getValue())); } } + + @PreDestroy + public void shutdown() { + applicationExecutor.shutdown(); + } } diff --git a/src/main/resources/config/application.yaml b/src/main/resources/config/application.yaml index 46687732c..2e05b56e6 100644 --- a/src/main/resources/config/application.yaml +++ b/src/main/resources/config/application.yaml @@ -40,3 +40,7 @@ powsybl-ws: name: networkmodifications queryBegin: '&' customQuery: ${powsybl-ws.database.customQueryBegin}reWriteBatchedInserts=true + +# maximum concurrent applications of modifications with preloadingStrategy=ALL_COLLECTIONS_NEEDED_FOR_BUS_VIEW +# to avoid out of memory issues +max-large-concurrent-applications: 2 diff --git a/src/test/java/org/gridsuite/modification/server/VoltageInitReportTest.java b/src/test/java/org/gridsuite/modification/server/VoltageInitReportTest.java index 93af94671..a20f4fe85 100644 --- a/src/test/java/org/gridsuite/modification/server/VoltageInitReportTest.java +++ b/src/test/java/org/gridsuite/modification/server/VoltageInitReportTest.java @@ -62,7 +62,7 @@ void testVoltageInitDuplicationLogs(final ApplicationStatus resultStatus, final final NetworkStoreService networkStoreService = new NetworkStoreServicePublic(restClient, PreloadingStrategy.NONE, (restClient_, preloadingStrategy, executorService) -> new CachedNetworkStoreClient(new OfflineNetworkStoreClient())); final EquipmentInfosService equipmentInfosService = Mockito.mock(EquipmentInfosService.class); - final NetworkModificationApplicator networkModificationApplicator = new NetworkModificationApplicator(networkStoreService, equipmentInfosService, reportService, null); + final NetworkModificationApplicator networkModificationApplicator = new NetworkModificationApplicator(networkStoreService, equipmentInfosService, reportService, null, 2); networkModificationApplicator.setCollectionThreshold(5); final Network network = Network.read(Paths.get(this.getClass().getClassLoader().getResource("fourSubstations_testsOpenReac.xiidm").toURI()));