Save ext id and desc for phys flows

- clear up unknowns on logicals if a physical flow is added and datatypes propagated up

#CTCTOWATLZ-2582
finos#6269

waltz-data/src/main/java/org/finos/waltz/data/physical_specification/PhysicalSpecificationDao.java

@@ -315,10 +315,61 @@ public int propagateDataTypesToLogicalFlows(String userName, long specificationI
                     .select(requiredDecorators)
                     .execute();
 
+
+            removeUnknownFromLogicalFlowWherePossible(tx, specificationId, userName);
+
             return insertCount;
         });
     }
 
+    private void removeUnknownFromLogicalFlowWherePossible(DSLContext tx, long specificationId, String userName) {
+
+        SelectHavingConditionStep<Record1<Long>> flowsWithOtherDataTypes = tx
+                .select(lfd.LOGICAL_FLOW_ID)
+                .from(lfd)
+                .innerJoin(lf).on(lf.ID.eq(lfd.LOGICAL_FLOW_ID))
+                .innerJoin(pf).on(pf.LOGICAL_FLOW_ID.eq(lf.ID))
+                .innerJoin(dt).on(dt.ID.eq(lfd.DECORATOR_ENTITY_ID))
+                .where(pf.SPECIFICATION_ID.eq(specificationId))
+                .and(dt.UNKNOWN.isFalse())
+                .and(lfd.DECORATOR_ENTITY_KIND.eq(EntityKind.DATA_TYPE.name()))
+                .groupBy(lfd.LOGICAL_FLOW_ID)
+                .having(count(lfd.DECORATOR_ENTITY_ID).gt(0));
+
+        SelectConditionStep<Record1<Long>> unknownDecoratorsThatCanBeRemoved = tx.select(lfd.ID)
+                .from(lfd)
+                .innerJoin(dt).on(lfd.DECORATOR_ENTITY_ID.eq(dt.ID)
+                        .and(lfd.DECORATOR_ENTITY_KIND.eq(EntityKind.DATA_TYPE.name())))
+                .where(lfd.LOGICAL_FLOW_ID.in(flowsWithOtherDataTypes))
+                .and(dt.UNKNOWN.isTrue());
+
+        SelectJoinStep<Record6<String, Long, String, String, String, String>> requiredChangeLogs = tx
+                .select(val(EntityKind.LOGICAL_DATA_FLOW.name()),
+                        flowsWithOtherDataTypes.field(0, Long.class), // logical flow id
+                        val("Removed 'Unknown' data type as known data types were propagated from underlying physical flow/s"),
+                        val(userName),
+                        val(Severity.INFORMATION.name()),
+                        val(Operation.REMOVE.name()))
+                .from(flowsWithOtherDataTypes);
+
+        int changelogsInserted = tx
+                .insertInto(CHANGE_LOG)
+                .columns(
+                        CHANGE_LOG.PARENT_KIND,
+                        CHANGE_LOG.PARENT_ID,
+                        CHANGE_LOG.MESSAGE,
+                        CHANGE_LOG.USER_ID,
+                        CHANGE_LOG.SEVERITY,
+                        CHANGE_LOG.OPERATION)
+                .select(requiredChangeLogs)
+                .execute();
+
+        int removedUnknowns = tx
+                .deleteFrom(lfd)
+                .where(lfd.ID.in(unknownDecoratorsThatCanBeRemoved))
+                .execute();
+    }
+
 
     public int updateFormat(long specId, DataFormatKindValue format) {
         return dsl

waltz-model/src/main/java/org/finos/waltz/model/physical_flow/FlowAttributes.java

@@ -22,12 +22,13 @@
 import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import org.finos.waltz.model.Criticality;
 import org.finos.waltz.model.DescriptionProvider;
+import org.finos.waltz.model.ExternalIdProvider;
 import org.immutables.value.Value;
 
 @Value.Immutable
 @JsonSerialize(as = ImmutableFlowAttributes.class)
 @JsonDeserialize(as = ImmutableFlowAttributes.class)
-public abstract class FlowAttributes implements DescriptionProvider {
+public abstract class FlowAttributes implements DescriptionProvider, ExternalIdProvider {
 
     public abstract TransportKindValue transport();
 

waltz-ng/client/physical-flows/svelte/PhysicalFlowCharacteristicsStep.svelte

@@ -13,6 +13,7 @@
     import {expandedSections, nestedEnums, physicalFlow} from "./physical-flow-editor-store";
     import Icon from "../../common/svelte/Icon.svelte";
     import {onMount} from "svelte";
+    import Markdown from "../../common/svelte/Markdown.svelte";
 
     let workingCopy = {};
     let transportKinds = [];
@@ -30,14 +31,16 @@
                 ? {basisOffset: offset.code}
                 : {basisOffset: "OTHER", customBasisOffset: offsetString}
 
-            workingCopy = Object.assign({}, $physicalFlow, offsetInfo)
+            workingCopy = Object.assign({}, $physicalFlow, offsetInfo);
+
         } else {
             workingCopy = {
                 transport: null,
                 frequency: null,
                 criticality: null,
                 basisOffset: "0",
-                description: ""
+                description: "",
+                externalId: null
             };
         }
     })
@@ -54,7 +57,9 @@
                 transport: workingCopy.transport,
                 frequency: workingCopy.frequency,
                 basisOffset,
-                criticality: workingCopy.criticality
+                criticality: workingCopy.criticality,
+                description: workingCopy.description,
+                externalId: workingCopy.externalId
             });
 
         openNextSection()
@@ -106,8 +111,18 @@
                         Frequency: {toFrequencyKindName($nestedEnums, $physicalFlow.frequency)}</li>
                     <li>
                         Criticality: {toCriticalityName($nestedEnums, $physicalFlow.criticality)}</li>
-                    <li>Basis
-                        Offset: {_.get(basisOffsetByCode, [$physicalFlow.basisOffset, "name"], $physicalFlow.basisOffset)}</li>
+                    <li>
+                        Basis
+                        Offset: {_.get(basisOffsetByCode, [$physicalFlow.basisOffset, "name"], $physicalFlow.basisOffset)}
+                    </li>
+                    <li>
+                        External ID: <span
+                        class:text-muted={!$physicalFlow.externalId}>{$physicalFlow.externalId || "Not provided"}</span>
+                    </li>
+                    <li>
+                        Description:
+                        <Markdown text={$physicalFlow.description}/>
+                    </li>
                 </ul>
             </div>
 

waltz-ng/client/physical-flows/svelte/PhysicalFlowRegistrationView.svelte

@@ -3,6 +3,7 @@
     import ViewLink from "../../common/svelte/ViewLink.svelte";
     import EntityLink from "../../common/svelte/EntityLink.svelte";
     import toasts from "../../svelte-stores/toast-store";
+    import pageInfo from "../../svelte-stores/page-navigation-store";
 
     import {
         dataTypes,
@@ -11,9 +12,9 @@
         nestedEnums,
         physicalFlow,
         physicalSpecification,
-        ViewMode,
+        skipDataTypes,
         viewMode,
-        skipDataTypes
+        ViewMode
     } from "./physical-flow-editor-store";
 
     import _ from "lodash";
@@ -61,14 +62,17 @@
             description: $physicalSpecification.description,
             format: $physicalSpecification.format,
             lastUpdatedBy: "waltz",
+            externalId: $physicalSpecification.externalId,
             id: $physicalSpecification.id ? $physicalSpecification.id : null
         }
 
         const flowAttributes = {
             transport: $physicalFlow.transport,
             frequency: $physicalFlow.frequency,
             basisOffset: $physicalFlow.basisOffset,
-            criticality: $physicalFlow.criticality
+            criticality: $physicalFlow.criticality,
+            description: $physicalFlow.description,
+            externalId: $physicalFlow.externalId
         }
 
         const command = {
@@ -80,7 +84,14 @@
 
         physicalFlowStore.create(command)
             .then(() => toasts.success("Successfully added physical flow"))
-            .then(() => history.back())
+            .then(() => {
+                $pageInfo = {
+                    state: "main.app.view",
+                    params: {
+                        id: primaryEntityRef.id
+                    }
+                };
+            })
             .catch(e => displayError("Could not create physical flow", e));
     }
 

waltz-service/src/main/java/org/finos/waltz/service/data_type/DataTypeDecoratorService.java

@@ -185,6 +185,7 @@ public int removeDataTypeDecorator(String userName, EntityReference entityRefere
 
         audit(format("Removed data types: %s", dataTypeIds.toString()),
                 entityReference, userName);
+
         recalculateDataTypeUsageForApplications(entityReference);
 
         return result;

waltz-service/src/main/java/org/finos/waltz/service/physical_flow/PhysicalFlowService.java

@@ -223,7 +223,7 @@ public PhysicalFlowCreateCommandResponse create(PhysicalFlowCreateCommand comman
                         .withLastUpdatedAt(now)
                         .withCreated(UserTimestamp.mkForUser(username, now))));
 
-        PhysicalFlow flow = ImmutablePhysicalFlow.builder()
+        ImmutablePhysicalFlow.Builder flowBuilder = ImmutablePhysicalFlow.builder()
                 .specificationId(specId)
                 .basisOffset(command.flowAttributes().basisOffset())
                 .frequency(command.flowAttributes().frequency())
@@ -233,12 +233,18 @@ public PhysicalFlowCreateCommandResponse create(PhysicalFlowCreateCommand comman
                 .logicalFlowId(command.logicalFlowId())
                 .lastUpdatedBy(username)
                 .lastUpdatedAt(now)
-                .created(UserTimestamp.mkForUser(username, now))
-                .build();
+                .created(UserTimestamp.mkForUser(username, now));
+
+        command
+                .flowAttributes()
+                .externalId()
+                .ifPresent(flowBuilder::externalId);
+
+        PhysicalFlow flow = flowBuilder.build();
 
         // ensure existing not in database
         List<PhysicalFlow> byAttributesAndSpecification = physicalFlowDao.findByAttributesAndSpecification(flow);
-        if(byAttributesAndSpecification.size() > 0) {
+        if (byAttributesAndSpecification.size() > 0) {
             return ImmutablePhysicalFlowCreateCommandResponse.builder()
                     .originalCommand(command)
                     .outcome(CommandOutcome.FAILURE)