task/fix develop merge issues

.nuget/packages.config

@@ -1,4 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
+
 <packages>
   <package id="NUnit.Console" version="3.9.0" />
   <package id="NUnit.ConsoleRunner" version="3.9.0" />

Application/ResearchDataManagementPlatform/RDMP.nuspec

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd">
+  <metadata>
+    <id>ResearchDataManagementPlatform</id>
+    <version>$version$</version>
+    <authors>Health Informatics Service, University of Dundee</authors>
+    <description>Research Data Management Platform</description>
+  </metadata>
+  <files>
+    <file src="bin\$configuration$\net6.0-windows\win-x64\*" target="lib\net6.0-windows7.0" />
+    <file src="bin\$configuration$\net6.0-windows\win-x64\Curation\**\*" target="lib\net6.0-windows7.0" />
+    <file src="bin\$configuration$\net6.0-windows\win-x64\Icon\**\*" target="lib\net6.0-windows7.0" />
+    <file src="bin\$configuration$\net6.0-windows\win-x64\WindowManagement\**\*" target="lib\net6.0-windows7.0" />
+  </files>
+</package>

Documentation/CodeTutorials/Packages.md

@@ -16,6 +16,7 @@
 | ObjectListView.Repack.NET6Plus | [GitHub](https://github.com/nasisakk/ObjectListViewRepack) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | |
 | Scintilla.NET | [GitHub](https://github.com/VPKSoft/Scintilla.NET) | [MIT](https://opensource.org/licenses/MIT) | |
 | VPKSoft.ScintillaLexers.NET | [GitHub](https://github.com/VPKSoft/ScintillaLexers) | [MIT](https://opensource.org/licenses/MIT) | |
+| Xam.Plugins.Settings | [GitHub](https://github.com/jamesmontemagno/SettingsPlugin)| [MIT](https://opensource.org/licenses/MIT)| Read/Write user settings for main client application |
 | WeCantSpell.Hunspell | [GitHub](https://github.com/aarondandy/WeCantSpell.Hunspell/) | [GPL-2 and others](https://github.com/aarondandy/WeCantSpell.Hunspell/blob/main/license.txt) | |
 | [DockPanelSuite.ThemeVS2015](http://dockpanelsuite.com/) | [GitHub](https://github.com/dockpanelsuite/dockpanelsuite) | [MIT](https://opensource.org/licenses/MIT)  | Provides Window layout and docking for RDMP. | There are no powershell initialization files in the package which can be run by the NuGet installer.|
 | [FAM FAM FAM Icons](https://web.archive.org/web/20070824000227/http://www.famfamfam.com/lab/icons/silk/) | N\A | [CC 2.5](https://creativecommons.org/licenses/by/2.5/) | Icons for user interfaces |

Rdmp.Core.Tests/Curation/ImportTests/GatherAndShareTests.cs

@@ -138,7 +138,7 @@ public void GatherAndShare_Plugin_Test()
             $"Imaginary1{PackPluginRunner.PluginPackageSuffix}"));
         File.WriteAllBytes(f1.FullName, new byte[] { 0x1, 0x2 });
 
-        var plugin = new Plugin(CatalogueRepository, new FileInfo(
+        var plugin = new Rdmp.Core.Curation.Data.Plugin(CatalogueRepository, new FileInfo(
             $"Imaginary{PackPluginRunner.PluginPackageSuffix}"), new Version(1, 1, 1), new Version(1, 1, 1));
         var lma1 = new LoadModuleAssembly(CatalogueRepository, f1, plugin);
 

Rdmp.Core.Tests/Curation/ImportTests/PluginClassTests.cs

@@ -72,7 +72,7 @@ public void TestPlugin_OrphanImport_Sharing()
             $"Blah3.{PackPluginRunner.PluginPackageSuffix}"));
         File.WriteAllBytes(fi3.FullName, new byte[] { 0x3, 0x4 });
 
-        var p = new Plugin(Repository, fi, new Version(1, 1, 1), new Version(1, 1, 1, 1));
+        var p = new Rdmp.Core.Curation.Data.Plugin(Repository, fi, new Version(1, 1, 1), new Version(1, 1, 1, 1));
         var lma = new LoadModuleAssembly(Repository, fi2, p);
         var lma2 = new LoadModuleAssembly(Repository, fi3, p);
 
@@ -116,7 +116,7 @@ public void TestPlugin_OrphanImport_Sharing()
     [TestCase("Rdmp.Dicom", "Rdmp.Dicom")]
     public void Test_Plugin_ShortName(string fullname, string expected)
     {
-        var p = WhenIHaveA<Plugin>();
+        var p = WhenIHaveA<Rdmp.Core.Curation.Data.Plugin>();
         p.Name = fullname;
         Assert.AreEqual(expected, p.GetShortName());
     }

Rdmp.Core.Tests/DataExport/DataAccess/SelectedColumnsTests.cs

@@ -34,12 +34,8 @@ public void CreateAndAssociateColumns()
         try
         {
             _ = new SelectedDataSets(DataExportRepository, config, ds, null);
-
             var cols = config.GetAllExtractableColumnsFor(ds);
 
-            Assert.AreEqual(1, cols.Length);
-            Assert.AreEqual(extractableColumn, cols.Single());
-
             cols = config.GetAllExtractableColumnsFor(ds);
 
             Assert.AreEqual(1, cols.Length);

Rdmp.Core.Tests/Rdmp.Core.Tests.csproj

@@ -69,7 +69,7 @@
             <PrivateAssets>all</PrivateAssets>
             <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
         </PackageReference>
-        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.7.1" />
+        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.7.2" />
         <PackageReference Include="NSubstitute" Version="5.0.0" />
         <PackageReference Include="NUnit" Version="3.13.3" />
         <PackageReference Include="NUnit3TestAdapter" Version="4.5.0">

Rdmp.Core/CohortCommitting/Pipeline/Sources/CohortIdentificationConfigurationSource.cs

@@ -122,10 +122,12 @@ private DataTable GetDataTable(IDataLoadEventListener listener)
             throw new Exception(
                 "CohortIdentificationCriteria execution resulted in an empty dataset (there were no cohorts matched by the query?)");
 
-        var dt = execution.Identifiers;
+        DataTable dt = execution.Identifiers;
+        dt.BeginLoadData();
         foreach (DataColumn column in dt.Columns)
             column.ReadOnly = false;
 
+        dt.EndLoadData();
         return dt;
     }
 

Rdmp.Core/CohortCommitting/Pipeline/Sources/PatientIdentifierColumnSource.cs

@@ -54,7 +54,7 @@ private DataTable GetDataTable(int timeout, int? topX)
 
         var colName = _extractionInformation.GetRuntimeName();
 
-        var dt = new DataTable();
+        DataTable dt = new DataTable();
         dt.BeginLoadData();
         dt.Columns.Add(colName);
 

Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandClearUserSettings.cs

@@ -21,4 +21,4 @@ public override void Execute()
         UserSettings.ClearUserSettings();
     }
 
-}
+}

Rdmp.Core/CommandLine/DatabaseCreation/CataloguePipelinesAndReferencesCreation.cs

@@ -109,7 +109,6 @@ private void CreateServers(PlatformDatabaseCreationOptions options)
         Console.WriteLine("Successfully configured RAW server");
     }
 
-
     public void CreatePipelines(PlatformDatabaseCreationOptions options)
     {
         var bulkInsertCsvPipe =

Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreationOptions.cs

@@ -45,7 +45,6 @@ public class PlatformDatabaseCreationOptions
 
     [Option('l', "Create Logging Server", Default = true, HelpText = "Create the default logging server in the Catalogue database once created. Is superseeded by 'Skip Pipelines'")]
     public bool CreateLoggingServer { get; set; }
-
     [Option('e', "ExampleDatasets", Default = false,
         HelpText = "Create example datasets, projects, extraction configurations and cohort queries")]
     public bool ExampleDatasets { get; set; }

Rdmp.Core/Curation/ANOEngineering/ForwardEngineerANOCatalogueEngine.cs

@@ -361,7 +361,6 @@ private ColumnInfo GetNewColumnInfoForOld(ColumnInfo col, bool isOptional = fals
 
         var toReturn = FindNewColumnNamed(syntaxHelper, col, col.GetRuntimeName(), isOptional) ??
                        FindNewColumnNamed(syntaxHelper, col, $"ANO{col.GetRuntimeName()}", isOptional);
-
         if (toReturn == null)
             return isOptional
                 ? null

Rdmp.Core/Curation/Data/Aggregation/AggregateConfiguration.cs

@@ -39,7 +39,7 @@ namespace Rdmp.Core.Curation.Data.Aggregation;
 /// <para>3. 'Joinable PatientIndex Table' - Produce a patient identifier fact table for joining to other Cohort Aggregates during cohort building (See JoinableCohortAggregateConfiguration)</para>
 /// <para>The above labels are informal terms.  Use IsCohortIdentificationAggregate and IsJoinablePatientIndexTable to determine what type a given
 /// AggregateConfiguration is. </para>
-/// 
+///
 /// <para>If your Aggregate is part of cohort identification (Identifier List or Patient Index Table) then its name will start with cic_X_ where X is the ID of the cohort identification
 /// configuration.  Depending on the user interface though this might not appear (See ToString implementation).</para>
 /// </summary>
@@ -257,7 +257,7 @@ private static bool IsBasicallyZero(object v) => v == null || v == DBNull.Value
     /// When an AggregateConfiguration is used in a cohort identification capacity it can have one or more 'patient index tables' defined e.g.
     /// 'Give me all prescriptions for morphine' (Prescribing) 'within 6 months of patient being discharged from hospital' (SMR01).  In this case
     /// a join is done against the secondary dataset.
-    /// 
+    ///
     /// <para>This property returns all such 'patient index table' AggregateConfigurations which are currently being used by this AggregateConfiguration
     /// for building its join.</para>
     /// </summary>

Rdmp.Core/Curation/Data/Catalogue.cs

@@ -519,10 +519,17 @@ public int? LoadMetadata_ID
 
     /// <inheritdoc/>
     [NoMappingToDatabase]
-    public ExternalDatabaseServer LiveLoggingServer =>
-        LiveLoggingServer_ID == null
-            ? null
-            : Repository.GetObjectByID<ExternalDatabaseServer>((int)LiveLoggingServer_ID);
+     public ExternalDatabaseServer LiveLoggingServer {
+        get{
+            if(LiveLoggingServer_ID != null){  
+                ExternalDatabaseServer[] dbs = Repository.GetAllObjectsWhere<ExternalDatabaseServer>("id",(int)LiveLoggingServer_ID);
+                if(dbs.Length > 0){
+                    return dbs.First();
+                }                         
+            }
+            return null;
+        }
+     }
 
     /// <inheritdoc/>
     [NoMappingToDatabase]

Rdmp.Core/Curation/Data/DataLoad/ProcessTask.cs

@@ -358,7 +358,6 @@ public IArgument[] CreateArgumentsForClassIfNotExists(Type t) =>
             //convert the result back from generic to specific (us)
             .ToArray();
 
-
     /// <inheritdoc/>
     public IArgument[] CreateArgumentsForClassIfNotExists<T>() => CreateArgumentsForClassIfNotExists(typeof(T));
 

Rdmp.Core/DataExport/Data/DeployedExtractionFilterParameter.cs

@@ -141,4 +141,4 @@ public DeployedExtractionFilterParameter ShallowClone(DeployedExtractionFilter i
         CopyShallowValuesTo(clone);
         return clone;
     }
-}
+}

Rdmp.Core/DataExport/DataExtraction/ExtractionTimeValidator.cs

@@ -50,7 +50,7 @@ public void Validate(DataTable dt, string validationColumnToPopulateIfAny)
     {
         if (!_initialized)
             Initialize(dt);
-
+        dt.BeginLoadData();
         foreach (DataRow r in dt.Rows)
         {
             //additive validation results, Results is a class that wraps DictionaryOfFailure which is an array of columns and each element is another array of consequences (with a row count for each consequence)

Rdmp.Core/DataLoad/Engine/LoadExecution/Components/Standard/MigrateStagingToLive.cs

@@ -26,9 +26,6 @@ public MigrateStagingToLive(HICDatabaseConfiguration databaseConfiguration,
         HICLoadConfigurationFlags loadConfigurationFlags)
     {
         _databaseConfiguration = databaseConfiguration;
-
-        Description = "Migrate Staging to Live";
-        SkipComponent = !loadConfigurationFlags.DoMigrateFromStagingToLive;
     }
 
     public override ExitCodeType Run(IDataLoadJob job, GracefulCancellationToken cancellationToken)

Rdmp.Core/DataLoad/Engine/Pipeline/Components/Anonymisation/ColumnInfoToANOTableConverter.cs

@@ -152,7 +152,7 @@ private void MigrateExistingData(Func<string, bool> shouldApplySql, DbConnection
                 throw new Exception("User decided not to perform update on table");
             cmdUpdateMainTable.ExecuteNonQuery();
         }
-        finally
+          finally
         {
             //always drop the temp anomap
             using var dropMappingTable = DatabaseCommandHelper.GetCommand("DROP TABLE TempANOMap", con);

Rdmp.Core/DataLoad/Engine/Pipeline/Components/CleanStrings.cs

@@ -31,7 +31,6 @@ public DataTable ProcessPipelineData(DataTable toProcess, IDataLoadEventListener
         GracefulCancellationToken cancellationToken)
     {
         timer.Start();
-
     StartAgain:
         foreach (DataRow row in toProcess.Rows)
         {
@@ -85,7 +84,6 @@ public DataTable ProcessPipelineData(DataTable toProcess, IDataLoadEventListener
         job.OnProgress(this,
             new ProgressEventArgs(_taskDescription, new ProgressMeasurement(_rowsProcessed, ProgressType.Records),
                 timer.Elapsed));
-
         return toProcess;
     }
 

Rdmp.Core/DataLoad/Modules/Attachers/FixedWidthFormatFile.cs

@@ -77,7 +77,7 @@ public FixedWidthFormatFile(FileInfo pathToFormatFile)
     public DataTable GetDataTableFromFlatFile(FileInfo f)
     {
         //setup the table
-        var toReturn = new DataTable();
+        DataTable toReturn = new DataTable();
 
         toReturn.BeginLoadData();
         foreach (var fixedWidthColumn in FormatColumns)

Rdmp.Core/DataLoad/Modules/DataFlowOperations/CohortSampler.cs

@@ -107,7 +107,7 @@ public DataTable ProcessPipelineData(DataTable toProcess, IDataLoadEventListener
             throw new Exception(
                 $"Cohort only contains {chosen.Count} unique identifiers.  This is less than the requested sample size of {SampleSize} and {nameof(FailIfNotEnoughIdentifiers)} is true");
 
-        var dtToReturn = new DataTable();
+        DataTable dtToReturn = new DataTable();
         dtToReturn.BeginLoadData();
         dtToReturn.Columns.Add(expectedFieldName);
 

Rdmp.Core/DataLoad/Modules/DataFlowOperations/Transposer.cs

@@ -65,7 +65,7 @@ public void Check(ICheckNotifier notifier)
 
     private DataTable GenerateTransposedTable(DataTable inputTable)
     {
-        var outputTable = new DataTable();
+        DataTable outputTable = new DataTable();
         outputTable.BeginLoadData();
         // Add columns by looping rows
 

Rdmp.Core/DataLoad/Modules/DataFlowSources/ExcelDataFlowSource.cs

@@ -124,7 +124,6 @@ public DataTable GetAllData(ISheet worksheet, IDataLoadEventListener listener)
     {
         var toReturn = new DataTable();
         toReturn.BeginLoadData();
-
         var rowEnumerator = worksheet.GetRowEnumerator();
         var nColumns = -1;
 
@@ -188,7 +187,6 @@ public DataTable GetAllData(ISheet worksheet, IDataLoadEventListener listener)
             if (!gotAtLeastOneGoodValue)
                 toReturn.Rows.Remove(r);
         }
-
         return toReturn;
     }
 

Rdmp.Core/DataLoad/Modules/DataFlowSources/SubComponents/FlatFileToDataTablePusher.cs

@@ -306,7 +306,6 @@ public DataTable StronglyTypeTable(DataTable workingTable, ExplicitTypingCollect
             foreach (DataRow row in workingTable.Rows)
                 dtCloned.Rows.Add(row.ItemArray.Select((v, idx) =>
                     deciders.TryGetValue(idx, out var decider) && v is string s ? decider.Parse(s) : v).ToArray());
-
             return dtCloned;
         }
 

Rdmp.Core/DataLoad/Modules/FileOperations/FileUnzipper.cs

@@ -46,35 +46,34 @@
 
     public ExitCodeType Fetch(IDataLoadJob job, GracefulCancellationToken cancellationToken)
     {
-        foreach (var fileInfo in job.LoadDirectory.ForLoading.GetFiles("*.zip"))
-        {
+        foreach (var fileInfo in job.LoadDirectory.ForLoading.GetFiles("*.zip")){
             //do it as regex rather than in GetFiles above because that method probably doesn't do regex
             if (ZipArchivePattern != null && !string.IsNullOrWhiteSpace(ZipArchivePattern.ToString()) &&
                 !ZipArchivePattern.IsMatch(fileInfo.Name)) continue;
             using var zipFile = ZipFile.Open(fileInfo.FullName, ZipArchiveMode.Read);
             //fire event telling user we found some files in the zip file
             job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, zipFile.Entries.Aggregate(
                 "Identified the following zip entries:", (s, n) =>
                     $"{n.Name},").TrimEnd(',')));
 
 
             foreach (var entry in zipFile.Entries)
             {
                 if (entry.Length == 0)
                     continue;
 
                 //if we are matching everything or we are matching on a regex that matches the entry name
                 if (ZipEntryPattern != null && !string.IsNullOrWhiteSpace(ZipEntryPattern.ToString()) &&
                     !ZipEntryPattern.IsMatch(entry.Name)) continue;
                 //extract it
                 var existingFile = job.LoadDirectory.ForLoading.GetFiles(entry.Name).FirstOrDefault();
 
                 if (existingFile != null && existingFile.Length == entry.Length)
                     continue;
 
                 UnzipWithEvents(entry, job.LoadDirectory, job);
             }
         }
 
         return ExitCodeType.Success;
     }

Rdmp.Core/Logging/DataLoadInfo.cs

@@ -348,5 +348,7 @@ public void LogProgress(ProgressEventType pevent, string Source, string Descript
         {
             Monitor.Pulse(_logWaiter);
         }
+        lock (_logWaiter)
+            Monitor.Pulse(_logWaiter);
     }
-}
+}

Rdmp.Core/MapsDirectlyToDatabaseTable/Versioning/Patcher.cs

@@ -78,7 +78,7 @@ public virtual Patch GetInitialCreateScriptContents(DiscoveredDatabase db)
                 {
                     var sr = new StreamReader(assembly.GetManifestResourceStream(candidates[0]));
 
-                    var sql = sr.ReadToEnd();
+                var sql = sr.ReadToEnd();
 
                     if (!sql.Contains(Patch.VersionKey))
                         sql = GetHeader(db.Server.DatabaseType, InitialScriptName, new Version(1, 0, 0)) + sql;

Rdmp.Core/Providers/CatalogueChildProvider.cs

@@ -187,8 +187,8 @@ public class CatalogueChildProvider : ICoreChildProvider
 
     public JoinableCohortAggregateConfigurationUse[] AllJoinableCohortAggregateConfigurationUse { get; private set; }
     public AllPluginsNode AllPluginsNode { get; private set; }
-    public Plugin[] AllPlugins { get; private set; }
-    public Plugin[] AllCompatiblePlugins { get; private set; }
+    public Curation.Data.Plugin[] AllPlugins { get; private set; }
+    public Curation.Data.Plugin[] AllCompatiblePlugins { get; private set; }
 
     public HashSet<StandardPipelineUseCaseNode> PipelineUseCases { get; set; } = new();
 
@@ -432,7 +432,7 @@ public CatalogueChildProvider(ICatalogueRepository repository, IChildProvider[]
         ReportProgress("After Governance");
 
         AllPluginsNode = new AllPluginsNode();
-        AllPlugins = GetAllObjects<Plugin>(repository);
+        AllPlugins = GetAllObjects<Curation.Data.Plugin>(repository);
         AllCompatiblePlugins = _catalogueRepository.PluginManager.GetCompatiblePlugins();
 
         AddChildren(AllPluginsNode);

Rdmp.Core/Rdmp.Core.csproj

@@ -371,9 +371,9 @@
     <PackageReference Include="Microsoft.XmlSerializer.Generator" Version="7.0.0" />
     <PackageReference Include="MongoDB.Driver" Version="2.20.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-    <PackageReference Include="NPOI" Version="2.6.0" />
     <PackageReference Include="NLog" Version="5.2.3" />
-    <PackageReference Include="SixLabors.ImageSharp" Version="3.0.1" />
+    <PackageReference Include="NPOI" Version="2.6.0" />
+    <PackageReference Include="SixLabors.ImageSharp" Version="3.0.2" />
     <PackageReference Include="SixLabors.ImageSharp.Drawing" Version="1.0.0" />
     <PackageReference Include="Spectre.Console" Version="0.47.0" />
     <PackageReference Include="System.Diagnostics.Debug" Version="4.3.0" />
@@ -394,6 +394,7 @@
     <PackageReference Include="System.Threading.ThreadPool" Version="4.3.0" />
     <PackageReference Include="Terminal.Gui" Version="1.13.5" />
     <PackageReference Include="YamlDotNet" Version="13.1.1" />
+    <PackageReference Include="Xam.Plugins.Settings" Version="3.1.1" />
   </ItemGroup>
   <ItemGroup>
     <DotNetCliToolReference Include="Microsoft.XmlSerializer.Generator" Version="7.0.0" />

Rdmp.Core/Repositories/CatalogueRepository.cs

@@ -29,7 +29,7 @@
 using Rdmp.Core.Repositories.Managers;
 using Rdmp.Core.ReusableLibraryCode;
 using Rdmp.Core.ReusableLibraryCode.Comments;
-
+ 
 namespace Rdmp.Core.Repositories;
 
 /// <inheritdoc cref="ICatalogueRepository"/>
@@ -96,7 +96,7 @@ public CatalogueRepository(DbConnectionStringBuilder catalogueConnectionString)
         Constructors.Add(typeof(StandardRegex), (rep, r) => new StandardRegex((ICatalogueRepository)rep, r));
         Constructors.Add(typeof(AnyTableSqlParameter),
             (rep, r) => new AnyTableSqlParameter((ICatalogueRepository)rep, r));
-        Constructors.Add(typeof(Plugin), (rep, r) => new Plugin((ICatalogueRepository)rep, r));
+        Constructors.Add(typeof(Curation.Data.Plugin), (rep, r) => new Curation.Data.Plugin((ICatalogueRepository)rep, r));
         Constructors.Add(typeof(ANOTable), (rep, r) => new ANOTable((ICatalogueRepository)rep, r));
         Constructors.Add(typeof(AggregateConfiguration),
             (rep, r) => new AggregateConfiguration((ICatalogueRepository)rep, r));

Rdmp.Core/Repositories/MEF.cs

@@ -146,7 +146,6 @@ public static string GetCSharpNameForType(Type t)
         if (t.GenericTypeArguments.Length != 1)
             throw new NotSupportedException(
                 "Generic type has more than 1 token (e.g. T1,T2) so no idea what MEF would call it");
-
         var genericTypeName = t.GetGenericTypeDefinition().Name;
 
         Debug.Assert(genericTypeName.EndsWith("`1", StringComparison.Ordinal));

Rdmp.Core/Repositories/Managers/IPluginManager.cs

@@ -13,5 +13,5 @@ namespace Rdmp.Core.Repositories.Managers;
 /// </summary>
 public interface IPluginManager
 {
-    Plugin[] GetCompatiblePlugins();
+    Curation.Data.Plugin[] GetCompatiblePlugins();
 }