task/fix develop merge issues

.nuget/packages.config

@@ -1,4 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
+
 <packages>
   <package id="NUnit.Console" version="3.9.0" />
   <package id="NUnit.ConsoleRunner" version="3.9.0" />

Rdmp.Core.Tests/Curation/ImportTests/GatherAndShareTests.cs

@@ -138,7 +138,7 @@ public void GatherAndShare_Plugin_Test()
             $"Imaginary1{PackPluginRunner.PluginPackageSuffix}"));
         File.WriteAllBytes(f1.FullName, new byte[] { 0x1, 0x2 });
 
-        var plugin = new Plugin(CatalogueRepository, new FileInfo(
+        var plugin = new Rdmp.Core.Curation.Data.Plugin(CatalogueRepository, new FileInfo(
             $"Imaginary{PackPluginRunner.PluginPackageSuffix}"), new Version(1, 1, 1), new Version(1, 1, 1));
         var lma1 = new LoadModuleAssembly(CatalogueRepository, f1, plugin);
 

Rdmp.Core.Tests/Curation/ImportTests/PluginClassTests.cs

@@ -72,7 +72,7 @@ public void TestPlugin_OrphanImport_Sharing()
             $"Blah3.{PackPluginRunner.PluginPackageSuffix}"));
         File.WriteAllBytes(fi3.FullName, new byte[] { 0x3, 0x4 });
 
-        var p = new Plugin(Repository, fi, new Version(1, 1, 1), new Version(1, 1, 1, 1));
+        var p = new Rdmp.Core.Curation.Data.Plugin(Repository, fi, new Version(1, 1, 1), new Version(1, 1, 1, 1));
         var lma = new LoadModuleAssembly(Repository, fi2, p);
         var lma2 = new LoadModuleAssembly(Repository, fi3, p);
 
@@ -116,7 +116,7 @@ public void TestPlugin_OrphanImport_Sharing()
     [TestCase("Rdmp.Dicom", "Rdmp.Dicom")]
     public void Test_Plugin_ShortName(string fullname, string expected)
     {
-        var p = WhenIHaveA<Plugin>();
+        var p = WhenIHaveA<Rdmp.Core.Curation.Data.Plugin>();
         p.Name = fullname;
         Assert.AreEqual(expected, p.GetShortName());
     }

Rdmp.Core.Tests/DataExport/Data/ExtractableCohortAuditLogBuilderTests.cs

@@ -82,7 +82,7 @@ public void AuditLogReFetch_WhenAuditLogIsRubbish()
         [Test]
         public void AuditLogReFetch_WhenSourceIsDeleted()
         {
-                var builder = new ExtractableCohortAuditLogBuilder();
+                _ = new ExtractableCohortAuditLogBuilder();
 
                 var ei = WhenIHaveA<ExtractionInformation>();
                 var desc = ExtractableCohortAuditLogBuilder.GetDescription(ei);

Rdmp.Core.Tests/DataExport/DataAccess/SelectedColumnsTests.cs

@@ -34,12 +34,8 @@ public void CreateAndAssociateColumns()
         try
         {
             _ = new SelectedDataSets(DataExportRepository, config, ds, null);
-
             var cols = config.GetAllExtractableColumnsFor(ds);
 
-            Assert.AreEqual(1, cols.Length);
-            Assert.AreEqual(extractableColumn, cols.Single());
-
             cols = config.GetAllExtractableColumnsFor(ds);
 
             Assert.AreEqual(1, cols.Length);

Rdmp.Core.Tests/DataLoad/Engine/Integration/PipelineTests/Components/RemoveDuplicatesTests.cs

@@ -108,5 +108,6 @@ public void TestNulls()
         Assert.AreEqual(5,
             remover.ProcessPipelineData(dt, ThrowImmediatelyDataLoadEventListener.Quiet,
                 new GracefulCancellationToken()).Rows.Count);
+        dt.Dispose();
     }
 }

Rdmp.Core.Tests/Rdmp.Core.Tests.csproj

@@ -69,7 +69,7 @@
             <PrivateAssets>all</PrivateAssets>
             <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
         </PackageReference>
-        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.7.1" />
+        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.7.2" />
         <PackageReference Include="NSubstitute" Version="5.0.0" />
         <PackageReference Include="NUnit" Version="3.13.3" />
         <PackageReference Include="NUnit3TestAdapter" Version="4.5.0">

Rdmp.Core.Tests/Reports/DocumentationReportDatabaseEntitiesTests.cs

@@ -36,5 +36,6 @@ public void Test_DocumentationReportDatabaseEntities_Normal()
                 iconProvider.GetImage(Arg.Any<object>(), Arg.Any<OverlayKind>()).Returns(img);
 
                 reporter.GenerateReport(store, ThrowImmediatelyCheckNotifier.Quiet, iconProvider, false);
+                img.Dispose();
         }
 }

Rdmp.Core/CohortCommitting/Pipeline/Sources/CohortIdentificationConfigurationSource.cs

@@ -122,10 +122,12 @@ private DataTable GetDataTable(IDataLoadEventListener listener)
             throw new Exception(
                 "CohortIdentificationCriteria execution resulted in an empty dataset (there were no cohorts matched by the query?)");
 
-        var dt = execution.Identifiers;
+        DataTable dt = execution.Identifiers;
+        dt.BeginLoadData();
         foreach (DataColumn column in dt.Columns)
             column.ReadOnly = false;
 
+        dt.EndLoadData();
         return dt;
     }
 

Rdmp.Core/CohortCommitting/Pipeline/Sources/PatientIdentifierColumnSource.cs

@@ -54,7 +54,7 @@ private DataTable GetDataTable(int timeout, int? topX)
 
         var colName = _extractionInformation.GetRuntimeName();
 
-        var dt = new DataTable();
+        DataTable dt = new DataTable();
         dt.BeginLoadData();
         dt.Columns.Add(colName);
 

Rdmp.Core/CohortCreation/Execution/AggregationContainerTask.cs

@@ -87,7 +87,7 @@ returned by this operation they must be in all the sets under this (including th
 APPEAR in any of the sets that come after the FIRST.  This means that you get everyone in the first set
 EXCEPT anyone appearing in any of the other sets that follow the FIRST.",
             _ => throw new ArgumentOutOfRangeException(
-                $"Did not know what tool tip to return for set operation {ToString()}")
+                $"Did not know what tool tip to return for set operation {((CohortAggregateContainer)Child).Operation.ToString()}")
         };
     }
 }

Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandClearUserSettings.cs

@@ -21,4 +21,4 @@ public override void Execute()
         UserSettings.ClearUserSettings();
     }
 
-}
+}

Rdmp.Core/CommandLine/DatabaseCreation/CataloguePipelinesAndReferencesCreation.cs

@@ -109,7 +109,6 @@ private void CreateServers(PlatformDatabaseCreationOptions options)
         Console.WriteLine("Successfully configured RAW server");
     }
 
-
     public void CreatePipelines(PlatformDatabaseCreationOptions options)
     {
         var bulkInsertCsvPipe =

Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreationOptions.cs

@@ -45,7 +45,6 @@ public class PlatformDatabaseCreationOptions
 
     [Option('l', "Create Logging Server", Default = true, HelpText = "Create the default logging server in the Catalogue database once created. Is superseeded by 'Skip Pipelines'")]
     public bool CreateLoggingServer { get; set; }
-
     [Option('e', "ExampleDatasets", Default = false,
         HelpText = "Create example datasets, projects, extraction configurations and cohort queries")]
     public bool ExampleDatasets { get; set; }

Rdmp.Core/Curation/ANOEngineering/ForwardEngineerANOCatalogueEngine.cs

@@ -361,7 +361,6 @@ private ColumnInfo GetNewColumnInfoForOld(ColumnInfo col, bool isOptional = fals
 
         var toReturn = FindNewColumnNamed(syntaxHelper, col, col.GetRuntimeName(), isOptional) ??
                        FindNewColumnNamed(syntaxHelper, col, $"ANO{col.GetRuntimeName()}", isOptional);
-
         if (toReturn == null)
             return isOptional
                 ? null

Rdmp.Core/Curation/Data/Aggregation/AggregateConfiguration.cs

@@ -39,7 +39,7 @@ namespace Rdmp.Core.Curation.Data.Aggregation;
 /// <para>3. 'Joinable PatientIndex Table' - Produce a patient identifier fact table for joining to other Cohort Aggregates during cohort building (See JoinableCohortAggregateConfiguration)</para>
 /// <para>The above labels are informal terms.  Use IsCohortIdentificationAggregate and IsJoinablePatientIndexTable to determine what type a given
 /// AggregateConfiguration is. </para>
-/// 
+///
 /// <para>If your Aggregate is part of cohort identification (Identifier List or Patient Index Table) then its name will start with cic_X_ where X is the ID of the cohort identification
 /// configuration.  Depending on the user interface though this might not appear (See ToString implementation).</para>
 /// </summary>
@@ -257,7 +257,7 @@ private static bool IsBasicallyZero(object v) => v == null || v == DBNull.Value
     /// When an AggregateConfiguration is used in a cohort identification capacity it can have one or more 'patient index tables' defined e.g.
     /// 'Give me all prescriptions for morphine' (Prescribing) 'within 6 months of patient being discharged from hospital' (SMR01).  In this case
     /// a join is done against the secondary dataset.
-    /// 
+    ///
     /// <para>This property returns all such 'patient index table' AggregateConfigurations which are currently being used by this AggregateConfiguration
     /// for building its join.</para>
     /// </summary>

Rdmp.Core/Curation/Data/Catalogue.cs

@@ -519,10 +519,17 @@ public int? LoadMetadata_ID
 
     /// <inheritdoc/>
     [NoMappingToDatabase]
-    public ExternalDatabaseServer LiveLoggingServer =>
-        LiveLoggingServer_ID == null
-            ? null
-            : Repository.GetObjectByID<ExternalDatabaseServer>((int)LiveLoggingServer_ID);
+     public ExternalDatabaseServer LiveLoggingServer {
+        get{
+            if(LiveLoggingServer_ID != null){  
+                ExternalDatabaseServer[] dbs = Repository.GetAllObjectsWhere<ExternalDatabaseServer>("id",(int)LiveLoggingServer_ID);
+                if(dbs.Length > 0){
+                    return dbs.First();
+                }                         
+            }
+            return null;
+        }
+     }
 
     /// <inheritdoc/>
     [NoMappingToDatabase]

Rdmp.Core/Curation/Data/DataLoad/ProcessTask.cs

@@ -358,7 +358,6 @@ public IArgument[] CreateArgumentsForClassIfNotExists(Type t) =>
             //convert the result back from generic to specific (us)
             .ToArray();
 
-
     /// <inheritdoc/>
     public IArgument[] CreateArgumentsForClassIfNotExists<T>() => CreateArgumentsForClassIfNotExists(typeof(T));
 

Rdmp.Core/DataExport/Data/DeployedExtractionFilterParameter.cs

@@ -141,4 +141,4 @@ public DeployedExtractionFilterParameter ShallowClone(DeployedExtractionFilter i
         CopyShallowValuesTo(clone);
         return clone;
     }
-}
+}

Rdmp.Core/DataExport/DataExtraction/ExtractionTimeValidator.cs

@@ -50,7 +50,7 @@ public void Validate(DataTable dt, string validationColumnToPopulateIfAny)
     {
         if (!_initialized)
             Initialize(dt);
-
+        dt.BeginLoadData();
         foreach (DataRow r in dt.Rows)
         {
             //additive validation results, Results is a class that wraps DictionaryOfFailure which is an array of columns and each element is another array of consequences (with a row count for each consequence)
@@ -61,6 +61,7 @@ public void Validate(DataTable dt, string validationColumnToPopulateIfAny)
             if (validationColumnToPopulateIfAny != null)
                 r[validationColumnToPopulateIfAny] = consequenceOnLastRowProcessed;
         }
+        dt.EndLoadData();
     }
 
     private void Initialize(DataTable dt)

Rdmp.Core/DataLoad/Engine/Pipeline/Components/Anonymisation/ColumnInfoToANOTableConverter.cs

@@ -152,7 +152,7 @@ private void MigrateExistingData(Func<string, bool> shouldApplySql, DbConnection
                 throw new Exception("User decided not to perform update on table");
             cmdUpdateMainTable.ExecuteNonQuery();
         }
-        finally
+          finally
         {
             //always drop the temp anomap
             using var dropMappingTable = DatabaseCommandHelper.GetCommand("DROP TABLE TempANOMap", con);

Rdmp.Core/DataLoad/Engine/Pipeline/Components/CleanStrings.cs

@@ -31,7 +31,6 @@ public DataTable ProcessPipelineData(DataTable toProcess, IDataLoadEventListener
         GracefulCancellationToken cancellationToken)
     {
         timer.Start();
-
     StartAgain:
         foreach (DataRow row in toProcess.Rows)
         {
@@ -85,7 +84,6 @@ public DataTable ProcessPipelineData(DataTable toProcess, IDataLoadEventListener
         job.OnProgress(this,
             new ProgressEventArgs(_taskDescription, new ProgressMeasurement(_rowsProcessed, ProgressType.Records),
                 timer.Elapsed));
-
         return toProcess;
     }
 

Rdmp.Core/DataLoad/Modules/Attachers/FixedWidthFormatFile.cs

@@ -77,7 +77,7 @@ public FixedWidthFormatFile(FileInfo pathToFormatFile)
     public DataTable GetDataTableFromFlatFile(FileInfo f)
     {
         //setup the table
-        var toReturn = new DataTable();
+        DataTable toReturn = new DataTable();
 
         toReturn.BeginLoadData();
         foreach (var fixedWidthColumn in FormatColumns)

Rdmp.Core/DataLoad/Modules/DataFlowOperations/CohortSampler.cs

@@ -107,7 +107,7 @@ public DataTable ProcessPipelineData(DataTable toProcess, IDataLoadEventListener
             throw new Exception(
                 $"Cohort only contains {chosen.Count} unique identifiers.  This is less than the requested sample size of {SampleSize} and {nameof(FailIfNotEnoughIdentifiers)} is true");
 
-        var dtToReturn = new DataTable();
+        DataTable dtToReturn = new DataTable();
         dtToReturn.BeginLoadData();
         dtToReturn.Columns.Add(expectedFieldName);
 

Rdmp.Core/DataLoad/Modules/DataFlowOperations/Transposer.cs

@@ -65,7 +65,7 @@ public void Check(ICheckNotifier notifier)
 
     private DataTable GenerateTransposedTable(DataTable inputTable)
     {
-        var outputTable = new DataTable();
+        DataTable outputTable = new DataTable();
         outputTable.BeginLoadData();
         // Add columns by looping rows
 

Rdmp.Core/DataLoad/Modules/DataFlowSources/ExcelDataFlowSource.cs

@@ -124,7 +124,6 @@ public DataTable GetAllData(ISheet worksheet, IDataLoadEventListener listener)
     {
         var toReturn = new DataTable();
         toReturn.BeginLoadData();
-
         var rowEnumerator = worksheet.GetRowEnumerator();
         var nColumns = -1;
 
@@ -188,7 +187,6 @@ public DataTable GetAllData(ISheet worksheet, IDataLoadEventListener listener)
             if (!gotAtLeastOneGoodValue)
                 toReturn.Rows.Remove(r);
         }
-
         return toReturn;
     }
 

Rdmp.Core/DataLoad/Modules/DataFlowSources/SubComponents/FlatFileToDataTablePusher.cs

@@ -306,7 +306,6 @@ public DataTable StronglyTypeTable(DataTable workingTable, ExplicitTypingCollect
             foreach (DataRow row in workingTable.Rows)
                 dtCloned.Rows.Add(row.ItemArray.Select((v, idx) =>
                     deciders.TryGetValue(idx, out var decider) && v is string s ? decider.Parse(s) : v).ToArray());
-
             return dtCloned;
         }
 

Rdmp.Core/DataLoad/Modules/FileOperations/FileUnzipper.cs

@@ -46,8 +46,7 @@ public void Initialize(ILoadDirectory directory, DiscoveredDatabase dbInfo)
 
     public ExitCodeType Fetch(IDataLoadJob job, GracefulCancellationToken cancellationToken)
     {
-        foreach (var fileInfo in job.LoadDirectory.ForLoading.GetFiles("*.zip"))
-        {
+        foreach (var fileInfo in job.LoadDirectory.ForLoading.GetFiles("*.zip")){
             //do it as regex rather than in GetFiles above because that method probably doesn't do regex
             if (ZipArchivePattern != null && !string.IsNullOrWhiteSpace(ZipArchivePattern.ToString()) &&
                 !ZipArchivePattern.IsMatch(fileInfo.Name)) continue;

Rdmp.Core/Logging/DataLoadInfo.cs

@@ -345,8 +345,6 @@ public void LogProgress(ProgressEventType pevent, string Source, string Descript
         }
 
         lock (_logWaiter)
-        {
             Monitor.Pulse(_logWaiter);
-        }
     }
-}
+}

Rdmp.Core/MapsDirectlyToDatabaseTable/Versioning/Patcher.cs

@@ -78,7 +78,7 @@ public virtual Patch GetInitialCreateScriptContents(DiscoveredDatabase db)
                 {
                     var sr = new StreamReader(assembly.GetManifestResourceStream(candidates[0]));
 
-                    var sql = sr.ReadToEnd();
+                var sql = sr.ReadToEnd();
 
                     if (!sql.Contains(Patch.VersionKey))
                         sql = GetHeader(db.Server.DatabaseType, InitialScriptName, new Version(1, 0, 0)) + sql;

Rdmp.Core/Providers/CatalogueChildProvider.cs

@@ -187,8 +187,8 @@ public class CatalogueChildProvider : ICoreChildProvider
 
     public JoinableCohortAggregateConfigurationUse[] AllJoinableCohortAggregateConfigurationUse { get; private set; }
     public AllPluginsNode AllPluginsNode { get; private set; }
-    public Plugin[] AllPlugins { get; private set; }
-    public Plugin[] AllCompatiblePlugins { get; private set; }
+    public Curation.Data.Plugin[] AllPlugins { get; private set; }
+    public Curation.Data.Plugin[] AllCompatiblePlugins { get; private set; }
 
     public HashSet<StandardPipelineUseCaseNode> PipelineUseCases { get; set; } = new();
 
@@ -432,7 +432,7 @@ public CatalogueChildProvider(ICatalogueRepository repository, IChildProvider[]
         ReportProgress("After Governance");
 
         AllPluginsNode = new AllPluginsNode();
-        AllPlugins = GetAllObjects<Plugin>(repository);
+        AllPlugins = GetAllObjects<Curation.Data.Plugin>(repository);
         AllCompatiblePlugins = _catalogueRepository.PluginManager.GetCompatiblePlugins();
 
         AddChildren(AllPluginsNode);

Rdmp.Core/Rdmp.Core.csproj

@@ -371,9 +371,9 @@
     <PackageReference Include="Microsoft.XmlSerializer.Generator" Version="7.0.0" />
     <PackageReference Include="MongoDB.Driver" Version="2.20.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-    <PackageReference Include="NPOI" Version="2.6.0" />
     <PackageReference Include="NLog" Version="5.2.3" />
-    <PackageReference Include="SixLabors.ImageSharp" Version="3.0.1" />
+    <PackageReference Include="NPOI" Version="2.6.0" />
+    <PackageReference Include="SixLabors.ImageSharp" Version="3.0.2" />
     <PackageReference Include="SixLabors.ImageSharp.Drawing" Version="1.0.0" />
     <PackageReference Include="Spectre.Console" Version="0.47.0" />
     <PackageReference Include="System.Diagnostics.Debug" Version="4.3.0" />

Rdmp.Core/Repositories/CatalogueRepository.cs

@@ -29,7 +29,7 @@
 using Rdmp.Core.Repositories.Managers;
 using Rdmp.Core.ReusableLibraryCode;
 using Rdmp.Core.ReusableLibraryCode.Comments;
-
+ 
 namespace Rdmp.Core.Repositories;
 
 /// <inheritdoc cref="ICatalogueRepository"/>
@@ -96,7 +96,7 @@ public CatalogueRepository(DbConnectionStringBuilder catalogueConnectionString)
         Constructors.Add(typeof(StandardRegex), (rep, r) => new StandardRegex((ICatalogueRepository)rep, r));
         Constructors.Add(typeof(AnyTableSqlParameter),
             (rep, r) => new AnyTableSqlParameter((ICatalogueRepository)rep, r));
-        Constructors.Add(typeof(Plugin), (rep, r) => new Plugin((ICatalogueRepository)rep, r));
+        Constructors.Add(typeof(Curation.Data.Plugin), (rep, r) => new Curation.Data.Plugin((ICatalogueRepository)rep, r));
         Constructors.Add(typeof(ANOTable), (rep, r) => new ANOTable((ICatalogueRepository)rep, r));
         Constructors.Add(typeof(AggregateConfiguration),
             (rep, r) => new AggregateConfiguration((ICatalogueRepository)rep, r));

Rdmp.Core/Repositories/MEF.cs

@@ -146,7 +146,6 @@ public static string GetCSharpNameForType(Type t)
         if (t.GenericTypeArguments.Length != 1)
             throw new NotSupportedException(
                 "Generic type has more than 1 token (e.g. T1,T2) so no idea what MEF would call it");
-
         var genericTypeName = t.GetGenericTypeDefinition().Name;
 
         Debug.Assert(genericTypeName.EndsWith("`1", StringComparison.Ordinal));

Rdmp.Core/Repositories/Managers/IPluginManager.cs

@@ -13,5 +13,5 @@ namespace Rdmp.Core.Repositories.Managers;
 /// </summary>
 public interface IPluginManager
 {
-    Plugin[] GetCompatiblePlugins();
+    Curation.Data.Plugin[] GetCompatiblePlugins();
 }