diff --git a/src/Dynamicweb.DataIntegration.Providers.OrderProvider.csproj b/src/Dynamicweb.DataIntegration.Providers.OrderProvider.csproj index decdeec..fcd2065 100644 --- a/src/Dynamicweb.DataIntegration.Providers.OrderProvider.csproj +++ b/src/Dynamicweb.DataIntegration.Providers.OrderProvider.csproj @@ -1,6 +1,6 @@  - 10.0.2 + 10.0.3 1.0.0.0 Order Provider @@ -22,10 +22,8 @@ snupkg - - - - + + - \ No newline at end of file + diff --git a/src/OrderDestinationWriter.cs b/src/OrderDestinationWriter.cs new file mode 100644 index 0000000..ce12cac --- /dev/null +++ b/src/OrderDestinationWriter.cs @@ -0,0 +1,74 @@ +using Dynamicweb.DataIntegration.Integration; +using Dynamicweb.DataIntegration.ProviderHelpers; +using Dynamicweb.Logging; +using System.Collections.Generic; +using System.Data; +using System.Data.SqlClient; +using System.Linq; + +namespace Dynamicweb.DataIntegration.Providers.OrderProvider; + +internal class OrderDestinationWriter : BaseSqlWriter +{ + private new Mapping Mapping { get; } + private ILogger Logger { get; } + private SqlBulkCopy SqlBulkCopier { get; } + private int SkippedFailedRowsCount { get; set; } + private DataTable TableToWrite { get; set; } + private DataSet DataToWrite { get; } = new DataSet(); + private string TempTablePrefix { get; } + private bool SkipFailingRows { get; } + internal SqlCommand SqlCommand { get; } + internal int RowsToWriteCount { get; set; } + + public OrderDestinationWriter(Mapping mapping, SqlConnection connection, ILogger logger, bool skipFailingRows) + { + Mapping = mapping; + SqlCommand = connection.CreateCommand(); + SqlCommand.CommandTimeout = 1200; + Logger = logger; + SkipFailingRows = skipFailingRows; + TempTablePrefix = $"TempTableForBulkImport{mapping.GetId()}"; + SqlBulkCopier = new SqlBulkCopy(connection); + SqlBulkCopier.DestinationTableName = mapping.DestinationTable.Name + TempTablePrefix; + SqlBulkCopier.BulkCopyTimeout = 0; + Initialize(); + if (connection.State != ConnectionState.Open) + connection.Open(); + } + + protected new virtual void Initialize() + { + List destColumns = new(); + var columnMappings = Mapping.GetColumnMappings(); + foreach (ColumnMapping columnMapping in columnMappings.DistinctBy(obj => obj.DestinationColumn.Name)) + { + destColumns.Add((SqlColumn)columnMapping.DestinationColumn); + } + if (Mapping.DestinationTable != null && Mapping.DestinationTable.Name == "EcomAssortmentPermissions") + { + if (columnMappings.Find(m => string.Compare(m.DestinationColumn.Name, "AssortmentPermissionAccessUserID", true) == 0) == null) + destColumns.Add(new SqlColumn("AssortmentPermissionAccessUserID", typeof(string), SqlDbType.Int, null, -1, false, true, false)); + } + SQLTable.CreateTempTable(SqlCommand, Mapping.DestinationTable.SqlSchema, Mapping.DestinationTable.Name, TempTablePrefix, destColumns, Logger); + + TableToWrite = DataToWrite.Tables.Add(Mapping.DestinationTable.Name + TempTablePrefix); + foreach (SqlColumn column in destColumns) + { + TableToWrite.Columns.Add(column.Name, column.Type); + } + } + + internal void FinishWriting() + { + SkippedFailedRowsCount = SqlBulkCopierWriteToServer(SqlBulkCopier, TableToWrite, SkipFailingRows, Mapping, Logger); + if (TableToWrite.Rows.Count != 0) + { + RowsToWriteCount = RowsToWriteCount + TableToWrite.Rows.Count - SkippedFailedRowsCount; + Logger.Log("Added " + RowsToWriteCount + " rows to temporary table for " + Mapping.DestinationTable.Name + "."); + } + } + + internal void MoveDataToMainTable(SqlTransaction sqlTransaction, bool updateOnlyExistingRecords, bool insertOnlyNewRecords) => + MoveDataToMainTable(Mapping, SqlCommand, sqlTransaction, TempTablePrefix, updateOnlyExistingRecords, insertOnlyNewRecords); +} diff --git a/src/OrderProvider.cs b/src/OrderProvider.cs index 8cd2394..f1f1ac0 100644 --- a/src/OrderProvider.cs +++ b/src/OrderProvider.cs @@ -2,7 +2,6 @@ using Dynamicweb.DataIntegration.Integration; using Dynamicweb.DataIntegration.Integration.Interfaces; using Dynamicweb.DataIntegration.ProviderHelpers; -using Dynamicweb.DataIntegration.Providers.DynamicwebProvider; using Dynamicweb.Ecommerce.Orders; using Dynamicweb.Extensibility.AddIns; using Dynamicweb.Extensibility.Editors; @@ -17,119 +16,51 @@ using System.Xml; using System.Xml.Linq; -namespace Dynamicweb.DataIntegration.Providers.OrderProvider +namespace Dynamicweb.DataIntegration.Providers.OrderProvider; + +[AddInName("Dynamicweb.DataIntegration.Providers.Provider"), AddInLabel("Order Provider"), AddInDescription("Order provider"), AddInIgnore(false)] +public class OrderProvider : BaseSqlProvider, ISource, IDestination, IParameterOptions { - [AddInName("Dynamicweb.DataIntegration.Providers.Provider"), AddInLabel("Order Provider"), AddInDescription("Order provider"), AddInIgnore(false)] - public class OrderProvider : SqlProvider.SqlProvider, ISource, IDestination, IParameterOptions - { - private const string OrderCustomerAccessUserExternalId = "OrderCustomerAccessUserExternalId"; - private Job job = null; + private const string OrderCustomerAccessUserExternalId = "OrderCustomerAccessUserExternalId"; + private Job job = null; + private Schema Schema { get; set; } + private string SqlConnectionString { get; set; } - [AddInParameter("Export not yet exported Orders"), AddInParameterEditor(typeof(YesNoParameterEditor), ""), AddInParameterGroup("Source")] - public virtual bool ExportNotExportedOrders { get; set; } + [AddInParameter("Export not yet exported Orders"), AddInParameterEditor(typeof(YesNoParameterEditor), ""), AddInParameterGroup("Source")] + public virtual bool ExportNotExportedOrders { get; set; } - [AddInParameter("Only export orders without externalID"), AddInParameterEditor(typeof(YesNoParameterEditor), ""), AddInParameterGroup("Source")] - public virtual bool ExportOnlyOrdersWithoutExtID { get; set; } + [AddInParameter("Only export orders without externalID"), AddInParameterEditor(typeof(YesNoParameterEditor), ""), AddInParameterGroup("Source")] + public virtual bool ExportOnlyOrdersWithoutExtID { get; set; } - [AddInParameter("Export completed orders only"), AddInParameterEditor(typeof(YesNoParameterEditor), ""), AddInParameterGroup("Source")] - public virtual bool DoNotExportCarts { get; set; } + [AddInParameter("Export completed orders only"), AddInParameterEditor(typeof(YesNoParameterEditor), ""), AddInParameterGroup("Source")] + public virtual bool DoNotExportCarts { get; set; } - [AddInParameter("Order state after export"), AddInParameterEditor(typeof(GroupedDropDownParameterEditor), "none=true;noneText=Leave unchanged;"), AddInParameterGroup("Source")] - public virtual string OrderStateAfterExport { get; set; } + [AddInParameter("Order state after export"), AddInParameterEditor(typeof(GroupedDropDownParameterEditor), "none=true;noneText=Leave unchanged;"), AddInParameterGroup("Source")] + public virtual string OrderStateAfterExport { get; set; } - [AddInParameter("Remove missing order lines"), AddInParameterEditor(typeof(YesNoParameterEditor), ""), AddInParameterGroup("Destination")] - public bool RemoveMissingOrderLines { get; set; } + [AddInParameter("Remove missing order lines"), AddInParameterEditor(typeof(YesNoParameterEditor), ""), AddInParameterGroup("Destination")] + public bool RemoveMissingOrderLines { get; set; } - #region Hide Properties in the UI section + [AddInParameter("Discard duplicates"), AddInParameterEditor(typeof(YesNoParameterEditor), ""), AddInParameterGroup("Destination")] + public bool DiscardDuplicates { get; set; } - [AddInParameter("Source server"), AddInParameterEditor(typeof(TextParameterEditor), "htmlClass=connectionStringInput;"), AddInParameterGroup("hidden")] - public override string SourceServer - { - get { return Server; } - set { Server = value; } - } - [AddInParameter("Destination server"), AddInParameterEditor(typeof(TextParameterEditor), "htmlClass=connectionStringInput;"), AddInParameterGroup("hidden")] - public override string DestinationServer - { - get { return Server; } - set { Server = value; } - } - [AddInParameter("Use integrated security to connect to source server"), AddInParameterEditor(typeof(YesNoParameterEditor), "htmlClass=connectionStringInput;"), AddInParameterGroup("hidden")] - public override bool SourceServerSSPI - { - get; - set; - } - [AddInParameter("Use integrated security to connect to destination server"), AddInParameterEditor(typeof(YesNoParameterEditor), "htmlClass=connectionStringInput;"), AddInParameterGroup("hidden")] - public override bool DestinationServerSSPI - { - get; - set; - } - [AddInParameter("Sql source server username"), AddInParameterEditor(typeof(TextParameterEditor), "htmlClass=connectionStringInput;"), AddInParameterGroup("hidden")] - public override string SourceUsername - { - get { return Username; } - set { Username = value; } - } - [AddInParameter("Sql destination server username"), AddInParameterEditor(typeof(TextParameterEditor), "htmlClass=connectionStringInput;"), AddInParameterGroup("hidden")] - public override string DestinationUsername - { - get { return Username; } - set { Username = value; } - } - [AddInParameter("Sql source server password"), AddInParameterEditor(typeof(TextParameterEditor), "htmlClass=connectionStringInput;"), AddInParameterGroup("hidden")] - public override string SourcePassword - { - get { return Password; } - set { Password = value; } - } - [AddInParameter("Sql destination server password"), AddInParameterEditor(typeof(TextParameterEditor), "htmlClass=connectionStringInput;"), AddInParameterGroup("hidden")] - public override string DestinationPassword - { - get { return Password; } - set { Password = value; } - } - [AddInParameter("Sql source database"), AddInParameterEditor(typeof(TextParameterEditor), "htmlClass=connectionStringInput;"), AddInParameterGroup("hidden")] - public override string SourceDatabase - { - get { return Catalog; } - set { Catalog = value; } - } - [AddInParameter("Sql source connection string"), AddInParameterEditor(typeof(TextParameterEditor), "htmlClass=connectionStringInput;"), AddInParameterGroup("hidden")] - public override string SourceConnectionString - { - get { return ManualConnectionString; } - set { ManualConnectionString = value; } - } - [AddInParameter("Sql destination connection string"), AddInParameterEditor(typeof(TextParameterEditor), "htmlClass=connectionStringInput;"), AddInParameterGroup("hidden")] - public override string DestinationConnectionString - { - get { return ManualConnectionString; } - set { ManualConnectionString = value; } - } - [AddInParameter("Sql destination server password"), AddInParameterEditor(typeof(TextParameterEditor), "htmlClass=connectionStringInput;"), AddInParameterGroup("hidden")] - public override string DestinationDatabase - { - get { return Catalog; } - set { Catalog = value; } - } - [AddInParameter("Remove missing rows after import"), AddInParameterEditor(typeof(YesNoParameterEditor), ""), AddInParameterGroup("hidden")] - public override bool RemoveMissingAfterImport { get; set; } - #endregion + [AddInParameter("Persist successful rows and skip failing rows"), AddInParameterEditor(typeof(YesNoParameterEditor), ""), AddInParameterGroup("Destination"), AddInParameterOrder(100)] + public bool SkipFailingRows { get; set; } - protected override SqlConnection Connection - { - get { return connection ?? (connection = (SqlConnection)Database.CreateConnection()); } - set { connection = value; } - } + private SqlConnection connection; + private SqlConnection Connection + { + get { return connection ?? (connection = (SqlConnection)Database.CreateConnection()); } + set { connection = value; } + } - public OrderProvider(string connectionString) - { - SqlConnectionString = connectionString; - Connection = new SqlConnection(connectionString); - DiscardDuplicates = false; - } + + public OrderProvider(string connectionString) + { + SqlConnectionString = connectionString; + Connection = new SqlConnection(connectionString); + DiscardDuplicates = false; + } public override void LoadSettings(Job job) { @@ -137,452 +68,455 @@ public override void LoadSettings(Job job) OrderTablesInJob(job, true); } - public new void Close() - { - if (job != null && job.Result == JobResult.Completed) - OrderSourceReader.UpdateExportedOrdersInDb(OrderStateAfterExport, connection); + public new void Close() + { + if (job != null && job.Result == JobResult.Completed) + OrderSourceReader.UpdateExportedOrdersInDb(OrderStateAfterExport, connection); - base.Close(); - } + Connection.Close(); + } - public override Schema GetOriginalSourceSchema() - { - Schema result = base.GetOriginalSourceSchema(); - List tablestToKeep = new List { "EcomOrders", "EcomOrderLines", "EcomOrderLineFields", "EcomOrderLineFieldGroupRelation" }; - List tablesToRemove = new List
(); - foreach (Table table in result.GetTables()) - { - if (!tablestToKeep.Contains(table.Name)) - tablesToRemove.Add(table); - else if (table.Name == "EcomOrders") - { - table.AddColumn(new SqlColumn(OrderCustomerAccessUserExternalId, typeof(string), SqlDbType.NVarChar, table, -1, false, false, true)); - } - } - foreach (Table table in tablesToRemove) + public override Schema GetOriginalSourceSchema() + { + Schema result = GetSqlSourceSchema(Connection); + List tablestToKeep = new() { "EcomOrders", "EcomOrderLines", "EcomOrderLineFields", "EcomOrderLineFieldGroupRelation" }; + List
tablesToRemove = new(); + foreach (Table table in result.GetTables()) + { + if (!tablestToKeep.Contains(table.Name)) + tablesToRemove.Add(table); + else if (table.Name == "EcomOrders") { - result.RemoveTable(table); + table.AddColumn(new SqlColumn(OrderCustomerAccessUserExternalId, typeof(string), SqlDbType.NVarChar, table, -1, + false, false, true)); } + } + foreach (Table table in tablesToRemove) + { + result.RemoveTable(table); + } - var orderLinesTable = result.GetTables().FirstOrDefault(obj => string.Equals(obj.Name, "EcomOrderLines", StringComparison.OrdinalIgnoreCase)); - var ordersTable = result.GetTables().FirstOrDefault(obj => string.Equals(obj.Name, "EcomOrders", StringComparison.OrdinalIgnoreCase)); - if (orderLinesTable != null && ordersTable != null) + var orderLinesTable = result.GetTables().FirstOrDefault(obj => string.Equals(obj.Name, "EcomOrderLines", StringComparison.OrdinalIgnoreCase)); + var ordersTable = result.GetTables().FirstOrDefault(obj => string.Equals(obj.Name, "EcomOrders", StringComparison.OrdinalIgnoreCase)); + if (orderLinesTable != null && ordersTable != null) + { + foreach (var column in ordersTable.Columns) { - foreach (var column in ordersTable.Columns) + if (!column.Name.Equals(OrderCustomerAccessUserExternalId, StringComparison.OrdinalIgnoreCase)) { - if (!column.Name.Equals(OrderCustomerAccessUserExternalId, StringComparison.OrdinalIgnoreCase)) - { - _ = orderLinesTable.AddNewColumn($"{column.Name}", column.Type, -1, false, column.IsPrimaryKey); - } + _ = orderLinesTable.AddNewColumn($"{column.Name}", column.Type, -1, false, column.IsPrimaryKey); } } - - return result; } - public override Schema GetOriginalDestinationSchema() - { - return GetOriginalSourceSchema(); - } + return result; + } - public override void OverwriteSourceSchemaToOriginal() - { - Schema = GetOriginalSourceSchema(); - } + public override Schema GetOriginalDestinationSchema() + { + return GetOriginalSourceSchema(); + } - public override void OverwriteDestinationSchemaToOriginal() - { - Schema = GetOriginalSourceSchema(); - } + public override void OverwriteSourceSchemaToOriginal() + { + Schema = GetOriginalSourceSchema(); + } - public override Schema GetSchema() - { - if (Schema == null) - { - Schema = GetOriginalSourceSchema(); - } - return Schema; - } - public OrderProvider(XmlNode xmlNode) + public override void OverwriteDestinationSchemaToOriginal() + { + Schema = GetOriginalSourceSchema(); + } + + public override Schema GetSchema() + { + Schema ??= GetOriginalSourceSchema(); + return Schema; + } + public OrderProvider(XmlNode xmlNode) + { + foreach (XmlNode node in xmlNode.ChildNodes) { - foreach (XmlNode node in xmlNode.ChildNodes) + switch (node.Name) { - switch (node.Name) - { - case "SqlConnectionString": - if (node.HasChildNodes) - { - SqlConnectionString = node.FirstChild.Value; - Connection = new SqlConnection(SqlConnectionString); - } - break; - case "Schema": - Schema = new Schema(node); - break; - case "ExportNotYetExportedOrders": - if (node.HasChildNodes) - { - ExportNotExportedOrders = node.FirstChild.Value == "True"; - } - break; - case "ExportOnlyOrdersWithoutExtID": - if (node.HasChildNodes) - { - ExportOnlyOrdersWithoutExtID = node.FirstChild.Value == "True"; - } - break; - case "DoNotExportCarts": - if (node.HasChildNodes) - { - DoNotExportCarts = node.FirstChild.Value == "True"; - } - break; - case "OrderStateAfterExport": - if (node.HasChildNodes) - { - OrderStateAfterExport = node.FirstChild.Value; - } - break; - case "DiscardDuplicates": - if (node.HasChildNodes) - { - DiscardDuplicates = node.FirstChild.Value == "True"; - } - break; - case "RemoveMissingOrderLines": - if (node.HasChildNodes) - { - RemoveMissingOrderLines = node.FirstChild.Value == "True"; - } - break; - case "SkipFailingRows": - if (node.HasChildNodes) - { - SkipFailingRows = node.FirstChild.Value == "True"; - } - break; - } + case "SqlConnectionString": + if (node.HasChildNodes) + { + SqlConnectionString = node.FirstChild.Value; + Connection = new SqlConnection(SqlConnectionString); + } + break; + case "Schema": + Schema = new Schema(node); + break; + case "ExportNotYetExportedOrders": + if (node.HasChildNodes) + { + ExportNotExportedOrders = node.FirstChild.Value == "True"; + } + break; + case "ExportOnlyOrdersWithoutExtID": + if (node.HasChildNodes) + { + ExportOnlyOrdersWithoutExtID = node.FirstChild.Value == "True"; + } + break; + case "DoNotExportCarts": + if (node.HasChildNodes) + { + DoNotExportCarts = node.FirstChild.Value == "True"; + } + break; + case "OrderStateAfterExport": + if (node.HasChildNodes) + { + OrderStateAfterExport = node.FirstChild.Value; + } + break; + case "DiscardDuplicates": + if (node.HasChildNodes) + { + DiscardDuplicates = node.FirstChild.Value == "True"; + } + break; + case "RemoveMissingOrderLines": + if (node.HasChildNodes) + { + RemoveMissingOrderLines = node.FirstChild.Value == "True"; + } + break; + case "SkipFailingRows": + if (node.HasChildNodes) + { + SkipFailingRows = node.FirstChild.Value == "True"; + } + break; } } + } - public override string ValidateDestinationSettings() - { - return string.Empty; - } - public override string ValidateSourceSettings() - { - return null; - } - public new virtual void SaveAsXml(XmlTextWriter xmlTextWriter) - { - xmlTextWriter.WriteElementString("SqlConnectionString", SqlConnectionString); - xmlTextWriter.WriteElementString("ExportNotYetExportedOrders", ExportNotExportedOrders.ToString(CultureInfo.CurrentCulture)); - xmlTextWriter.WriteElementString("ExportOnlyOrdersWithoutExtID", ExportOnlyOrdersWithoutExtID.ToString(CultureInfo.CurrentCulture)); - xmlTextWriter.WriteElementString("DoNotExportCarts ", DoNotExportCarts.ToString(CultureInfo.CurrentCulture)); - xmlTextWriter.WriteElementString("OrderStateAfterExport ", OrderStateAfterExport); - xmlTextWriter.WriteElementString("DiscardDuplicates", DiscardDuplicates.ToString(CultureInfo.CurrentCulture)); - xmlTextWriter.WriteElementString("RemoveMissingOrderLines", RemoveMissingOrderLines.ToString(CultureInfo.CurrentCulture)); - xmlTextWriter.WriteElementString("SkipFailingRows", SkipFailingRows.ToString()); - GetSchema().SaveAsXml(xmlTextWriter); - } - public override void UpdateSourceSettings(ISource source) - { - OrderProvider newProvider = (OrderProvider)source; - ExportNotExportedOrders = newProvider.ExportNotExportedOrders; - ExportOnlyOrdersWithoutExtID = newProvider.ExportOnlyOrdersWithoutExtID; - DoNotExportCarts = newProvider.DoNotExportCarts; - OrderStateAfterExport = newProvider.OrderStateAfterExport; - DiscardDuplicates = newProvider.DiscardDuplicates; - RemoveMissingOrderLines = newProvider.RemoveMissingOrderLines; - SkipFailingRows = newProvider.SkipFailingRows; - base.UpdateSourceSettings(source); - } - public override void UpdateDestinationSettings(IDestination destination) - { - ISource newProvider = (ISource)destination; - UpdateSourceSettings(newProvider); - } - public override string Serialize() + public override string ValidateDestinationSettings() + { + return string.Empty; + } + + public override string ValidateSourceSettings() + { + return null; + } + + public new virtual void SaveAsXml(XmlTextWriter xmlTextWriter) + { + xmlTextWriter.WriteElementString("SqlConnectionString", SqlConnectionString); + xmlTextWriter.WriteElementString("ExportNotYetExportedOrders", ExportNotExportedOrders.ToString(CultureInfo.CurrentCulture)); + xmlTextWriter.WriteElementString("ExportOnlyOrdersWithoutExtID", ExportOnlyOrdersWithoutExtID.ToString(CultureInfo.CurrentCulture)); + xmlTextWriter.WriteElementString("DoNotExportCarts ", DoNotExportCarts.ToString(CultureInfo.CurrentCulture)); + xmlTextWriter.WriteElementString("OrderStateAfterExport ", OrderStateAfterExport); + xmlTextWriter.WriteElementString("DiscardDuplicates", DiscardDuplicates.ToString(CultureInfo.CurrentCulture)); + xmlTextWriter.WriteElementString("RemoveMissingOrderLines", RemoveMissingOrderLines.ToString(CultureInfo.CurrentCulture)); + xmlTextWriter.WriteElementString("SkipFailingRows", SkipFailingRows.ToString()); + GetSchema().SaveAsXml(xmlTextWriter); + } + + public override void UpdateSourceSettings(ISource source) + { + OrderProvider newProvider = (OrderProvider)source; + ExportNotExportedOrders = newProvider.ExportNotExportedOrders; + ExportOnlyOrdersWithoutExtID = newProvider.ExportOnlyOrdersWithoutExtID; + DoNotExportCarts = newProvider.DoNotExportCarts; + OrderStateAfterExport = newProvider.OrderStateAfterExport; + DiscardDuplicates = newProvider.DiscardDuplicates; + RemoveMissingOrderLines = newProvider.RemoveMissingOrderLines; + SkipFailingRows = newProvider.SkipFailingRows; + SqlConnectionString = newProvider.SqlConnectionString; + } + + public override void UpdateDestinationSettings(IDestination destination) + { + ISource newProvider = (ISource)destination; + UpdateSourceSettings(newProvider); + } + + public override string Serialize() + { + XDocument document = new XDocument(new XDeclaration("1.0", "utf-8", string.Empty)); + XElement root = new XElement("Parameters"); + root.Add(CreateParameterNode(GetType(), "Export not yet exported Orders", ExportNotExportedOrders.ToString(CultureInfo.CurrentCulture))); + root.Add(CreateParameterNode(GetType(), "Only export orders without externalID", ExportOnlyOrdersWithoutExtID.ToString(CultureInfo.CurrentCulture))); + root.Add(CreateParameterNode(GetType(), "Export completed orders only", DoNotExportCarts.ToString(CultureInfo.CurrentCulture))); + root.Add(CreateParameterNode(GetType(), "Order state after export", OrderStateAfterExport)); + root.Add(CreateParameterNode(GetType(), "Discard duplicates", DiscardDuplicates.ToString())); + root.Add(CreateParameterNode(GetType(), "Remove missing order lines", RemoveMissingOrderLines.ToString())); + root.Add(CreateParameterNode(GetType(), "Persist successful rows and skip failing rows", SkipFailingRows.ToString())); + document.Add(root); + + return document.ToString(); + } + + public OrderProvider() + { + DiscardDuplicates = false; + } + + public new ISourceReader GetReader(Mapping mapping) + { + return new OrderSourceReader(mapping, Connection, ExportNotExportedOrders, ExportOnlyOrdersWithoutExtID, DoNotExportCarts); + } + + protected internal static void OrderTablesInJob(Job job, bool isSourceLookup) + { + MappingCollection tables = new MappingCollection(); + + var mappings = GetMappingsByName(job.Mappings, "EcomOrders", isSourceLookup); + if (mappings != null) { - XDocument document = new XDocument(new XDeclaration("1.0", "utf-8", string.Empty)); - XElement root = new XElement("Parameters"); - root.Add(CreateParameterNode(GetType(), "Export not yet exported Orders", ExportNotExportedOrders.ToString(CultureInfo.CurrentCulture))); - root.Add(CreateParameterNode(GetType(), "Only export orders without externalID", ExportOnlyOrdersWithoutExtID.ToString(CultureInfo.CurrentCulture))); - root.Add(CreateParameterNode(GetType(), "Export completed orders only", DoNotExportCarts.ToString(CultureInfo.CurrentCulture))); - root.Add(CreateParameterNode(GetType(), "Order state after export", OrderStateAfterExport)); - root.Add(CreateParameterNode(GetType(), "Discard duplicates", DiscardDuplicates.ToString())); - root.Add(CreateParameterNode(GetType(), "Remove missing order lines", RemoveMissingOrderLines.ToString())); - root.Add(CreateParameterNode(GetType(), "Persist successful rows and skip failing rows", SkipFailingRows.ToString())); - document.Add(root); - - return document.ToString(); + tables.AddRange(mappings); } - public OrderProvider() + mappings = GetMappingsByName(job.Mappings, "EcomOrderLines", isSourceLookup); + if (mappings != null) { - DiscardDuplicates = false; + tables.AddRange(mappings); } - public new ISourceReader GetReader(Mapping mapping) + + mappings = GetMappingsByName(job.Mappings, "EcomOrderLineFields", isSourceLookup); + if (mappings != null) { - return new OrderSourceReader(mapping, Connection, ExportNotExportedOrders, ExportOnlyOrdersWithoutExtID, DoNotExportCarts); + tables.AddRange(mappings); } - protected internal static void OrderTablesInJob(Job job, bool isSourceLookup) + mappings = GetMappingsByName(job.Mappings, "EcomOrderLineFieldGroupRelation", isSourceLookup); + if (mappings != null) { - MappingCollection tables = new MappingCollection(); - - var mappings = GetMappingsByName(job.Mappings, "EcomOrders", isSourceLookup); - if (mappings != null) - { - tables.AddRange(mappings); - } - - mappings = GetMappingsByName(job.Mappings, "EcomOrderLines", isSourceLookup); - if (mappings != null) - { - tables.AddRange(mappings); - } - - mappings = GetMappingsByName(job.Mappings, "EcomOrderLineFields", isSourceLookup); - if (mappings != null) - { - tables.AddRange(mappings); - } + tables.AddRange(mappings); + } - mappings = GetMappingsByName(job.Mappings, "EcomOrderLineFieldGroupRelation", isSourceLookup); - if (mappings != null) - { - tables.AddRange(mappings); - } + job.Mappings = tables; + } - job.Mappings = tables; + internal static List GetMappingsByName(MappingCollection collection, string name, bool isSourceLookup) + { + if (isSourceLookup) + { + return collection.FindAll(map => map.SourceTable != null && map.SourceTable.Name == name); } - - internal static List GetMappingsByName(MappingCollection collection, string name, bool isSourceLookup) + else { - if (isSourceLookup) - { - return collection.FindAll(map => map.SourceTable != null && map.SourceTable.Name == name); - } - else - { - return collection.FindAll(map => map.DestinationTable != null && map.DestinationTable.Name == name); - } + return collection.FindAll(map => map.DestinationTable != null && map.DestinationTable.Name == name); } + } - public override bool RunJob(Job job) + public override bool RunJob(Job job) + { + OrderTablesInJob(job, false); + + var writers = new List(); + SqlTransaction sqlTransaction = null; + Dictionary sourceRow = null; + try { - OrderTablesInJob(job, false); + ReplaceMappingConditionalsWithValuesFromRequest(job); + if (Connection.State != ConnectionState.Open) + Connection.Open(); - var writers = new List(); - SqlTransaction sqlTransaction = null; - Dictionary sourceRow = null; - try + AddMappingsToJobThatNeedsToBeThereForMoveToMainTables(job); + foreach (Mapping mapping in job.Mappings) { - ReplaceMappingConditionalsWithValuesFromRequest(job); - if (Connection.State != ConnectionState.Open) - Connection.Open(); - - AddMappingsToJobThatNeedsToBeThereForMoveToMainTables(job); - foreach (Mapping mapping in job.Mappings) + if (mapping.Active) { - if (mapping.Active) + Logger.Log("Starting import to temporary table for " + mapping.DestinationTable.Name + "."); + using (var reader = job.Source.GetReader(mapping)) { - Logger.Log("Starting import to temporary table for " + mapping.DestinationTable.Name + "."); - using (var reader = job.Source.GetReader(mapping)) + var writer = new OrderDestinationWriter(mapping, Connection, Logger, SkipFailingRows); + var columnMappings = mapping.GetColumnMappings(); + while (!reader.IsDone()) { - var writer = new DynamicwebBulkInsertDestinationWriter(mapping, Connection, false, false, Logger, null, DiscardDuplicates, false, SkipFailingRows); - var columnMappings = mapping.GetColumnMappings(); - while (!reader.IsDone()) - { - sourceRow = reader.GetNext(); - ProcessInputRow(mapping, sourceRow); - ProcessRow(mapping, columnMappings, sourceRow); - writer.Write(sourceRow); - } - writer.FinishWriting(); - writers.Add(writer); + sourceRow = reader.GetNext(); + ProcessInputRow(mapping, sourceRow); + ProcessRow(mapping, columnMappings, sourceRow); + writer.Write(sourceRow); } - Logger.Log("Finished import to temporary table for " + mapping.DestinationTable.Name + "."); + writer.FinishWriting(); + writers.Add(writer); } + Logger.Log("Finished import to temporary table for " + mapping.DestinationTable.Name + "."); } + } - sourceRow = null; - RemoveColumnMappingsFromJobThatShouldBeSkippedInMoveToMainTables(job); - sqlTransaction = Connection.BeginTransaction(); - foreach (DynamicwebBulkInsertDestinationWriter writer in writers) - { - writer.MoveDataToMainTable(sqlTransaction); - } + sourceRow = null; + RemoveColumnMappingsFromJobThatShouldBeSkippedInMoveToMainTables(job); + sqlTransaction = Connection.BeginTransaction(); + foreach (OrderDestinationWriter writer in writers) + { + writer.MoveDataToMainTable(sqlTransaction, false, false); + } - RemoveMissingRows(writers, sqlTransaction); + RemoveMissingRows(writers, sqlTransaction); - sqlTransaction.Commit(); - Ecommerce.Services.Orders.ClearCache(); - } - catch (Exception ex) - { - string msg = ex.Message; - LogManager.System.GetLogger(LogCategory.Application, "Dataintegration").Error($"{GetType().Name} error: {ex.Message} Stack: {ex.StackTrace}", ex); + sqlTransaction.Commit(); + Ecommerce.Services.Orders.ClearCache(); + } + catch (Exception ex) + { + string msg = ex.Message; + LogManager.System.GetLogger(LogCategory.Application, "Dataintegration").Error($"{GetType().Name} error: {ex.Message} Stack: {ex.StackTrace}", ex); - if (sourceRow != null) - msg += GetFailedSourceRowMessage(sourceRow); + if (sourceRow != null) + msg += GetFailedSourceRowMessage(sourceRow); - Logger.Log("Import job failed: " + msg); - if (sqlTransaction != null) - sqlTransaction.Rollback(); - return false; - } - finally + Logger.Log("Import job failed: " + msg); + if (sqlTransaction != null) + sqlTransaction.Rollback(); + return false; + } + finally + { + foreach (var writer in writers) { - foreach (var writer in writers) - { - writer.Close(); - } - job.Source.Close(); - Connection.Dispose(); - sourceRow = null; + writer.Close(); } - return true; + job.Source.Close(); + Connection.Dispose(); + sourceRow = null; } + return true; + } - IEnumerable IParameterOptions.GetParameterOptions(string parameterName) - { - var result = new List(); + IEnumerable IParameterOptions.GetParameterOptions(string parameterName) + { + var result = new List(); - foreach (OrderState state in Ecommerce.Services.OrderStates.GetStatesByOrderType(OrderType.Order)) - { - if (state.IsDeleted) - continue; - string name = state.GetName(Ecommerce.Services.Languages.GetDefaultLanguageId()); - string group = Ecommerce.Services.OrderFlows.GetFlowById(state.OrderFlowId)?.Name; - var value = new GroupedDropDownParameterEditor.DropDownItem(name, group, state.Id); - result.Add(new(name, value) { Group = group }); - } - return result; + foreach (OrderState state in Ecommerce.Services.OrderStates.GetStatesByOrderType(OrderType.Order)) + { + if (state.IsDeleted) + continue; + string name = state.GetName(Ecommerce.Services.Languages.GetDefaultLanguageId()); + string group = Ecommerce.Services.OrderFlows.GetFlowById(state.OrderFlowId)?.Name; + var value = new GroupedDropDownParameterEditor.DropDownItem(name, group, state.Id); + result.Add(new(name, value) { Group = group }); } + return result; + } - private string SourceColumnNameForDestinationOrderCustomerAccessUserId = string.Empty; + private string SourceColumnNameForDestinationOrderCustomerAccessUserId = string.Empty; - private void AddMappingsToJobThatNeedsToBeThereForMoveToMainTables(Job job) + private void AddMappingsToJobThatNeedsToBeThereForMoveToMainTables(Job job) + { + Mapping mapping = job.Mappings.Find(m => m.DestinationTable.Name == "EcomOrders"); + if (mapping != null) { - Mapping mapping = job.Mappings.Find(m => m.DestinationTable.Name == "EcomOrders"); - if (mapping != null) + var columnMappings = mapping.GetColumnMappings(); + if (columnMappings.Find(cm => string.Compare(cm.DestinationColumn.Name, OrderCustomerAccessUserExternalId, true) == 0) != null) { - var columnMappings = mapping.GetColumnMappings(); - if (columnMappings.Find(cm => string.Compare(cm.DestinationColumn.Name, OrderCustomerAccessUserExternalId, true) == 0) != null) + var OrderCustomerAccessUserIdMapping = columnMappings.Find(cm => string.Compare(cm.DestinationColumn.Name, "OrderCustomerAccessUserId", true) == 0); + if (OrderCustomerAccessUserIdMapping == null) { - var OrderCustomerAccessUserIdMapping = columnMappings.Find(cm => string.Compare(cm.DestinationColumn.Name, "OrderCustomerAccessUserId", true) == 0); - if (OrderCustomerAccessUserIdMapping == null) - { - Column randomColumn = new Column(Guid.NewGuid().ToString(), typeof(string), mapping.SourceTable, false, true); - SourceColumnNameForDestinationOrderCustomerAccessUserId = randomColumn.Name; - mapping.AddMapping(randomColumn, job.Destination.GetSchema().GetTables().Find(t => t.Name == "EcomOrders").Columns.Find(c => string.Compare(c.Name, "OrderCustomerAccessUserId", true) == 0), true); - } - else + Column randomColumn = new Column(Guid.NewGuid().ToString(), typeof(string), mapping.SourceTable, false, true); + SourceColumnNameForDestinationOrderCustomerAccessUserId = randomColumn.Name; + mapping.AddMapping(randomColumn, job.Destination.GetSchema().GetTables().Find(t => t.Name == "EcomOrders").Columns.Find(c => string.Compare(c.Name, "OrderCustomerAccessUserId", true) == 0), true); + } + else + { + if (OrderCustomerAccessUserIdMapping.SourceColumn != null) { - if (OrderCustomerAccessUserIdMapping.SourceColumn != null) - { - SourceColumnNameForDestinationOrderCustomerAccessUserId = OrderCustomerAccessUserIdMapping.SourceColumn.Name; - } + SourceColumnNameForDestinationOrderCustomerAccessUserId = OrderCustomerAccessUserIdMapping.SourceColumn.Name; } } } } + } - private void RemoveColumnMappingsFromJobThatShouldBeSkippedInMoveToMainTables(Job job) + private void RemoveColumnMappingsFromJobThatShouldBeSkippedInMoveToMainTables(Job job) + { + Mapping cleanMapping = job.Mappings.Find(m => m.DestinationTable.Name == "EcomOrders"); + if (cleanMapping != null) { - Mapping cleanMapping = job.Mappings.Find(m => m.DestinationTable.Name == "EcomOrders"); - if (cleanMapping != null) - { - ColumnMappingCollection columnMapping = cleanMapping.GetColumnMappings(true); - columnMapping.RemoveAll(cm => cm.DestinationColumn != null && string.Compare(cm.DestinationColumn.Name, OrderCustomerAccessUserExternalId, true) == 0); - } + ColumnMappingCollection columnMapping = cleanMapping.GetColumnMappings(true); + columnMapping.RemoveAll(cm => cm.DestinationColumn != null && string.Compare(cm.DestinationColumn.Name, OrderCustomerAccessUserExternalId, true) == 0); } + } - private Hashtable _existingUsers = null; - /// - /// Collection of , key value pairs - /// - private Hashtable ExistingUsers + private Hashtable _existingUsers = null; + /// + /// Collection of , key value pairs + /// + private Hashtable ExistingUsers + { + get { - get + if (_existingUsers == null) { - if (_existingUsers == null) + _existingUsers = new Hashtable(); + SqlDataAdapter usersDataAdapter = new SqlDataAdapter("select AccessUserExternalID, AccessUserID from AccessUser where AccessUserExternalID is not null and AccessUserExternalID <> ''", Connection); + new SqlCommandBuilder(usersDataAdapter); + DataSet dataSet = new DataSet(); + usersDataAdapter.Fill(dataSet); + DataTable dataTable = dataSet.Tables[0]; + if (dataTable != null) { - _existingUsers = new Hashtable(); - SqlDataAdapter usersDataAdapter = new SqlDataAdapter("select AccessUserExternalID, AccessUserID from AccessUser where AccessUserExternalID is not null and AccessUserExternalID <> ''", Connection); - new SqlCommandBuilder(usersDataAdapter); - DataSet dataSet = new DataSet(); - usersDataAdapter.Fill(dataSet); - DataTable dataTable = dataSet.Tables[0]; - if (dataTable != null) + string key; + foreach (DataRow row in dataTable.Rows) { - string key; - foreach (DataRow row in dataTable.Rows) + key = row["AccessUserExternalID"].ToString(); + if (!_existingUsers.ContainsKey(key)) { - key = row["AccessUserExternalID"].ToString(); - if (!_existingUsers.ContainsKey(key)) - { - _existingUsers.Add(key, row["AccessUserID"].ToString()); - } + _existingUsers.Add(key, row["AccessUserID"].ToString()); } } } - return _existingUsers; } + return _existingUsers; } + } - private void ProcessRow(Mapping mapping, ColumnMappingCollection columnMappings, Dictionary row) + private void ProcessRow(Mapping mapping, ColumnMappingCollection columnMappings, Dictionary row) + { + if (mapping != null && mapping.DestinationTable != null && mapping.DestinationTable.Name == "EcomOrders" && !string.IsNullOrEmpty(SourceColumnNameForDestinationOrderCustomerAccessUserId)) { - if (mapping != null && mapping.DestinationTable != null && mapping.DestinationTable.Name == "EcomOrders" && !string.IsNullOrEmpty(SourceColumnNameForDestinationOrderCustomerAccessUserId)) + object accessUserId = DBNull.Value; + var OrderCustomerAccessUserExternalIdMapping = columnMappings.Find(cm => string.Compare(cm.DestinationColumn.Name, OrderCustomerAccessUserExternalId, true) == 0); + if (OrderCustomerAccessUserExternalIdMapping != null && OrderCustomerAccessUserExternalIdMapping.SourceColumn != null) { - object accessUserId = DBNull.Value; - var OrderCustomerAccessUserExternalIdMapping = columnMappings.Find(cm => string.Compare(cm.DestinationColumn.Name, OrderCustomerAccessUserExternalId, true) == 0); - if (OrderCustomerAccessUserExternalIdMapping != null && OrderCustomerAccessUserExternalIdMapping.SourceColumn != null) + if (row.ContainsKey(OrderCustomerAccessUserExternalIdMapping.SourceColumn.Name)) { - if (row.ContainsKey(OrderCustomerAccessUserExternalIdMapping.SourceColumn.Name)) + string externalID = Convert.ToString(row[OrderCustomerAccessUserExternalIdMapping.SourceColumn.Name]); + if (!string.IsNullOrEmpty(externalID) && ExistingUsers.ContainsKey(externalID)) { - string externalID = Convert.ToString(row[OrderCustomerAccessUserExternalIdMapping.SourceColumn.Name]); - if (!string.IsNullOrEmpty(externalID) && ExistingUsers.ContainsKey(externalID)) - { - accessUserId = ExistingUsers[externalID]; - } + accessUserId = ExistingUsers[externalID]; } } - if (!row.ContainsKey(SourceColumnNameForDestinationOrderCustomerAccessUserId)) - { - row.Add(SourceColumnNameForDestinationOrderCustomerAccessUserId, accessUserId); - } - else - { - row[SourceColumnNameForDestinationOrderCustomerAccessUserId] = accessUserId; - } + } + if (!row.ContainsKey(SourceColumnNameForDestinationOrderCustomerAccessUserId)) + { + row.Add(SourceColumnNameForDestinationOrderCustomerAccessUserId, accessUserId); + } + else + { + row[SourceColumnNameForDestinationOrderCustomerAccessUserId] = accessUserId; } } + } - private void RemoveMissingRows(IEnumerable writers, SqlTransaction sqlTransaction) + private void RemoveMissingRows(IEnumerable writers, SqlTransaction sqlTransaction) + { + if (RemoveMissingOrderLines) { - if (RemoveMissingOrderLines) + OrderDestinationWriter writer = writers.FirstOrDefault(w => string.Compare(w.Mapping?.DestinationTable?.Name, "EcomOrderLines", true) == 0); + if (writer != null && writer.RowsToWriteCount > 0 && writer.SqlCommand != null) { - DynamicwebBulkInsertDestinationWriter writer = writers.FirstOrDefault(w => string.Compare(w.Mapping?.DestinationTable?.Name, "EcomOrderLines", true) == 0); - if (writer != null && writer.RowsToWriteCount > 0 && writer.SqlCommand != null) + string tempTableName = $"EcomOrderLinesTempTableForBulkImport{writer.Mapping.GetId()}"; + writer.SqlCommand.Transaction = sqlTransaction; + if (writer.Mapping.GetColumnMappings().Any(m => m.Active && m.DestinationColumn != null && string.Compare(m.DestinationColumn.Name, "OrderLineOrderID", true) == 0)) { - string tempTableName = $"EcomOrderLinesTempTableForBulkImport{writer.Mapping.GetId()}"; - writer.SqlCommand.Transaction = sqlTransaction; - if (writer.Mapping.GetColumnMappings().Any(m => m.Active && m.DestinationColumn != null && string.Compare(m.DestinationColumn.Name, "OrderLineOrderID", true) == 0)) - { - writer.SqlCommand.CommandText = $"DELETE t1 FROM EcomOrderLines AS t1 LEFT JOIN {tempTableName} t2 " + - "ON t1.OrderLineID = t2.OrderLineID AND t1.OrderLineOrderID = t2.OrderLineOrderID " + - $"WHERE t2.OrderLineID IS NULL AND t1.OrderLineOrderID IN (SELECT DISTINCT(OrderLineOrderID) from {tempTableName})"; - } - else - { - writer.SqlCommand.CommandText = $"DELETE FROM EcomOrderLines WHERE OrderLineID NOT IN (SELECT OrderLineID FROM {tempTableName}) " + - $"AND OrderLineOrderID IN(SELECT DISTINCT(OrderLineOrderID) FROM EcomOrderLines WHERE OrderLineID IN(SELECT OrderLineID FROM {tempTableName}))"; - } - writer.SqlCommand.ExecuteNonQuery(); + writer.SqlCommand.CommandText = $"DELETE t1 FROM EcomOrderLines AS t1 LEFT JOIN {tempTableName} t2 " + + "ON t1.OrderLineID = t2.OrderLineID AND t1.OrderLineOrderID = t2.OrderLineOrderID " + + $"WHERE t2.OrderLineID IS NULL AND t1.OrderLineOrderID IN (SELECT DISTINCT(OrderLineOrderID) from {tempTableName})"; + } + else + { + writer.SqlCommand.CommandText = $"DELETE FROM EcomOrderLines WHERE OrderLineID NOT IN (SELECT OrderLineID FROM {tempTableName}) " + + $"AND OrderLineOrderID IN(SELECT DISTINCT(OrderLineOrderID) FROM EcomOrderLines WHERE OrderLineID IN(SELECT OrderLineID FROM {tempTableName}))"; } + writer.SqlCommand.ExecuteNonQuery(); } } } diff --git a/src/OrderSourceReader.cs b/src/OrderSourceReader.cs index a124ee7..b745597 100644 --- a/src/OrderSourceReader.cs +++ b/src/OrderSourceReader.cs @@ -1,5 +1,4 @@ using Dynamicweb.DataIntegration.Integration; -using Dynamicweb.DataIntegration.Providers.SqlProvider; using Dynamicweb.Ecommerce.Orders; using System; using System.Collections.Generic; @@ -8,15 +7,14 @@ namespace Dynamicweb.DataIntegration.Providers.OrderProvider { - internal class OrderSourceReader : SqlSourceReader + internal class OrderSourceReader : BaseSqlReader { private static MappingConditionalCollection _ordersConditions = null; private static List _ordersToExport = null; private ColumnMappingCollection _columnMappings = null; - public OrderSourceReader(Mapping mapping, SqlConnection connection, bool exportNotExportedOrders, bool exportOnlyOrdersWithoutExtID, bool doNotExportCarts) + public OrderSourceReader(Mapping mapping, SqlConnection connection, bool exportNotExportedOrders, bool exportOnlyOrdersWithoutExtID, bool doNotExportCarts) : base(mapping, connection) { - base.DoInitialization(mapping, connection); _columnMappings = mapping.GetColumnMappings(); _command = new SqlCommand { Connection = connection }; if (connection.State.ToString() != "Open")