Con 3755 import payments pefrormance

azure/finance.template.json

@@ -586,4 +586,4 @@
             "value": "[variables('workerAppServiceName')]"
         }
     }
-}
+}

src/SFA.DAS.EAS.Employer_Financial.Database/SFA.DAS.EAS.Employer_Financial.Database.sqlproj

@@ -198,6 +198,7 @@
     <Build Include="Tables\TransactionLine_EOF.sql" />
     <Build Include="StoredProcedures\CreateDraftExpiredFunds.sql" />
     <Build Include="StoredProcedures\GetDraftExpiredFunds.sql" />
+    <Build Include="StoredProcedures\CreateAccountTransfersV1.sql" />
   </ItemGroup>
   <ItemGroup>
     <None Include="Database.publish.xml" />

src/SFA.DAS.EAS.Employer_Financial.Database/StoredProcedures/CreateAccountTransfersV1.sql

@@ -0,0 +1,46 @@
+CREATE PROCEDURE [employer_financial].[CreateAccountTransfersV1]
+	@transfers [employer_financial].[AccountTransferTable] READONLY
+AS
+	INSERT INTO [employer_financial].[AccountTransfers] 
+	(
+		SenderAccountId, 
+		SenderAccountName,
+		ReceiverAccountId, 
+		ReceiverAccountName,
+		ApprenticeshipId, 
+		CourseName,	
+		CourseLevel,
+		PeriodEnd,
+		Amount, 
+		Type, 		
+		CreatedDate,
+		RequiredPaymentId
+	)
+	Select 
+		tr.SenderAccountId, 
+		actSender.Name, 
+		tr.ReceiverAccountId, 
+		actReceiver.Name, 
+		tr.ApprenticeshipId, 
+		IsNull(Ir.CourseName, 'Unknown Course'), 
+		Ir.CourseLevel, 
+		tr.PeriodEnd, 
+		tr.Amount, 
+		tr.Type, GetDate(), 
+		tr.RequiredPaymentId  
+		FROM 
+			(SELECT
+				p.AccountId,
+				p.PeriodEnd,
+				p.ApprenticeshipId,
+				meta.ApprenticeshipCourseName as CourseName
+				,meta.ApprenticeshipCourseLevel as CourseLevel
+				,COUNT(DISTINCT Uln) as ApprenticeCount 
+				,SUM(p.Amount) as PaymentTotal		
+				FROM [employer_financial].[Payment] p
+				INNER JOIN [employer_financial].[PaymentMetaData] meta ON p.PaymentMetaDataId = meta.Id
+				Inner join @transfers tr on p.AccountId = tr.ReceiverAccountId and p.PeriodEnd = tr.PeriodEnd and p.ApprenticeshipId = tr.ApprenticeshipId
+				GROUP BY meta.ApprenticeshipCourseName, meta.ApprenticeshipCourseLevel, p.AccountId, p.PeriodEnd, p.ApprenticeshipId)  Ir
+		Inner join @transfers tr on Ir.AccountId = tr.ReceiverAccountId and Ir.PeriodEnd = tr.PeriodEnd and Ir.ApprenticeshipId = tr.ApprenticeshipId
+		Inner join [employer_financial].[Account] actSender on actSender.Id = tr.SenderAccountId
+		Inner join [employer_financial].[Account] actReceiver on actReceiver.Id = tr.ReceiverAccountId

src/SFA.DAS.EAS.Employer_Financial.Database/Tables/Payment.sql

@@ -62,4 +62,10 @@ CREATE NONCLUSTERED INDEX [IX_Payment_AccountIdUkprnPeriodEndUln] ON [employer_f
 GO
 
 CREATE NONCLUSTERED INDEX [IX_Payment_AccountIdCollectionPeriodMonthCollectionPeriodYear] ON [employer_financial].[Payment] ([AccountId], [CollectionPeriodMonth], [CollectionPeriodYear]) INCLUDE ([Amount], [ApprenticeshipId], [CollectionPeriodId], [DeliveryPeriodMonth], [DeliveryPeriodYear], [FundingSource], [PaymentMetaDataId], [PeriodEnd], [Ukprn], [Uln]) WITH (ONLINE = ON)
+GO
+
+CREATE NONCLUSTERED INDEX [IX_Payment_Ukprn] ON [employer_financial].[Payment] ([Ukprn]) WITH (ONLINE = ON)
+GO
+
+CREATE NONCLUSTERED INDEX [IX_Payment_UkprnAccountId] ON [employer_financial].[Payment] ([AccountId], [Ukprn]) WITH (ONLINE = ON)
 GO

src/SFA.DAS.EAS.Employer_Financial.Database/Tables/TransactionLine.sql

@@ -44,3 +44,6 @@ GO
 
 CREATE UNIQUE NONCLUSTERED INDEX [UK_TransactionLine_AccountId_TransactionType_TransactionDate] ON [employer_financial].[TransactionLine] ([AccountId] ASC, [TransactionType] ASC, [TransactionDate] ASC) WHERE [TransactionType] = /*ExpiredFund*/ 5
 GO
+
+CREATE INDEX [IX_TransactionLine_Account_DateCreated] ON [employer_financial].[TransactionLine] (AccountId, DateCreated) WITH (ONLINE = ON)
+GO

src/SFA.DAS.EmployerFinance.MessageHandlers/CommandHandlers/ImportAccountPaymentsCommandHandler.cs

@@ -1,4 +1,5 @@
-using System.Threading.Tasks;
+using System;
+using System.Threading.Tasks;
 using MediatR;
 using NServiceBus;
 using SFA.DAS.EmployerFinance.Commands.CreateTransferTransactions;
@@ -22,28 +23,32 @@ public ImportAccountPaymentsCommandHandler(IMediator mediator, ILog logger)
 
         public async Task Handle(ImportAccountPaymentsCommand message, IMessageHandlerContext context)
         {
-            _logger.Info($"Processing refresh payment command for Account ID: {message.AccountId} PeriodEnd: {message.PeriodEndRef}");
+            var correlationId = Guid.NewGuid();
+            _logger.Info($"Processing refresh payment command for Account ID: {message.AccountId} PeriodEnd: {message.PeriodEndRef} CorrelationId: {correlationId}");
 
             await _mediator.SendAsync(new RefreshPaymentDataCommand
             {
                 AccountId = message.AccountId,
-                PeriodEnd = message.PeriodEndRef
-            });
+                PeriodEnd = message.PeriodEndRef,
+                CorrelationId = correlationId
+        });
 
-            _logger.Info($"Processing refresh account transfers command for AccountId:{message.AccountId} PeriodEnd:{message.PeriodEndRef}");
+            _logger.Info($"Processing refresh account transfers command for AccountId:{message.AccountId} PeriodEnd:{message.PeriodEndRef}, CorrelationId: {correlationId}");
 
             await _mediator.SendAsync(new RefreshAccountTransfersCommand
             {
                 ReceiverAccountId = message.AccountId,
-                PeriodEnd = message.PeriodEndRef
+                PeriodEnd = message.PeriodEndRef,
+                CorrelationId = correlationId
             });
 
             _logger.Info($"Processing create account transfer transactions command for AccountId:{message.AccountId} PeriodEnd:{message.PeriodEndRef}");
 
             await _mediator.SendAsync(new CreateTransferTransactionsCommand
             {
                 ReceiverAccountId = message.AccountId,
-                PeriodEnd = message.PeriodEndRef
+                PeriodEnd = message.PeriodEndRef,
+                CorrelationId = correlationId
             });
         }
     }

src/SFA.DAS.EmployerFinance.MessageHandlers/Program.cs

@@ -4,8 +4,8 @@
 using SFA.DAS.AutoConfiguration;
 using SFA.DAS.EmployerFinance.MessageHandlers.DependencyResolution;
 using SFA.DAS.EmployerFinance.Startup;
-using System.Configuration;
 using Microsoft.ApplicationInsights.Extensibility;
+using System.Configuration;
 
 namespace SFA.DAS.EmployerFinance.MessageHandlers
 {
@@ -14,6 +14,7 @@ public class Program
         public static void Main()
         {
             TelemetryConfiguration.Active.InstrumentationKey = ConfigurationManager.AppSettings["APPINSIGHTS_INSTRUMENTATIONKEY"];
+
             MainAsync().GetAwaiter().GetResult();
         }
 

src/SFA.DAS.EmployerFinance.UnitTests/Commands/RefreshAccountTransfersTests/WhenIRefreshAnAccountsTransfers.cs

@@ -60,7 +60,8 @@ public void Arrange()
                 ReceiverAccountId = ReceiverAccountId,
                 ReceiverAccountName = ReceiverAccountName,
                 ApprenticeshipId = 1245,
-                Amount = 1200
+                Amount = 1200,
+                PeriodEnd = "1718-R01"
             };
 
             _transfers = new List<AccountTransfer>
@@ -85,7 +86,7 @@ public void Arrange()
             _validator.Setup(x => x.Validate(It.IsAny<RefreshAccountTransfersCommand>()))
                 .Returns(new ValidationResult());
 
-            _paymentService.Setup(x => x.GetAccountTransfers(It.IsAny<string>(), It.IsAny<long>()))
+            _paymentService.Setup(x => x.GetAccountTransfers(It.IsAny<string>(), It.IsAny<long>(), It.IsAny<Guid>()))
                 .ReturnsAsync(_transfers);
 
             _transferRepository.Setup(x => x.GetTransferPaymentDetails(It.IsAny<AccountTransfer>()))
@@ -108,7 +109,7 @@ public async Task ThenTheTransfersShouldBeRetrieved()
             await _handler.Handle(_command);
 
             //Assert
-            _paymentService.Verify(x => x.GetAccountTransfers(_command.PeriodEnd, _command.ReceiverAccountId), Times.Once);
+            _paymentService.Verify(x => x.GetAccountTransfers(_command.PeriodEnd, _command.ReceiverAccountId, It.IsAny<Guid>()), Times.Once);
         }
 
         [Test]
@@ -121,9 +122,7 @@ public async Task ThenTheRetrievedTransfersShouldBeSaved()
             _transferRepository.Verify(x => x.CreateAccountTransfers(It.Is<IEnumerable<AccountTransfer>>(t =>
                 t.All(at => at.ApprenticeshipId.Equals(_accountTransfer.ApprenticeshipId) &&
                             at.SenderAccountId.Equals(_accountTransfer.SenderAccountId) &&
-                            at.SenderAccountName.Equals(_accountTransfer.SenderAccountName) &&
                             at.ReceiverAccountId.Equals(_accountTransfer.ReceiverAccountId) &&
-                            at.ReceiverAccountName.Equals(_accountTransfer.ReceiverAccountName) &&
                             at.Amount.Equals(_accountTransfer.Amount)))), Times.Once);
         }
 
@@ -180,7 +179,7 @@ public async Task ThenThePaymentShouldNotBeRetrievedIfTheCommandIsInvalid()
             }
 
             //Assert
-            _paymentService.Verify(x => x.GetAccountTransfers(_command.PeriodEnd, _command.ReceiverAccountId), Times.Never);
+            _paymentService.Verify(x => x.GetAccountTransfers(_command.PeriodEnd, _command.ReceiverAccountId, Guid.NewGuid()), Times.Never);
         }
 
         [Test]
@@ -214,7 +213,7 @@ public async Task ThenTheTransfersShouldNotBeSavedIfTheCommandIsInvalid()
         public void ThenIfWeCannotGetTransfersWeShouldNotTryToProcessThem()
         {
             //Assert
-            _paymentService.Setup(x => x.GetAccountTransfers(It.IsAny<string>(), It.IsAny<long>()))
+            _paymentService.Setup(x => x.GetAccountTransfers(It.IsAny<string>(), It.IsAny<long>(), It.IsAny<Guid>()))
                 .Throws<WebException>();
 
             //Act + Assert
@@ -223,91 +222,16 @@ public void ThenIfWeCannotGetTransfersWeShouldNotTryToProcessThem()
             _transferRepository.Verify(x => x.CreateAccountTransfers(_transfers), Times.Never);
         }
 
-        [Test]
-        public void ThenIfWeCannotGetTransfersWeShouldLogAnError()
-        {
-            //Assert
-            var exception = new Exception();
-            _paymentService.Setup(x => x.GetAccountTransfers(It.IsAny<string>(), It.IsAny<long>()))
-                .Throws(exception);
-
-            //Act + Assert
-            Assert.ThrowsAsync<Exception>(() => _handler.Handle(_command));
-
-            _logger.Verify(x => x.Error(exception, It.IsAny<string>()), Times.Once);
-        }
-
-        [Test]
-
-        public async Task ThenPaymentDetailsShouldBeCalledForEachTransfer()
-        {
-            //Act
-            await _handler.Handle(_command);
-
-            //Assert
-            foreach (var transfer in _transfers)
-            {
-                _transferRepository.Verify(x => x.GetTransferPaymentDetails(It.Is<AccountTransfer>(t =>
-                                t.ApprenticeshipId.Equals(_accountTransfer.ApprenticeshipId) &&
-                                t.SenderAccountId.Equals(_accountTransfer.SenderAccountId) &&
-                                t.SenderAccountName.Equals(_accountTransfer.SenderAccountName) &&
-                                t.ReceiverAccountId.Equals(_accountTransfer.ReceiverAccountId) &&
-                                t.ReceiverAccountName.Equals(_accountTransfer.ReceiverAccountName) &&
-                                t.Amount.Equals(_accountTransfer.Amount))), Times.Once);
-            }
-        }
-
-        [Test]
-
-        public async Task ThenATransferShouldBeCreatedWithTheCorrectPaymentDetails()
-        {
-            //Act
-            await _handler.Handle(_command);
-
-            //Assert
-            _transferRepository.Verify(x => x.CreateAccountTransfers(It.Is<IEnumerable<AccountTransfer>>(
-                transfers =>
-                    transfers.All(t => t.CourseName.Equals(_details.CourseName) &&
-                                       t.CourseLevel.Equals(_details.CourseLevel) &&
-                                       t.ApprenticeCount.Equals(_details.ApprenticeCount)))), Times.Once);
-        }
-
-        [Test]
-
-        public async Task ThenATransferShouldBeCreatedWithTheCorrectReceiverAccountName()
-        {
-            //Act
-            await _handler.Handle(_command);
-
-            //Assert
-            _transferRepository.Verify(x => x.CreateAccountTransfers(It.Is<IEnumerable<AccountTransfer>>(
-                transfers =>
-                    transfers.All(t => t.ReceiverAccountName.Equals(ReceiverAccountName)))), Times.Once);
-        }
-
-        [Test]
-
-        public async Task ThenIfTransferAmountAndPaymentTotalsDoNotMatchAWarningIsLogged()
-        {
-            //Assign
-            _details.PaymentTotal = 10; //Should be 1200
-
-            //Act
-            await _handler.Handle(_command);
-
-            //Assert
-            _logger.Verify(x => x.Warn("Transfer total does not match transfer payments total"));
-        }
-
         [Test]
         public async Task ThenATransferPamentsForTheSameApprenticeAndCourseShouldBeAggregated()
         {
-            //Assert
-            //Duplicate the transfer to simulate two transfers from different delivery periods
-            //(We will not be catching duplicate transfers that exactly match as there is no ID or value in the transfer that remains unique to help us)
-            _transfers.Add(_accountTransfer);
+        //TODO: will need to updated if approach agreed.
+       //Assert
+       //Duplicate the transfer to simulate two transfers from different delivery periods
+       //(We will not be catching duplicate transfers that exactly match as there is no ID or value in the transfer that remains unique to help us)
+       _transfers.Add(_accountTransfer);
 
-            _paymentService.Setup(x => x.GetAccountTransfers(_command.PeriodEnd, _command.ReceiverAccountId))
+            _paymentService.Setup(x => x.GetAccountTransfers(_command.PeriodEnd, _command.ReceiverAccountId, Guid.NewGuid()))
                 .ReturnsAsync(_transfers);
 
             //Act
@@ -335,7 +259,7 @@ public async Task ThenATransferPamentsForTheSameApprenticeButDifferentCourseShou
             });
 
 
-            _paymentService.Setup(x => x.GetAccountTransfers(_command.PeriodEnd, _command.ReceiverAccountId))
+            _paymentService.Setup(x => x.GetAccountTransfers(_command.PeriodEnd, _command.ReceiverAccountId, Guid.NewGuid()))
                 .ReturnsAsync(_transfers);
 
 

src/SFA.DAS.EmployerFinance.UnitTests/Commands/RefreshPaymentDataTests/WhenIHAveCompletedProcessing.cs

@@ -180,7 +180,7 @@ public RefreshPaymentDataCommandHandlerTestsFixture SetExistingPayments(List<Gui
 
         public RefreshPaymentDataCommandHandlerTestsFixture SetIncomingPayments(List<PaymentDetails> paymentDetails)
         {
-            _paymentService.Setup(x => x.GetAccountPayments(It.IsAny<string>(), It.IsAny<long>()))
+            _paymentService.Setup(x => x.GetAccountPayments(It.IsAny<string>(), It.IsAny<long>(), It.IsAny<Guid>()))
                 .ReturnsAsync(paymentDetails);
 
             return this;

src/SFA.DAS.EmployerFinance.UnitTests/Commands/RefreshPaymentDataTests/WhenIReceiveTheCommand.cs

@@ -47,7 +47,8 @@ public void Arrange()
             _command = new RefreshPaymentDataCommand
             {
                 AccountId = AccountId,
-                PeriodEnd = PeriodEnd
+                PeriodEnd = PeriodEnd,
+                CorrelationId = Guid.NewGuid()
             };
 
             _existingPaymentIds = new List<Guid>
@@ -74,7 +75,7 @@ public void Arrange()
             }};
 
             _paymentService = new Mock<IPaymentService>();
-            _paymentService.Setup(x => x.GetAccountPayments(It.IsAny<string>(), It.IsAny<long>()))
+            _paymentService.Setup(x => x.GetAccountPayments(It.IsAny<string>(), It.IsAny<long>(), It.IsAny<Guid>()))
                            .ReturnsAsync(_paymentDetails);
 
             _mediator = new Mock<IMediator>();
@@ -118,14 +119,14 @@ public async Task ThenThePaymentsApiIsCalledToGetPaymentData()
             await _handler.Handle(_command);
 
             //Assert
-            _paymentService.Verify(x => x.GetAccountPayments(_command.PeriodEnd, _command.AccountId));
+            _paymentService.Verify(x => x.GetAccountPayments(_command.PeriodEnd, _command.AccountId, It.IsAny<Guid>()));
         }
 
         [Test]
         public async Task ThenTheRepositoryIsNotCalledIfTheCommandIsValidAndThereAreNotPayments()
         {
             //Arrange
-            _paymentService.Setup(x => x.GetAccountPayments(It.IsAny<string>(), It.IsAny<long>()))
+            _paymentService.Setup(x => x.GetAccountPayments(It.IsAny<string>(), It.IsAny<long>(), It.IsAny<Guid>()))
                            .ReturnsAsync(new List<PaymentDetails>());
             //Act
             await _handler.Handle(_command);
@@ -164,7 +165,7 @@ public async Task ThenTheRepositoryIsCalledToSeeIfTheDataHasAlreadyBeenSavedAndI
                 new PaymentDetails { Id = _existingPaymentIds[1]}
             };
 
-            _paymentService.Setup(x => x.GetAccountPayments(It.IsAny<string>(), It.IsAny<long>()))
+            _paymentService.Setup(x => x.GetAccountPayments(It.IsAny<string>(), It.IsAny<long>(), It.IsAny<Guid>()))
                 .ReturnsAsync(_paymentDetails);
 
             //Act
@@ -180,7 +181,7 @@ public async Task ThenTheRepositoryIsCalledToSeeIfTheDataHasAlreadyBeenSavedAndI
         public async Task ThenWhenAnExceptionIsThrownFromTheApiClientNothingIsProcessedAndAnErrorIsLogged()
         {
             //Assert
-            _paymentService.Setup(x => x.GetAccountPayments(It.IsAny<string>(), It.IsAny<long>()))
+            _paymentService.Setup(x => x.GetAccountPayments(It.IsAny<string>(), It.IsAny<long>(), It.IsAny<Guid>()))
                            .ThrowsAsync(new WebException());
 
             //Act
@@ -191,7 +192,7 @@ public async Task ThenWhenAnExceptionIsThrownFromTheApiClientNothingIsProcessedA
             _mediator.Verify(x => x.PublishAsync(It.IsAny<ProcessPaymentEvent>()), Times.Never);
 
             _logger.Verify(x => x.Error(It.IsAny<WebException>(),
-                $"Unable to get payment information for AccountId = '{AccountId}' and PeriodEnd = '{_command.PeriodEnd}'"));
+                $"Unable to get payment information for AccountId = '{AccountId}' and PeriodEnd = '{_command.PeriodEnd}' CorrelationId: {_command.CorrelationId}"));
         }
 
         [Test]
@@ -216,7 +217,7 @@ public async Task ShouldOnlyAddPaymentsWhichAreNotAlreadyAdded()
                 new PaymentDetails { Id = newPaymentGuid}
             };
 
-            _paymentService.Setup(x => x.GetAccountPayments(It.IsAny<string>(), It.IsAny<long>()))
+            _paymentService.Setup(x => x.GetAccountPayments(It.IsAny<string>(), It.IsAny<long>(), It.IsAny<Guid>()))
                            .ReturnsAsync(_paymentDetails);
 
             //Act
@@ -242,7 +243,7 @@ public async Task ShouldOnlyAddNonFullyFundedPayments()
                 new PaymentDetails { Id = fullyFundedPaymentGuid, FundingSource = FundingSource.FullyFundedSfa}
             };
 
-            _paymentService.Setup(x => x.GetAccountPayments(It.IsAny<string>(), It.IsAny<long>()))
+            _paymentService.Setup(x => x.GetAccountPayments(It.IsAny<string>(), It.IsAny<long>(), It.IsAny<Guid>()))
                 .ReturnsAsync(_paymentDetails);
 
             //Act