Skip to content

Commit

Permalink
Implmented logging
Browse files Browse the repository at this point in the history
  • Loading branch information
FirestarJes committed Sep 2, 2024
1 parent a7a953e commit 9cb4451
Show file tree
Hide file tree
Showing 14 changed files with 97 additions and 25 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
using Microsoft.EntityFrameworkCore.Query;
using Microsoft.EntityFrameworkCore.Query.SqlExpressions;
using Microsoft.EntityFrameworkCore.Storage;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;

namespace Energinet.DataHub.SettlementReport.CalculationResults.Infrastructure.Experimental;
Expand All @@ -31,10 +32,11 @@ public abstract class DatabricksContextBase : IDisposable

protected DatabricksContextBase(
DatabricksSqlWarehouseQueryExecutor databricksSqlWarehouseQueryExecutor,
IOptions<DeltaTableOptions> options)
IOptions<DeltaTableOptions> options,
ILoggerFactory loggerFactory)
{
_dbContext = new DbContextCore(OnModelCreating);
_executor = new DatabricksSqlQueryExecutor(_dbContext, databricksSqlWarehouseQueryExecutor, options);
_executor = new DatabricksSqlQueryExecutor(_dbContext, databricksSqlWarehouseQueryExecutor, options, loggerFactory);
}

public void Dispose()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,20 +25,18 @@ namespace Energinet.DataHub.SettlementReport.CalculationResults.Infrastructure.E
public sealed class DatabricksSqlQueryExecutor
{
private readonly DatabricksSqlWarehouseQueryExecutor _databricksSqlWarehouseQueryExecutor;
private readonly ILogger<DatabricksSqlQueryExecutor> _logger;
private readonly DatabricksSqlQueryBuilder _sqlQueryBuilder;
private readonly DatabricksSqlRowHydrator _sqlRowHydrator;

public DatabricksSqlQueryExecutor(
DbContext dbContext,
DatabricksSqlWarehouseQueryExecutor databricksSqlWarehouseQueryExecutor,
IOptions<DeltaTableOptions> options,
ILogger<DatabricksSqlQueryExecutor> logger)
ILoggerFactory loggerFactory)
{
_sqlQueryBuilder = new DatabricksSqlQueryBuilder(dbContext, options);
_sqlRowHydrator = new DatabricksSqlRowHydrator();
_sqlRowHydrator = new DatabricksSqlRowHydrator(loggerFactory);
_databricksSqlWarehouseQueryExecutor = databricksSqlWarehouseQueryExecutor;
_logger = logger;
}

public async Task<int> CountAsync(DatabricksSqlQueryable query, CancellationToken cancellationToken = default)
Expand Down Expand Up @@ -66,7 +64,7 @@ internal string BuildDebugString(DatabricksSqlQueryable query)
private IAsyncEnumerable<TElement> ExecuteCoreAsync<TElement>(DatabricksStatement databricksStatement, CancellationToken cancellationToken = default)
{
var rows = _databricksSqlWarehouseQueryExecutor.ExecuteStatementParallelAsync(databricksStatement, Format.ApacheArrow, cancellationToken);
return _sqlRowHydrator.HydrateAsync<TElement>(rows, _logger, cancellationToken);
return _sqlRowHydrator.HydrateAsync<TElement>(rows, cancellationToken);
}

private sealed class CountResult
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,15 @@ public sealed class DatabricksSqlRowHydrator
UnmappedMemberHandling = JsonUnmappedMemberHandling.Disallow,
}.ConfigureForNodaTime(DateTimeZoneProviders.Tzdb);

private readonly ILogger<DatabricksSqlRowHydrator> _logger;

public DatabricksSqlRowHydrator(ILoggerFactory loggerFactory)
{
_logger = loggerFactory.CreateLogger<DatabricksSqlRowHydrator>();
}

public async IAsyncEnumerable<TElement> HydrateAsync<TElement>(
IAsyncEnumerable<dynamic> rows,
ILogger<DatabricksSqlQueryExecutor> logger,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
var propertyMap = _typeInfoCache.GetOrAdd(typeof(TElement), CreateTypeInfoCache);
Expand All @@ -54,7 +60,7 @@ public async IAsyncEnumerable<TElement> HydrateAsync<TElement>(
rowCounter++;
}

logger.LogInformation("Hydration for {RowCounter} rows took: {ElapsedMilliseconds}ms", rowCounter, sw.ElapsedMilliseconds);
_logger.LogInformation("Hydration for {RowCounter} rows took: {ElapsedMilliseconds}ms", rowCounter, sw.ElapsedMilliseconds);
}

private TElement Hydrate<TElement>(ExpandoObject expandoObject, IReadOnlyDictionary<string, (PropertyInfo Property, TypeConverter Converter)> propertyMap, Stopwatch sw)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
using Energinet.DataHub.SettlementReport.CalculationResults.Infrastructure.Experimental;
using Energinet.DataHub.SettlementReport.Common.Infrastructure.Options;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;

namespace Energinet.DataHub.SettlementReport.CalculationResults.Infrastructure.Persistence.Databricks;
Expand All @@ -26,8 +27,9 @@ public sealed class SettlementReportDatabricksContext : DatabricksContextBase, I

public SettlementReportDatabricksContext(
IOptions<DeltaTableOptions> deltaTableOptions,
DatabricksSqlWarehouseQueryExecutor sqlWarehouseQueryExecutor)
: base(sqlWarehouseQueryExecutor, deltaTableOptions)
DatabricksSqlWarehouseQueryExecutor sqlWarehouseQueryExecutor,
ILoggerFactory loggerFactory)
: base(sqlWarehouseQueryExecutor, deltaTableOptions, loggerFactory)
{
_deltaTableOptions = deltaTableOptions;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
using Energinet.DataHub.SettlementReport.Common.Infrastructure.Options;
using Energinet.DataHub.SettlementReport.Common.Interfaces.Models;
using Energinet.DataHub.Wholesale.CalculationResults.IntegrationTests.Fixtures;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Moq;
using NodaTime;
Expand Down Expand Up @@ -57,32 +58,40 @@ public SettlementReportFileRequestHandlerIntegrationTests(

Fixture.Inject(mockedOptions);

var mockedLogging = new Mock<ILogger>();
var mockedLoggerFactory = new Mock<ILoggerFactory>();
mockedLoggerFactory.Setup(x => x.CreateLogger(It.IsAny<string>())).Returns(mockedLogging.Object);
var sqlWarehouseQueryExecutor = _databricksSqlStatementApiFixture.GetDatabricksExecutor();
var databricksContext = new SettlementReportDatabricksContext(mockedOptions.Object, sqlWarehouseQueryExecutor);
var databricksContext = new SettlementReportDatabricksContext(mockedOptions.Object, sqlWarehouseQueryExecutor, mockedLoggerFactory.Object);

var settlementReportDataRepository = new SettlementReportEnergyResultRepository(databricksContext);
var settlementReportWholesaleRepository = new SettlementReportWholesaleRepository(databricksContext);
var settlementReportChargeLinkPeriodsRepository = new SettlementReportChargeLinkPeriodsRepository(databricksContext);

var settlementReportMeteringPointMasterDataRepository = new SettlementReportMeteringPointMasterDataRepository(new SettlementReportDatabricksContext(
mockedOptions.Object,
sqlWarehouseQueryExecutor));
sqlWarehouseQueryExecutor,
mockedLoggerFactory.Object));

var settlementReportMeteringPointTimeSeriesResultRepository = new SettlementReportMeteringPointTimeSeriesResultRepository(new SettlementReportDatabricksContext(
mockedOptions.Object,
sqlWarehouseQueryExecutor));
sqlWarehouseQueryExecutor,
mockedLoggerFactory.Object));

var settlementReportMonthlyAmountRepository = new SettlementReportMonthlyAmountRepository(new SettlementReportDatabricksContext(
mockedOptions.Object,
sqlWarehouseQueryExecutor));
sqlWarehouseQueryExecutor,
mockedLoggerFactory.Object));

var settlementReportChargePriceRepository = new SettlementReportChargePriceRepository(new SettlementReportDatabricksContext(
mockedOptions.Object,
sqlWarehouseQueryExecutor));
sqlWarehouseQueryExecutor,
mockedLoggerFactory.Object));

var settlementReportMonthlyAmountTotalRepository = new SettlementReportMonthlyAmountTotalRepository(new SettlementReportDatabricksContext(
mockedOptions.Object,
sqlWarehouseQueryExecutor));
sqlWarehouseQueryExecutor,
mockedLoggerFactory.Object));

Fixture.Inject<ISettlementReportFileGeneratorFactory>(new SettlementReportFileGeneratorFactory(
settlementReportDataRepository,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.1" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.10.0" />
<PackageReference Include="SimplerSoftware.EntityFrameworkCore.SqlServer.NodaTime" Version="8.0.1" />
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
using Energinet.DataHub.SettlementReport.Common.Interfaces.Models;
using Energinet.DataHub.Wholesale.CalculationResults.IntegrationTests.Fixtures;
using FluentAssertions;
using Microsoft.Extensions.Logging;
using Moq;
using Xunit;

namespace Energinet.DataHub.Wholesale.CalculationResults.IntegrationTests.Infrastructure.SettlementReports_v2;
Expand All @@ -32,14 +34,19 @@ public class SettlementReportChargeLinkPeriodsRepositoryTests : TestBase<Settlem

public SettlementReportChargeLinkPeriodsRepositoryTests(MigrationsFreeDatabricksSqlStatementApiFixture databricksSqlStatementApiFixture)
{
var mockedLogging = new Mock<ILogger>();

var mockedLoggerFactory = new Mock<ILoggerFactory>();
mockedLoggerFactory.Setup(x => x.CreateLogger(It.IsAny<string>())).Returns(mockedLogging.Object);
_databricksSqlStatementApiFixture = databricksSqlStatementApiFixture;

_databricksSqlStatementApiFixture.DatabricksSchemaManager.DeltaTableOptions.Value.SettlementReportSchemaName =
databricksSqlStatementApiFixture.DatabricksSchemaManager.DeltaTableOptions.Value.SCHEMA_NAME;

Fixture.Inject<ISettlementReportDatabricksContext>(new SettlementReportDatabricksContext(
_databricksSqlStatementApiFixture.DatabricksSchemaManager.DeltaTableOptions,
_databricksSqlStatementApiFixture.GetDatabricksExecutor()));
_databricksSqlStatementApiFixture.GetDatabricksExecutor(),
mockedLoggerFactory.Object));
}

[Fact(Skip = "Performance testing")]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
using Energinet.DataHub.SettlementReport.CalculationResults.Interfaces.SettlementReports_v2.Models;
using Energinet.DataHub.SettlementReport.Common.Interfaces.Models;
using Energinet.DataHub.Wholesale.CalculationResults.IntegrationTests.Fixtures;
using Microsoft.Extensions.Logging;
using Moq;
using Xunit;

namespace Energinet.DataHub.Wholesale.CalculationResults.IntegrationTests.Infrastructure.SettlementReports_v2;
Expand All @@ -40,6 +42,10 @@ public class SettlementReportChargePricesRepositoryTests : TestBase<SettlementRe

public SettlementReportChargePricesRepositoryTests(MigrationsFreeDatabricksSqlStatementApiFixture databricksSqlStatementApiFixture)
{
var mockedLogging = new Mock<ILogger>();

var mockedLoggerFactory = new Mock<ILoggerFactory>();
mockedLoggerFactory.Setup(x => x.CreateLogger(It.IsAny<string>())).Returns(mockedLogging.Object);
_databricksSqlStatementApiFixture = databricksSqlStatementApiFixture;

_databricksSqlStatementApiFixture.DatabricksSchemaManager.DeltaTableOptions.Value.SettlementReportSchemaName =
Expand All @@ -50,7 +56,8 @@ public SettlementReportChargePricesRepositoryTests(MigrationsFreeDatabricksSqlSt

Fixture.Inject<ISettlementReportDatabricksContext>(new SettlementReportDatabricksContext(
_databricksSqlStatementApiFixture.DatabricksSchemaManager.DeltaTableOptions,
_databricksSqlStatementApiFixture.GetDatabricksExecutor()));
_databricksSqlStatementApiFixture.GetDatabricksExecutor(),
mockedLoggerFactory.Object));
}

[Fact(Skip = "Performance testing")]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
using Energinet.DataHub.SettlementReport.CalculationResults.Interfaces.SettlementReports_v2.Models;
using Energinet.DataHub.SettlementReport.Common.Interfaces.Models;
using Energinet.DataHub.Wholesale.CalculationResults.IntegrationTests.Fixtures;
using Microsoft.Extensions.Logging;
using Moq;
using Xunit;

namespace Energinet.DataHub.Wholesale.CalculationResults.IntegrationTests.Infrastructure.SettlementReports_v2;
Expand All @@ -31,14 +33,19 @@ public class SettlementReportEnergyResultRepositoryTests : TestBase<SettlementRe

public SettlementReportEnergyResultRepositoryTests(MigrationsFreeDatabricksSqlStatementApiFixture databricksSqlStatementApiFixture)
{
var mockedLogging = new Mock<ILogger>();

var mockedLoggerFactory = new Mock<ILoggerFactory>();
mockedLoggerFactory.Setup(x => x.CreateLogger(It.IsAny<string>())).Returns(mockedLogging.Object);
_databricksSqlStatementApiFixture = databricksSqlStatementApiFixture;

_databricksSqlStatementApiFixture.DatabricksSchemaManager.DeltaTableOptions.Value.SettlementReportSchemaName =
databricksSqlStatementApiFixture.DatabricksSchemaManager.DeltaTableOptions.Value.SCHEMA_NAME;

Fixture.Inject<ISettlementReportDatabricksContext>(new SettlementReportDatabricksContext(
_databricksSqlStatementApiFixture.DatabricksSchemaManager.DeltaTableOptions,
_databricksSqlStatementApiFixture.GetDatabricksExecutor()));
_databricksSqlStatementApiFixture.GetDatabricksExecutor(),
mockedLoggerFactory.Object));
}

[Fact(Skip = "Performance testing")]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
using Energinet.DataHub.SettlementReport.Common.Interfaces.Models;
using Energinet.DataHub.Wholesale.CalculationResults.IntegrationTests.Fixtures;
using FluentAssertions;
using Microsoft.Extensions.Logging;
using Moq;
using Xunit;

namespace Energinet.DataHub.Wholesale.CalculationResults.IntegrationTests.Infrastructure.SettlementReports_v2;
Expand All @@ -32,14 +34,19 @@ public class SettlementReportMeteringPointMasterDataRepositoryTests : TestBase<S

public SettlementReportMeteringPointMasterDataRepositoryTests(MigrationsFreeDatabricksSqlStatementApiFixture databricksSqlStatementApiFixture)
{
var mockedLogging = new Mock<ILogger>();

var mockedLoggerFactory = new Mock<ILoggerFactory>();
mockedLoggerFactory.Setup(x => x.CreateLogger(It.IsAny<string>())).Returns(mockedLogging.Object);
_databricksSqlStatementApiFixture = databricksSqlStatementApiFixture;

_databricksSqlStatementApiFixture.DatabricksSchemaManager.DeltaTableOptions.Value.SettlementReportSchemaName =
databricksSqlStatementApiFixture.DatabricksSchemaManager.DeltaTableOptions.Value.SCHEMA_NAME;

Fixture.Inject<ISettlementReportDatabricksContext>(new SettlementReportDatabricksContext(
_databricksSqlStatementApiFixture.DatabricksSchemaManager.DeltaTableOptions,
_databricksSqlStatementApiFixture.GetDatabricksExecutor()));
_databricksSqlStatementApiFixture.GetDatabricksExecutor(),
mockedLoggerFactory.Object));
}

[Fact(Skip = "Performance testing")]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
using Energinet.DataHub.SettlementReport.CalculationResults.Interfaces.SettlementReports_v2.Models;
using Energinet.DataHub.SettlementReport.Common.Interfaces.Models;
using Energinet.DataHub.Wholesale.CalculationResults.IntegrationTests.Fixtures;
using Microsoft.Extensions.Logging;
using Moq;
using Xunit;

namespace Energinet.DataHub.Wholesale.CalculationResults.IntegrationTests.Infrastructure.SettlementReports_v2;
Expand All @@ -32,14 +34,19 @@ public class SettlementReportMeteringPointTimeSeriesResultRepositoryTests : Test

public SettlementReportMeteringPointTimeSeriesResultRepositoryTests(MigrationsFreeDatabricksSqlStatementApiFixture databricksSqlStatementApiFixture)
{
var mockedLogging = new Mock<ILogger>();

var mockedLoggerFactory = new Mock<ILoggerFactory>();
mockedLoggerFactory.Setup(x => x.CreateLogger(It.IsAny<string>())).Returns(mockedLogging.Object);
_databricksSqlStatementApiFixture = databricksSqlStatementApiFixture;

_databricksSqlStatementApiFixture.DatabricksSchemaManager.DeltaTableOptions.Value.SettlementReportSchemaName =
databricksSqlStatementApiFixture.DatabricksSchemaManager.DeltaTableOptions.Value.SCHEMA_NAME;

Fixture.Inject<ISettlementReportDatabricksContext>(new SettlementReportDatabricksContext(
_databricksSqlStatementApiFixture.DatabricksSchemaManager.DeltaTableOptions,
_databricksSqlStatementApiFixture.GetDatabricksExecutor()));
_databricksSqlStatementApiFixture.GetDatabricksExecutor(),
mockedLoggerFactory.Object));
}

[Fact]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
using Energinet.DataHub.SettlementReport.CalculationResults.Interfaces.SettlementReports_v2.Models;
using Energinet.DataHub.SettlementReport.Common.Interfaces.Models;
using Energinet.DataHub.Wholesale.CalculationResults.IntegrationTests.Fixtures;
using Microsoft.Extensions.Logging;
using Moq;
using Xunit;

namespace Energinet.DataHub.Wholesale.CalculationResults.IntegrationTests.Infrastructure.SettlementReports_v2;
Expand All @@ -31,14 +33,18 @@ public class SettlementReportMonthlyAmountRepositoryTests : TestBase<SettlementR

public SettlementReportMonthlyAmountRepositoryTests(MigrationsFreeDatabricksSqlStatementApiFixture databricksSqlStatementApiFixture)
{
var mockedLogging = new Mock<ILogger>();
var mockedLoggerFactory = new Mock<ILoggerFactory>();
mockedLoggerFactory.Setup(x => x.CreateLogger(It.IsAny<string>())).Returns(mockedLogging.Object);
_databricksSqlStatementApiFixture = databricksSqlStatementApiFixture;

_databricksSqlStatementApiFixture.DatabricksSchemaManager.DeltaTableOptions.Value.SettlementReportSchemaName =
databricksSqlStatementApiFixture.DatabricksSchemaManager.DeltaTableOptions.Value.SCHEMA_NAME;

Fixture.Inject<ISettlementReportDatabricksContext>(new SettlementReportDatabricksContext(
_databricksSqlStatementApiFixture.DatabricksSchemaManager.DeltaTableOptions,
_databricksSqlStatementApiFixture.GetDatabricksExecutor()));
_databricksSqlStatementApiFixture.GetDatabricksExecutor(),
mockedLoggerFactory.Object));
}

[Fact(Skip = "Performance testing")]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
using Energinet.DataHub.SettlementReport.CalculationResults.Interfaces.SettlementReports_v2.Models;
using Energinet.DataHub.SettlementReport.Common.Interfaces.Models;
using Energinet.DataHub.Wholesale.CalculationResults.IntegrationTests.Fixtures;
using Microsoft.Extensions.Logging;
using Moq;
using Xunit;

namespace Energinet.DataHub.Wholesale.CalculationResults.IntegrationTests.Infrastructure.SettlementReports_v2;
Expand All @@ -31,14 +33,19 @@ public class SettlementReportMonthlyAmountTotalRepositoryTests : TestBase<Settle

public SettlementReportMonthlyAmountTotalRepositoryTests(MigrationsFreeDatabricksSqlStatementApiFixture databricksSqlStatementApiFixture)
{
var mockedLogging = new Mock<ILogger>();
var mockedLoggerFactory = new Mock<ILoggerFactory>();
mockedLoggerFactory.Setup(x => x.CreateLogger(It.IsAny<string>())).Returns(mockedLogging.Object);

_databricksSqlStatementApiFixture = databricksSqlStatementApiFixture;

_databricksSqlStatementApiFixture.DatabricksSchemaManager.DeltaTableOptions.Value.SettlementReportSchemaName =
databricksSqlStatementApiFixture.DatabricksSchemaManager.DeltaTableOptions.Value.SCHEMA_NAME;

Fixture.Inject<ISettlementReportDatabricksContext>(new SettlementReportDatabricksContext(
_databricksSqlStatementApiFixture.DatabricksSchemaManager.DeltaTableOptions,
_databricksSqlStatementApiFixture.GetDatabricksExecutor()));
_databricksSqlStatementApiFixture.GetDatabricksExecutor(),
mockedLoggerFactory.Object));
}

[Fact(Skip = "Performance testing")]
Expand Down
Loading

0 comments on commit 9cb4451

Please sign in to comment.