From 676346517f82ae852597f81f2814ccae5ec95fda Mon Sep 17 00:00:00 2001 From: Dalet Date: Sun, 5 Mar 2023 22:02:10 +0900 Subject: [PATCH 1/6] Integrations tests with Testcontainers --- LeaderboardBackend.Test/Bans.cs | 19 +++- LeaderboardBackend.Test/Categories.cs | 14 ++- .../Fixtures/PostgresDatabaseFixture.cs | 106 ++++++++++++++++++ .../Fixtures/TestConfig.cs | 23 ++++ LeaderboardBackend.Test/Judgements.cs | 16 ++- .../LeaderboardBackend.Test.csproj | 1 + LeaderboardBackend.Test/Leaderboards.cs | 16 ++- LeaderboardBackend.Test/Modships.cs | 16 ++- LeaderboardBackend.Test/Runs.cs | 11 +- .../TestApi/TestApiFactory.cs | 91 +++++++++++---- LeaderboardBackend.Test/Users.cs | 16 ++- README.md | 4 +- docker-compose.yml | 10 -- example.env | 2 - 14 files changed, 296 insertions(+), 49 deletions(-) create mode 100644 LeaderboardBackend.Test/Fixtures/PostgresDatabaseFixture.cs create mode 100644 LeaderboardBackend.Test/Fixtures/TestConfig.cs diff --git a/LeaderboardBackend.Test/Bans.cs b/LeaderboardBackend.Test/Bans.cs index 1ba15c8d..535d7b5e 100644 --- a/LeaderboardBackend.Test/Bans.cs +++ b/LeaderboardBackend.Test/Bans.cs @@ -22,11 +22,24 @@ internal class Bans private static User s_modUser = null!; private static User s_normalUser = null!; - [SetUp] - public static async Task SetUp() + [OneTimeSetUp] + public void OneTimeSetup() { - s_factory = new TestApiFactory(); + s_factory = new(); s_apiClient = s_factory.CreateTestApiClient(); + } + + [OneTimeTearDown] + public void OneTimeTeardown() + { + s_factory.Dispose(); + } + + [SetUp] + public async Task SetUp() + { + s_factory.ResetDatabase(); + s_adminJwt = (await s_apiClient.LoginAdminUser()).Token; // Set up users and leaderboard diff --git a/LeaderboardBackend.Test/Categories.cs b/LeaderboardBackend.Test/Categories.cs index 0793b457..87adbe30 100644 --- a/LeaderboardBackend.Test/Categories.cs +++ b/LeaderboardBackend.Test/Categories.cs @@ -17,10 +17,22 @@ internal class Categories private static string? s_jwt; [OneTimeSetUp] - public static async Task SetUp() + public void OneTimeSetUp() { s_factory = new TestApiFactory(); s_apiClient = s_factory.CreateTestApiClient(); + } + + [OneTimeTearDown] + public void OneTimeTearDown() + { + s_factory.Dispose(); + } + + [SetUp] + public async Task SetUp() + { + s_factory.ResetDatabase(); s_jwt = (await s_apiClient.LoginAdminUser()).Token; } diff --git a/LeaderboardBackend.Test/Fixtures/PostgresDatabaseFixture.cs b/LeaderboardBackend.Test/Fixtures/PostgresDatabaseFixture.cs new file mode 100644 index 00000000..6f64cd29 --- /dev/null +++ b/LeaderboardBackend.Test/Fixtures/PostgresDatabaseFixture.cs @@ -0,0 +1,106 @@ +using System; +using System.Threading.Tasks; +using DotNet.Testcontainers.Builders; +using DotNet.Testcontainers.Configurations; +using DotNet.Testcontainers.Containers; +using LeaderboardBackend.Test.Fixtures; +using Npgsql; +using NUnit.Framework; + +// Fixtures apply to all tests in its namespace +// It has no namespace on purpose, so that the fixture applies to all tests in this assembly + +[SetUpFixture] // https://docs.nunit.org/articles/nunit/writing-tests/attributes/setupfixture.html +internal class PostgresDatabaseFixture +{ + public static TestcontainerDatabase? PostgresContainer { get; private set; } + public static bool HasCreatedTemplate { get; private set; } = false; + + private static string TemplateDatabase => PostgresContainer?.Database + "_template"; + + [OneTimeSetUp] + public static async Task OneTimeSetup() + { + if (TestConfig.DatabaseBackend != DatabaseBackend.TestContainer) + { + return; + } + + PostgresContainer = new ContainerBuilder() + .WithDatabase(new PostgreSqlTestcontainerConfiguration + { + Database = "db", + Username = "dbuser", + Password = "weft5gst768sr", + }) + .WithTmpfsMount("/var/lib/postgresql/data") // db files in-memory + .Build(); + await PostgresContainer.StartAsync(); + } + + [OneTimeTearDown] + public static async Task OneTimeTearDown() + { + if (PostgresContainer is null) + { + return; + } + + await PostgresContainer.DisposeAsync(); + } + + public static void CreateTemplateFromCurrentDb() + { + ThrowIfNotInitialized(); + + NpgsqlConnection.ClearAllPools(); // can't drop a DB if connections remain open + using NpgsqlDataSource conn = CreateConnectionToTemplate(); + conn.CreateCommand(@$" + DROP DATABASE IF EXISTS {TemplateDatabase}; + CREATE DATABASE {TemplateDatabase} + WITH TEMPLATE {PostgresContainer!.Database} + OWNER '{PostgresContainer!.Username}'; + ") + .ExecuteNonQuery(); + HasCreatedTemplate = true; + } + + + // It is faster to recreate the db from an already seeded template + // compared to dropping the db and recreating it from scratch + public static void ResetDatabaseToTemplate() + { + ThrowIfNotInitialized(); + if (!HasCreatedTemplate) + { + throw new InvalidOperationException("Database template has not been created."); + } + NpgsqlConnection.ClearAllPools(); // can't drop a DB if connections remain open + using NpgsqlDataSource conn = CreateConnectionToTemplate(); + conn.CreateCommand(@$" + DROP DATABASE IF EXISTS {PostgresContainer!.Database}; + CREATE DATABASE {PostgresContainer!.Database} + WITH TEMPLATE {TemplateDatabase} + OWNER '{PostgresContainer!.Username}'; + ") + .ExecuteNonQuery(); + } + + private static NpgsqlDataSource CreateConnectionToTemplate() + { + ThrowIfNotInitialized(); + NpgsqlConnectionStringBuilder connStrBuilder = new(PostgresContainer!.ConnectionString) + { + Database = "template1" + }; + return NpgsqlDataSource.Create(connStrBuilder); + } + + private static void ThrowIfNotInitialized() + { + if (PostgresContainer is null) + { + throw new InvalidOperationException("Postgres container is not initialized."); + } + } +} diff --git a/LeaderboardBackend.Test/Fixtures/TestConfig.cs b/LeaderboardBackend.Test/Fixtures/TestConfig.cs new file mode 100644 index 00000000..e766321d --- /dev/null +++ b/LeaderboardBackend.Test/Fixtures/TestConfig.cs @@ -0,0 +1,23 @@ +using System; + +namespace LeaderboardBackend.Test.Fixtures; + +internal static class TestConfig +{ + public static DatabaseBackend DatabaseBackend { get; private set; } = DatabaseBackend.TestContainer; + + static TestConfig() + { + string? backendVar = Environment.GetEnvironmentVariable("INTEGRATION_TESTS_DB_BACKEND"); + if (Enum.TryParse(backendVar, out DatabaseBackend dbBackend)) + { + DatabaseBackend = dbBackend; + } + } +} + +internal enum DatabaseBackend +{ + InMemory, + TestContainer, +} diff --git a/LeaderboardBackend.Test/Judgements.cs b/LeaderboardBackend.Test/Judgements.cs index 3c30327c..47f0144c 100644 --- a/LeaderboardBackend.Test/Judgements.cs +++ b/LeaderboardBackend.Test/Judgements.cs @@ -24,11 +24,23 @@ internal class Judgements private const string VALID_PASSWORD = "c00l_pAssword"; private const string VALID_EMAIL = "test@email.com"; - [SetUp] - public static async Task SetUp() + [OneTimeSetUp] + public void OneTimeSetup() { s_factory = new TestApiFactory(); s_apiClient = s_factory.CreateTestApiClient(); + } + + [OneTimeTearDown] + public void OneTimeTearDown() + { + s_factory.Dispose(); + } + + [SetUp] + public async Task SetUp() + { + s_factory.ResetDatabase(); // Set up a default Leaderboard and a mod user for that leaderboard to use as the Jwt for // tests diff --git a/LeaderboardBackend.Test/LeaderboardBackend.Test.csproj b/LeaderboardBackend.Test/LeaderboardBackend.Test.csproj index 1a68fee9..7d24fee1 100644 --- a/LeaderboardBackend.Test/LeaderboardBackend.Test.csproj +++ b/LeaderboardBackend.Test/LeaderboardBackend.Test.csproj @@ -16,6 +16,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/LeaderboardBackend.Test/Leaderboards.cs b/LeaderboardBackend.Test/Leaderboards.cs index 91c96823..e76b5976 100644 --- a/LeaderboardBackend.Test/Leaderboards.cs +++ b/LeaderboardBackend.Test/Leaderboards.cs @@ -19,11 +19,23 @@ internal class Leaderboards private static TestApiFactory s_factory = null!; private static string? s_jwt; - [SetUp] - public static async Task SetUp() + [OneTimeSetUp] + public void OneTimeSetUp() { s_factory = new TestApiFactory(); s_apiClient = s_factory.CreateTestApiClient(); + } + + [OneTimeTearDown] + public void OneTimeTearDown() + { + s_factory.Dispose(); + } + + [SetUp] + public async Task SetUp() + { + s_factory.ResetDatabase(); s_jwt = (await s_apiClient.LoginAdminUser()).Token; } diff --git a/LeaderboardBackend.Test/Modships.cs b/LeaderboardBackend.Test/Modships.cs index bb938c1b..db5c2e93 100644 --- a/LeaderboardBackend.Test/Modships.cs +++ b/LeaderboardBackend.Test/Modships.cs @@ -18,11 +18,23 @@ internal class Modships private static TestApiFactory s_factory = null!; private static string s_jwt = null!; - [SetUp] - public static async Task SetUp() + [OneTimeSetUp] + public void OneTimeSetUp() { s_factory = new TestApiFactory(); s_apiClient = s_factory.CreateTestApiClient(); + } + + [OneTimeTearDown] + public void OneTimeTearDown() + { + s_factory.Dispose(); + } + + [SetUp] + public async Task SetUp() + { + s_factory.ResetDatabase(); s_jwt = (await s_apiClient.LoginAdminUser()).Token; } diff --git a/LeaderboardBackend.Test/Runs.cs b/LeaderboardBackend.Test/Runs.cs index c2aa45a0..0eaf7ced 100644 --- a/LeaderboardBackend.Test/Runs.cs +++ b/LeaderboardBackend.Test/Runs.cs @@ -19,11 +19,18 @@ internal class Runs private static string s_jwt = null!; private static long s_categoryId; - [SetUp] - public static async Task SetUp() + [OneTimeSetUp] + public void OneTimeSetUp() { s_factory = new TestApiFactory(); s_apiClient = s_factory.CreateTestApiClient(); + } + + [SetUp] + public async Task SetUp() + { + s_factory.ResetDatabase(); + s_jwt = (await s_apiClient.LoginAdminUser()).Token; Leaderboard createdLeaderboard = await s_apiClient.Post( diff --git a/LeaderboardBackend.Test/TestApi/TestApiFactory.cs b/LeaderboardBackend.Test/TestApi/TestApiFactory.cs index 42e1619b..8cdb67a2 100644 --- a/LeaderboardBackend.Test/TestApi/TestApiFactory.cs +++ b/LeaderboardBackend.Test/TestApi/TestApiFactory.cs @@ -1,5 +1,7 @@ +using System; using System.Net.Http; using LeaderboardBackend.Models.Entities; +using LeaderboardBackend.Test.Fixtures; using LeaderboardBackend.Test.Lib; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Mvc.Testing; @@ -19,36 +21,55 @@ protected override void ConfigureWebHost(IWebHostBuilder builder) base.ConfigureWebHost(builder); - builder.ConfigureServices(services => - { - int testPort = DotNetEnv.Env.GetInt("POSTGRES_TEST_PORT", -1); - if (testPort >= 0) + builder.ConfigureServices(services => + { + if (TestConfig.DatabaseBackend == DatabaseBackend.TestContainer) { + if (PostgresDatabaseFixture.PostgresContainer is null) + { + throw new InvalidOperationException("Postgres container is not initialized."); + } + services.Configure(conf => { - if (conf.Pg is not null) + conf.UseInMemoryDb = false; + conf.Pg = new PostgresConfig { - conf.Pg.Port = (ushort)testPort; - } + Db = PostgresDatabaseFixture.PostgresContainer.Database, + Port = (ushort)PostgresDatabaseFixture.PostgresContainer.Port, + Host = PostgresDatabaseFixture.PostgresContainer.Hostname, + User = PostgresDatabaseFixture.PostgresContainer.Username, + Password = PostgresDatabaseFixture.PostgresContainer.Password + }; }); } - - // Reset the database on every test run - using ServiceProvider scope = services.BuildServiceProvider(); - ApplicationContext dbContext = scope.GetRequiredService(); - dbContext.Database.EnsureDeleted(); - - if (dbContext.Database.IsInMemory()) - { - dbContext.Database.EnsureCreated(); - } else { - dbContext.Database.Migrate(); + services.Configure(conf => + { + conf.UseInMemoryDb = true; + }); } - Seed(dbContext); - }); + using IServiceScope scope = services.BuildServiceProvider().CreateScope(); + ApplicationContext dbContext = scope.ServiceProvider.GetRequiredService(); + + switch (TestConfig.DatabaseBackend) + { + case DatabaseBackend.TestContainer when !PostgresDatabaseFixture.HasCreatedTemplate: + dbContext.Database.Migrate(); + Seed(dbContext); + PostgresDatabaseFixture.CreateTemplateFromCurrentDb(); + break; + case DatabaseBackend.InMemory: + if (dbContext.Database.EnsureCreated()) + { + Seed(dbContext); + } + + break; + } + }); } public TestApiClient CreateTestApiClient() @@ -56,10 +77,8 @@ public TestApiClient CreateTestApiClient() HttpClient client = CreateClient(); return new TestApiClient(client); } - private static void Seed(ApplicationContext dbContext) { - Leaderboard leaderboard = new() { Name = "Mario Goes to Jail", @@ -80,4 +99,32 @@ private static void Seed(ApplicationContext dbContext) dbContext.SaveChanges(); } + + /// + /// Deletes and recreates the database + /// + public void ResetDatabase() + { + switch (TestConfig.DatabaseBackend) + { + case DatabaseBackend.InMemory: + ResetInMemoryDb(); + break; + case DatabaseBackend.TestContainer: + PostgresDatabaseFixture.ResetDatabaseToTemplate(); + break; + default: + throw new NotImplementedException("Database reset is not implemented."); + } + } + + private void ResetInMemoryDb() + { + using IServiceScope scope = Services.CreateScope(); + ApplicationContext dbContext = scope.ServiceProvider.GetRequiredService(); + dbContext.Database.EnsureDeleted(); + dbContext.Database.EnsureCreated(); + + Seed(dbContext); + } } diff --git a/LeaderboardBackend.Test/Users.cs b/LeaderboardBackend.Test/Users.cs index 9750476a..b9d44ab8 100644 --- a/LeaderboardBackend.Test/Users.cs +++ b/LeaderboardBackend.Test/Users.cs @@ -19,13 +19,25 @@ internal class Users private const string VALID_PASSWORD = "c00l_pAssword"; private const string VALID_EMAIL = "test@email.com"; - [SetUp] - public static void SetUp() + [OneTimeSetUp] + public void OneTimeSetUp() { s_factory = new TestApiFactory(); s_apiClient = s_factory.CreateTestApiClient(); } + [OneTimeTearDown] + public void OneTimeTearDown() + { + s_factory.Dispose(); + } + + [SetUp] + public void SetUp() + { + s_factory.ResetDatabase(); + } + [Test] public static void GetUser_NotFound() { diff --git a/README.md b/README.md index 00ac3764..fa93dae7 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ This repo is a proof-of-concept for switching to a C# with ASP.NET Core stack. T * JSON REST API intended for the leaderboards.gg site * C# with ASP.NET Core, .NET 7 -* Docker containers for PostgreSQL hosting and management run via [Docker Compose](https://docs.docker.com/compose/install/) +* Docker containers for PostgreSQL hosting and management run via [Docker Compose](https://docs.docker.com/compose/install/), and also for integration tests with [Testcontainers](https://dotnet.testcontainers.org/) # Project Setup @@ -145,6 +145,8 @@ dotnet run --project LeaderboardBackend --urls https://localhost:7128 // dotnet The value provided to `--urls` has to match what's listed under `applicationUrl` in `LeaderboardBackend/Properties/launchSettings.json`. As of this writing, it's `https://localhost:7128`. #### Test the App +Docker is required to run integration tests against a real database. +However, if you would like to run the tests with an in-memory database, you can set the following environment variable: `INTEGRATION_TESTS_DB_BACKEND=InMemory`. To run the tests, run the following commands from the root of the project: diff --git a/docker-compose.yml b/docker-compose.yml index 52687bf5..fa61c55c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,16 +12,6 @@ services: ports: - ${ApplicationContext__PG__PORT}:5432 - db-test: - image: postgres - restart: always - environment: - POSTGRES_USER: ${ApplicationContext__PG__USER} - POSTGRES_PASSWORD: ${ApplicationContext__PG__PASSWORD} - POSTGRES_DB: ${ApplicationContext__PG__DB} - ports: - - ${POSTGRES_TEST_PORT}:5432 - adminer: image: adminer restart: always diff --git a/example.env b/example.env index 766475c7..da71ddda 100644 --- a/example.env +++ b/example.env @@ -11,7 +11,5 @@ ApplicationContext__PG__PORT=5432 JWT__KEY=coolsecretkeywow JWT__ISSUER=leaderboards.gg -POSTGRES_TEST_PORT=5433 - ADMINER_PORT=1337 From 9ac48f7387cc453de53274b3d95385d42f248156 Mon Sep 17 00:00:00 2001 From: Dalet Date: Sun, 5 Mar 2023 23:05:51 +0900 Subject: [PATCH 2/6] improve tests speed by reducing setup improved test speed by ~35% on my machine --- LeaderboardBackend.Test/Bans.cs | 18 +++++++----------- LeaderboardBackend.Test/Categories.cs | 12 ++++-------- LeaderboardBackend.Test/Judgements.cs | 18 +++++++----------- LeaderboardBackend.Test/Leaderboards.cs | 14 +++++--------- LeaderboardBackend.Test/Modships.cs | 7 ++++--- 5 files changed, 27 insertions(+), 42 deletions(-) diff --git a/LeaderboardBackend.Test/Bans.cs b/LeaderboardBackend.Test/Bans.cs index 535d7b5e..0fd217e4 100644 --- a/LeaderboardBackend.Test/Bans.cs +++ b/LeaderboardBackend.Test/Bans.cs @@ -23,21 +23,11 @@ internal class Bans private static User s_normalUser = null!; [OneTimeSetUp] - public void OneTimeSetup() + public async Task OneTimeSetup() { s_factory = new(); s_apiClient = s_factory.CreateTestApiClient(); - } - - [OneTimeTearDown] - public void OneTimeTeardown() - { - s_factory.Dispose(); - } - [SetUp] - public async Task SetUp() - { s_factory.ResetDatabase(); s_adminJwt = (await s_apiClient.LoginAdminUser()).Token; @@ -74,6 +64,12 @@ await s_apiClient.Post( s_modJwt = (await s_apiClient.LoginUser("mod@email.com", "Passw0rd!")).Token; } + [OneTimeTearDown] + public void OneTimeTeardown() + { + s_factory.Dispose(); + } + [Test] public static async Task CreateSiteBan_Ok() { diff --git a/LeaderboardBackend.Test/Categories.cs b/LeaderboardBackend.Test/Categories.cs index 87adbe30..cc5735fc 100644 --- a/LeaderboardBackend.Test/Categories.cs +++ b/LeaderboardBackend.Test/Categories.cs @@ -17,10 +17,13 @@ internal class Categories private static string? s_jwt; [OneTimeSetUp] - public void OneTimeSetUp() + public async Task OneTimeSetUp() { s_factory = new TestApiFactory(); s_apiClient = s_factory.CreateTestApiClient(); + + s_factory.ResetDatabase(); + s_jwt = (await s_apiClient.LoginAdminUser()).Token; } [OneTimeTearDown] @@ -29,13 +32,6 @@ public void OneTimeTearDown() s_factory.Dispose(); } - [SetUp] - public async Task SetUp() - { - s_factory.ResetDatabase(); - s_jwt = (await s_apiClient.LoginAdminUser()).Token; - } - [Test] public static void GetCategory_Unauthorized() { diff --git a/LeaderboardBackend.Test/Judgements.cs b/LeaderboardBackend.Test/Judgements.cs index 47f0144c..0c2e881a 100644 --- a/LeaderboardBackend.Test/Judgements.cs +++ b/LeaderboardBackend.Test/Judgements.cs @@ -25,21 +25,11 @@ internal class Judgements private const string VALID_EMAIL = "test@email.com"; [OneTimeSetUp] - public void OneTimeSetup() + public async Task OneTimeSetup() { s_factory = new TestApiFactory(); s_apiClient = s_factory.CreateTestApiClient(); - } - - [OneTimeTearDown] - public void OneTimeTearDown() - { - s_factory.Dispose(); - } - [SetUp] - public async Task SetUp() - { s_factory.ResetDatabase(); // Set up a default Leaderboard and a mod user for that leaderboard to use as the Jwt for @@ -89,6 +79,12 @@ public async Task SetUp() s_jwt = (await s_apiClient.LoginUser(VALID_EMAIL, VALID_PASSWORD)).Token; } + [OneTimeTearDown] + public void OneTimeTearDown() + { + s_factory.Dispose(); + } + [Test] public async Task CreateJudgement_OK() { diff --git a/LeaderboardBackend.Test/Leaderboards.cs b/LeaderboardBackend.Test/Leaderboards.cs index e76b5976..7dd669c9 100644 --- a/LeaderboardBackend.Test/Leaderboards.cs +++ b/LeaderboardBackend.Test/Leaderboards.cs @@ -20,10 +20,13 @@ internal class Leaderboards private static string? s_jwt; [OneTimeSetUp] - public void OneTimeSetUp() + public async Task OneTimeSetUp() { s_factory = new TestApiFactory(); s_apiClient = s_factory.CreateTestApiClient(); + + s_factory.ResetDatabase(); + s_jwt = (await s_apiClient.LoginAdminUser()).Token; } [OneTimeTearDown] @@ -32,18 +35,11 @@ public void OneTimeTearDown() s_factory.Dispose(); } - [SetUp] - public async Task SetUp() - { - s_factory.ResetDatabase(); - s_jwt = (await s_apiClient.LoginAdminUser()).Token; - } - [Test] public static void GetLeaderboard_NotFound() { RequestFailureException e = Assert.ThrowsAsync(async () => - await s_apiClient.Get($"/api/leaderboards/2", new()))!; + await s_apiClient.Get($"/api/leaderboards/{long.MaxValue}", new()))!; Assert.AreEqual(HttpStatusCode.NotFound, e.Response.StatusCode); } diff --git a/LeaderboardBackend.Test/Modships.cs b/LeaderboardBackend.Test/Modships.cs index db5c2e93..4d7dff01 100644 --- a/LeaderboardBackend.Test/Modships.cs +++ b/LeaderboardBackend.Test/Modships.cs @@ -19,10 +19,12 @@ internal class Modships private static string s_jwt = null!; [OneTimeSetUp] - public void OneTimeSetUp() + public async Task OneTimeSetUp() { s_factory = new TestApiFactory(); s_apiClient = s_factory.CreateTestApiClient(); + + s_jwt = (await s_apiClient.LoginAdminUser()).Token; } [OneTimeTearDown] @@ -32,10 +34,9 @@ public void OneTimeTearDown() } [SetUp] - public async Task SetUp() + public void SetUp() { s_factory.ResetDatabase(); - s_jwt = (await s_apiClient.LoginAdminUser()).Token; } [Test] From f0164cd24fbde8d6d4b6cdea58b9ccb790974a7e Mon Sep 17 00:00:00 2001 From: Dalet Date: Mon, 6 Mar 2023 00:01:15 +0900 Subject: [PATCH 3/6] test job: optional .env and separate restore, build, and test steps --- .github/workflows/test-backend.yml | 14 ++++++-------- .../LeaderboardBackend.Jobs.csproj | 2 +- .../LeaderboardBackend.Test.csproj | 2 +- LeaderboardBackend/LeaderboardBackend.csproj | 2 +- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test-backend.yml b/.github/workflows/test-backend.yml index 461ba8e8..cfc9f7a3 100644 --- a/.github/workflows/test-backend.yml +++ b/.github/workflows/test-backend.yml @@ -13,11 +13,9 @@ jobs: uses: actions/setup-dotnet@v1 with: dotnet-version: '7.0.x' - - name: Create .env file - run: cp example.env .env - - name: Spin up containers - run: docker compose up -d - - name: Run Tests on In Memory Database - run: ApplicationContext__UseInMemoryDb=true dotnet test LeaderboardBackend.Test - - name: Run Tests on Test Postgres Database - run: ApplicationContext__UseInMemoryDb=false dotnet test LeaderboardBackend.Test + - name: Restore NuGet packages + run: dotnet restore + - name: Build + run: dotnet build --no-restore + - name: Run tests + run: dotnet test LeaderboardBackend.Test --no-restore diff --git a/LeaderboardBackend.Jobs/LeaderboardBackend.Jobs.csproj b/LeaderboardBackend.Jobs/LeaderboardBackend.Jobs.csproj index 73850be8..7ad9a1d2 100644 --- a/LeaderboardBackend.Jobs/LeaderboardBackend.Jobs.csproj +++ b/LeaderboardBackend.Jobs/LeaderboardBackend.Jobs.csproj @@ -13,7 +13,7 @@ - + Always diff --git a/LeaderboardBackend.Test/LeaderboardBackend.Test.csproj b/LeaderboardBackend.Test/LeaderboardBackend.Test.csproj index 7d24fee1..475ea7e0 100644 --- a/LeaderboardBackend.Test/LeaderboardBackend.Test.csproj +++ b/LeaderboardBackend.Test/LeaderboardBackend.Test.csproj @@ -24,7 +24,7 @@ - + Always diff --git a/LeaderboardBackend/LeaderboardBackend.csproj b/LeaderboardBackend/LeaderboardBackend.csproj index 85e04ba1..fcb3a41e 100644 --- a/LeaderboardBackend/LeaderboardBackend.csproj +++ b/LeaderboardBackend/LeaderboardBackend.csproj @@ -38,7 +38,7 @@ - + Always From 45b1b3ea858747dfd598435872e870afab3df3e5 Mon Sep 17 00:00:00 2001 From: Dalet Date: Wed, 8 Mar 2023 23:59:07 +0900 Subject: [PATCH 4/6] upgrade Testcontainers to v3.0.0 --- .../Fixtures/PostgresDatabaseFixture.cs | 39 ++++++++++--------- .../LeaderboardBackend.Test.csproj | 3 +- .../TestApi/TestApiFactory.cs | 8 ++-- 3 files changed, 26 insertions(+), 24 deletions(-) diff --git a/LeaderboardBackend.Test/Fixtures/PostgresDatabaseFixture.cs b/LeaderboardBackend.Test/Fixtures/PostgresDatabaseFixture.cs index 6f64cd29..6fc97180 100644 --- a/LeaderboardBackend.Test/Fixtures/PostgresDatabaseFixture.cs +++ b/LeaderboardBackend.Test/Fixtures/PostgresDatabaseFixture.cs @@ -1,11 +1,9 @@ using System; using System.Threading.Tasks; -using DotNet.Testcontainers.Builders; -using DotNet.Testcontainers.Configurations; -using DotNet.Testcontainers.Containers; using LeaderboardBackend.Test.Fixtures; using Npgsql; using NUnit.Framework; +using Testcontainers.PostgreSql; // Fixtures apply to all tests in its namespace // It has no namespace on purpose, so that the fixture applies to all tests in this assembly @@ -13,10 +11,13 @@ [SetUpFixture] // https://docs.nunit.org/articles/nunit/writing-tests/attributes/setupfixture.html internal class PostgresDatabaseFixture { - public static TestcontainerDatabase? PostgresContainer { get; private set; } + public static PostgreSqlContainer? PostgresContainer { get; private set; } + public static string? Database { get; private set; } + public static string? Username { get; private set; } + public static string? Password { get; private set; } + public static int Port { get; private set; } public static bool HasCreatedTemplate { get; private set; } = false; - - private static string TemplateDatabase => PostgresContainer?.Database + "_template"; + private static string TemplateDatabase => Database! + "_template"; [OneTimeSetUp] public static async Task OneTimeSetup() @@ -26,16 +27,16 @@ public static async Task OneTimeSetup() return; } - PostgresContainer = new ContainerBuilder() - .WithDatabase(new PostgreSqlTestcontainerConfiguration - { - Database = "db", - Username = "dbuser", - Password = "weft5gst768sr", - }) + PostgresContainer = new PostgreSqlBuilder() .WithTmpfsMount("/var/lib/postgresql/data") // db files in-memory .Build(); await PostgresContainer.StartAsync(); + + NpgsqlConnectionStringBuilder connStrBuilder = new(PostgresContainer.GetConnectionString()); + Username = connStrBuilder.Username!; + Password = connStrBuilder.Password!; + Database = connStrBuilder.Database!; + Port = connStrBuilder.Port; } [OneTimeTearDown] @@ -58,8 +59,8 @@ public static void CreateTemplateFromCurrentDb() conn.CreateCommand(@$" DROP DATABASE IF EXISTS {TemplateDatabase}; CREATE DATABASE {TemplateDatabase} - WITH TEMPLATE {PostgresContainer!.Database} - OWNER '{PostgresContainer!.Username}'; + WITH TEMPLATE {Database} + OWNER '{Username}'; ") .ExecuteNonQuery(); HasCreatedTemplate = true; @@ -78,10 +79,10 @@ public static void ResetDatabaseToTemplate() NpgsqlConnection.ClearAllPools(); // can't drop a DB if connections remain open using NpgsqlDataSource conn = CreateConnectionToTemplate(); conn.CreateCommand(@$" - DROP DATABASE IF EXISTS {PostgresContainer!.Database}; - CREATE DATABASE {PostgresContainer!.Database} + DROP DATABASE IF EXISTS {Database}; + CREATE DATABASE {Database} WITH TEMPLATE {TemplateDatabase} - OWNER '{PostgresContainer!.Username}'; + OWNER '{Username}'; ") .ExecuteNonQuery(); } @@ -89,7 +90,7 @@ WITH TEMPLATE {TemplateDatabase} private static NpgsqlDataSource CreateConnectionToTemplate() { ThrowIfNotInitialized(); - NpgsqlConnectionStringBuilder connStrBuilder = new(PostgresContainer!.ConnectionString) + NpgsqlConnectionStringBuilder connStrBuilder = new(PostgresContainer!.GetConnectionString()) { Database = "template1" }; diff --git a/LeaderboardBackend.Test/LeaderboardBackend.Test.csproj b/LeaderboardBackend.Test/LeaderboardBackend.Test.csproj index 475ea7e0..37350fbd 100644 --- a/LeaderboardBackend.Test/LeaderboardBackend.Test.csproj +++ b/LeaderboardBackend.Test/LeaderboardBackend.Test.csproj @@ -16,7 +16,8 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + + diff --git a/LeaderboardBackend.Test/TestApi/TestApiFactory.cs b/LeaderboardBackend.Test/TestApi/TestApiFactory.cs index 8cdb67a2..ca62ce5c 100644 --- a/LeaderboardBackend.Test/TestApi/TestApiFactory.cs +++ b/LeaderboardBackend.Test/TestApi/TestApiFactory.cs @@ -35,11 +35,11 @@ protected override void ConfigureWebHost(IWebHostBuilder builder) conf.UseInMemoryDb = false; conf.Pg = new PostgresConfig { - Db = PostgresDatabaseFixture.PostgresContainer.Database, - Port = (ushort)PostgresDatabaseFixture.PostgresContainer.Port, + Db = PostgresDatabaseFixture.Database!, + Port = (ushort)PostgresDatabaseFixture.Port, Host = PostgresDatabaseFixture.PostgresContainer.Hostname, - User = PostgresDatabaseFixture.PostgresContainer.Username, - Password = PostgresDatabaseFixture.PostgresContainer.Password + User = PostgresDatabaseFixture.Username!, + Password = PostgresDatabaseFixture.Password! }; }); } From a67bed6785066bc915a551765e4da00a06165e55 Mon Sep 17 00:00:00 2001 From: Dalet Date: Sat, 11 Mar 2023 13:47:57 +0900 Subject: [PATCH 5/6] README: update the database setup section --- README.md | 44 ++++++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index fa93dae7..beff62dd 100644 --- a/README.md +++ b/README.md @@ -65,31 +65,43 @@ After installing a code editor: ## Running the Application +The application reads configuration from environment variables, a `.env` file, and a `appsettings.json` file. + +The easiest is to make a `.env` file. As a starting point, you can copy the template file `example.env` at the project root and rename it to `.env`. +```bash +cp example.env .env +``` + ### Running the Database(s) +We use Postgres, but have the ability to run Entity Framework's in-memory DB as well instead, which allows quicker debugging. -We use Postgres, but have the ability to run .NET's in-memory DB as well instead, which allows quicker debugging. If you would like to use the latter, you can set `ApplicationContext__UseInMemoryDb` in your `.env` file to `true`, or not make a `.env` file at all, then skip to the next section. If you would like to use the former, follow these instructions: +**Running with Postgres is highly encouraged**, as the in-memory DB will not behave the same as Postgres, +and [some features will not work at all](https://learn.microsoft.com/en-us/ef/core/testing/choosing-a-testing-strategy#in-memory-as-a-database-fake) (for example: transactions and constraints). -As mentioned above, we run Docker containers for the DB. After [installing Docker Compose](https://docs.docker.com/compose/install/), run these commands in the project root: +#### Postgres with Docker compose +As mentioned above, we run Docker containers for the DB. After [installing Docker Compose](https://docs.docker.com/compose/install/), run this command in the project root: ```bash -cp example.env .env -sudo docker-compose (or docker compose) up -d +docker compose up -d ``` +If you're using Linux, you might need to run Docker with `sudo`. This starts up: - Adminer which is a GUI manager for the databases -- The main Postgres DB -- The test Postgres DB +- A Postgres DB + +If you're using the default values provided in `example.env`, input these values in Adminer for access: -Using the default values provided in `example.env`, input these values in Adminer for access: +| Field | Value | Comment +| -------- | ------------------ | ------- +| System | PostgreSQL | +| Server | `db` | You cannot set it to `localhost` because it must match the Docker container name +| Username | `admin` | Corresponds to the `ApplicationContext__PG__USER` config +| Password | `example` | Corresponds to `ApplicationContext__PG__PASSWORD` +| Database | `leaderboardsmain` | Corresponds to `ApplicationContext__PG__DB` -| Field | Value | -| --- | --- | -| System | PostgreSQL | -| Server | db (for main) / db-test (for test) | -| Username | admin | -| Password | example | -| Database | leaderboardsmain / leaderboardstest | +#### In-memory database +To use the in-memory database, set `ApplicationContext__UseInMemoryDb` in your `.env` file to `true`. ### Visual Studio @@ -145,8 +157,8 @@ dotnet run --project LeaderboardBackend --urls https://localhost:7128 // dotnet The value provided to `--urls` has to match what's listed under `applicationUrl` in `LeaderboardBackend/Properties/launchSettings.json`. As of this writing, it's `https://localhost:7128`. #### Test the App -Docker is required to run integration tests against a real database. -However, if you would like to run the tests with an in-memory database, you can set the following environment variable: `INTEGRATION_TESTS_DB_BACKEND=InMemory`. +Docker is required to run integration tests against a real Postgres database. +However, if you would still like to run the tests with an in-memory database, you can set the following environment variable: `INTEGRATION_TESTS_DB_BACKEND=InMemory`. To run the tests, run the following commands from the root of the project: From fa755378621a50c56dbc008011b22f84b4c49d4c Mon Sep 17 00:00:00 2001 From: Dalet Date: Sat, 25 Mar 2023 14:18:38 +0900 Subject: [PATCH 6/6] run migration only with real db --- LeaderboardBackend/Program.cs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/LeaderboardBackend/Program.cs b/LeaderboardBackend/Program.cs index 4063c298..1ba38cdf 100644 --- a/LeaderboardBackend/Program.cs +++ b/LeaderboardBackend/Program.cs @@ -180,11 +180,6 @@ ApplicationContextConfig config = scope.ServiceProvider .GetRequiredService>().Value; - if (config.MigrateDb) - { - context.Database.Migrate(); - } - if (config.UseInMemoryDb) { // If in memory DB, the only way to have an admin user is to seed it at startup. @@ -199,6 +194,10 @@ context.Users.Add(admin); await context.SaveChangesAsync(); } + else if (config.MigrateDb) + { + context.Database.Migrate(); + } } app.UseAuthentication();