diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b7e9b331..e62c5afc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -56,4 +56,12 @@ jobs: - uses: bluefireteam/melos-action@v3 - run: melos run format-check - run: melos run analyze -- ${{ inputs.fatal_warnings && '--fatal-infos' || '--no-fatal-warnings --no-fatal-infos' }} + - name: Prepare server + working-directory: wrestling_scoreboard_server + run: | + cp .env.example .env + source .env + docker pull postgres:latest + docker run --detach --name wrestling_scoreboard_database -p ${DATABASE_HOST}:${DATABASE_PORT}:5432 -e POSTGRES_USER=${DATABASE_USER} -e POSTGRES_DATABASE=${DATABASE_NAME} -e POSTGRES_PASSWORD=${DATABASE_PASSWORD} postgres - run: melos run test + - run: docker container rm --force wrestling_scoreboard_database diff --git a/README.md b/README.md index 89bdce22..ea25d194 100644 --- a/README.md +++ b/README.md @@ -23,9 +23,7 @@ Tags: scoreboard, wrestling, scoring, bracket, mat, team, fight, competition, to The App consists of three components, the client, the server and the database. You can download the client and the server for your preferred platforms from the [Releases section](https://github.com/Oberhauser-Dev/wrestling_scoreboard/releases). -For setting up the database and hosting a server, see: -- [Database](wrestling_scoreboard_server/database/README.md) -- [Server](wrestling_scoreboard_server/README.md#setup) +For setting up the database and hosting a server, see the [Server Setup](wrestling_scoreboard_server/README.md#setup). ## Development diff --git a/wrestling_scoreboard_server/.env.example b/wrestling_scoreboard_server/.env.example index 46618e91..f52638a2 100644 --- a/wrestling_scoreboard_server/.env.example +++ b/wrestling_scoreboard_server/.env.example @@ -8,10 +8,10 @@ JWT_ISSUER='Wrestling Scoreboard (oberhauser.dev)' JWT_SECRET=my-ultra-secure-and-ultra-long-secret JWT_EXPIRES_IN_DAYS=90 -DATABASE_HOST='localhost' +DATABASE_HOST='127.0.0.1' DATABASE_PORT=5432 DATABASE_USER='wrestling' -DATABASE_PASSWORD='' +DATABASE_PASSWORD='my-password' DATABASE_NAME='wrestling_scoreboard' # One of: ALL, FINEST, FINER, FINE, CONFIG, INFO, WARNING, SEVERE, SHOUT, OFF diff --git a/wrestling_scoreboard_server/README.md b/wrestling_scoreboard_server/README.md index ece1509e..8ad2f9da 100644 --- a/wrestling_scoreboard_server/README.md +++ b/wrestling_scoreboard_server/README.md @@ -4,17 +4,27 @@ Wrestling software server for managing team matches and competitions. ## Setup -See [database docs](./database/README.md), to set up the Postgres database. - -It is recommended to start the app with user privileges, here `www`. Avoid using root. - Download the latest server version from the [releases section](https://github.com/Oberhauser-Dev/wrestling_scoreboard/releases) -and extract it into e.g. inside `$HOME/.local/share/wrestling_scoreboard_server` +and extract it into e.g. inside `$HOME/.local/share/wrestling_scoreboard_server`. + +It is recommended to start the app with user privileges, here `www`. Avoid using `root`, especially if the server is open to the public. ### Environment variables: Create file `.env` in the `wrestling_scoreboard_server` directory. -A pre-configuration can be found in `.env.example` file. Change the values to your needs. +A pre-configuration can be found in `.env.example` file (`cp .env.example .env`). Change the values to your needs. + +### Database + +You can run the database e.g. in a docker container: + +```shell +source .env +docker pull postgres:latest +docker run --detach --name wrestling_scoreboard_database -p ${DATABASE_HOST}:${DATABASE_PORT}:5432 -e POSTGRES_USER=${DATABASE_USER} -e POSTGRES_DATABASE=${DATABASE_NAME} -e POSTGRES_PASSWORD=${DATABASE_PASSWORD} postgres +``` + +For a manual / more detailed setup of the Postgres database, see the [database docs](./database/README.md). ### Run server @@ -51,11 +61,12 @@ systemctl --user start wrestling-scoreboard-server.service ``` Additionally, enable session for user `www` on boot: + ```bash sudo loginctl enable-linger www ``` -To view server logs: +To view server logs: `journalctl --user -u wrestling-scoreboard-server` ### Web server diff --git a/wrestling_scoreboard_server/database/README.md b/wrestling_scoreboard_server/database/README.md index e73f38bf..900fd5c7 100644 --- a/wrestling_scoreboard_server/database/README.md +++ b/wrestling_scoreboard_server/database/README.md @@ -10,9 +10,9 @@ Then add this to your system PATH variable `C:/Program Files/PostgreSQL/ migrate() async { - final db = PostgresDb(); - - String semver; - try { - final res = await db.connection.execute('SELECT semver FROM migration LIMIT 1;'); - final row = res.singleOrNull; - semver = row?.toColumnMap()['semver'] ?? '0.0.0'; - } catch (_) { - semver = '0.0.0'; - } - final databaseVersion = Version.parse(semver); - - final dir = Directory('database/migration'); - final List entities = await dir.list().toList(); - final migrationMap = entities.map((entity) { - final migrationVersion = entity.uri.pathSegments.last.split('_')[0]; - return MapEntry(Version.parse(migrationVersion.replaceFirst('v', '')), entity); - }).toList(); - migrationMap.sort((a, b) => a.key.compareTo(b.key)); - int migrationStartIndex = 0; - while (migrationStartIndex < migrationMap.length) { - if (databaseVersion.compareTo(migrationMap[migrationStartIndex].key) < 0) { - break; - } - migrationStartIndex++; - } - final migrationRange = migrationMap.sublist(migrationStartIndex, migrationMap.length); - if (migrationRange.isNotEmpty) { - for (var migration in migrationRange) { - await _executeSqlFile(migration.value.path); - } - await db.connection.execute("UPDATE migration SET semver = '${migrationRange.last.key.toString()}';"); - } - } - /// Reset all tables Future reset(Request request, User? user) async { try { - await _restoreDefault(); - Iterable entityControllers = - dataTypes.map((t) => ShelfController.getControllerFromDataType(t)).nonNulls; - // Remove data - await Future.forEach(entityControllers, (e) => e.deleteMany()); - // No need to restore the database semantic version for migration, - // as it is no entity controller and therefore keeps it default value. - - // Recreate default admin - await SecuredUserController().createSingle( - User(username: 'admin', createdAt: DateTime.now(), privilege: UserPrivilege.admin, password: 'admin') - .toSecuredUser()); + await PostgresDb().reset(); return Response.ok('{"status": "success"}'); } catch (err) { return Response.internalServerError(body: '{"err": "$err"}'); @@ -69,29 +17,8 @@ class DatabaseController { /// Export a database to dump Future export(Request request, User? user) async { - final db = PostgresDb(); - final args = [ - // '--file', - // './database/dump/${DateTime.now().toIso8601String().replaceAll(':', '-').replaceAll(RegExp(r'\.[0-9]{3}'), '')}-' - // 'PostgreSQL-wrestling_scoreboard-dump.sql', - '--username', - db.dbUser, - '--host', - db.postgresHost, - '--port', - db.postgresPort.toString(), - '--dbname', - db.postgresDatabaseName, - '--schema', - 'public' - ]; - final process = await Process.run('pg_dump', args, environment: {'PGPASSWORD': db.dbPW}); - - if (process.exitCode == 0) { - return Response.ok(process.stdout); - } else { - return Response.internalServerError(body: '{"err": "${process.stderr}"}'); - } + final sqlStr = await PostgresDb().export(); + return Response.ok(sqlStr); } /// Restore a database dump @@ -101,7 +28,7 @@ class DatabaseController { File file = File('${Directory.systemTemp.path}/${message.hashCode.toUnsigned(20).toRadixString(16).padLeft(5, '0')}.sql'); await file.writeAsString(message); - await _restore(file.path); + await PostgresDb().restore(file.path); return Response.ok('{"status": "success"}'); } catch (err) { return Response.internalServerError(body: '{"err": "$err"}'); @@ -111,45 +38,10 @@ class DatabaseController { /// Restore the default database dump Future restoreDefault(Request request, User? user) async { try { - await _restoreDefault(); + await PostgresDb().restoreDefault(); return Response.ok('{"status": "success"}'); } catch (err) { return Response.internalServerError(body: '{"err": "$err"}'); } } - - Future _restoreDefault() => _restore('./database/dump/PostgreSQL-wrestling_scoreboard-dump.sql'); - - Future _restore(String dumpPath) async { - final db = PostgresDb(); - await db.connection.execute('DROP SCHEMA IF EXISTS public CASCADE;'); - await _executeSqlFile(dumpPath); - } - - Future _executeSqlFile(String sqlFilePath) async { - final db = PostgresDb(); - await db.close(); - - final args = [ - '--file', - sqlFilePath, - '--username', - db.dbUser, - '--host', - db.postgresHost, - '--port', - db.postgresPort.toString(), - db.postgresDatabaseName, - ]; - final processResult = await Process.run('psql', args, environment: {'PGPASSWORD': db.dbPW}); - await db.open(); - - Iterable entityControllers = - dataTypes.map((t) => ShelfController.getControllerFromDataType(t)).nonNulls; - await Future.forEach(entityControllers, (e) => e.init()); - - if (processResult.exitCode != 0) { - throw processResult.stderr; - } - } } diff --git a/wrestling_scoreboard_server/lib/controllers/entity_controller.dart b/wrestling_scoreboard_server/lib/controllers/entity_controller.dart index 9339ff3d..cf9db915 100644 --- a/wrestling_scoreboard_server/lib/controllers/entity_controller.dart +++ b/wrestling_scoreboard_server/lib/controllers/entity_controller.dart @@ -215,19 +215,24 @@ abstract class EntityController { String tableName; String primaryKeyName; - late Future getSingleRawStmt; - late Future deleteSingleStmt; + late psql.Statement getSingleRawStmt; + late psql.Statement deleteSingleStmt; - EntityController({required this.tableName, this.primaryKeyName = 'id'}) { - init(); + EntityController({required this.tableName, this.primaryKeyName = 'id'}); + + static Future initAll() async { + // Reinit all prepared statements + Iterable entityControllers = + dataTypes.map((t) => ShelfController.getControllerFromDataType(t)).nonNulls; + await Future.forEach(entityControllers, (e) => e.init()); } - void init() { + Future init() async { getSingleRawStmt = - PostgresDb().connection.prepare(psql.Sql.named('SELECT * FROM $tableName WHERE $primaryKeyName = @id;')); + await PostgresDb().connection.prepare(psql.Sql.named('SELECT * FROM $tableName WHERE $primaryKeyName = @id;')); deleteSingleStmt = - PostgresDb().connection.prepare(psql.Sql.named('DELETE FROM $tableName WHERE $primaryKeyName = @id;')); + await PostgresDb().connection.prepare(psql.Sql.named('DELETE FROM $tableName WHERE $primaryKeyName = @id;')); } Future getSingle(int id, {required bool obfuscate}) async { @@ -240,7 +245,7 @@ abstract class EntityController { int id, { required bool obfuscate, }) async { - final resStream = (await getSingleRawStmt).bind({'id': id}); + final resStream = getSingleRawStmt.bind({'id': id}); final many = await resStream.toColumnMap().toList(); if (many.isEmpty) throw InvalidParameterException('$T with id "$id" not found'); var data = many.first; @@ -331,7 +336,7 @@ abstract class EntityController { Future deleteSingle(int id) async { try { - await (await deleteSingleStmt).bind({'id': id}).toList(); + await deleteSingleStmt.bind({'id': id}).toList(); } on psql.PgException catch (_) { return false; } diff --git a/wrestling_scoreboard_server/lib/controllers/organizational_controller.dart b/wrestling_scoreboard_server/lib/controllers/organizational_controller.dart index f748472f..74aa2a5a 100644 --- a/wrestling_scoreboard_server/lib/controllers/organizational_controller.dart +++ b/wrestling_scoreboard_server/lib/controllers/organizational_controller.dart @@ -9,13 +9,13 @@ abstract class OrganizationalController extends ShelfC OrganizationalController({required super.tableName}); - late Future getSingleOfOrgRawStmt; + late psql.Statement getSingleOfOrgRawStmt; @override - void init() { - getSingleOfOrgRawStmt = PostgresDb().connection.prepare( + Future init() async { + await super.init(); + getSingleOfOrgRawStmt = await PostgresDb().connection.prepare( psql.Sql.named('SELECT * FROM $tableName WHERE organization_id = @orgId AND org_sync_id = @orgSyncId;')); - super.init(); } /// Get a single data object via a foreign id (sync id), given by an organization. @@ -30,7 +30,7 @@ abstract class OrganizationalController extends ShelfC orgSyncId = orgSyncId.trim(); _logger.warning('$T with orgSyncId "$orgSyncId" was trimmed'); } - final resStream = (await getSingleOfOrgRawStmt).bind({'orgSyncId': orgSyncId, 'orgId': orgId}); + final resStream = getSingleOfOrgRawStmt.bind({'orgSyncId': orgSyncId, 'orgId': orgId}); final many = await resStream.toColumnMap().toList(); if (many.isEmpty) throw InvalidParameterException('$T with orgSyncId "$orgSyncId" not found'); return many.first; diff --git a/wrestling_scoreboard_server/lib/server.dart b/wrestling_scoreboard_server/lib/server.dart index 89e99d55..7c7542fd 100644 --- a/wrestling_scoreboard_server/lib/server.dart +++ b/wrestling_scoreboard_server/lib/server.dart @@ -9,16 +9,15 @@ import 'package:shelf/shelf.dart'; import 'package:shelf/shelf_io.dart' as shelf_io; import 'package:shelf_router/shelf_router.dart' as shelf_router; import 'package:shelf_static/shelf_static.dart' as shelf_static; -import 'package:wrestling_scoreboard_server/controllers/database_controller.dart'; import 'package:wrestling_scoreboard_server/controllers/websocket_handler.dart'; -import 'package:wrestling_scoreboard_server/services/environment.dart'; -import 'package:wrestling_scoreboard_server/services/pubspec.dart'; import 'package:wrestling_scoreboard_server/routes/api_route.dart'; +import 'package:wrestling_scoreboard_server/services/environment.dart'; import 'package:wrestling_scoreboard_server/services/postgres_db.dart'; +import 'package:wrestling_scoreboard_server/services/pubspec.dart'; import 'middleware/cors.dart'; -Future init() async { +Future init() async { // Init logger Logger.root.level = env.logLevel ?? Level.INFO; Logger.root.onRecord.listen( @@ -48,8 +47,9 @@ Future init() async { final port = env.port ?? 8080; // Must open the database before initializing any routes. - await PostgresDb().open(); - await DatabaseController().migrate(); + final db = PostgresDb(); + await db.open(); + await db.migrate(); final webSocketLog = Logger('Websocket'); diff --git a/wrestling_scoreboard_server/lib/services/postgres_db.dart b/wrestling_scoreboard_server/lib/services/postgres_db.dart index 22e098cc..5c63a756 100644 --- a/wrestling_scoreboard_server/lib/services/postgres_db.dart +++ b/wrestling_scoreboard_server/lib/services/postgres_db.dart @@ -1,8 +1,15 @@ +import 'dart:io'; + import 'package:postgres/postgres.dart' as psql; +import 'package:pub_semver/pub_semver.dart'; +import 'package:wrestling_scoreboard_common/common.dart'; +import 'package:wrestling_scoreboard_server/controllers/entity_controller.dart'; +import 'package:wrestling_scoreboard_server/controllers/user_controller.dart'; import 'environment.dart'; const _isReleaseMode = bool.fromEnvironment("dart.vm.product"); +const defaultDatabasePath = './database/dump/PostgreSQL-wrestling_scoreboard-dump.sql'; class PostgresDb { final String postgresHost = env.databaseHost ?? 'localhost'; @@ -12,7 +19,12 @@ class PostgresDb { final String postgresDatabaseName = env.databaseName ?? 'wrestling_scoreboard'; static final PostgresDb _singleton = PostgresDb._internal(); - late psql.Connection connection; + + psql.Connection? _connection; + + psql.Connection get connection => _connection != null + ? _connection! + : throw Exception('Database connection has not yet been initialized. Plz call `open()` first.'); factory PostgresDb() { return _singleton; @@ -20,9 +32,8 @@ class PostgresDb { PostgresDb._internal(); - /// Only call this once! Future open() async { - connection = await psql.Connection.open( + _connection ??= await psql.Connection.open( psql.Endpoint( host: postgresHost, port: postgresPort, @@ -37,5 +48,140 @@ class PostgresDb { Future close() async { await connection.close(); + _connection = null; + } +} + +extension DatabaseExt on PostgresDb { + /*Future createIfNotExists([String dumpPath = defaultDatabasePath]) async { + await open(usePostgresDb: true); + final res = await connection.execute("SELECT 1 FROM pg_database WHERE datname='$postgresDatabaseName';"); + if (res.isEmpty) { + await connection.execute("CREATE DATABASE $postgresDatabaseName;"); + await close(); + await open(usePostgresDb: false); + await restore(dumpPath); + } + await close(); + }*/ + + Future migrate() async { + String semver; + try { + final res = await connection.execute('SELECT semver FROM migration LIMIT 1;'); + final row = res.singleOrNull; + semver = row?.toColumnMap()['semver'] ?? '0.0.0'; + } catch (_) { + // DB has not yet been initialized + semver = '0.0.0-pre'; + } + final databaseVersion = Version.parse(semver); + + final dir = Directory('database/migration'); + final List entities = await dir.list().toList(); + final migrationMap = entities.map((entity) { + final migrationVersion = entity.uri.pathSegments.last.split('_')[0]; + return MapEntry(Version.parse(migrationVersion.replaceFirst('v', '')), entity); + }).toList(); + migrationMap.sort((a, b) => a.key.compareTo(b.key)); + int migrationStartIndex = 0; + while (migrationStartIndex < migrationMap.length) { + if (databaseVersion.compareTo(migrationMap[migrationStartIndex].key) < 0) { + break; + } + migrationStartIndex++; + } + final migrationRange = migrationMap.sublist(migrationStartIndex, migrationMap.length); + if (migrationRange.isNotEmpty) { + for (var migration in migrationRange) { + await executeSqlFile(migration.value.path); + } + await connection.execute("UPDATE migration SET semver = '${migrationRange.last.key.toString()}';"); + } + await _prepare(); + } + + Future restoreDefault() async { + await restore(defaultDatabasePath); + await _prepare(); + } + + Future restore(String dumpPath) async { + await connection.execute('DROP SCHEMA IF EXISTS public CASCADE;'); + await executeSqlFile(dumpPath); + } + + Future reset() async { + await restoreDefault(); + Iterable entityControllers = + dataTypes.map((t) => ShelfController.getControllerFromDataType(t)).nonNulls; + // Remove data + await Future.forEach(entityControllers, (e) => e.deleteMany()); + // No need to restore the database semantic version for migration, + // as it is no entity controller and therefore keeps it default value. + + // Recreate default admin + await SecuredUserController().createSingle( + User(username: 'admin', createdAt: DateTime.now(), privilege: UserPrivilege.admin, password: 'admin') + .toSecuredUser()); + } + + Future executeSqlFile(String sqlFilePath) async { + await close(); + + final args = [ + '--file', + sqlFilePath, + '--username', + dbUser, + '--host', + postgresHost, + '--port', + postgresPort.toString(), + postgresDatabaseName, + ]; + final processResult = await Process.run('psql', args, environment: {'PGPASSWORD': dbPW}); + + if (processResult.exitCode != 0) { + throw processResult.stderr; + } + + await open(); + } + + /// Prepare should only be called after migration or on an up-to-date database, + /// as DataObjects may not in sync with the tables. + Future _prepare() async { + await EntityController.initAll(); + } + + Future export() async { + final args = [ + // '--file', + // './database/dump/${DateTime.now().toIso8601String().replaceAll(':', '-').replaceAll(RegExp(r'\.[0-9]{3}'), '')}-' + // 'PostgreSQL-wrestling_scoreboard-dump.sql', + '--username', + dbUser, + '--host', + postgresHost, + '--port', + postgresPort.toString(), + '--dbname', + postgresDatabaseName, + '--schema', + 'public', + ]; + final process = await Process.run('pg_dump', args, environment: {'PGPASSWORD': dbPW}); + + if (process.exitCode == 0) { + return process.stdout; + } else { + throw Exception(process.stderr); + } + } + + static sanitizeSql(String sqlString) { + sqlString = sqlString.replaceAll('\r\n', '\n'); + return sqlString.split('\n').where((line) => !line.startsWith('-- Dumped')).join('\n'); } } diff --git a/wrestling_scoreboard_server/test/server_test.dart b/wrestling_scoreboard_server/test/server_test.dart index 1a7e2834..e04de9f0 100644 --- a/wrestling_scoreboard_server/test/server_test.dart +++ b/wrestling_scoreboard_server/test/server_test.dart @@ -1,9 +1,27 @@ +import 'dart:io'; + import 'package:test/test.dart'; +import 'package:wrestling_scoreboard_server/server.dart' as server; +import 'package:wrestling_scoreboard_server/services/postgres_db.dart'; void main() { - myTestMethod() => 42; + test('DB import and export match', () async { + final db = PostgresDb(); + await db.open(); + final defaultDump = await File(defaultDatabasePath).readAsString(); + await db.restore(defaultDatabasePath); + final databaseExport = await db.export(); + expect(DatabaseExt.sanitizeSql(databaseExport), DatabaseExt.sanitizeSql(defaultDump)); + }); - test('Run some test', () { - expect(myTestMethod(), 42); + test('Start server and migrate', () async { + final db = PostgresDb(); + await db.open(); + await db.restore('./database/migration/v0.0.0_Setup-DB.sql'); + await server.init(); + // At this point the database should already be migrated + final defaultDump = await File(defaultDatabasePath).readAsString(); + final databaseExport = await db.export(); + expect(DatabaseExt.sanitizeSql(databaseExport), DatabaseExt.sanitizeSql(defaultDump)); }); }