From ba38841ca6ecde4cead5d266e91c40365aaa2a80 Mon Sep 17 00:00:00 2001 From: Adrian Clay Date: Tue, 3 Oct 2023 11:46:20 +0100 Subject: [PATCH] NIAD-2781: Operating Document - Database requirements (#321) * Add support for runtime specified SNOMED files in snomed-schema Dockerfile Hardcoding the specific version of SNOMED makes this file fragile. * Prevent SNOMED data from being bundled within built Docker image We want to publish this image to Dockerhub, so need to prevent any SNOMED data being accidently included within that published image * Add database section to OPERATING.md Covers: - What database is used - Why it is needed - How to run the commands to initialise the database schemas Also update the release script to push the snomed-database-loader to Dockerhub so it can be referenced by the OPERATING.md instructions * Link to the migratestructuredrecord endpoint from the DB OPERATING --- .dockerignore | 6 ++++ CHANGELOG.md | 5 +++ Jenkinsfile | 7 ++++- OPERATING.md | 54 +++++++++++++++++++++++++++++++++ docker/snomed-schema/Dockerfile | 2 +- release-scripts/release.sh | 3 +- 6 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..9a7eb0a22 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +## Avoid accidently redistributing SNOMED data within the built docker images +# The SNOMED database loader generates temporary files which contain SNOMED CT data +snomed-database-loader/tmp_* +# The SNOMED database is distributed as a ZIP +snomed-database-loader/*.zip + diff --git a/CHANGELOG.md b/CHANGELOG.md index 9de1d647c..3ffad4577 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased] +### Added + +* Documented database requirements in [OPERATING.md](/OPERATING.md#database-requirements) +* New docker image published as [nhsdev/nia-ps-db-migration](https://hub.docker.com/r/nhsdev/nia-ps-db-migration) + ### Fixed * Fix issue where continue message was not accepted by EMIS. diff --git a/Jenkinsfile b/Jenkinsfile index 9147c0584..6959bdf4d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -59,7 +59,12 @@ pipeline { docker-compose -f docker/docker-compose.yml up -d ps_db docker-compose -f docker/docker-compose.yml up db_migration aws s3 cp s3://snomed-schema/uk_sct2mo_36.0.0_20230412000001Z.zip ./snomed-database-loader/uk_sct2mo_36.0.0_20230412000001Z.zip - docker-compose -f docker/docker-compose.yml up snomed_schema + # As Jenkins is running inside of Docker too, can't just reference the snomed file as a volume as part of the docker run command + # Instead copy the file into a named volume first as a separate docker command + docker volume create --name snomed + cat ./snomed-database-loader/uk_sct2mo_36.0.0_20230412000001Z.zip | docker run --rm --interactive -v snomed:/snomed alpine sh -c "cat > /snomed/uk_sct2mo_36.0.0_20230412000001Z.zip" + docker-compose -f docker/docker-compose.yml run --rm --volume snomed:/snomed snomed_schema /snomed/uk_sct2mo_36.0.0_20230412000001Z.zip + docker volume rm snomed docker-compose -f docker/docker-compose.yml up snomed_immunization ''' } diff --git a/OPERATING.md b/OPERATING.md index 552426187..5256c2b5d 100644 --- a/OPERATING.md +++ b/OPERATING.md @@ -22,6 +22,60 @@ yyyy-mm-dd HH:mm:ss.SSS Level=DEBUG Logger=u.n.a.p.t.s.BundleMapperService Conve ## Database requirements +* The adaptor requires a [PostgreSQL] database +* The adaptor stores the identifiers, status, and metadata for each patient transfer +* The adaptor uses the database as a source of SNOMED information +* Deleting the database, or its records will cause any in-progress transfers to fail +* In addition to the [/Patient/$gpc.migratestructuredrecord][migratestructuredrecord] endpoint, the database can be used to monitor for any failed or incomplete transfers + +[PostgreSQL]: https://www.postgresql.org/ +[migratestructuredrecord]: README.md#patientgpcmigratestructuredrecord + +### Updating the application schema + +The adaptor uses Liquibase to perform DB migrations. +New versions of the Adaptor may require DB changes, which will necessitate the execution of the migration script before the new version of the application can be executed. + +The DB migrations is build as a Docker image, hosted on DockerHub under [nhsdev/nia-ps-db-migration](https://hub.docker.com/r/nhsdev/nia-ps-db-migration). + +Required environment variables: + +- POSTGRES_PASSWORD e.g. super5ecret +- PS_DB_OWNER_NAME e.g. postgres +- PS_DB_URL e.g. jdbc:postgresql://hostname:port +- GPC_FACADE_USER_DB_PASSWORD e.g. another5ecret, used when creating the user `gpc_user` +- GP2GP_TRANSLATOR_USER_DB_PASSWORD e.g. yetanother5ecret, used when creating the user `gp2gp_user` + +*When passing passwords into this script it is the responsibility of the supplier to ensure that passwords are being kept secure by using appropriate controls within their infrastructure.* + +### Updating the SNOMED Database + +The adaptor requires an up to date copy of the SNOMED DB as part of translating FHIR `CodableConcepts`. + +The SNOMED loader script is built as a Docker image, hosted on DockerHub under [nhsdev/nia-ps-snomed-schema](https://hub.docker.com/r/nhsdev/nia-ps-snomed-schema). + +Running the loader script will delete any existing SNOMED data, and then proceed to populate it using the provided extract. + +Required environment variables: + +- PS_DB_OWNER_NAME e.g. postgres +- POSTGRES_PASSWORD e.g. super5ecret +- PS_DB_HOST e.g. hostname.domain.com +- PS_DB_PORT e.g. 5432 + +The docker container has a required argument which is the path to a zipped SnomedCT RF2 file. +The container does not come bundled with any Snomed data itself. +You will need to provide this file to the container. + +*When passing passwords into this script it is the responsibility of the supplier to ensure that passwords are being kept secure by using appropriate controls within their infrastructure.* + +Example usage: +```sh +$ docker run --rm -e PS_DB_OWNER_NAME=postgres -e POSTGRES_PASSWORD=super5ecret -e PS_DB_HOST=postgres -e PS_DB_PORT=5432 \ + -v /path/to/uk_sct2mo_36.3.0_20230705000001Z.zip:/snomed/uk_sct2mo_36.3.0_20230705000001Z.zip \ + nhsdev/nia-ps-snomed-schema /snomed/uk_sct2mo_36.3.0_20230705000001Z.zip +``` + ## Message broker requirements ## Object storage diff --git a/docker/snomed-schema/Dockerfile b/docker/snomed-schema/Dockerfile index 71b1dc513..4eb601621 100644 --- a/docker/snomed-schema/Dockerfile +++ b/docker/snomed-schema/Dockerfile @@ -3,4 +3,4 @@ FROM postgres:14.0 RUN apt-get update && apt-get install -y unzip COPY /snomed-database-loader/ /snomed-database-loader/ WORKDIR /snomed-database-loader -CMD ./load_release-postgresql.sh uk_sct2mo_36.0.0_20230412000001Z.zip \ No newline at end of file +ENTRYPOINT [ "./load_release-postgresql.sh" ] \ No newline at end of file diff --git a/release-scripts/release.sh b/release-scripts/release.sh index f105206a7..e3869d999 100644 --- a/release-scripts/release.sh +++ b/release-scripts/release.sh @@ -11,4 +11,5 @@ cd .. docker buildx build -f docker/gp2gp-translator/Dockerfile . --platform linux/arm64/v8,linux/amd64 --tag nhsdev/nia-ps-adaptor:${BUILD_TAG} --push docker buildx build -f docker/gpc-facade/Dockerfile . --platform linux/arm64/v8,linux/amd64 --tag nhsdev/nia-ps-facade:${BUILD_TAG} --push -docker buildx build -f docker/db-migration/Dockerfile . --platform linux/arm64/v8,linux/amd64 --tag nhsdev/nia-ps-db-migration:${BUILD_TAG} --push \ No newline at end of file +docker buildx build -f docker/db-migration/Dockerfile . --platform linux/arm64/v8,linux/amd64 --tag nhsdev/nia-ps-db-migration:${BUILD_TAG} --push +docker buildx build -f docker/snomed-schema/Dockerfile . --platform linux/arm64/v8,linux/amd64 --tag nhsdev/nia-ps-snomed-schema:${BUILD_TAG} --push \ No newline at end of file