diff --git a/poetry.lock b/poetry.lock index ae7d97e1..2e1776bc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "aiohttp" @@ -2956,18 +2956,18 @@ files = [ [[package]] name = "pydantic" -version = "2.6.3" +version = "2.5.2" description = "Data validation using Python type hints" optional = false -python-versions = ">=3.8" +python-versions = ">=3.7" files = [ - {file = "pydantic-2.6.3-py3-none-any.whl", hash = "sha256:72c6034df47f46ccdf81869fddb81aade68056003900a8724a4f160700016a2a"}, - {file = "pydantic-2.6.3.tar.gz", hash = "sha256:e07805c4c7f5c6826e33a1d4c9d47950d7eaf34868e2690f8594d2e30241f11f"}, + {file = "pydantic-2.5.2-py3-none-any.whl", hash = "sha256:80c50fb8e3dcecfddae1adbcc00ec5822918490c99ab31f6cf6140ca1c1429f0"}, + {file = "pydantic-2.5.2.tar.gz", hash = "sha256:ff177ba64c6faf73d7afa2e8cad38fd456c0dbe01c9954e71038001cd15a6edd"}, ] [package.dependencies] annotated-types = ">=0.4.0" -pydantic-core = "2.16.3" +pydantic-core = "2.14.5" typing-extensions = ">=4.6.1" [package.extras] @@ -2975,90 +2975,116 @@ email = ["email-validator (>=2.0.0)"] [[package]] name = "pydantic-core" -version = "2.16.3" +version = "2.14.5" description = "" optional = false -python-versions = ">=3.8" +python-versions = ">=3.7" files = [ - {file = "pydantic_core-2.16.3-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:75b81e678d1c1ede0785c7f46690621e4c6e63ccd9192af1f0bd9d504bbb6bf4"}, - {file = "pydantic_core-2.16.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9c865a7ee6f93783bd5d781af5a4c43dadc37053a5b42f7d18dc019f8c9d2bd1"}, - {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:162e498303d2b1c036b957a1278fa0899d02b2842f1ff901b6395104c5554a45"}, - {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2f583bd01bbfbff4eaee0868e6fc607efdfcc2b03c1c766b06a707abbc856187"}, - {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b926dd38db1519ed3043a4de50214e0d600d404099c3392f098a7f9d75029ff8"}, - {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:716b542728d4c742353448765aa7cdaa519a7b82f9564130e2b3f6766018c9ec"}, - {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc4ad7f7ee1a13d9cb49d8198cd7d7e3aa93e425f371a68235f784e99741561f"}, - {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bd87f48924f360e5d1c5f770d6155ce0e7d83f7b4e10c2f9ec001c73cf475c99"}, - {file = "pydantic_core-2.16.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0df446663464884297c793874573549229f9eca73b59360878f382a0fc085979"}, - {file = "pydantic_core-2.16.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4df8a199d9f6afc5ae9a65f8f95ee52cae389a8c6b20163762bde0426275b7db"}, - {file = "pydantic_core-2.16.3-cp310-none-win32.whl", hash = "sha256:456855f57b413f077dff513a5a28ed838dbbb15082ba00f80750377eed23d132"}, - {file = "pydantic_core-2.16.3-cp310-none-win_amd64.whl", hash = "sha256:732da3243e1b8d3eab8c6ae23ae6a58548849d2e4a4e03a1924c8ddf71a387cb"}, - {file = "pydantic_core-2.16.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:519ae0312616026bf4cedc0fe459e982734f3ca82ee8c7246c19b650b60a5ee4"}, - {file = "pydantic_core-2.16.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b3992a322a5617ded0a9f23fd06dbc1e4bd7cf39bc4ccf344b10f80af58beacd"}, - {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d62da299c6ecb04df729e4b5c52dc0d53f4f8430b4492b93aa8de1f541c4aac"}, - {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2acca2be4bb2f2147ada8cac612f8a98fc09f41c89f87add7256ad27332c2fda"}, - {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1b662180108c55dfbf1280d865b2d116633d436cfc0bba82323554873967b340"}, - {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e7c6ed0dc9d8e65f24f5824291550139fe6f37fac03788d4580da0d33bc00c97"}, - {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a6b1bb0827f56654b4437955555dc3aeeebeddc47c2d7ed575477f082622c49e"}, - {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e56f8186d6210ac7ece503193ec84104da7ceb98f68ce18c07282fcc2452e76f"}, - {file = "pydantic_core-2.16.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:936e5db01dd49476fa8f4383c259b8b1303d5dd5fb34c97de194560698cc2c5e"}, - {file = "pydantic_core-2.16.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:33809aebac276089b78db106ee692bdc9044710e26f24a9a2eaa35a0f9fa70ba"}, - {file = "pydantic_core-2.16.3-cp311-none-win32.whl", hash = "sha256:ded1c35f15c9dea16ead9bffcde9bb5c7c031bff076355dc58dcb1cb436c4721"}, - {file = "pydantic_core-2.16.3-cp311-none-win_amd64.whl", hash = "sha256:d89ca19cdd0dd5f31606a9329e309d4fcbb3df860960acec32630297d61820df"}, - {file = "pydantic_core-2.16.3-cp311-none-win_arm64.whl", hash = "sha256:6162f8d2dc27ba21027f261e4fa26f8bcb3cf9784b7f9499466a311ac284b5b9"}, - {file = "pydantic_core-2.16.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:0f56ae86b60ea987ae8bcd6654a887238fd53d1384f9b222ac457070b7ac4cff"}, - {file = "pydantic_core-2.16.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c9bd22a2a639e26171068f8ebb5400ce2c1bc7d17959f60a3b753ae13c632975"}, - {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4204e773b4b408062960e65468d5346bdfe139247ee5f1ca2a378983e11388a2"}, - {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f651dd19363c632f4abe3480a7c87a9773be27cfe1341aef06e8759599454120"}, - {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aaf09e615a0bf98d406657e0008e4a8701b11481840be7d31755dc9f97c44053"}, - {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8e47755d8152c1ab5b55928ab422a76e2e7b22b5ed8e90a7d584268dd49e9c6b"}, - {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:500960cb3a0543a724a81ba859da816e8cf01b0e6aaeedf2c3775d12ee49cade"}, - {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:cf6204fe865da605285c34cf1172879d0314ff267b1c35ff59de7154f35fdc2e"}, - {file = "pydantic_core-2.16.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d33dd21f572545649f90c38c227cc8631268ba25c460b5569abebdd0ec5974ca"}, - {file = "pydantic_core-2.16.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:49d5d58abd4b83fb8ce763be7794d09b2f50f10aa65c0f0c1696c677edeb7cbf"}, - {file = "pydantic_core-2.16.3-cp312-none-win32.whl", hash = "sha256:f53aace168a2a10582e570b7736cc5bef12cae9cf21775e3eafac597e8551fbe"}, - {file = "pydantic_core-2.16.3-cp312-none-win_amd64.whl", hash = "sha256:0d32576b1de5a30d9a97f300cc6a3f4694c428d956adbc7e6e2f9cad279e45ed"}, - {file = "pydantic_core-2.16.3-cp312-none-win_arm64.whl", hash = "sha256:ec08be75bb268473677edb83ba71e7e74b43c008e4a7b1907c6d57e940bf34b6"}, - {file = "pydantic_core-2.16.3-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:b1f6f5938d63c6139860f044e2538baeee6f0b251a1816e7adb6cbce106a1f01"}, - {file = "pydantic_core-2.16.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2a1ef6a36fdbf71538142ed604ad19b82f67b05749512e47f247a6ddd06afdc7"}, - {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:704d35ecc7e9c31d48926150afada60401c55efa3b46cd1ded5a01bdffaf1d48"}, - {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d937653a696465677ed583124b94a4b2d79f5e30b2c46115a68e482c6a591c8a"}, - {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c9803edf8e29bd825f43481f19c37f50d2b01899448273b3a7758441b512acf8"}, - {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:72282ad4892a9fb2da25defeac8c2e84352c108705c972db82ab121d15f14e6d"}, - {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f752826b5b8361193df55afcdf8ca6a57d0232653494ba473630a83ba50d8c9"}, - {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4384a8f68ddb31a0b0c3deae88765f5868a1b9148939c3f4121233314ad5532c"}, - {file = "pydantic_core-2.16.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:a4b2bf78342c40b3dc830880106f54328928ff03e357935ad26c7128bbd66ce8"}, - {file = "pydantic_core-2.16.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:13dcc4802961b5f843a9385fc821a0b0135e8c07fc3d9949fd49627c1a5e6ae5"}, - {file = "pydantic_core-2.16.3-cp38-none-win32.whl", hash = "sha256:e3e70c94a0c3841e6aa831edab1619ad5c511199be94d0c11ba75fe06efe107a"}, - {file = "pydantic_core-2.16.3-cp38-none-win_amd64.whl", hash = "sha256:ecdf6bf5f578615f2e985a5e1f6572e23aa632c4bd1dc67f8f406d445ac115ed"}, - {file = "pydantic_core-2.16.3-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:bda1ee3e08252b8d41fa5537413ffdddd58fa73107171a126d3b9ff001b9b820"}, - {file = "pydantic_core-2.16.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:21b888c973e4f26b7a96491c0965a8a312e13be108022ee510248fe379a5fa23"}, - {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be0ec334369316fa73448cc8c982c01e5d2a81c95969d58b8f6e272884df0074"}, - {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b5b6079cc452a7c53dd378c6f881ac528246b3ac9aae0f8eef98498a75657805"}, - {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7ee8d5f878dccb6d499ba4d30d757111847b6849ae07acdd1205fffa1fc1253c"}, - {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7233d65d9d651242a68801159763d09e9ec96e8a158dbf118dc090cd77a104c9"}, - {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6119dc90483a5cb50a1306adb8d52c66e447da88ea44f323e0ae1a5fcb14256"}, - {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:578114bc803a4c1ff9946d977c221e4376620a46cf78da267d946397dc9514a8"}, - {file = "pydantic_core-2.16.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d8f99b147ff3fcf6b3cc60cb0c39ea443884d5559a30b1481e92495f2310ff2b"}, - {file = "pydantic_core-2.16.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4ac6b4ce1e7283d715c4b729d8f9dab9627586dafce81d9eaa009dd7f25dd972"}, - {file = "pydantic_core-2.16.3-cp39-none-win32.whl", hash = "sha256:e7774b570e61cb998490c5235740d475413a1f6de823169b4cf94e2fe9e9f6b2"}, - {file = "pydantic_core-2.16.3-cp39-none-win_amd64.whl", hash = "sha256:9091632a25b8b87b9a605ec0e61f241c456e9248bfdcf7abdf344fdb169c81cf"}, - {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:36fa178aacbc277bc6b62a2c3da95226520da4f4e9e206fdf076484363895d2c"}, - {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:dcca5d2bf65c6fb591fff92da03f94cd4f315972f97c21975398bd4bd046854a"}, - {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a72fb9963cba4cd5793854fd12f4cfee731e86df140f59ff52a49b3552db241"}, - {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b60cc1a081f80a2105a59385b92d82278b15d80ebb3adb200542ae165cd7d183"}, - {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:cbcc558401de90a746d02ef330c528f2e668c83350f045833543cd57ecead1ad"}, - {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:fee427241c2d9fb7192b658190f9f5fd6dfe41e02f3c1489d2ec1e6a5ab1e04a"}, - {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f4cb85f693044e0f71f394ff76c98ddc1bc0953e48c061725e540396d5c8a2e1"}, - {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:b29eeb887aa931c2fcef5aa515d9d176d25006794610c264ddc114c053bf96fe"}, - {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:a425479ee40ff021f8216c9d07a6a3b54b31c8267c6e17aa88b70d7ebd0e5e5b"}, - {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:5c5cbc703168d1b7a838668998308018a2718c2130595e8e190220238addc96f"}, - {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:99b6add4c0b39a513d323d3b93bc173dac663c27b99860dd5bf491b240d26137"}, - {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75f76ee558751746d6a38f89d60b6228fa174e5172d143886af0f85aa306fd89"}, - {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:00ee1c97b5364b84cb0bd82e9bbf645d5e2871fb8c58059d158412fee2d33d8a"}, - {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:287073c66748f624be4cef893ef9174e3eb88fe0b8a78dc22e88eca4bc357ca6"}, - {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:ed25e1835c00a332cb10c683cd39da96a719ab1dfc08427d476bce41b92531fc"}, - {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:86b3d0033580bd6bbe07590152007275bd7af95f98eaa5bd36f3da219dcd93da"}, - {file = "pydantic_core-2.16.3.tar.gz", hash = "sha256:1cac689f80a3abab2d3c0048b29eea5751114054f032a941a32de4c852c59cad"}, + {file = "pydantic_core-2.14.5-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:7e88f5696153dc516ba6e79f82cc4747e87027205f0e02390c21f7cb3bd8abfd"}, + {file = "pydantic_core-2.14.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4641e8ad4efb697f38a9b64ca0523b557c7931c5f84e0fd377a9a3b05121f0de"}, + {file = "pydantic_core-2.14.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:774de879d212db5ce02dfbf5b0da9a0ea386aeba12b0b95674a4ce0593df3d07"}, + {file = "pydantic_core-2.14.5-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ebb4e035e28f49b6f1a7032920bb9a0c064aedbbabe52c543343d39341a5b2a3"}, + {file = "pydantic_core-2.14.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b53e9ad053cd064f7e473a5f29b37fc4cc9dc6d35f341e6afc0155ea257fc911"}, + {file = "pydantic_core-2.14.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8aa1768c151cf562a9992462239dfc356b3d1037cc5a3ac829bb7f3bda7cc1f9"}, + {file = "pydantic_core-2.14.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eac5c82fc632c599f4639a5886f96867ffced74458c7db61bc9a66ccb8ee3113"}, + {file = "pydantic_core-2.14.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d2ae91f50ccc5810b2f1b6b858257c9ad2e08da70bf890dee02de1775a387c66"}, + {file = "pydantic_core-2.14.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6b9ff467ffbab9110e80e8c8de3bcfce8e8b0fd5661ac44a09ae5901668ba997"}, + {file = "pydantic_core-2.14.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:61ea96a78378e3bd5a0be99b0e5ed00057b71f66115f5404d0dae4819f495093"}, + {file = "pydantic_core-2.14.5-cp310-none-win32.whl", hash = "sha256:bb4c2eda937a5e74c38a41b33d8c77220380a388d689bcdb9b187cf6224c9720"}, + {file = "pydantic_core-2.14.5-cp310-none-win_amd64.whl", hash = "sha256:b7851992faf25eac90bfcb7bfd19e1f5ffa00afd57daec8a0042e63c74a4551b"}, + {file = "pydantic_core-2.14.5-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:4e40f2bd0d57dac3feb3a3aed50f17d83436c9e6b09b16af271b6230a2915459"}, + {file = "pydantic_core-2.14.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ab1cdb0f14dc161ebc268c09db04d2c9e6f70027f3b42446fa11c153521c0e88"}, + {file = "pydantic_core-2.14.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aae7ea3a1c5bb40c93cad361b3e869b180ac174656120c42b9fadebf685d121b"}, + {file = "pydantic_core-2.14.5-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:60b7607753ba62cf0739177913b858140f11b8af72f22860c28eabb2f0a61937"}, + {file = "pydantic_core-2.14.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2248485b0322c75aee7565d95ad0e16f1c67403a470d02f94da7344184be770f"}, + {file = "pydantic_core-2.14.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:823fcc638f67035137a5cd3f1584a4542d35a951c3cc68c6ead1df7dac825c26"}, + {file = "pydantic_core-2.14.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:96581cfefa9123accc465a5fd0cc833ac4d75d55cc30b633b402e00e7ced00a6"}, + {file = "pydantic_core-2.14.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a33324437018bf6ba1bb0f921788788641439e0ed654b233285b9c69704c27b4"}, + {file = "pydantic_core-2.14.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9bd18fee0923ca10f9a3ff67d4851c9d3e22b7bc63d1eddc12f439f436f2aada"}, + {file = "pydantic_core-2.14.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:853a2295c00f1d4429db4c0fb9475958543ee80cfd310814b5c0ef502de24dda"}, + {file = "pydantic_core-2.14.5-cp311-none-win32.whl", hash = "sha256:cb774298da62aea5c80a89bd58c40205ab4c2abf4834453b5de207d59d2e1651"}, + {file = "pydantic_core-2.14.5-cp311-none-win_amd64.whl", hash = "sha256:e87fc540c6cac7f29ede02e0f989d4233f88ad439c5cdee56f693cc9c1c78077"}, + {file = "pydantic_core-2.14.5-cp311-none-win_arm64.whl", hash = "sha256:57d52fa717ff445cb0a5ab5237db502e6be50809b43a596fb569630c665abddf"}, + {file = "pydantic_core-2.14.5-cp312-cp312-macosx_10_7_x86_64.whl", hash = "sha256:e60f112ac88db9261ad3a52032ea46388378034f3279c643499edb982536a093"}, + {file = "pydantic_core-2.14.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6e227c40c02fd873c2a73a98c1280c10315cbebe26734c196ef4514776120aeb"}, + {file = "pydantic_core-2.14.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f0cbc7fff06a90bbd875cc201f94ef0ee3929dfbd5c55a06674b60857b8b85ed"}, + {file = "pydantic_core-2.14.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:103ef8d5b58596a731b690112819501ba1db7a36f4ee99f7892c40da02c3e189"}, + {file = "pydantic_core-2.14.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c949f04ecad823f81b1ba94e7d189d9dfb81edbb94ed3f8acfce41e682e48cef"}, + {file = "pydantic_core-2.14.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c1452a1acdf914d194159439eb21e56b89aa903f2e1c65c60b9d874f9b950e5d"}, + {file = "pydantic_core-2.14.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cb4679d4c2b089e5ef89756bc73e1926745e995d76e11925e3e96a76d5fa51fc"}, + {file = "pydantic_core-2.14.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:cf9d3fe53b1ee360e2421be95e62ca9b3296bf3f2fb2d3b83ca49ad3f925835e"}, + {file = "pydantic_core-2.14.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:70f4b4851dbb500129681d04cc955be2a90b2248d69273a787dda120d5cf1f69"}, + {file = "pydantic_core-2.14.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:59986de5710ad9613ff61dd9b02bdd2f615f1a7052304b79cc8fa2eb4e336d2d"}, + {file = "pydantic_core-2.14.5-cp312-none-win32.whl", hash = "sha256:699156034181e2ce106c89ddb4b6504c30db8caa86e0c30de47b3e0654543260"}, + {file = "pydantic_core-2.14.5-cp312-none-win_amd64.whl", hash = "sha256:5baab5455c7a538ac7e8bf1feec4278a66436197592a9bed538160a2e7d11e36"}, + {file = "pydantic_core-2.14.5-cp312-none-win_arm64.whl", hash = "sha256:e47e9a08bcc04d20975b6434cc50bf82665fbc751bcce739d04a3120428f3e27"}, + {file = "pydantic_core-2.14.5-cp37-cp37m-macosx_10_7_x86_64.whl", hash = "sha256:af36f36538418f3806048f3b242a1777e2540ff9efaa667c27da63d2749dbce0"}, + {file = "pydantic_core-2.14.5-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:45e95333b8418ded64745f14574aa9bfc212cb4fbeed7a687b0c6e53b5e188cd"}, + {file = "pydantic_core-2.14.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e47a76848f92529879ecfc417ff88a2806438f57be4a6a8bf2961e8f9ca9ec7"}, + {file = "pydantic_core-2.14.5-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d81e6987b27bc7d101c8597e1cd2bcaa2fee5e8e0f356735c7ed34368c471550"}, + {file = "pydantic_core-2.14.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:34708cc82c330e303f4ce87758828ef6e457681b58ce0e921b6e97937dd1e2a3"}, + {file = "pydantic_core-2.14.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:652c1988019752138b974c28f43751528116bcceadad85f33a258869e641d753"}, + {file = "pydantic_core-2.14.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e4d090e73e0725b2904fdbdd8d73b8802ddd691ef9254577b708d413bf3006e"}, + {file = "pydantic_core-2.14.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5c7d5b5005f177764e96bd584d7bf28d6e26e96f2a541fdddb934c486e36fd59"}, + {file = "pydantic_core-2.14.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:a71891847f0a73b1b9eb86d089baee301477abef45f7eaf303495cd1473613e4"}, + {file = "pydantic_core-2.14.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a717aef6971208f0851a2420b075338e33083111d92041157bbe0e2713b37325"}, + {file = "pydantic_core-2.14.5-cp37-none-win32.whl", hash = "sha256:de790a3b5aa2124b8b78ae5faa033937a72da8efe74b9231698b5a1dd9be3405"}, + {file = "pydantic_core-2.14.5-cp37-none-win_amd64.whl", hash = "sha256:6c327e9cd849b564b234da821236e6bcbe4f359a42ee05050dc79d8ed2a91588"}, + {file = "pydantic_core-2.14.5-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:ef98ca7d5995a82f43ec0ab39c4caf6a9b994cb0b53648ff61716370eadc43cf"}, + {file = "pydantic_core-2.14.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c6eae413494a1c3f89055da7a5515f32e05ebc1a234c27674a6956755fb2236f"}, + {file = "pydantic_core-2.14.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dcf4e6d85614f7a4956c2de5a56531f44efb973d2fe4a444d7251df5d5c4dcfd"}, + {file = "pydantic_core-2.14.5-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6637560562134b0e17de333d18e69e312e0458ee4455bdad12c37100b7cad706"}, + {file = "pydantic_core-2.14.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:77fa384d8e118b3077cccfcaf91bf83c31fe4dc850b5e6ee3dc14dc3d61bdba1"}, + {file = "pydantic_core-2.14.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:16e29bad40bcf97aac682a58861249ca9dcc57c3f6be22f506501833ddb8939c"}, + {file = "pydantic_core-2.14.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:531f4b4252fac6ca476fbe0e6f60f16f5b65d3e6b583bc4d87645e4e5ddde331"}, + {file = "pydantic_core-2.14.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:074f3d86f081ce61414d2dc44901f4f83617329c6f3ab49d2bc6c96948b2c26b"}, + {file = "pydantic_core-2.14.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:c2adbe22ab4babbca99c75c5d07aaf74f43c3195384ec07ccbd2f9e3bddaecec"}, + {file = "pydantic_core-2.14.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0f6116a558fd06d1b7c2902d1c4cf64a5bd49d67c3540e61eccca93f41418124"}, + {file = "pydantic_core-2.14.5-cp38-none-win32.whl", hash = "sha256:fe0a5a1025eb797752136ac8b4fa21aa891e3d74fd340f864ff982d649691867"}, + {file = "pydantic_core-2.14.5-cp38-none-win_amd64.whl", hash = "sha256:079206491c435b60778cf2b0ee5fd645e61ffd6e70c47806c9ed51fc75af078d"}, + {file = "pydantic_core-2.14.5-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:a6a16f4a527aae4f49c875da3cdc9508ac7eef26e7977952608610104244e1b7"}, + {file = "pydantic_core-2.14.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:abf058be9517dc877227ec3223f0300034bd0e9f53aebd63cf4456c8cb1e0863"}, + {file = "pydantic_core-2.14.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:49b08aae5013640a3bfa25a8eebbd95638ec3f4b2eaf6ed82cf0c7047133f03b"}, + {file = "pydantic_core-2.14.5-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c2d97e906b4ff36eb464d52a3bc7d720bd6261f64bc4bcdbcd2c557c02081ed2"}, + {file = "pydantic_core-2.14.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3128e0bbc8c091ec4375a1828d6118bc20404883169ac95ffa8d983b293611e6"}, + {file = "pydantic_core-2.14.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:88e74ab0cdd84ad0614e2750f903bb0d610cc8af2cc17f72c28163acfcf372a4"}, + {file = "pydantic_core-2.14.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c339dabd8ee15f8259ee0f202679b6324926e5bc9e9a40bf981ce77c038553db"}, + {file = "pydantic_core-2.14.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3387277f1bf659caf1724e1afe8ee7dbc9952a82d90f858ebb931880216ea955"}, + {file = "pydantic_core-2.14.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ba6b6b3846cfc10fdb4c971980a954e49d447cd215ed5a77ec8190bc93dd7bc5"}, + {file = "pydantic_core-2.14.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ca61d858e4107ce5e1330a74724fe757fc7135190eb5ce5c9d0191729f033209"}, + {file = "pydantic_core-2.14.5-cp39-none-win32.whl", hash = "sha256:ec1e72d6412f7126eb7b2e3bfca42b15e6e389e1bc88ea0069d0cc1742f477c6"}, + {file = "pydantic_core-2.14.5-cp39-none-win_amd64.whl", hash = "sha256:c0b97ec434041827935044bbbe52b03d6018c2897349670ff8fe11ed24d1d4ab"}, + {file = "pydantic_core-2.14.5-pp310-pypy310_pp73-macosx_10_7_x86_64.whl", hash = "sha256:79e0a2cdbdc7af3f4aee3210b1172ab53d7ddb6a2d8c24119b5706e622b346d0"}, + {file = "pydantic_core-2.14.5-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:678265f7b14e138d9a541ddabbe033012a2953315739f8cfa6d754cc8063e8ca"}, + {file = "pydantic_core-2.14.5-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:95b15e855ae44f0c6341ceb74df61b606e11f1087e87dcb7482377374aac6abe"}, + {file = "pydantic_core-2.14.5-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:09b0e985fbaf13e6b06a56d21694d12ebca6ce5414b9211edf6f17738d82b0f8"}, + {file = "pydantic_core-2.14.5-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3ad873900297bb36e4b6b3f7029d88ff9829ecdc15d5cf20161775ce12306f8a"}, + {file = "pydantic_core-2.14.5-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:2d0ae0d8670164e10accbeb31d5ad45adb71292032d0fdb9079912907f0085f4"}, + {file = "pydantic_core-2.14.5-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:d37f8ec982ead9ba0a22a996129594938138a1503237b87318392a48882d50b7"}, + {file = "pydantic_core-2.14.5-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:35613015f0ba7e14c29ac6c2483a657ec740e5ac5758d993fdd5870b07a61d8b"}, + {file = "pydantic_core-2.14.5-pp37-pypy37_pp73-macosx_10_7_x86_64.whl", hash = "sha256:ab4ea451082e684198636565224bbb179575efc1658c48281b2c866bfd4ddf04"}, + {file = "pydantic_core-2.14.5-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ce601907e99ea5b4adb807ded3570ea62186b17f88e271569144e8cca4409c7"}, + {file = "pydantic_core-2.14.5-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb2ed8b3fe4bf4506d6dab3b93b83bbc22237e230cba03866d561c3577517d18"}, + {file = "pydantic_core-2.14.5-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:70f947628e074bb2526ba1b151cee10e4c3b9670af4dbb4d73bc8a89445916b5"}, + {file = "pydantic_core-2.14.5-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:4bc536201426451f06f044dfbf341c09f540b4ebdb9fd8d2c6164d733de5e634"}, + {file = "pydantic_core-2.14.5-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f4791cf0f8c3104ac668797d8c514afb3431bc3305f5638add0ba1a5a37e0d88"}, + {file = "pydantic_core-2.14.5-pp38-pypy38_pp73-macosx_10_7_x86_64.whl", hash = "sha256:038c9f763e650712b899f983076ce783175397c848da04985658e7628cbe873b"}, + {file = "pydantic_core-2.14.5-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:27548e16c79702f1e03f5628589c6057c9ae17c95b4c449de3c66b589ead0520"}, + {file = "pydantic_core-2.14.5-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c97bee68898f3f4344eb02fec316db93d9700fb1e6a5b760ffa20d71d9a46ce3"}, + {file = "pydantic_core-2.14.5-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b9b759b77f5337b4ea024f03abc6464c9f35d9718de01cfe6bae9f2e139c397e"}, + {file = "pydantic_core-2.14.5-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:439c9afe34638ace43a49bf72d201e0ffc1a800295bed8420c2a9ca8d5e3dbb3"}, + {file = "pydantic_core-2.14.5-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:ba39688799094c75ea8a16a6b544eb57b5b0f3328697084f3f2790892510d144"}, + {file = "pydantic_core-2.14.5-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:ccd4d5702bb90b84df13bd491be8d900b92016c5a455b7e14630ad7449eb03f8"}, + {file = "pydantic_core-2.14.5-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:81982d78a45d1e5396819bbb4ece1fadfe5f079335dd28c4ab3427cd95389944"}, + {file = "pydantic_core-2.14.5-pp39-pypy39_pp73-macosx_10_7_x86_64.whl", hash = "sha256:7f8210297b04e53bc3da35db08b7302a6a1f4889c79173af69b72ec9754796b8"}, + {file = "pydantic_core-2.14.5-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:8c8a8812fe6f43a3a5b054af6ac2d7b8605c7bcab2804a8a7d68b53f3cd86e00"}, + {file = "pydantic_core-2.14.5-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:206ed23aecd67c71daf5c02c3cd19c0501b01ef3cbf7782db9e4e051426b3d0d"}, + {file = "pydantic_core-2.14.5-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c2027d05c8aebe61d898d4cffd774840a9cb82ed356ba47a90d99ad768f39789"}, + {file = "pydantic_core-2.14.5-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:40180930807ce806aa71eda5a5a5447abb6b6a3c0b4b3b1b1962651906484d68"}, + {file = "pydantic_core-2.14.5-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:615a0a4bff11c45eb3c1996ceed5bdaa2f7b432425253a7c2eed33bb86d80abc"}, + {file = "pydantic_core-2.14.5-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f5e412d717366e0677ef767eac93566582518fe8be923361a5c204c1a62eaafe"}, + {file = "pydantic_core-2.14.5-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:513b07e99c0a267b1d954243845d8a833758a6726a3b5d8948306e3fe14675e3"}, + {file = "pydantic_core-2.14.5.tar.gz", hash = "sha256:6d30226dfc816dd0fdf120cae611dd2215117e4f9b124af8c60ab9093b6e8e71"}, ] [package.dependencies] @@ -4113,17 +4139,17 @@ dash = ">=2.15.0" [[package]] name = "ssb-datadoc-model" -version = "4.1.2" +version = "4.2.0" description = "Data Model for use in Statistics Norway's Metadata system" optional = false python-versions = ">=3.10" files = [ - {file = "ssb_datadoc_model-4.1.2-py3-none-any.whl", hash = "sha256:bde3cbbadd351f2c729eab8c2649a4bb96d941dc163d77c4a67dd318da84c066"}, - {file = "ssb_datadoc_model-4.1.2.tar.gz", hash = "sha256:21347330b583ea344f7018d8f6f03a5b669ce344cf0ebab018f37f0c66b51a83"}, + {file = "ssb_datadoc_model-4.2.0-py3-none-any.whl", hash = "sha256:402157eafd855387cc9015d07e6912f8e882a753afcc2165cc753b7ce158cd71"}, + {file = "ssb_datadoc_model-4.2.0.tar.gz", hash = "sha256:474204c21b3acae0871ac457f51a0e75c15a3806d5d382a5e0ecce1082b9d04c"}, ] [package.dependencies] -pydantic = ">=2.4.0" +pydantic = ">=2.5.2" [[package]] name = "ssb-klass-python" @@ -4649,4 +4675,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.10,<4.0" -content-hash = "91730cd623b3215fbd7217a93d4036e4ef1a43f3716295f9ec2fc63dd90e6bbc" +content-hash = "76358066a497e342d515bd98d8ace7cd022b03e569ef3e5902857ad05e82c39c" diff --git a/pyproject.toml b/pyproject.toml index a31a6dad..95e4011f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,10 +26,10 @@ Changelog = "https://github.com/statisticsnorway/datadoc/releases" python = ">=3.10,<4.0" pyarrow = ">=8.0.0" dash = ">=2.15.0" -pydantic = ">2" +pydantic = "==2.5.2" dash-bootstrap-components = ">=1.1.0" pandas = ">=1.4.2" -ssb-datadoc-model = "==4.1.2" +ssb-datadoc-model = "==4.2.0" dapla-toolbelt = ">=1.3.3" gunicorn = ">=21.2.0" flask-healthz = ">=0.0.3" @@ -94,6 +94,7 @@ show_missing = true fail_under = 80 [tool.mypy] +plugins = ["pydantic.mypy"] strict = false warn_unreachable = true pretty = true diff --git a/src/datadoc/app.py b/src/datadoc/app.py index b039ed6c..4fc0ae10 100644 --- a/src/datadoc/app.py +++ b/src/datadoc/app.py @@ -16,6 +16,7 @@ from datadoc import state from datadoc.backend.datadoc_metadata import DataDocMetadata from datadoc.backend.statistic_subject_mapping import StatisticSubjectMapping +from datadoc.backend.unit_types import UnitTypes from datadoc.enums import SupportedLanguages from datadoc.frontend.callbacks.register_callbacks import register_callbacks from datadoc.frontend.callbacks.register_callbacks import ( @@ -138,6 +139,10 @@ def collect_data_from_external_sources() -> None: config.get_statistical_subject_source_url(), ) + state.unit_types = UnitTypes( + config.get_unit_code(), + ) + def main(dataset_path: str | None = None) -> None: """Entrypoint when running as a script.""" diff --git a/src/datadoc/backend/dapla_dataset_path_info.py b/src/datadoc/backend/dapla_dataset_path_info.py index 354032a3..b109b466 100644 --- a/src/datadoc/backend/dapla_dataset_path_info.py +++ b/src/datadoc/backend/dapla_dataset_path_info.py @@ -13,7 +13,7 @@ import arrow -from datadoc.enums import DatasetState +from datadoc.enums import DataSetState from datadoc.enums import SupportedLanguages if TYPE_CHECKING: @@ -336,7 +336,7 @@ def _extract_period_string_from_index(self, index: int) -> str | None: def _extract_norwegian_dataset_state_path_part( self, - dataset_state: DatasetState, + dataset_state: DataSetState, ) -> set: norwegian_dataset_state_path_part = dataset_state.get_value_for_language( SupportedLanguages.NORSK_BOKMÅL, @@ -372,19 +372,19 @@ def contains_data_until(self) -> datetime.date | None: @property def dataset_state( self, - ) -> DatasetState | None: + ) -> DataSetState | None: """Extract the dataset state from the path. Examples: >>> DaplaDatasetPathInfo('klargjorte_data/person_data_v1.parquet').dataset_state - + >>> DaplaDatasetPathInfo('utdata/min_statistikk/person_data_v1.parquet').dataset_state - + >>> DaplaDatasetPathInfo('my_special_data/person_data_v1.parquet').dataset_state None """ dataset_path_parts = set(self.dataset_path.parts) - for s in DatasetState: + for s in DataSetState: # We assume that files are saved in the Norwegian language as specified by SSB. norwegian_dataset_state_path_part_variations = ( self._extract_norwegian_dataset_state_path_part(s) diff --git a/src/datadoc/backend/datadoc_metadata.py b/src/datadoc/backend/datadoc_metadata.py index 42507790..bd330dd3 100644 --- a/src/datadoc/backend/datadoc_metadata.py +++ b/src/datadoc/backend/datadoc_metadata.py @@ -19,9 +19,8 @@ from datadoc.backend.dataset_parser import DatasetParser from datadoc.backend.model_backwards_compatibility import upgrade_metadata from datadoc.enums import Assessment -from datadoc.enums import DatasetState -from datadoc.enums import DatasetStatus -from datadoc.enums import VariableRole +from datadoc.enums import DataSetState +from datadoc.enums import DataSetStatus from datadoc.frontend.fields.display_dataset import ( OBLIGATORY_DATASET_METADATA_IDENTIFIERS, ) @@ -49,17 +48,14 @@ def __init__( ) -> None: """Read in a dataset if supplied, otherwise naively instantiate the class.""" self._statistic_subject_mapping = statistic_subject_mapping - self.dataset_path = dataset_path self.metadata_document: pathlib.Path | CloudPath | None = None self.container: model.MetadataContainer | None = None - self.dataset: pathlib.Path | CloudPath | None = None + self.dataset_path: pathlib.Path | CloudPath | None = None self.short_name: str | None = None - self.meta: model.DatadocJsonSchema = model.DatadocJsonSchema( - percentage_complete=0, - dataset=model.Dataset(), - variables=[], - ) + self.dataset = model.Dataset() + self.variables: list = [] + self.variables_lookup: dict[str, model.Variable] = {} if metadata_document_path: # In this case the user has specified an independent metadata document for editing @@ -67,13 +63,13 @@ def __init__( self.metadata_document = self._open_path(metadata_document_path) self.extract_metadata_from_existing_document(self.metadata_document) elif dataset_path: - self.dataset = self._open_path(dataset_path) + self.dataset_path = self._open_path(dataset_path) # The short_name is set as the dataset filename without file extension - self.short_name = self.dataset.stem + self.short_name = self.dataset_path.stem # Build the metadata document path based on the dataset path # Example: /path/to/dataset.parquet -> /path/to/dataset__DOC.json - self.metadata_document = self.dataset.parent / ( - self.dataset.stem + METADATA_DOCUMENT_FILE_SUFFIX + self.metadata_document = self.dataset_path.parent / ( + self.dataset_path.stem + METADATA_DOCUMENT_FILE_SUFFIX ) self.extract_metadata_from_files() @@ -97,19 +93,19 @@ def extract_metadata_from_files(self) -> None: """ if self.metadata_document is not None and self.metadata_document.exists(): self.extract_metadata_from_existing_document(self.metadata_document) - elif self.dataset is not None: - self.extract_metadata_from_dataset(self.dataset) - self.meta.dataset.id = uuid.uuid4() + elif self.dataset_path is not None: + self.extract_metadata_from_dataset(self.dataset_path) + self.dataset.id = uuid.uuid4() # Set default values for variables where appropriate v: model.Variable - for v in self.meta.variables: + for v in self.variables: if v.variable_role is None: - v.variable_role = VariableRole.MEASURE + v.variable_role = model.VariableRole.MEASURE if v.direct_person_identifying is None: v.direct_person_identifying = False - if not self.meta.dataset.id: - self.meta.dataset.id = uuid.uuid4() - self.variables_lookup = {v.short_name: v for v in self.meta.variables} + if not self.dataset.id: + self.dataset.id = uuid.uuid4() + self.variables_lookup = {v.short_name: v for v in self.variables} def extract_metadata_from_existing_document( self, @@ -135,9 +131,15 @@ def extract_metadata_from_existing_document( datadoc_metadata = upgrade_metadata( datadoc_metadata, ) - self.meta = model.DatadocJsonSchema.model_validate_json( + + meta = model.DatadocMetadata.model_validate_json( json.dumps(datadoc_metadata), ) + if meta.dataset is not None: + self.dataset = meta.dataset + if meta.variables is not None: + self.variables = meta.variables + except json.JSONDecodeError: logger.warning( "Could not open existing metadata file %s. \ @@ -173,17 +175,17 @@ def extract_metadata_from_dataset( dapla_dataset_path_info.statistic_short_name, ) - self.meta.dataset = model.Dataset( + self.dataset = model.Dataset( short_name=self.short_name, dataset_state=dapla_dataset_path_info.dataset_state, - dataset_status=DatasetStatus.DRAFT, + dataset_status=DataSetStatus.DRAFT, assessment=self.get_assessment_by_state( dapla_dataset_path_info.dataset_state, ), version=dapla_dataset_path_info.dataset_version, contains_data_from=str(dapla_dataset_path_info.contains_data_from), contains_data_until=str(dapla_dataset_path_info.contains_data_until), - data_source_path=self.dataset, + data_source_path=self.dataset_path, metadata_created_by=user_info.get_user_info_for_current_platform().short_email, # TODO @mmwinther: Remove multiple_language_support once the model is updated. # https://github.com/statisticsnorway/ssb-datadoc-model/issues/41 @@ -193,21 +195,21 @@ def extract_metadata_from_dataset( nn=subject_field, ), ) - self.meta.variables = self.ds_schema.get_fields() + self.variables = self.ds_schema.get_fields() @staticmethod - def get_assessment_by_state(state: DatasetState | None) -> Assessment | None: + def get_assessment_by_state(state: DataSetState | None) -> Assessment | None: """Find assessment derived by dataset state.""" if state is None: return None match (state): case ( - DatasetState.INPUT_DATA - | DatasetState.PROCESSED_DATA - | DatasetState.STATISTICS + DataSetState.INPUT_DATA + | DataSetState.PROCESSED_DATA + | DataSetState.STATISTICS ): return Assessment.PROTECTED - case DatasetState.OUTPUT_DATA: + case DataSetState.OUTPUT_DATA: return Assessment.OPEN case _: return None @@ -216,17 +218,23 @@ def write_metadata_document(self) -> None: """Write all currently known metadata to file.""" timestamp: datetime = get_timestamp_now() - if self.meta.dataset.metadata_created_date is None: - self.meta.dataset.metadata_created_date = timestamp - self.meta.dataset.metadata_last_updated_date = timestamp - self.meta.dataset.metadata_last_updated_by = ( + if self.dataset.metadata_created_date is None: + self.dataset.metadata_created_date = timestamp + self.dataset.metadata_last_updated_date = timestamp + self.dataset.metadata_last_updated_by = ( user_info.get_user_info_for_current_platform().short_email ) - self.meta.dataset.file_path = str(self.dataset) + self.dataset.file_path = str(self.dataset_path) + + datadoc: model.DatadocMetadata = model.DatadocMetadata( + percentage_complete=self.percent_complete, + dataset=self.dataset, + variables=self.variables, + ) if self.container: - self.container.datadoc = self.meta + self.container.datadoc = datadoc else: - self.container = model.MetadataContainer(datadoc=self.meta) + self.container = model.MetadataContainer(datadoc=datadoc) if self.metadata_document: self.metadata_document.write_text(self.container.model_dump_json(indent=4)) logger.info("Saved metadata document %s", self.metadata_document) @@ -246,11 +254,11 @@ def percent_complete(self) -> int: num_set_fields = len( [ k - for k, v in self.meta.dataset.model_dump().items() + for k, v in self.dataset.model_dump().items() if k in OBLIGATORY_DATASET_METADATA_IDENTIFIERS and v is not None ], ) - for variable in self.meta.variables: + for variable in self.variables: num_all_fields += len(OBLIGATORY_VARIABLES_METADATA) num_set_fields += len( [ diff --git a/src/datadoc/backend/dataset_parser.py b/src/datadoc/backend/dataset_parser.py index 3d2d8d4e..b8cf382b 100644 --- a/src/datadoc/backend/dataset_parser.py +++ b/src/datadoc/backend/dataset_parser.py @@ -209,7 +209,7 @@ def get_fields(self) -> list[Variable]: # If this is not correct, the user may fix it via the UI name=LanguageStringType( **{ - state.current_metadata_language: sas_reader.columns[ # type: ignore [attr-defined] + state.current_metadata_language.value: sas_reader.columns[ # type: ignore [attr-defined] i ].label, }, diff --git a/src/datadoc/backend/model_backwards_compatibility.py b/src/datadoc/backend/model_backwards_compatibility.py index 95012aae..b0ed6282 100644 --- a/src/datadoc/backend/model_backwards_compatibility.py +++ b/src/datadoc/backend/model_backwards_compatibility.py @@ -82,7 +82,7 @@ def handle_version_1_0_0(supplied_metadata: dict[str, Any]) -> dict[str, Any]: supplied_metadata["dataset"]["data_source"] = LanguageStringType( en=supplied_metadata["dataset"]["data_source"], ) - supplied_metadata["document_version"] = "2.0.0" + supplied_metadata["document_version"] = "2.1.0" return supplied_metadata @@ -118,7 +118,7 @@ def handle_version_0_1_1(supplied_metadata: dict[str, Any]) -> dict[str, Any]: handler=handle_version_1_0_0, ) BackwardsCompatibleVersion( - version="2.0.0", + version="2.1.0", handler=handle_current_version, ) diff --git a/src/datadoc/backend/unit_types.py b/src/datadoc/backend/unit_types.py index c91e4677..2249111f 100644 --- a/src/datadoc/backend/unit_types.py +++ b/src/datadoc/backend/unit_types.py @@ -1,66 +1,144 @@ from __future__ import annotations import logging +from dataclasses import dataclass from typing import TYPE_CHECKING from datadoc.backend.external_sources.external_sources import GetExternalSource +from datadoc.enums import SupportedLanguages if TYPE_CHECKING: import pandas as pd - from klass.classes.codes import KlassCodes from klass.classes.classification import KlassClassification logger = logging.getLogger(__name__) +@dataclass +class UnitType: + """Data structure for the a unit type.""" + + titles: dict[str, str] + unit_code: str + + def get_title(self, language: SupportedLanguages) -> str: + """Get the title in the given language.""" + try: + return self.titles[ + ( + # Adjust to language codes in the UnitTypes structure. + "nb" + if language + in [ + SupportedLanguages.NORSK_BOKMÅL, + SupportedLanguages.NORSK_NYNORSK, + ] + else "en" + ) + ] + except KeyError: + logger.exception( + "Could not find title for subject %s and language: %s", + self, + language.name, + ) + return "" + + class UnitTypes(GetExternalSource): """Class for retrieving classifications from Klass.""" - def __init__(self, classification_id: int) -> None: + def __init__(self, classification_id: int | None) -> None: """Retrieves a list of classifications given a classification id. Initializes the classifications list and starts fetching the classifications. """ - self._classifications: list[str] = [] + self.supported_languages = [ + SupportedLanguages.NORSK_BOKMÅL.value, + SupportedLanguages.ENGLISH.value, + ] + + self._classifications: list[UnitType] = [] self.classification_id = classification_id - self.classifications_dataframe: pd.DataFrame | None = None + self.classifications_dataframes: dict[str, pd.DataFrame] | None = None super().__init__() def _fetch_data_from_external_source( self, - ) -> KlassCodes | None: + ) -> dict[str, pd.DataFrame] | None: """Fetches the classifications from Klass by classification id. returns a pandas dataframe with the class data for the given classification id. """ try: - return KlassClassification(str(self.classification_id)).get_codes() + classifications_dataframes = {} + for i in self.supported_languages: + classifications_dataframes[i] = ( + KlassClassification( + str(self.classification_id), + i, + ) + .get_codes() + .data + ) + except Exception: logger.exception( "Exception while getting classifications from Klass", ) return None + else: + return classifications_dataframes - def _extract_unit_types_from_dataframe( + def _extract_titles( self, - classifications_dataframe: pd.DataFrame, - ) -> list[str]: + dataframes: dict[SupportedLanguages, pd.DataFrame], + ) -> list[dict[str, str]]: + list_of_titles = [] + languages = list(dataframes) + for i in range(len(dataframes[SupportedLanguages.NORSK_BOKMÅL])): + titles = {} + for j in languages: + if "name" in dataframes[j]: + titles[str(j)] = dataframes[j].loc[:, "name"][i] + else: + titles[str(j)] = None + list_of_titles.append(titles) + return list_of_titles + + def _create_unit_types_from_dataframe( + self, + classifications_dataframes: dict[SupportedLanguages, pd.DataFrame], + ) -> list[UnitType]: """Method that finds the name column in the dataframe, and returns all values in a list.""" - if "name" in classifications_dataframe.columns: - return classifications_dataframe.loc[:, "name"].to_list() - return [] + classification_names = self._extract_titles(classifications_dataframes) + classification_codes: list + if "code" in classifications_dataframes[SupportedLanguages.NORSK_BOKMÅL]: + classification_codes = ( + classifications_dataframes[SupportedLanguages.NORSK_BOKMÅL] + .loc[:, "code"] + .to_list() + ) + else: + classification_codes = [None] * len(classification_names) + unit_types = [] + for a, b in zip(classification_names, classification_codes): + unit_types.append( + UnitType(a, b), + ) + return unit_types def _get_classification_dataframe_if_loaded(self) -> bool: """Checks if the data from Klass is loaded, then gets the classifications from the dataframe.""" - if self.check_if_external_data_is_loaded(): - self.classifications_dataframe = self.retrieve_external_data() - if self.classifications_dataframe is not None: - self._classifications = self._extract_unit_types_from_dataframe( - self.classifications_dataframe, + if not self._classifications: + self.classifications_dataframes = self.retrieve_external_data() + if self.classifications_dataframes is not None: + self._classifications = self._create_unit_types_from_dataframe( + self.classifications_dataframes, ) logger.debug( "Thread finished. found %s classifications", @@ -73,7 +151,7 @@ def _get_classification_dataframe_if_loaded(self) -> bool: return False @property - def classifications(self) -> list[str]: + def classifications(self) -> list[UnitType]: """Getter for primary subjects.""" self._get_classification_dataframe_if_loaded() logger.debug("Got %s classifications subjects", len(self._classifications)) diff --git a/src/datadoc/config.py b/src/datadoc/config.py index f81432b6..1b72b009 100644 --- a/src/datadoc/config.py +++ b/src/datadoc/config.py @@ -128,3 +128,8 @@ def get_dapla_service() -> DaplaService | None: def get_oidc_token() -> str | None: """Get the JWT token from the environment.""" return _get_config_item("OIDC_TOKEN") + + +def get_unit_code() -> int | None: + """The code for the Unit Type code list in Klass.""" + return int(_get_config_item("DATADOC_UNIT_CODE") or 702) diff --git a/src/datadoc/enums.py b/src/datadoc/enums.py index bbadc2b7..1c941736 100644 --- a/src/datadoc/enums.py +++ b/src/datadoc/enums.py @@ -90,174 +90,61 @@ class Assessment(LanguageStringsEnum): OPEN = LanguageStringType(en=model.Assessment.OPEN.value, nn="ÅPEN", nb="ÅPEN") -class DatasetStatus(LanguageStringsEnum): +class DataSetStatus(LanguageStringsEnum): """Lifecycle status of a dataset.""" DRAFT = LanguageStringType( - en=model.DatasetStatus.DRAFT.value, + en=model.DataSetStatus.DRAFT.value, nn="UTKAST", nb="UTKAST", ) INTERNAL = LanguageStringType( - en=model.DatasetStatus.INTERNAL.value, + en=model.DataSetStatus.INTERNAL.value, nn="INTERN", nb="INTERN", ) EXTERNAL = LanguageStringType( - en=model.DatasetStatus.EXTERNAL.value, + en=model.DataSetStatus.EXTERNAL.value, nn="EKSTERN", nb="EKSTERN", ) DEPRECATED = LanguageStringType( - en=model.DatasetStatus.DEPRECATED.value, + en=model.DataSetStatus.DEPRECATED.value, nn="UTGÅTT", nb="UTGÅTT", ) -class DatasetState(LanguageStringsEnum): +class DataSetState(LanguageStringsEnum): """Processing state of a dataset.""" SOURCE_DATA = LanguageStringType( - en=model.DatasetState.SOURCE_DATA.value, + en=model.DataSetState.SOURCE_DATA.value, nn="KILDEDATA", nb="KILDEDATA", ) INPUT_DATA = LanguageStringType( - en=model.DatasetState.INPUT_DATA.value, + en=model.DataSetState.INPUT_DATA.value, nn="INNDATA", nb="INNDATA", ) PROCESSED_DATA = LanguageStringType( - en=model.DatasetState.PROCESSED_DATA.value, + en=model.DataSetState.PROCESSED_DATA.value, nn="KLARGJORTE DATA", nb="KLARGJORTE DATA", ) STATISTICS = LanguageStringType( - en=model.DatasetState.STATISTICS.value, + en=model.DataSetState.STATISTICS.value, nn="STATISTIKK", nb="STATISTIKK", ) OUTPUT_DATA = LanguageStringType( - en=model.DatasetState.OUTPUT_DATA.value, + en=model.DataSetState.OUTPUT_DATA.value, nn="UTDATA", nb="UTDATA", ) -class UnitType(LanguageStringsEnum): - """Statistical unit types. - - Ref: https://www.ssb.no/metadata/definisjoner-av-statistiske-enheter. - """ - - ARBEIDSULYKKE = LanguageStringType( - en="WORK ACCIDENT", - nn=model.UnitType.ARBEIDSULYKKE.value, - nb=model.UnitType.ARBEIDSULYKKE.value, - ) - BOLIG = LanguageStringType( - en="HOUSING", - nn=model.UnitType.BOLIG.value, - nb=model.UnitType.BOLIG.value, - ) - BYGNING = LanguageStringType( - en="BUILDING", - nn=model.UnitType.BYGNING.value, - nb=model.UnitType.BYGNING.value, - ) - EIENDOM = LanguageStringType( - en="PROPERTY", - nn=model.UnitType.EIENDOM.value, - nb=model.UnitType.EIENDOM.value, - ) - FAMILIE = LanguageStringType( - en="FAMILY", - nn=model.UnitType.FAMILIE.value, - nb=model.UnitType.FAMILIE.value, - ) - FORETAK = LanguageStringType( - en="COMPANY", - nn=model.UnitType.FORETAK.value, - nb=model.UnitType.FORETAK.value, - ) - FYLKE = LanguageStringType( - en="REGION", - nn=model.UnitType.FYLKE.value, - nb=model.UnitType.FYLKE.value, - ) - HAVNEANLOEP = LanguageStringType( - en="PORT CALL", - nn=model.UnitType.HAVNEANLOEP.value, - nb=model.UnitType.HAVNEANLOEP.value, - ) - HUSHOLDNING = LanguageStringType( - en="HOUSEHOLD", - nn=model.UnitType.HUSHOLDNING.value, - nb=model.UnitType.HUSHOLDNING.value, - ) - KJOERETOEY = LanguageStringType( - en="VEHICLE", - nn=model.UnitType.KJOERETOEY.value, - nb=model.UnitType.KJOERETOEY.value, - ) - KOMMUNE = LanguageStringType( - en="COUNTY", - nn=model.UnitType.KOMMUNE.value, - nb=model.UnitType.KOMMUNE.value, - ) - KURS = LanguageStringType( - en="COURSE", - nn=model.UnitType.KURS.value, - nb=model.UnitType.KURS.value, - ) - LOVBRUDD = LanguageStringType( - en="CRIME", - nn=model.UnitType.LOVBRUDD.value, - nb=model.UnitType.LOVBRUDD.value, - ) - PERSON = LanguageStringType( - en="PERSON", - nn=model.UnitType.PERSON.value, - nb=model.UnitType.PERSON.value, - ) - STAT = LanguageStringType( - en="STATE", - nn=model.UnitType.STAT.value, - nb=model.UnitType.STAT.value, - ) - STORFE = LanguageStringType( - en="CATTLE", - nn=model.UnitType.STORFE.value, - nb=model.UnitType.STORFE.value, - ) - TRAFIKKULYKKE = LanguageStringType( - en="TRAFFIC ACCIDENT", - nn=model.UnitType.TRAFIKKULYKKE.value, - nb=model.UnitType.TRAFIKKULYKKE.value, - ) - TRANSAKSJON = LanguageStringType( - en="TRANSACTION", - nn=model.UnitType.TRANSAKSJON.value, - nb=model.UnitType.TRANSAKSJON.value, - ) - VARE_TJENESTE = LanguageStringType( - en="GOOD/SERVICE", - nn=model.UnitType.VARE_TJENESTE.value, - nb=model.UnitType.VARE_TJENESTE.value, - ) - VERDIPAPIR = LanguageStringType( - en="SERVICE", - nn=model.UnitType.VERDIPAPIR.value, - nb=model.UnitType.VERDIPAPIR.value, - ) - VIRKSOMHET = LanguageStringType( - en="BUSINESS", - nn=model.UnitType.VIRKSOMHET.value, - nb=model.UnitType.VIRKSOMHET.value, - ) - - class TemporalityTypeType(LanguageStringsEnum): """Temporality of a dataset. diff --git a/src/datadoc/frontend/callbacks/dataset.py b/src/datadoc/frontend/callbacks/dataset.py index 1c0257fb..a6d4b365 100644 --- a/src/datadoc/frontend/callbacks/dataset.py +++ b/src/datadoc/frontend/callbacks/dataset.py @@ -4,6 +4,7 @@ import logging import traceback +from typing import TYPE_CHECKING from pydantic import ValidationError @@ -23,6 +24,9 @@ from datadoc.frontend.fields.display_dataset import DatasetIdentifiers from datadoc.utils import METADATA_DOCUMENT_FILE_SUFFIX +if TYPE_CHECKING: + from datadoc_model.model import LanguageStringType + logger = logging.getLogger(__name__) VALIDATION_ERROR = "Validation error: " @@ -40,7 +44,7 @@ def open_file(file_path: str | None = None) -> DataDocMetadata: dataset = file_path or get_dataset_path() logger.info("Opening dataset %s", dataset) - return DataDocMetadata(state.statistic_subject_mapping, dataset_path=dataset) + return DataDocMetadata(state.statistic_subject_mapping, dataset_path=str(dataset)) def open_dataset_handling( @@ -79,16 +83,16 @@ def process_keyword(value: str) -> list[str]: def process_special_cases( - value: MetadataInputTypes, + value: MetadataInputTypes | LanguageStringType, metadata_identifier: str, -) -> MetadataInputTypes: +) -> MetadataInputTypes | LanguageStringType: """Pre-process metadata where needed. Some types of metadata need processing before being saved to the model. Handle these cases here, other values are returned unchanged. """ - updated_value: MetadataInputTypes + updated_value: MetadataInputTypes | LanguageStringType if metadata_identifier == DatasetIdentifiers.KEYWORD.value and isinstance( value, str, @@ -98,7 +102,7 @@ def process_special_cases( updated_value, _ = parse_and_validate_dates( str(value), getattr( - state.metadata.meta.dataset, + state.metadata.dataset, DatasetIdentifiers.CONTAINS_DATA_UNTIL.value, ), ) @@ -107,7 +111,7 @@ def process_special_cases( elif metadata_identifier == DatasetIdentifiers.CONTAINS_DATA_UNTIL.value: _, updated_value = parse_and_validate_dates( getattr( - state.metadata.meta.dataset, + state.metadata.dataset, DatasetIdentifiers.CONTAINS_DATA_FROM.value, ), str(value), @@ -121,7 +125,7 @@ def process_special_cases( str, ): updated_value = find_existing_language_string( - state.metadata.meta.dataset, + state.metadata.dataset, value, metadata_identifier, ) @@ -133,7 +137,7 @@ def process_special_cases( def accept_dataset_metadata_input( - value: MetadataInputTypes, + value: MetadataInputTypes | LanguageStringType, metadata_identifier: str, ) -> tuple[bool, str]: """Handle user inputs of dataset metadata values.""" @@ -146,7 +150,7 @@ def accept_dataset_metadata_input( value = process_special_cases(value, metadata_identifier) # Update the value in the model setattr( - state.metadata.meta.dataset, + state.metadata.dataset, metadata_identifier, value, ) @@ -172,7 +176,7 @@ def update_dataset_metadata_language() -> list[MetadataInputTypes]: This allows editing of strings in the chosen language. """ return [ - m.value_getter(state.metadata.meta.dataset, m.identifier) + m.value_getter(state.metadata.dataset, m.identifier) for m in DISPLAYED_DATASET_METADATA ] diff --git a/src/datadoc/frontend/callbacks/register_callbacks.py b/src/datadoc/frontend/callbacks/register_callbacks.py index b2df1692..24ae8232 100644 --- a/src/datadoc/frontend/callbacks/register_callbacks.py +++ b/src/datadoc/frontend/callbacks/register_callbacks.py @@ -74,8 +74,6 @@ def callback_update_progress( def callback_save_metadata_file(n_clicks: int) -> bool: """Save the metadata document to disk.""" if n_clicks and n_clicks > 0: - # Write the final completion percentage to the model - state.metadata.meta.percentage_complete = state.metadata.percent_complete state.metadata.write_metadata_document() return True diff --git a/src/datadoc/frontend/callbacks/utils.py b/src/datadoc/frontend/callbacks/utils.py index cc91b9af..33ed5d53 100644 --- a/src/datadoc/frontend/callbacks/utils.py +++ b/src/datadoc/frontend/callbacks/utils.py @@ -16,9 +16,11 @@ from datadoc import state if TYPE_CHECKING: + import pathlib from enum import Enum import pydantic + from cloudpathlib import CloudPath from datadoc.enums import SupportedLanguages @@ -85,9 +87,9 @@ def find_existing_language_string( return language_strings -def get_dataset_path() -> str | None: +def get_dataset_path() -> pathlib.Path | CloudPath | str | None: """Extract the path to the dataset from the potential sources.""" - if state.metadata.dataset is not None: + if state.metadata.dataset_path is not None: return state.metadata.dataset_path path_from_env = config.get_datadoc_dataset_path() logger.info( diff --git a/src/datadoc/frontend/callbacks/variables.py b/src/datadoc/frontend/callbacks/variables.py index 36e52d1c..f0f31390 100644 --- a/src/datadoc/frontend/callbacks/variables.py +++ b/src/datadoc/frontend/callbacks/variables.py @@ -24,7 +24,7 @@ from datadoc.utils import get_display_values if TYPE_CHECKING: - from datadoc_model import model + from datadoc_model.model import LanguageStringType logger = logging.getLogger(__name__) @@ -78,9 +78,9 @@ def get_metadata_field( def handle_multi_language_metadata( metadata_field: str, - new_value: MetadataInputTypes, + new_value: MetadataInputTypes | LanguageStringType, updated_row_id: str, -) -> MetadataInputTypes | model.LanguageStringType: +) -> MetadataInputTypes | LanguageStringType: """Handle updates to fields which support multiple languages.""" if new_value is None: # This edge case occurs when the user removes the text in an input field @@ -115,7 +115,9 @@ def accept_variable_datatable_metadata_input( for row_index in range(len(data)): # Update all the variables for this column to ensure we read in the value - new_value: MetadataInputTypes = data[row_index][metadata_field] + new_value: MetadataInputTypes | LanguageStringType = data[row_index][ + metadata_field + ] updated_row_id = data[row_index][VariableIdentifiers.SHORT_NAME.value] try: @@ -205,7 +207,7 @@ def update_variable_table_language( v, state.current_metadata_language, ) - for v in state.metadata.meta.variables + for v in state.metadata.variables ], False, # Don't show validation error "", # No validation explanation needed diff --git a/src/datadoc/frontend/components/variables_tab.py b/src/datadoc/frontend/components/variables_tab.py index 06dd07eb..7e71505f 100644 --- a/src/datadoc/frontend/components/variables_tab.py +++ b/src/datadoc/frontend/components/variables_tab.py @@ -26,7 +26,7 @@ def build_variables_tab() -> dbc.Tab: # Populate fields with known values data=[ get_display_values(v, state.current_metadata_language) - for v in state.metadata.meta.variables + for v in state.metadata.variables ], # Define columns based on the information in DISPLAY_VARIABLES columns=[ diff --git a/src/datadoc/frontend/fields/display_base.py b/src/datadoc/frontend/fields/display_base.py index e657d5fd..f8b285e0 100644 --- a/src/datadoc/frontend/fields/display_base.py +++ b/src/datadoc/frontend/fields/display_base.py @@ -68,8 +68,8 @@ def get_metadata_and_stringify(metadata: BaseModel, identifier: str) -> str: def get_multi_language_metadata(metadata: BaseModel, identifier: str) -> str | None: - """Get a metadata value supportng multiple languages from the model.""" - value: LanguageStringType = getattr(metadata, identifier) + """Get a metadata value supporting multiple languages from the model.""" + value: LanguageStringType | None = getattr(metadata, identifier) if value is None: return value return str(getattr(value, state.current_metadata_language)) diff --git a/src/datadoc/frontend/fields/display_dataset.py b/src/datadoc/frontend/fields/display_dataset.py index 3b618c91..3c49a3e3 100644 --- a/src/datadoc/frontend/fields/display_dataset.py +++ b/src/datadoc/frontend/fields/display_dataset.py @@ -53,6 +53,19 @@ def get_statistical_subject_options( ] +def get_unit_type_options( + language: SupportedLanguages, +) -> list[dict[str, str]]: + """Collect the unit type options for the given language.""" + return [ + { + "label": unit_type.get_title(language), + "value": unit_type.unit_code, + } + for unit_type in state.unit_types.classifications + ] + + class DatasetIdentifiers(str, Enum): """As defined here: https://statistics-norway.atlassian.net/l/c/aoSfEWJU.""" @@ -107,7 +120,7 @@ class DatasetIdentifiers(str, Enum): description="Livssyklus for datasettet", options_getter=functools.partial( get_enum_options_for_language, - enums.DatasetStatus, + enums.DataSetStatus, ), ), DatasetIdentifiers.DATASET_STATE: DisplayDatasetMetadataDropdown( @@ -117,7 +130,7 @@ class DatasetIdentifiers(str, Enum): obligatory=True, options_getter=functools.partial( get_enum_options_for_language, - enums.DatasetState, + enums.DataSetState, ), ), DatasetIdentifiers.NAME: DisplayDatasetMetadata( @@ -164,10 +177,8 @@ class DatasetIdentifiers(str, Enum): identifier=DatasetIdentifiers.UNIT_TYPE.value, display_name="Enhetstype", description="Primær enhetstype for datafil, datatabell eller datasett. Se Vi jobber med en avklaring av behov for flere enhetstyper her.", - options_getter=functools.partial( - get_enum_options_for_language, - enums.UnitType, - ), + multiple_language_support=False, + options_getter=get_unit_type_options, ), DatasetIdentifiers.TEMPORALITY_TYPE: DisplayDatasetMetadataDropdown( identifier=DatasetIdentifiers.TEMPORALITY_TYPE.value, diff --git a/src/datadoc/state.py b/src/datadoc/state.py index 7966a694..2219bb33 100644 --- a/src/datadoc/state.py +++ b/src/datadoc/state.py @@ -15,6 +15,7 @@ if TYPE_CHECKING: from datadoc.backend.datadoc_metadata import DataDocMetadata from datadoc.backend.statistic_subject_mapping import StatisticSubjectMapping + from datadoc.backend.unit_types import UnitTypes from datadoc.enums import SupportedLanguages @@ -24,3 +25,5 @@ current_metadata_language: SupportedLanguages statistic_subject_mapping: StatisticSubjectMapping + +unit_types: UnitTypes diff --git a/tests/backend/test_dapla_dataset_path_info.py b/tests/backend/test_dapla_dataset_path_info.py index 6e729b8a..732ba619 100644 --- a/tests/backend/test_dapla_dataset_path_info.py +++ b/tests/backend/test_dapla_dataset_path_info.py @@ -11,7 +11,7 @@ from datadoc.backend.dapla_dataset_path_info import ISO_YEAR_MONTH_DAY from datadoc.backend.dapla_dataset_path_info import SSB_BIMESTER from datadoc.backend.dapla_dataset_path_info import DaplaDatasetPathInfo -from datadoc.enums import DatasetState +from datadoc.enums import DataSetState from tests.utils import TEST_BUCKET_PARQUET_FILEPATH_WITH_SHORTNAME from tests.utils import TEST_PARQUET_FILEPATH @@ -149,18 +149,18 @@ def test_extract_period_info_no_period_info_in_path(data: str): @pytest.mark.parametrize( ("path_parts_to_insert", "expected_result"), [ - ("kildedata", DatasetState.SOURCE_DATA), - ("inndata", DatasetState.INPUT_DATA), - ("roskildedata/klargjorte-data", DatasetState.PROCESSED_DATA), - ("klargjorte_data", DatasetState.PROCESSED_DATA), - ("klargjorte-data", DatasetState.PROCESSED_DATA), - ("statistikk", DatasetState.STATISTICS), + ("kildedata", DataSetState.SOURCE_DATA), + ("inndata", DataSetState.INPUT_DATA), + ("roskildedata/klargjorte-data", DataSetState.PROCESSED_DATA), + ("klargjorte_data", DataSetState.PROCESSED_DATA), + ("klargjorte-data", DataSetState.PROCESSED_DATA), + ("statistikk", DataSetState.STATISTICS), ("", None), ], ) def test_get_dataset_state( full_dataset_state_path: pathlib.Path, - expected_result: DatasetState, + expected_result: DataSetState, ): actual_state = DaplaDatasetPathInfo(full_dataset_state_path).dataset_state assert actual_state == expected_result diff --git a/tests/backend/test_datadoc_metadata.py b/tests/backend/test_datadoc_metadata.py index 0df316e5..65c7fef5 100644 --- a/tests/backend/test_datadoc_metadata.py +++ b/tests/backend/test_datadoc_metadata.py @@ -14,7 +14,7 @@ import pytest from cloudpathlib.local import LocalGSClient from cloudpathlib.local import LocalGSPath -from datadoc_model.model import DatadocJsonSchema +from datadoc_model.model import DatadocMetadata from datadoc_model.model import Dataset from datadoc_model.model import Variable @@ -23,8 +23,8 @@ from datadoc.backend.user_info import PLACEHOLDER_EMAIL_ADDRESS from datadoc.backend.user_info import TestUserInfo from datadoc.enums import Assessment -from datadoc.enums import DatasetState -from datadoc.enums import DatasetStatus +from datadoc.enums import DataSetState +from datadoc.enums import DataSetStatus from datadoc.enums import DataType from datadoc.enums import VariableRole from tests.utils import TEST_BUCKET_PARQUET_FILEPATH @@ -62,19 +62,20 @@ def generate_periodic_file( def test_existing_metadata_file( metadata: DataDocMetadata, ): - assert metadata.meta.dataset.name.en == "successfully_read_existing_file" + assert metadata.dataset.name.en == "successfully_read_existing_file" # type: ignore [union-attr] def test_metadata_document_percent_complete(metadata: DataDocMetadata): - dataset = Dataset(dataset_state=DatasetState.OUTPUT_DATA) + dataset = Dataset(dataset_state=DataSetState.OUTPUT_DATA) variable_1 = Variable(data_type=DataType.BOOLEAN) variable_2 = Variable(data_type=DataType.INTEGER) - document = DatadocJsonSchema( + document = DatadocMetadata( percentage_complete=0, dataset=dataset, variables=[variable_1, variable_2], ) - metadata.meta = document + metadata.dataset = document.dataset # type: ignore [assignment] + metadata.variables = document.variables # type: ignore [assignment] assert metadata.percent_complete == 17 # noqa: PLR2004 @@ -87,10 +88,10 @@ def test_write_metadata_document( metadata.write_metadata_document() written_document = tmp_path / TEST_EXISTING_METADATA_FILE_NAME assert Path.exists(written_document) - assert metadata.meta.dataset.metadata_created_date == dummy_timestamp - assert metadata.meta.dataset.metadata_created_by == PLACEHOLDER_EMAIL_ADDRESS - assert metadata.meta.dataset.metadata_last_updated_date == dummy_timestamp - assert metadata.meta.dataset.metadata_last_updated_by == PLACEHOLDER_EMAIL_ADDRESS + assert metadata.dataset.metadata_created_date == dummy_timestamp + assert metadata.dataset.metadata_created_by == PLACEHOLDER_EMAIL_ADDRESS + assert metadata.dataset.metadata_last_updated_date == dummy_timestamp + assert metadata.dataset.metadata_last_updated_by == PLACEHOLDER_EMAIL_ADDRESS with Path.open(written_document) as f: written_metadata = json.loads(f.read()) @@ -124,17 +125,17 @@ def test_write_metadata_document_existing_document( dummy_timestamp: datetime, metadata: DataDocMetadata, ): - original_created_date: datetime = metadata.meta.dataset.metadata_created_date - original_created_by = metadata.meta.dataset.metadata_created_by + original_created_date = metadata.dataset.metadata_created_date + original_created_by = metadata.dataset.metadata_created_by metadata.write_metadata_document() - assert metadata.meta.dataset.metadata_created_by == original_created_by - assert metadata.meta.dataset.metadata_created_date == original_created_date - assert metadata.meta.dataset.metadata_last_updated_by == PLACEHOLDER_EMAIL_ADDRESS - assert metadata.meta.dataset.metadata_last_updated_date == dummy_timestamp + assert metadata.dataset.metadata_created_by == original_created_by + assert metadata.dataset.metadata_created_date == original_created_date + assert metadata.dataset.metadata_last_updated_by == PLACEHOLDER_EMAIL_ADDRESS + assert metadata.dataset.metadata_last_updated_date == dummy_timestamp def test_metadata_id(metadata: DataDocMetadata): - assert isinstance(metadata.meta.dataset.id, UUID) + assert isinstance(metadata.dataset.id, UUID) @pytest.mark.parametrize( @@ -148,11 +149,11 @@ def test_existing_metadata_none_id( with Path.open(Path(existing_metadata_file)) as f: pre_open_id: None = json.load(f)["datadoc"]["dataset"]["id"] assert pre_open_id is None - assert isinstance(metadata.meta.dataset.id, UUID) + assert isinstance(metadata.dataset.id, UUID) metadata.write_metadata_document() with Path.open(Path(existing_metadata_file)) as f: post_write_id = json.load(f)["datadoc"]["dataset"]["id"] - assert post_write_id == str(metadata.meta.dataset.id) + assert post_write_id == str(metadata.dataset.id) @pytest.mark.parametrize( @@ -168,8 +169,8 @@ def test_existing_metadata_valid_id( with Path.open(Path(existing_metadata_file)) as f: pre_open_id = json.load(f)["datadoc"]["dataset"]["id"] assert pre_open_id is not None - assert isinstance(metadata.meta.dataset.id, UUID) - assert str(metadata.meta.dataset.id) == pre_open_id + assert isinstance(metadata.dataset.id, UUID) + assert str(metadata.dataset.id) == pre_open_id metadata.write_metadata_document() with Path.open(Path(existing_metadata_file)) as f: post_write_id = json.load(f)["datadoc"]["dataset"]["id"] @@ -178,12 +179,12 @@ def test_existing_metadata_valid_id( def test_variable_role_default_value(metadata: DataDocMetadata): assert all( - v.variable_role == VariableRole.MEASURE.value for v in metadata.meta.variables + v.variable_role == VariableRole.MEASURE.value for v in metadata.variables ) def test_direct_person_identifying_default_value(metadata: DataDocMetadata): - assert all(not v.direct_person_identifying for v in metadata.meta.variables) + assert all(not v.direct_person_identifying for v in metadata.variables) def test_save_file_path_metadata_field( @@ -193,7 +194,7 @@ def test_save_file_path_metadata_field( metadata.write_metadata_document() with Path.open(Path(existing_metadata_file)) as f: saved_file_path = json.load(f)["datadoc"]["dataset"]["file_path"] - assert saved_file_path == str(metadata.dataset) + assert saved_file_path == str(metadata.dataset_path) def test_save_file_path_dataset_and_no_metadata( @@ -203,7 +204,7 @@ def test_save_file_path_dataset_and_no_metadata( metadata.write_metadata_document() with (tmp_path / TEST_EXISTING_METADATA_FILE_NAME).open() as f: saved_file_path = json.load(f)["datadoc"]["dataset"]["file_path"] - assert saved_file_path == str(metadata.dataset) + assert saved_file_path == str(metadata.dataset_path) @pytest.mark.parametrize( @@ -223,8 +224,8 @@ def test_period_metadata_fields_saved( subject_mapping_fake_statistical_structure, str(generate_periodic_file), ) - assert metadata.meta.dataset.contains_data_from == expected_from - assert metadata.meta.dataset.contains_data_until == expected_until + assert metadata.dataset.contains_data_from == expected_from + assert metadata.dataset.contains_data_until == expected_until @pytest.mark.parametrize( @@ -257,11 +258,11 @@ def test_open_file( ( TEST_PROCESSED_DATA_POPULATION_DIRECTORY / "person_testdata_p2021-12-31_p2021-12-31_v1.parquet", - DatasetStatus.INTERNAL.value, + DataSetStatus.INTERNAL.value, ), ( TEST_PARQUET_FILEPATH, - DatasetStatus.DRAFT.value, + DataSetStatus.DRAFT.value, ), ( "", @@ -272,14 +273,14 @@ def test_open_file( def test_dataset_status_default_value( subject_mapping_fake_statistical_structure: StatisticSubjectMapping, dataset_path: str, - expected_type: DatasetStatus | None, + expected_type: DataSetStatus | None, ): datadoc_metadata = DataDocMetadata( subject_mapping_fake_statistical_structure, str(dataset_path), ) - assert datadoc_metadata.meta.dataset.dataset_status == expected_type + assert datadoc_metadata.dataset.dataset_status == expected_type @pytest.mark.parametrize( @@ -319,7 +320,7 @@ def test_dataset_assessment_default_value( statistic_subject_mapping=StatisticSubjectMapping(source_url=""), dataset_path=str(copy_dataset_to_path), ) - assert datadoc_metadata.meta.dataset.assessment == expected_type + assert datadoc_metadata.dataset.assessment == expected_type @pytest.mark.parametrize( @@ -343,4 +344,4 @@ def test_extract_subject_field_value_from_statistic_structure_xml( ) # TODO @mmwinther: Remove multiple_language_support once the model is updated. # https://github.com/statisticsnorway/ssb-datadoc-model/issues/41 - assert metadata.meta.dataset.subject_field.en == expected_subject_code + assert metadata.dataset.subject_field.en == expected_subject_code # type: ignore [union-attr] diff --git a/tests/backend/test_model_backwards_compatibility.py b/tests/backend/test_model_backwards_compatibility.py index 7fb69ce7..1b8c9b34 100644 --- a/tests/backend/test_model_backwards_compatibility.py +++ b/tests/backend/test_model_backwards_compatibility.py @@ -19,7 +19,7 @@ def test_existing_metadata_current_model_version(): - current_model_version = "2.0.0" + current_model_version = "2.1.0" fresh_metadata = {"document_version": current_model_version} upgraded_metadata = upgrade_metadata(fresh_metadata) assert upgraded_metadata == fresh_metadata @@ -44,4 +44,4 @@ def test_backwards_compatibility( file_metadata = json.loads(f.read()) # Just test a single value to make sure we have a working model - assert metadata.meta.dataset.name.en == file_metadata["dataset"]["name"]["en"] + assert metadata.dataset.name.en == file_metadata["dataset"]["name"]["en"] # type: ignore [union-attr] diff --git a/tests/backend/test_unit_types.py b/tests/backend/test_unit_types.py index e69b3d5c..85998334 100644 --- a/tests/backend/test_unit_types.py +++ b/tests/backend/test_unit_types.py @@ -1,74 +1,119 @@ -import functools -import pathlib - -import pandas as pd import pytest +from datadoc.backend.unit_types import UnitType from datadoc.backend.unit_types import UnitTypes from tests.utils import TEST_RESOURCES_DIRECTORY -TEST_UNIT_TYPES_DIR = "unit_types" - - -@pytest.fixture() -def _mock_fetch_dataframe( - mocker, - unit_types_csv_filepath: pathlib.Path, -) -> None: - def fake_unit_types() -> pd.DataFrame: - return pd.read_csv(unit_types_csv_filepath) - - mocker.patch( - "datadoc.backend.unit_types.UnitTypes._fetch_data_from_external_source", - functools.partial(fake_unit_types), - ) - - -@pytest.fixture() -def unit_types_fake_structure( - _mock_fetch_dataframe, -) -> UnitTypes: - return UnitTypes(100) +UNIT_TYPES_DIR = "unit_types" @pytest.mark.parametrize( - ("unit_types_csv_filepath", "expected"), + ( + "unit_types_csv_filepath_nb", + "unit_types_csv_filepath_nn", + "unit_types_csv_filepath_en", + "expected", + ), [ ( - TEST_RESOURCES_DIRECTORY / TEST_UNIT_TYPES_DIR / "unit_types.csv", + TEST_RESOURCES_DIRECTORY / UNIT_TYPES_DIR / "unit_types_nb.csv", + TEST_RESOURCES_DIRECTORY / UNIT_TYPES_DIR / "unit_types_nn.csv", + TEST_RESOURCES_DIRECTORY / UNIT_TYPES_DIR / "unit_types_en.csv", [ - "Adresse", - "Arbeidsulykke", - "Bolig", - "Bygning", - "Eiendom", - "Familie", - "Foretak", - "Fylke (forvaltning)", - "Fylke (geografisk)", - "Havneanløp", - "Husholdning", - "Kjøretøy", - "Kommune (forvaltning)", - "Kommune (geografisk)", - "Kurs", - "Lovbrudd", - "Person", - "Skip", - "Statlig virksomhet", - "Storfe", - "Trafikkulykke", - "Transaksjon", - "Valg", - "Vare/tjeneste", - "Verdipapir", - "Virksomhet", + UnitType( + titles={ + "nb": "Adresse", + "nn": "Adresse", + "en": "Adresse", + }, + unit_code="01", + ), + UnitType( + titles={ + "nb": "Arbeidsulykke", + "nn": "Arbeidsulykke", + "en": "Arbeidsulykke", + }, + unit_code="02", + ), + UnitType( + titles={ + "nb": "Bolig", + "nn": "Bolig", + "en": "Bolig", + }, + unit_code="03", + ), ], ), ( - TEST_RESOURCES_DIRECTORY / TEST_UNIT_TYPES_DIR / "empty.csv", + TEST_RESOURCES_DIRECTORY / UNIT_TYPES_DIR / "empty.csv", + TEST_RESOURCES_DIRECTORY / UNIT_TYPES_DIR / "empty.csv", + TEST_RESOURCES_DIRECTORY / UNIT_TYPES_DIR / "empty.csv", [], ), + ( + TEST_RESOURCES_DIRECTORY / UNIT_TYPES_DIR / "unit_types_nb.csv", + TEST_RESOURCES_DIRECTORY / UNIT_TYPES_DIR / "empty.csv", + TEST_RESOURCES_DIRECTORY / UNIT_TYPES_DIR / "empty.csv", + [ + UnitType( + titles={ + "nb": "Adresse", + "nn": None, + "en": None, + }, + unit_code="01", + ), + UnitType( + titles={ + "nb": "Arbeidsulykke", + "nn": None, + "en": None, + }, + unit_code="02", + ), + UnitType( + titles={ + "nb": "Bolig", + "nn": None, + "en": None, + }, + unit_code="03", + ), + ], + ), + ( + TEST_RESOURCES_DIRECTORY / UNIT_TYPES_DIR / "no_code.csv", + TEST_RESOURCES_DIRECTORY / UNIT_TYPES_DIR / "no_code.csv", + TEST_RESOURCES_DIRECTORY / UNIT_TYPES_DIR / "no_code.csv", + [ + UnitType( + titles={ + "nb": "Adresse", + "nn": "Adresse", + "en": "Adresse", + }, + unit_code=None, + ), + UnitType( + titles={ + "nb": "Arbeidsulykke", + "nn": "Arbeidsulykke", + "en": "Arbeidsulykke", + }, + unit_code=None, + ), + UnitType( + titles={ + "nb": "Bolig", + "nn": "Bolig", + "en": "Bolig", + }, + unit_code=None, + ), + ], + ), ], ) @pytest.mark.usefixtures("_mock_fetch_dataframe") diff --git a/tests/conftest.py b/tests/conftest.py index 6057cdbc..c2e288d9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,6 +12,7 @@ from pathlib import Path from typing import TYPE_CHECKING +import pandas as pd import pytest from bs4 import BeautifulSoup from bs4 import ResultSet @@ -20,6 +21,7 @@ from datadoc import state from datadoc.backend.datadoc_metadata import DataDocMetadata from datadoc.backend.statistic_subject_mapping import StatisticSubjectMapping +from datadoc.backend.unit_types import UnitTypes from datadoc.backend.user_info import TestUserInfo from tests.backend.test_statistic_subject_mapping import ( STATISTICAL_SUBJECT_STRUCTURE_DIR, @@ -31,6 +33,8 @@ from .utils import TEST_PARQUET_FILEPATH from .utils import TEST_RESOURCES_DIRECTORY +UNIT_TYPES_DIR = "unit_types" + if TYPE_CHECKING: from pytest_mock import MockerFixture @@ -187,6 +191,60 @@ def fake_statistical_structure() -> ResultSet: ) +@pytest.fixture() +def subject_mapping_http_exception( + requests_mock, + exception_to_raise, +) -> StatisticSubjectMapping: + requests_mock.get( + "http://test.some.url.com", + exc=exception_to_raise, + ) + return StatisticSubjectMapping("http://test.some.url.com") + + +@pytest.fixture() +def unit_types_csv_filepath_nb() -> pathlib.Path: + return TEST_RESOURCES_DIRECTORY / UNIT_TYPES_DIR / "unit_types_nb.csv" + + +@pytest.fixture() +def unit_types_csv_filepath_nn() -> pathlib.Path: + return TEST_RESOURCES_DIRECTORY / UNIT_TYPES_DIR / "unit_types_nn.csv" + + +@pytest.fixture() +def unit_types_csv_filepath_en() -> pathlib.Path: + return TEST_RESOURCES_DIRECTORY / UNIT_TYPES_DIR / "unit_types_en.csv" + + +@pytest.fixture() +def _mock_fetch_dataframe( + mocker, + unit_types_csv_filepath_nb: pathlib.Path, + unit_types_csv_filepath_nn: pathlib.Path, + unit_types_csv_filepath_en: pathlib.Path, +) -> None: + def fake_unit_types() -> dict[str, pd.DataFrame]: + return { + "nb": pd.read_csv(unit_types_csv_filepath_nb, converters={"code": str}), + "nn": pd.read_csv(unit_types_csv_filepath_nn, converters={"code": str}), + "en": pd.read_csv(unit_types_csv_filepath_en, converters={"code": str}), + } + + mocker.patch( + "datadoc.backend.unit_types.UnitTypes._fetch_data_from_external_source", + functools.partial(fake_unit_types), + ) + + +@pytest.fixture() +def unit_types_fake_structure( + _mock_fetch_dataframe, +) -> UnitTypes: + return UnitTypes(100) + + @pytest.fixture() def copy_dataset_to_path( tmp_path: pathlib.Path, diff --git a/tests/frontend/callbacks/test_callbacks_utils.py b/tests/frontend/callbacks/test_callbacks_utils.py index d12ffa17..32b0ef64 100644 --- a/tests/frontend/callbacks/test_callbacks_utils.py +++ b/tests/frontend/callbacks/test_callbacks_utils.py @@ -60,11 +60,10 @@ def test_update_global_language_state(): "model_enum", [ model.Assessment, - model.DatasetStatus, - model.DatasetState, + model.DataSetStatus, + model.DataSetState, model.DataType, model.VariableRole, - model.UnitType, model.TemporalityTypeType, ], ) @@ -86,8 +85,8 @@ class TestEnum(Enum): "enum", [ enums.Assessment, - enums.DatasetState, - enums.DatasetStatus, + enums.DataSetState, + enums.DataSetStatus, enums.TemporalityTypeType, enums.DataType, enums.VariableRole, diff --git a/tests/frontend/callbacks/test_dataset_callbacks.py b/tests/frontend/callbacks/test_dataset_callbacks.py index 54f5f225..562beec4 100644 --- a/tests/frontend/callbacks/test_dataset_callbacks.py +++ b/tests/frontend/callbacks/test_dataset_callbacks.py @@ -14,7 +14,7 @@ from datadoc import enums from datadoc import state from datadoc.backend.datadoc_metadata import DataDocMetadata -from datadoc.enums import DatasetState +from datadoc.enums import DataSetState from datadoc.enums import LanguageStringsEnum from datadoc.enums import SupportedLanguages from datadoc.frontend.callbacks.dataset import accept_dataset_metadata_input @@ -28,6 +28,7 @@ if TYPE_CHECKING: from datadoc.backend.statistic_subject_mapping import StatisticSubjectMapping + from datadoc.backend.unit_types import UnitTypes from datadoc.frontend.callbacks.utils import MetadataInputTypes DATASET_CALLBACKS_MODULE = "datadoc.frontend.callbacks.dataset" @@ -54,13 +55,13 @@ def file_path(): ), ( DatasetIdentifiers.DATASET_STATUS, - enums.DatasetStatus.INTERNAL, - enums.DatasetStatus.INTERNAL.value, + enums.DataSetStatus.INTERNAL, + enums.DataSetStatus.INTERNAL.value, ), ( DatasetIdentifiers.DATASET_STATE, - enums.DatasetState.INPUT_DATA, - enums.DatasetState.INPUT_DATA.value, + enums.DataSetState.INPUT_DATA, + enums.DataSetState.INPUT_DATA.value, ), ( DatasetIdentifiers.NAME, @@ -93,11 +94,6 @@ def file_path(): "Version description", enums.LanguageStringType(nb="Version description"), ), - ( - DatasetIdentifiers.UNIT_TYPE, - enums.UnitType.ARBEIDSULYKKE, - enums.UnitType.ARBEIDSULYKKE.value, - ), ( DatasetIdentifiers.TEMPORALITY_TYPE, enums.TemporalityTypeType.ACCUMULATED, @@ -142,7 +138,7 @@ def test_accept_dataset_metadata_input_valid_data( assert output[0] is False assert output[1] == "" assert ( - getattr(state.metadata.meta.dataset, metadata_identifier.value) + getattr(state.metadata.dataset, metadata_identifier.value) == expected_model_value ) @@ -211,7 +207,7 @@ def test_update_dataset_metadata_language_strings( language_object: model.LanguageStringType, ): state.metadata = metadata - state.metadata.meta.dataset.name = language_object + state.metadata.dataset.name = language_object state.current_metadata_language = SupportedLanguages.NORSK_BOKMÅL output = update_dataset_metadata_language() assert english_name not in output @@ -224,35 +220,37 @@ def test_update_dataset_metadata_language_strings( def test_update_dataset_metadata_language_enums(): state.metadata = DataDocMetadata(str(TEST_PARQUET_FILEPATH)) - state.metadata.meta.dataset.dataset_state = DatasetState.PROCESSED_DATA + state.metadata.dataset.dataset_state = DataSetState.PROCESSED_DATA state.current_metadata_language = SupportedLanguages.NORSK_BOKMÅL output = update_dataset_metadata_language() - assert DatasetState.PROCESSED_DATA.language_strings.nb not in output - assert DatasetState.PROCESSED_DATA.name in output + assert DataSetState.PROCESSED_DATA.language_strings.nb not in output + assert DataSetState.PROCESSED_DATA.name in output state.current_metadata_language = SupportedLanguages.ENGLISH output = update_dataset_metadata_language() - assert DatasetState.PROCESSED_DATA.language_strings.nb not in output - assert DatasetState.PROCESSED_DATA.name in output + assert DataSetState.PROCESSED_DATA.language_strings.nb not in output + assert DataSetState.PROCESSED_DATA.name in output @pytest.mark.parametrize( "enum_for_options", [ enums.Assessment, - enums.DatasetState, - enums.DatasetStatus, + enums.DataSetState, + enums.DataSetStatus, enums.TemporalityTypeType, ], ) @pytest.mark.parametrize("language", list(SupportedLanguages)) def test_change_language_dataset_metadata_options_enums( subject_mapping_fake_statistical_structure: StatisticSubjectMapping, + unit_types_fake_structure: UnitTypes, metadata: DataDocMetadata, enum_for_options: LanguageStringsEnum, language: SupportedLanguages, ): state.metadata = metadata state.statistic_subject_mapping = subject_mapping_fake_statistical_structure + state.unit_types = unit_types_fake_structure value = change_language_dataset_metadata(language) for options in cast(list[list[dict[str, str]]], value[0:-1]): diff --git a/tests/frontend/callbacks/test_variables_callbacks.py b/tests/frontend/callbacks/test_variables_callbacks.py index 78bae012..0c0248a3 100644 --- a/tests/frontend/callbacks/test_variables_callbacks.py +++ b/tests/frontend/callbacks/test_variables_callbacks.py @@ -117,7 +117,7 @@ def test_accept_variable_table_metadata_input_incorrect_data_type( active_cell: dict[str, MetadataInputTypes], ): state.metadata = metadata - previous_metadata = deepcopy(state.metadata.meta.variables) + previous_metadata = deepcopy(state.metadata.variables) output = accept_variable_datatable_metadata_input( DATA_INVALID, active_cell, @@ -127,7 +127,7 @@ def test_accept_variable_table_metadata_input_incorrect_data_type( assert output[0] == DATA_ORIGINAL assert output[1] is True assert "validation error for Variable" in output[2] - assert state.metadata.meta.variables == previous_metadata + assert state.metadata.variables == previous_metadata def test_find_existing_language_string_pre_existing_strings( @@ -158,7 +158,7 @@ def test_update_variable_table_language( ): state.metadata = metadata test_variable = random.choice( # noqa: S311 not for cryptographic purposes - [v.short_name for v in state.metadata.meta.variables], + [v.short_name for v in state.metadata.variables], ) state.metadata.variables_lookup[test_variable].name = language_object output = update_variable_table_language( @@ -310,13 +310,13 @@ def test_accept_variable_metadata_input_valid( assert ( accept_variable_metadata_input( value, - metadata.meta.variables[0].short_name, + metadata.variables[0].short_name, metadata_field=metadata_field.value, ) is None ) assert ( - getattr(state.metadata.meta.variables[0], metadata_field.value) + getattr(state.metadata.variables[0], metadata_field.value) == expected_model_value ) @@ -327,7 +327,7 @@ def test_accept_variable_metadata_input_invalid( state.metadata = metadata message = accept_variable_metadata_input( "not a url", - metadata.meta.variables[0].short_name, + metadata.variables[0].short_name, metadata_field=VariableIdentifiers.DEFINITION_URI.value, ) assert message is not None diff --git a/tests/frontend/fields/test_display_dataset.py b/tests/frontend/fields/test_display_dataset.py index de59323e..d11c7d86 100644 --- a/tests/frontend/fields/test_display_dataset.py +++ b/tests/frontend/fields/test_display_dataset.py @@ -3,9 +3,11 @@ from datadoc import state from datadoc.enums import SupportedLanguages from datadoc.frontend.fields.display_dataset import get_statistical_subject_options +from datadoc.frontend.fields.display_dataset import get_unit_type_options from tests.backend.test_statistic_subject_mapping import ( STATISTICAL_SUBJECT_STRUCTURE_DIR, ) +from tests.backend.test_unit_types import UNIT_TYPES_DIR from tests.utils import TEST_RESOURCES_DIRECTORY @@ -43,3 +45,25 @@ def test_get_statistical_subject_options( state.statistic_subject_mapping = subject_mapping_fake_statistical_structure state.statistic_subject_mapping.wait_for_external_result() assert get_statistical_subject_options(SupportedLanguages.NORSK_BOKMÅL) == expected + + +@pytest.mark.parametrize( + ("unit_types_csv_filepath_nb", "expected"), + [ + ( + TEST_RESOURCES_DIRECTORY / UNIT_TYPES_DIR / "unit_types_nb.csv", + [ + {"label": "Adresse", "value": "01"}, + {"label": "Arbeidsulykke", "value": "02"}, + {"label": "Bolig", "value": "03"}, + ], + ), + ], +) +def test_get_unit_type_options( + unit_types_fake_structure, + expected, +): + state.unit_types = unit_types_fake_structure + state.unit_types.wait_for_external_result() + assert get_unit_type_options(SupportedLanguages.NORSK_BOKMÅL) == expected diff --git a/tests/resources/existing_metadata_file/invalid_id_field/person_data_v1__DOC.json b/tests/resources/existing_metadata_file/invalid_id_field/person_data_v1__DOC.json index 1d3a86fb..653c2a9c 100644 --- a/tests/resources/existing_metadata_file/invalid_id_field/person_data_v1__DOC.json +++ b/tests/resources/existing_metadata_file/invalid_id_field/person_data_v1__DOC.json @@ -2,7 +2,7 @@ "document_version": "0.0.1", "datadoc": { "percentage_complete": 6, - "document_version": "2.0.0", + "document_version": "2.1.0", "dataset": { "short_name": "person_data_v1", "assessment": "OPEN", diff --git a/tests/resources/existing_metadata_file/person_data_v1__DOC.json b/tests/resources/existing_metadata_file/person_data_v1__DOC.json index 663e0d1f..ed7dcdc3 100644 --- a/tests/resources/existing_metadata_file/person_data_v1__DOC.json +++ b/tests/resources/existing_metadata_file/person_data_v1__DOC.json @@ -2,7 +2,7 @@ "document_version": "0.0.1", "datadoc": { "percentage_complete": 98, - "document_version": "2.0.0", + "document_version": "2.1.0", "dataset": { "short_name": "person_data_v1", "assessment": "SENSITIVE", diff --git a/tests/resources/existing_metadata_file/valid_id_field/person_data_v1__DOC.json b/tests/resources/existing_metadata_file/valid_id_field/person_data_v1__DOC.json index fd42a80c..90a178bd 100644 --- a/tests/resources/existing_metadata_file/valid_id_field/person_data_v1__DOC.json +++ b/tests/resources/existing_metadata_file/valid_id_field/person_data_v1__DOC.json @@ -2,7 +2,7 @@ "document_version": "0.0.1", "datadoc": { "percentage_complete": 6, - "document_version": "2.0.0", + "document_version": "2.1.0", "dataset": { "short_name": "person_data_v1", "assessment": "OPEN", diff --git a/tests/resources/klargjorte_data/person_testdata_p2021-12-31_p2021-12-31_v1__DOC.json b/tests/resources/klargjorte_data/person_testdata_p2021-12-31_p2021-12-31_v1__DOC.json index c1eaafe9..e8e4bc98 100644 --- a/tests/resources/klargjorte_data/person_testdata_p2021-12-31_p2021-12-31_v1__DOC.json +++ b/tests/resources/klargjorte_data/person_testdata_p2021-12-31_p2021-12-31_v1__DOC.json @@ -2,7 +2,7 @@ "document_version": "0.0.1", "datadoc": { "percentage_complete": 97, - "document_version": "2.0.0", + "document_version": "2.1.0", "dataset": { "short_name": "person_testdata_p2021-12-31_p2021-12-31", "assessment": "PROTECTED", diff --git a/tests/resources/unit_types/no_code.csv b/tests/resources/unit_types/no_code.csv new file mode 100644 index 00000000..bde1cc27 --- /dev/null +++ b/tests/resources/unit_types/no_code.csv @@ -0,0 +1,4 @@ +parentCode,level,name, +,1,Adresse, +,1,Arbeidsulykke, +,1,Bolig, diff --git a/tests/resources/unit_types/unit_types_en.csv b/tests/resources/unit_types/unit_types_en.csv new file mode 100644 index 00000000..b3aee784 --- /dev/null +++ b/tests/resources/unit_types/unit_types_en.csv @@ -0,0 +1,4 @@ +code,parentCode,level,name, +01,,1,Adresse, +02,,1,Arbeidsulykke, +03,,1,Bolig, diff --git a/tests/resources/unit_types/unit_types_nb.csv b/tests/resources/unit_types/unit_types_nb.csv new file mode 100644 index 00000000..b3aee784 --- /dev/null +++ b/tests/resources/unit_types/unit_types_nb.csv @@ -0,0 +1,4 @@ +code,parentCode,level,name, +01,,1,Adresse, +02,,1,Arbeidsulykke, +03,,1,Bolig, diff --git a/tests/resources/unit_types/unit_types_nn.csv b/tests/resources/unit_types/unit_types_nn.csv new file mode 100644 index 00000000..b3aee784 --- /dev/null +++ b/tests/resources/unit_types/unit_types_nn.csv @@ -0,0 +1,4 @@ +code,parentCode,level,name, +01,,1,Adresse, +02,,1,Arbeidsulykke, +03,,1,Bolig, diff --git a/tests/test_smoke.py b/tests/test_smoke.py index 78c8feac..91f5300d 100644 --- a/tests/test_smoke.py +++ b/tests/test_smoke.py @@ -4,8 +4,10 @@ from datadoc.app import get_app -def test_get_app(subject_mapping_fake_statistical_structure): +def test_get_app(subject_mapping_fake_statistical_structure, unit_types_fake_structure): state.statistic_subject_mapping = subject_mapping_fake_statistical_structure + state.unit_types = unit_types_fake_structure + app, _ = get_app() assert app.config["name"] == "Datadoc" assert len(app.callback_map.items()) > 0