From 6802bd6710df510d5daaef25ca5ec4f0dbd24b2f Mon Sep 17 00:00:00 2001 From: yuanzhou Date: Mon, 9 Nov 2020 22:32:30 -0500 Subject: [PATCH 01/37] Add refactor deplou --- docker/docker-compose.refactor.yml | 17 +++++++++++++++++ docker/uuid-api-docker.sh | 6 +++--- 2 files changed, 20 insertions(+), 3 deletions(-) create mode 100644 docker/docker-compose.refactor.yml diff --git a/docker/docker-compose.refactor.yml b/docker/docker-compose.refactor.yml new file mode 100644 index 0000000..d560e17 --- /dev/null +++ b/docker/docker-compose.refactor.yml @@ -0,0 +1,17 @@ +version: "3.7" + +services: + + uuid-api: + environment: + - HOST_GID=${HOST_GID:-1000} + - HOST_UID=${HOST_UID:-1000} + init: true + restart: always + volumes: + # Mount the VERSION file and BUILD file + - "../VERSION:/usr/src/app/src/VERSION" + - "../BUILD:/usr/src/app/src/BUILD" + # Mount the source code to container + - "../src:/usr/src/app/src" + \ No newline at end of file diff --git a/docker/uuid-api-docker.sh b/docker/uuid-api-docker.sh index d1b20b2..5c0d5e8 100755 --- a/docker/uuid-api-docker.sh +++ b/docker/uuid-api-docker.sh @@ -50,8 +50,8 @@ function export_version() { echo "UUID_API_VERSION: $UUID_API_VERSION" } -if [[ "$1" != "localhost" && "$1" != "dev" && "$1" != "test" && "$1" != "stage" && "$1" != "prod" ]]; then - echo "Unknown build environment '$1', specify one of the following: localhost|dev|test|stage|prod" +if [[ "$1" != "localhost" && "$1" != "dev" && "$1" != "test" && "$1" != "stage" && "$1" != "prod" && "$1" != "refactor" ]]; then + echo "Unknown build environment '$1', specify one of the following: localhost|dev|test|stage|prod|refactor" else if [[ "$2" != "check" && "$2" != "config" && "$2" != "build" && "$2" != "start" && "$2" != "stop" && "$2" != "down" ]]; then echo "Unknown command '$2', specify one of the following: check|config|build|start|stop|down" @@ -103,7 +103,7 @@ else # Only mount the VERSION file and BUILD file for localhost and dev # On test/stage/prod, copy the VERSION file and BUILD file to image - if [[ "$1" != "localhost" && "$1" != "dev" ]]; then + if [[ "$1" != "localhost" && "$1" != "dev" && "$1" != "refactor" ]]; then # Delete old VERSION and BUILD files if found if [ -f "uuid-api/src/VERSION" ]; then rm -rf uuid-api/src/VERSION From dccb83b36b171a501c119de57a6f7a50162ef23f Mon Sep 17 00:00:00 2001 From: yuanzhou Date: Mon, 9 Nov 2020 23:04:24 -0500 Subject: [PATCH 02/37] Add mysql for refactor --- docker/docker-compose.refactor.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docker/docker-compose.refactor.yml b/docker/docker-compose.refactor.yml index d560e17..297914f 100644 --- a/docker/docker-compose.refactor.yml +++ b/docker/docker-compose.refactor.yml @@ -14,4 +14,19 @@ services: - "../BUILD:/usr/src/app/src/BUILD" # Mount the source code to container - "../src:/usr/src/app/src" + + # Only used by local development + hubmap-mysql: + build: ./hubmap-mysql + # Build the image with name and tag + image: hubmap-mysql:1.1 + hostname: hubmap-mysql + container_name: hubmap-mysql + environment: + MYSQL_ROOT_PASSWORD: 123 + # Use the same port mapping for dev and prod + ports: + - "3306:3306" + networks: + - gateway_hubmap \ No newline at end of file From cd12894f12eb9d81a1363111a679cd609df2aae6 Mon Sep 17 00:00:00 2001 From: yuanzhou Date: Tue, 10 Nov 2020 13:42:30 -0500 Subject: [PATCH 03/37] Mysql changes --- .../hubmap-mysql/uuids.sql | 0 docker/uuid-api-docker.sh | 6 ------ sql/dump-uuids.sh | 1 - sql/uuid.ddl | 15 --------------- 4 files changed, 22 deletions(-) rename sql/uuids-dev.sql => docker/hubmap-mysql/uuids.sql (100%) delete mode 100755 sql/dump-uuids.sh delete mode 100644 sql/uuid.ddl diff --git a/sql/uuids-dev.sql b/docker/hubmap-mysql/uuids.sql similarity index 100% rename from sql/uuids-dev.sql rename to docker/hubmap-mysql/uuids.sql diff --git a/docker/uuid-api-docker.sh b/docker/uuid-api-docker.sh index 5c0d5e8..65dcf7e 100755 --- a/docker/uuid-api-docker.sh +++ b/docker/uuid-api-docker.sh @@ -95,12 +95,6 @@ else # Copy over the source code to docker directory cp -r ../src uuid-api/ - # Also load the sample MySQL database for localhost build - if [ "$1" = "localhost" ]; then - # Copy over the sql database - cp -r ../sql/uuids-dev.sql hubmap-mysql/uuids-dev.sql - fi - # Only mount the VERSION file and BUILD file for localhost and dev # On test/stage/prod, copy the VERSION file and BUILD file to image if [[ "$1" != "localhost" && "$1" != "dev" && "$1" != "refactor" ]]; then diff --git a/sql/dump-uuids.sh b/sql/dump-uuids.sh deleted file mode 100755 index cc4ac40..0000000 --- a/sql/dump-uuids.sh +++ /dev/null @@ -1 +0,0 @@ -mysqldump --column-statistics=0 -h hubmap-db.cluster-cylxi65xrqts.us-east-1.rds.amazonaws.com -u uuid-user -p hm_uuid hm_uuids --where="hmuuid in ('ab4d0146e3ba01a3cd8983ee12dff146','f0592b8e070436d163128e0a31e40a2e','4239ed6141398a90cabb922fdad217fb','388d7819bf06e753ba5b65746a0cd075','a7160e7f0a405871465a23d8ee856a2c','3f712e77c1791a6e890e9d6966c45a2b','1869313f3c0516888015bb803825e287','7e1240df1efe4f0294e47358fa8ae017','2d2e54ce45fa90fec8d7e48be8505265','8a916e34265dd30d1c1c649542f4972f','d412b101576cfb85272ef36f74084ebf','1967c65f8d98eedf4afa064c41e057fa','c2cfc0ae651f79300f7fd2fe7fe911a2','8ba9062ee334c085594a38536efb1c37','83050dede4a7ebffb7cae5e7718595b5','32ae6d81bc9df2d62a550c62c02df39a','560598323e562f391f49608b60e17af5','4c726730a7ebdb5ca8fc91edfacdb9c7','c5edf18b0f03eb93db777996ee9f5d6d','7c1f400b11e53a8a7bde1e548f54535a','2303858a6fe1c0df94b3fa8ae696c8ee','3a2bdd52946a14081d448261c7b52bb2','796d05afe2a5a540f88682f46e856276','07a29e4c-ed43-11e8-b56a-0e8017bdda58','308f5ffc-ed43-11e8-b56a-0e8017bdda58','73bb26e4-ed43-11e8-8f19-0a7c1eab007a','def5fd76-ed43-11e8-b56a-0e8017bdda58','03b3d854-ed44-11e8-8bce-0e368f3075e8','5bd084c8-edc2-11e8-802f-0e368f3075e8','0947556dc2a3600f7a16e9c7aaf54a5a','4dfb043427be1f6286861c99bd94ed42','60430ead1a85b8322ef591cf0e493817','94c4b8cdffed74bbfcf3162468a99db9','48f8ad0a6475780ef4369081cbfe847d','3b137f31d8f4b7165049e2f0a6d67c8e','a1ae80517795535dd545b1076dbeb6d5','64521601f4f8f55a2c9a696d02fde162','0b887d3578db66ad5a98f9d50226ccae','edc87a64427607da3cc1bce950204341','a14b72f088e84b2b32f1a0e399e04c98','ca7cb524cf62889c1ee2cae92408588b','839798e18fd1fb3cdf85fcb84b2f1a84','4202257e415b612935ac7bf79d196fc7','5cea3bee7e3bd89e3aa9541e9d670fa3','23e255b1402b9ae92316d905436e029d','fb994c1a98d22c1065f086717c40a0fb','c5b4cfc4d7baa4565627a4cfad425582','b64988b7fae24affcd1ed7d2d7190bf1','c2ea6e6eb86e266540f5a6b7cdfed193','05dd8d7780c9a11f5fba46a0ab4fece2','5e575a3071207bd49f5d49b343329d35','664acbc43547a8d49841435218f70b5e','320c47465818fdeb63b0cd6d80b8a4a5','a13e4991dd708d2ceaa5318a1aa83e16','908459de2572ac7af4f472f6940212ee','881306c15de72b09bcf44307f6c47153','66137b969c7fad79a11bb09922785359','17eba3927b403cc8b241b210f1b88ec4','c17da61e33c0eae447a12cdfbfc90371','5b51806e19227ac033b157ae7483a46f','bfb040bd04130081fe2105d008952da4','8a13a0c01181ee9eedd092b16d8dc0b0','936560fdec4a7232b8ba1a97b081be28','ad9bc3417575de0b804041081d3f18f0','5e19e04e57c3358be95488894429528c','e43cac7054b7497f2973a4499b32869a','759c3133dd6e2a317af2773cc27c5dd1','30e9501b1827da22bd8658be37623848','85edb6771520d338cecb4655db509e3b','fa3f363da2ba393c893778251f4b64c8','9b8aa2554ef6a64219b012f0a95b918c','9408dc26b636463354265d42d329fcf9','c6ff1a50ddee31fa8d210d5e86ec34ac','a67d1751a6966571ad2d7c79feb3e2e3','c682061c9c087763678e85780aef8c01','195cab9b5e926dcef651ca0b1b0c06bc','95207e8124e5410fe0ff88e4eaacc942','62a68faca27a4342699ae4ea13f32609','0a5d6db9d51f4c5304a6549dc3719b31','dc457f08d5fc14dcced39e1b4848fa36','22b458f624b8b183d51987889e350d02','af0336c44ebdf8852249c8d8b607537a','9836a2b69403f01c02cdf92d59d3a4e7','83276c66021fe982bcbe0d1e187de92f','fd3fa069884680b236e53a539726fd05','d31f674c1b24bf38ae6781885fd51998','06cc13e8c58acb8c1b6234bd99ed6859','3c5a274d76d3cb395f991f920e581700','d1f35fce9c9ef1974cc90baf647aaa6b','4b34b75ea6eac35fad39072487564ee6','9195f7caaea425027a22159f6209f3b0','364da9dbb059962393357a04468425f3','abfc172fdddfc316691e2dd40e1e9816','cba31f41f0be15decc46b09070092e7e','9f3120c4d41ff9642a8ba690877e09a0','16d58b6222490a6cca96561bd9f11211','e3592f6038448299670e1854b160ad1b','f91c4e7662b0bedd90470e0911a23e84','a9d925e1ec388424b71a655be9cca958','d2b56cc797f35eff68cdb7e68c55d3bc','43cf0bd9e2c0c925702b1c8decbc0e04','a4813d1e9c7726d361ab04e4657fd245','425d4b757611f0ae927cb274da2a478f','3d53f3585e31d3468e9c958d76d4b39a','f275a6c310b8eb8b6e4bddb7a428ce17','4e17749a1fdb7ee1865aab234c45a53d','68c0ec0da34325a51d670f7b02f81600','7c3ec1afde2e9835baf3496947c60ba9','4656e89acc20b8eaee5641b23b629086','a2b86340418fee8aa295d8fb221171d3','afa93dadcd4498f4d363c32014485521','022cc9a77c1a1a833ed57516ca6f3425','6987562d04c43431d2699d9495edb30d','4b1bfd66454ae3aa02c4581f4759869d','5a9f088a1b08cb0a63a4e6a022fdb5a3','7117460581c7f6da7346aa4c44fe507e','8767cc477c04ae23d0c496862ce6956a','20ff52d8e1d90957c1312151f1596b00','dbcb3551b04ca3e111c7f37c44e5e845','7579646a457c5bf93a8baa92a3514aeb','487107a871058f5915a4ae7b45b1a915','b559ba4af7bcffa0d55f1ce8bbaf6f92','49625df0dc41ed0acbfa7e23250d7bd8','9f077a3b8bbada471a93f0c1cad402a1','ea9c7e4f3de01b3fdbc05f173967df5c','0313a46497e49bd8d61564e18f9d7cc6','128ad113535eb77a706b708ea30d7e68','7dc90ca785146beb143e7cce863449f7','39ffefa98e85cd5f77662ad1f5543571','61291be0571a1a55b1e68daaf6441ba1','fa1e71289817e5e31bb3e09a9864289a','68eb352a2f09069d265bec86512774a6','a7f6ad2cc7d8fd314ce84e85ce91000c','b3f9408bc2d2c0bdd9a30705f4d1d6a3','11f10caade409b1dec31eeb5f149c473','d530e05ef00877f6e5d2ab721efdb9d1','73dafa3bb81af03e576a605dd6cccac1','b508cad66d0a96d2fdde656d70bc0fd0','7cdcdbf711d655743c6e47a259472802','7d935f6eaad1c9ab9dc263b8ed79d5d4','bd9b751be0538741475e2c19552d898b','206abe57fef4cd8e24074fbb3fedab35','73272072c60e43da28db4385b60e863d','8b6d8aab3141da188b13d1a0e59e1337','191edfc049de83980e4dd835cc2189ef','c1167f17258ba23334432c1439d35132','2a819cd72c5c3caf931bf882cc9f0453','f5f0bcb30bd550d8f7c1ec0594b0bb23','cd24dddceb539910ab28d9a9f456bd51','4e4f24d461995c8d891f4241543e934a','7ee4d02b83545f9b56e778612269b537','a557a576b2d66c6599d3741de7321eb5','f38269433b8d6ada1ae3ab2fca2eefb3','c24829bdbbfc58191afcdd2a612bc163','0f7e77cf643e56dfe9c5108cf4bc3eb8','ff2ffa3b25b9378cb643c670b76fa723','1f04101fccd6b7f1e408f599679bfa73','e7d37cee4d268b4e12ef5b334ec78735','79f04b926e24509ed9f755e917960055','330860408c0c482df2273cbf5bd18d05','a2e4bb9f8c44d9c18d01f95427aaa383','51a59399225f5aefdd8ad64547201f39','8f0b7ba04ed105410b1111c4096b11e8','b5e8fbb23e7b5c8777a87d75a981a2e6','a52bcc3424fbdf1af6f17ec84581dcab','d3983f2bc9457a1561ff31777e4b99a1','888ae7dfa118c39080bb12114add833f','82ad2e3bdc45d73bdecce75aa537b0ee','1bac382829da6d95fe27a75a983a139c','f0613815bc38cceb96174961d9fba008','2b0d9eeb260b2cd3b7a8f1e702ff4fad','db0bb1324f02cd60018b74bd0f4ca26a','e4478fa658af76b03ee1e8d797090d48','2dd0ec30379745c769f97671c810ff54','6d5962e5a3f0d1f9bba0e8bb83f51ffa','de15622a194b17a541eb4a6bdead7d24','68c02e403040a36cbb1160b7e8e3cac2','dc9596acb0f5c48b2fb719cce6088fa4','f58b819820e3c67517f6c0c15e99e4b4','02b94c96512201a8d90ff82f167a19cc','4ce13d4345eeccc73a5ca927b1a9829a','cc22b13e776990619e0f56af3d5b0633','293166d00186e76323b6c5557c9c1872','423cb99d876a08245ce188a06e4e4465','e871935d348e12d484207b735301941c','a884099024a803700e5a4622ea3b5f1d','f4ce59a72c4615d2e50a5e82b889f68a','698b510ef71b98cf1ceab74ce6b2a690','1c215d2015ea970396372857671ead77','ef44552e78d1c66bde2416e7fd997898','d51aa880280be75088f816f59208ae5e','8693658fa105339b6320b1686ff5fe43','6e770a0c7c546e7e088050713ea2d26a','5b08c1959b715256457bde5e81702801','89097611b618d97570be283fd6d20047','9408af678890b6db3cd480771ebaa8ae','4aa1afa50cf88b589bf4258f7269cc50','c9a14fcf2d40d38916421ecd51f44ad5','fab28f0d5ee56b0b1754588cca9e3bfb','d57f668b8c4ead60a9c96177c28cf06f','2b1a5c8392c68cbca130de19a9b42a62','db2fad74a533aa3b1b2538336da05f48','51c66a84679bf57f079023d507feaffd','4953267d88759df1e0a8d47d9b20fdfa','d37025e7027bfca09473ddb2daac266b','b982e63e365245b38e6ad62ded487c09','5525c0793bf8006441422f0f023b8e15','ceb769e5e1b4fd93f5821f6abe44437e','247310bd96bd1936746dbafd33740545','a15fa443633d597ca18592211d8fdd89','fdd21bf0765db8d54cbe3d1cc3d20331','6891fa8480fb26f644d5cb4fdb0cd48f','53c5c132e70fda272712dcb193ef96c9','5eed1d36fcfd7ebbba74a6c063aea3a8','7dde784299770261b6b09f396642c22a','aa4066b39f501311afe389b500584981','9245d8a4314ad4891e123c0bf67fa746','2f08eec10bee8c8a1b8336357fa13234','a41210275f08ec8b8b973a4f10a73e5f','5dcaa6c87cc0ab92a41efe8e3baf9251','c209205c78da55e0a3d2e737d431ef99','dada49cbb3b038352bb28ddb85edb3af','2e48ead9343e2760deabc0cf8ba70a03','bfac577e556d4807b395eda9e703bb09','11cb455388172e3790c09136e673a91a','9c873a1cd94cf77e53e5f586c41c03ae','4be7446fa7dbf6b106192dbadff58951','c1a28a36b8dfd8eb1bbd11bbddcabd74','dd03eb31170caca4beae5c3f649029d2','a44806fc438db068b27a0d67da81c6f5','8db174dd5f09a7566ae6e06af1a4852f','b66320b229b7075c3a089a4d20746a76','bd3483cef35f344045cc91afb8cfb814','eea658ba171d084495d5940f67be2103','a88c703f49126df3ccc0d5cb72cdfa9a','be1051923e31ef890b435d51dcfe2ed6','29a66bb6edf6c26fbfca84887cc670d1','dd78d7c38d7d4b0a05d8b3a70ee17605','e9665b0f0d32c855a068c2cdc51abf30','13cf847289425291e88f58cf033cd94b','23f85c64c8c478ef8f143e94d8336b9b','cf57a9e5322ec39fc078b8557028cf53','bc31f1c7ce2c80c5fd45c9cf6f3da4b8','55c58af192c1f997013a9fb00efb1836','9667626f899f77c7e59b0a68eba8d697','b6e58e51e5241e71a417b059ed475bea','5c8eb6ca98aa73fb77a72be4e3b6a86e','e8d984aaef86fcfd972c6f39dfee799e','f4ac1b83a65bfcea6b17959913615bf1','3be5dbf421834e9c47ff92883abe8afb','4727352081852c4a80dcfb4abb1d99f1','82e7eb1a1f98e630a49c791ff0e3edac','fee6167bebf7ebca60b68fad976be9cf','1975d5d280c34f8ce92ea44555d93370','7ab3baa845bdf80ac3d1d7dd40ce174b','5118272c3318f558e978593a2d8b6f8e','c82962876a5825eac41b1a1a4b45ae7c','d6be7b5ec50dacd4e8faf45c78e4b7c9','4946bf21633b6e42de135a984ea9c3a4','acf9ec3aeff850a23a2aa4b29225b724','e3db0053cbef4cbf8cd9b0299a30682e','d0c53568267890ab83022764c6f8cbee','99cc46e77eca8395cb86b56edc7f39e0','197d5a46f343f5e0c9cf50b3381c7723','fa4f0706fdb32dd54415f327a6ab3e4c','d4a5062901e946d4f5a405858353e492','041c40fdc1010ec844705e255038224b','9aa6aa7223b6bc0601333105ddbcab3e','9a1c666a3fcb59a8310daf1538b4361f','e20703146c0d44b92d32739ea3f8aaa4','a5bcfba9d0cf74ffe2c525a4b5ca0c54','4079f6db0c33d3a32cc905ad16619cee','8d8e975b02d536ca6ab0aa08ff5cf6de','432f761acde3dd186319d6114478ba59','b93a3da8c323eb2a92a27c1ae5e89678','2f54339cef65e3f7ce2a7abba261af3c','f47f9c2414461f23dc96e6705bee06af','91e142ac5acc863a9d2cc3f6e4cab6cc','7ea1f4c5c206733962d30a307faff1fe','e2434d90d10f86f14cf44ff8fcf17f79','e36414d513a26c8f07a263e7113dcc0c','8175a67d1b2514460c478bd9710eef6b','e6ddc5a58836079ed3508a2c13e8983a','ad8ebcebba7d93fc0b3296def5ce4787','362076285506ceaf88faebc20d678ca0','2d78b506d393d1bf9d0ab1c9ec81c09f','0787d7124d2b4b935eebe55d658c6b0b','8e7a04a52680651561955c964135c6ee','3b57b4e1c005681ee0defe3d2dee11cd','9d0c6876fab0c7dfc2c67ef98eec3d86','8a2a0ccf64231381b1a805d121672e09','83c78127bcf446dd9e77f81ebd5bddaa','e67c1f9e87503ddf34aec56ecf399258','db88e976a8ba767cee91266d712a3400','9ccbfdbcfd4d8cddc1d0e19730db3501','bf08108b295f42c22f1d44e290a07096','40e0eaab4b828298882df0742369d9fa','e007c3e179a81af09f49334c4a22d271','57ece467f58757347489d28fb514cd72','9032b9f220c46a3f17addb6bc3a1125a','c8229394601f818bcd6b6c8fc19acea0','892f9299a3d94c48b4a18fea09cc5c53','c3983fbebc1392121517291b44dedd18','18c3e0e74b497fa573ba4129b7d0ae1a','73ec3f7a2047525a15a52924d4f1814f','6a9fe548a4d63e4503385bc1d1e870d7','12a0e990b719b20f469c6b9b54f84697','3203ec69becfa6c4c755287fb77b5308','bcc9a9cd21db99ef93d9b0fec890e101','8acce0552fcc99965b4d5d0e1f40f8d5','f1f6dfce4427bf4041f19c3c30322086','1c88bbc0a1b6c48134481672d16d838f','b719308b48ec0c385090d1f744af69c1','1b7044e1618b5103d32d3a4b3b438c25','c390092f35c283074e71929abaf65e76','2aedac2b218cb321f663607abf437e68','e3ad3e5fa368c8af5852061325a38b79','0b27182d98863ca08c30e3d6749e9e4f','3c130d2f5192cc22229911c8015a64dd','afcee7c667345f49a7388b9ddc9cf802','1cf8af23d9f0431366a145bf05cbb4b3','9319eacd9a9a6fe9535454ff32cf9701','4dc2ce6b188ad5ef7779aa622ba92190','5615b7fa2b1b58fb48f46310dd8cc1aa','fbc22bdaf4ebeb5da4c22627e6acf6ec','2b24fa85d1628919a41a241b1362e4c9','25979b5d4b8ae104540b2cba2ff19deb','02af759c030665f182e6b157c957f3e5','ec7203d0d805fdaaf8cb4f14e19b49dd','bc0dbe8cb5a41863740f75ef430493e4','cdb9d686238b2e39e035cd6c60eeee05','b83dd76cd64eeb8f0b3bc1d6c78870c5','e9105794d925c005e196b5a2830e681b','6e17855c3ecd5610530a4e8888ee89c8','346f5e97bb82b6ee9eaf88a66128842c','50e7b564cdcf33f589518aa7995a0164','08a2f7fa882357d26c960b66ca8d683b','8c56ebea8e65cf435698976d61b9aac6','21671c1891a9949fa04999ac3bdbe491','38e0200890d4ce2a0b058895e07659b0','29f25c10c6387500f776506784e9380c','5050812dd7a7f1f1dadcb2123909b355','83e48054cd3646dd291004a9ebcdcd3d','fdb37f39ff3e511aea2c207ec335d038','1089fe8a4ae8b6480b9a536eae02c430','464c2db1693a77919fc682a7886a13c8','bc69e8c128575ecc4effbcc5b38b42da','bf89ecec15504bc01f0910566e3f6e11','d792fbd80cc03965e1d83b9b67488b34','92315a59bf783476cd1f2cccbe9e72af','cf9c464f559e572194c2b7f9eff3bf7a','e452d1550db8ad45de197a3faba30c96','533e135bf9b844f3dd4ec99c21122ba2','6acb00967732624e0008ecbc7f588da3','00914a993a36962fd8c152f1b0d0ecc8','4dcbb6b4daeb2b0941942e1f26d4d795','9ee2b9e99513ab4426fcb56df01f5062','24521c04102180515c7477bf2840a6f1','5724b195239f9b3195cfde92ac6a002b','714cc72100389cc7d3bfcb4a9dbd80d6','10a3de7b5bc209d4d8d4cf6d3f59a501','6a4ab76c6be1ffa0e3a684699da72b33','18d3ab8ee037a44285aa9b52946c0534','11b9fbfd98fd3ad1fe6182790483d63a','6928eb9989dec67c67b704e5d79ddcad','454dfd895337a0765ab03aefe966c9cf','03c8bab975d47e1eb72fea59bce66847','ae8cb69060045b9ad29ed001c68bccf8','4c5b78fa6b1fcabdc1633da3b5de9c6c','a72196008d91aa85142c389f3b70472a','b61545057c96f025787d2bc1d47b5ccd','0b4d0e70559d4c89bae6f742bbc48971','ebc5f7f81d08df4dafd2a8706a2eb699','50b48828ea6bbf4611c64b17dffe0d5f','72737cc4d161d2c0d0d0219f0a01c2bc','342dcc61bf57ea2fe9fab26a6ef0e6b2','a1f9ac1fa9bcbbadc2a3e56f09f779f9','ea0214f956f2b22140d1be019475d9bb','ae13d63b53fd42606d859961ae1926ee','195788fc0fde1a93244a038745aeeab8','ba50094c10e7fb02c5c0fb52ce4cd137','a20e105da9bdd918826e80d1c38665a9','82a4f28241b6860b7203e60aa6af91ad','983cd2f13d767adcbcb44aa97a569c9a','a0e4ca28164edf14e08dab69034599fe','96778e71b734a7bb50218ce58dbfd5b7','398400024fda58e293cdb435db3c777e','673520f8fb45f2533c99819106e9d24d','42e10053358328c9079f1c8181287b6d','766511c1cfccca517ef60cf4b4e4c840','866e811c63867e0e5464c2bbcb71d249','397287454c9b2ec22ada423d31f7c193','c86f79aa3d3bec19a7f0db08bc28199d','60d6ccfaeceae23631838bd277fc76cf','38a95843d34333c3a4cef9c0f5235719','72de0b1f1a5c07db9c45d2466ef481b8','faa49365776fdf14a2a4299808c8f15d','3efa37a477e27d382b87388b799f717f','e7fee514c9d1b26cbbb89e96f63b2c61','67d29a816e4b47ca0dff056b24f0b089','14ce6b6d1c9bbbb5f3b98c68c6963c22','0ee530673a49069017498ef6309ac5d6','df2984f4f83af7438ede14813a93ea3e','cc0d0e9edf94c1228868492f4d57313f','9e2296e880a2f74203ec3ce7d9536851','9548fee30b48e47ffeb54d31f73b12ca','264045332cce86b210990070a663905b','6e3c7ae142f2ac7706d493b39fdb7131','76ca8965a3aecbe1dc3edef85d30e520','a80592541176399f60945dfd0ef0a9ba','7e1aefaf4c82b189ccb07af877093eff','14ac99c8ef9551a3368013c88cbf35fc','1d78ff597ab6bea21e2c4eedec9a9017','cea26e46eb466fc7b53ccc9d23d6d6e1','b53f1546a164d7bcdfe149b81d5fb0cc','da901ed6390bfd74d9997a42af7767c0','b46d3b632734fad4cab1c2e5a04079dd','f6e06cbb6ba8ebf9d9e092a0e5490542','bee0053d99e40c2d21d40b4a61e32a52','3dd775bf35c474edab81e57c6d2c4d45','151141149b5d48f0d6d974b5942fd48f','899747da78628edf3043bbe07e63bbce','c0e8a60ae59473cde2fd8ae3beadd650','222bbed027acb87e4ab778f23b094885','78de090f5be0fb88adb2c0bd99587f87','c6242bb47af2b897230d7dc54b2cc89a','c7f13a8f7d201661dfcd3cff24e05589','ba195429f60c3b2a2b47bc71a0e18469','f9f310730c8f6b6d6caafa8c2a370d39','8eb9d2cab5bc51c1dc40af28caa472ea','18c59507393134877004306b586838fd','aa8d768cf44215c2b02a04e862b92772','1f9f281fc088ca48814377c84a907595','ef59e8449b57c825ae92cc117e50c549','a6a8cda11ae804e4a573b2c6b373bc7f','4e33815b93f5a7ea54056f842887c34e','4a8cccaa80798f3227993087ddd15b8c','35eb2c73106e2b4e83ab3f9697c1d796','43c961b8336e9d89ea58c386ede24362','fc594ffa0a84f7daa133ebf621a2f52f','a04548f86c797ad0d3f323446ef7f56d','a77d2d5dda6ad3fb33f61aa90d14c9fa','6bdf8db27d92ee591a86beb83dffc003','67e479b967cc207486dba2d7cb67e293','552746b529ce32fdc71ef76cc5ad57cb','1234da7e4c4d1cfa28906eb8868b2677','f76c98038e54b8fcfba104b97f96e0ad','705ea06de55b4d0409b9b765e8496de4','ab2ca0c94b0a33d8b9f212db0688d684','f84d8b0cd604cb9515e766ba577b2eba','f8c8a1f1327f503ce80c20d63c956fcf','cd874e7937e43a2f30e8bc501bd16ea9','b91dfe7e209aff6f30b3bff8403a6a55','72f1f8206ac12be1459a93881bb17e09','c28a8071b83bf75956a6e91f802bc27c','a7fcf01519c735db0192d955980c1d70','30735f9718a63969a9b170f29b764f68','5c64ffa03bc54c7be1047dcddb0bcf64','12d5f609f4587f60ce994d7f5ebaa96d','b8c39aba8d59dd4cc8a2310e694eb03c','04eb70c54ac12fbe55ca8dcef6067568','5b6dbfb8054f888a5815cccb3288cbe1','47a3c8ab6c89b7f8d8eb07504aafa3ef','284e03850a2699d8c4a83c77b18808db','ef0ddfdb2384ab8777548ce964551b23','a88bde09086235ef8b2267da30b405f8','12824460f185889c4e3055bededef174','b50026c8129113dde0770d2e0d3ab060','abd84703211a6c1c59109b4766ddcb74','e1d848d9b62b5f20aa37d32b183bcaf0','0d3ea4c2d086ce0fac03603ad0373b0d','f77519da0320c42b210b9feab85106f8','b424887e3382f3f014979b458c29650f','c5aa590ed3707f1cc152927b63e6a5cf','a94eb599074f81c35e61cc10d81f0277','96eb77de82e1e37f371c1f7b87d35764','f62814efe26f796908748d6c6560b129','5ab51f1136ec2fe5ecb00922682ea863','1dde1bdbaf81546b933b376415cb721b','d38652890fbba62525deeeda4348fa5c','7e70db995fae6722174c920aaac07762','ef776be97c2b51a98af3792d8a2ab52a','44ea4cc65a156d7d64ddaab03cf0d0ea','96ed13b2e9274cca2ffe4377fbf1d395','c67614d72829d5fbc9ded2486e2059d7','d414d0e4cdf56f990084851640d00e8f','9ecf00e85a89ea75c3bd3f3a3f2574f9','9d69380019b1a67a0b86b436e45c393d','f574305e74e12c6997b054cf4f589d55','666a819d19f19a796ee3449c49003e90','f97f3e4eabc1290c37d47b9d7b6956e0','4ab0bb86f585a8ed33fe4e96a71623bb','d11309edee5ea1bdf6a38b78fdc1eaec','c670ed92fdf6646587e3bfb039ad7562','1265a8447f877c6aade66fcce62a2cf5','65b40cc0e859e89f412677ffd0a5325e','9b7d65a50d238fc6b54dd903a869f6c8','d230b563120d9c7bb8bf572bcf762feb','76d1b35f45981f3b525f4bf83c1d4e00','6879c5df064245a3a1c206764f9e2cc7','de3bba0297508db522fb742c8d4810b7','7c9a16ba7d6f2c7385af8075df002704','6b2798d5d8cdea42e493da6d1f41d06e','64428df5793edc0eeed3345cf3fa235e','74a9cdc682dac6c063b0888f2362be0a','7b1d172fd191a303c22233c7869861bd','70235b2ce237c089f5523a631b7a7f66','642cd2ca03bf65515ceb0ee73a13fa39','a23694f6c19be01836d946b686c46748','bc72c73b76cc675f79635b0234fdcb15','efc3bfb513b3bb43585870f17a1956b8','86e6c1490b18ff2b655a54dc9921a916','22fc478d549f8c4bc3adb6e4c5d144ac','9f3f2f4f92103d1cf522d51e1ebade28','0a19bfb66824dcc3f861cd9563588ba3','1484eb41c8cf90b59052aa2b67fae58f','3d65db6b100f3e08cd145bcffbca616f','01032a25826d3dccf58dbc749478cd30','331c5397e2f3b48514dd00bde4d1e095','4609a4059d208143467295b2692315ac','dfe12b0279d7cf37155b4f3ef51be391','458be4251b5b01477a71ce0f94efe717','f4f217a06a56d565fb4dcb44c7791edf','3e7ab6a4c1b841638e185b23996119ab','3e871980c57397e19e71e9ff771c274a','b001f240471d7b13c63a50433c257cd6','27668d6b200ddd3a868a3cbea063beb6','e08c04cfadda55713ac255016bfe9eb6','bc84f4c2a1a7b3d403cb1cacf53f0834','a1bec3d09eaa960ec6d98b0c4b6f7249')" diff --git a/sql/uuid.ddl b/sql/uuid.ddl deleted file mode 100644 index 9d9359b..0000000 --- a/sql/uuid.ddl +++ /dev/null @@ -1,15 +0,0 @@ -CREATE TABLE hm_uuids -( - HMUUID VARCHAR(35) PRIMARY KEY, - DOI_SUFFIX VARCHAR(10) NULL UNIQUE, - ENTITY_TYPE VARCHAR(20) NOT NULL, - PARENT_UUID VARCHAR(35) NULL, - TIME_GENERATED TIMESTAMP NOT NULL, - USER_ID VARCHAR(50) NOT NULL, - USER_EMAIL VARCHAR(50) NULL, - HUBMAP_ID VARCHAR(170) NULL -); - -CREATE UNIQUE INDEX UUID_IDX ON hm_uuids (HMUUID); -CREATE UNIQUE INDEX DOI_IDX ON hm_uuids (DOI_SUFFIX); -CREATE UNIQUE INDEX HM_ID_IDX on hm_uuids (HUBMAP_ID); From 50578da6c39c3ff80cd6a530107662c5cbda3b7f Mon Sep 17 00:00:00 2001 From: yuanzhou Date: Tue, 10 Nov 2020 13:44:00 -0500 Subject: [PATCH 04/37] Use uuids.sql --- docker/hubmap-mysql/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/hubmap-mysql/Dockerfile b/docker/hubmap-mysql/Dockerfile index 8377dff..b687f5a 100644 --- a/docker/hubmap-mysql/Dockerfile +++ b/docker/hubmap-mysql/Dockerfile @@ -5,4 +5,4 @@ LABEL description="HuBMAP MySQL 5.6 for UUID API Service" \ version="1.1" # Copy from host to image -COPY ./uuids-dev.sql /docker-entrypoint-initdb.d/uuids-dev.sql +COPY ./uuids.sql /docker-entrypoint-initdb.d/uuids.sql From f43678db8431f2d30f88fe76f65b80f3cb536332 Mon Sep 17 00:00:00 2001 From: yuanzhou Date: Tue, 17 Nov 2020 11:23:22 -0500 Subject: [PATCH 05/37] Remove refactor mysql --- docker/docker-compose.refactor.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/docker/docker-compose.refactor.yml b/docker/docker-compose.refactor.yml index 297914f..ee5f21f 100644 --- a/docker/docker-compose.refactor.yml +++ b/docker/docker-compose.refactor.yml @@ -14,19 +14,3 @@ services: - "../BUILD:/usr/src/app/src/BUILD" # Mount the source code to container - "../src:/usr/src/app/src" - - # Only used by local development - hubmap-mysql: - build: ./hubmap-mysql - # Build the image with name and tag - image: hubmap-mysql:1.1 - hostname: hubmap-mysql - container_name: hubmap-mysql - environment: - MYSQL_ROOT_PASSWORD: 123 - # Use the same port mapping for dev and prod - ports: - - "3306:3306" - networks: - - gateway_hubmap - \ No newline at end of file From fcb4ed96417a07d892af8674b76bd7fa1e7868af Mon Sep 17 00:00:00 2001 From: yuanzhou Date: Mon, 7 Dec 2020 11:04:41 -0500 Subject: [PATCH 06/37] Handle 404 for id query --- docker/docker-compose.localhost.yml | 28 ++++++++++++++-------------- src/app.py | 5 +++++ 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/docker/docker-compose.localhost.yml b/docker/docker-compose.localhost.yml index 01143e3..e4a9e92 100644 --- a/docker/docker-compose.localhost.yml +++ b/docker/docker-compose.localhost.yml @@ -13,20 +13,20 @@ services: # Mount the source code to container - "../src:/usr/src/app/src" - # Only used by local development - hubmap-mysql: - build: ./hubmap-mysql - # Build the image with name and tag - image: hubmap-mysql:1.1 - hostname: hubmap-mysql - container_name: hubmap-mysql - environment: - MYSQL_ROOT_PASSWORD: 123 - # Use the same port mapping for dev and prod - ports: - - "3306:3306" - networks: - - gateway_hubmap + # # Only used by local development + # hubmap-mysql: + # build: ./hubmap-mysql + # # Build the image with name and tag + # image: hubmap-mysql:1.1 + # hostname: hubmap-mysql + # container_name: hubmap-mysql + # environment: + # MYSQL_ROOT_PASSWORD: 123 + # # Use the same port mapping for dev and prod + # ports: + # - "3306:3306" + # networks: + # - gateway_hubmap diff --git a/src/app.py b/src/app.py index dd04e8a..dbf6977 100644 --- a/src/app.py +++ b/src/app.py @@ -129,6 +129,11 @@ def get_hmuuid(hmuuid): try: if request.method == "GET": info = worker.getIdInfo(hmuuid) + + # In Python, empty sequences (strings, lists, tuples) are false + if not info: + return Response ("Could not find the target id: " + hmuuid, 404) + return info else: return Response ("Invalid request use GET to retrieve UUID information", 500) From ed32a8736e23a68f8b1b4a5df2a14fe82da65f8a Mon Sep 17 00:00:00 2001 From: yuanzhou Date: Mon, 7 Dec 2020 11:15:29 -0500 Subject: [PATCH 07/37] Response 404 for empty id info --- src/app.py | 5 ----- src/uuid_worker.py | 6 +++++- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/app.py b/src/app.py index dbf6977..dd04e8a 100644 --- a/src/app.py +++ b/src/app.py @@ -129,11 +129,6 @@ def get_hmuuid(hmuuid): try: if request.method == "GET": info = worker.getIdInfo(hmuuid) - - # In Python, empty sequences (strings, lists, tuples) are false - if not info: - return Response ("Could not find the target id: " + hmuuid, 404) - return info else: return Response ("Invalid request use GET to retrieve UUID information", 500) diff --git a/src/uuid_worker.py b/src/uuid_worker.py index 52ca7ac..37ec706 100644 --- a/src/uuid_worker.py +++ b/src/uuid_worker.py @@ -446,7 +446,11 @@ def getIdInfo(self, hmid): with closing(dbConn.cursor()) as curs: curs.execute(sql) results = [dict((curs.description[i][0], value) for i, value in enumerate(row)) for row in curs.fetchall()] - + + # In Python, empty sequences (strings, lists, tuples) are false + if not results: + return Response ("Could not find the target id: " + hmid, 404) + return json.dumps(results, indent=4, sort_keys=True, default=str) def uuidExists(self, hmid): From 42379e0a1bd995a5029e1acbcca0dfc0d50cd000 Mon Sep 17 00:00:00 2001 From: yuanzhou Date: Mon, 7 Dec 2020 11:18:42 -0500 Subject: [PATCH 08/37] Fix tab --- src/uuid_worker.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/uuid_worker.py b/src/uuid_worker.py index 37ec706..955172f 100644 --- a/src/uuid_worker.py +++ b/src/uuid_worker.py @@ -446,11 +446,11 @@ def getIdInfo(self, hmid): with closing(dbConn.cursor()) as curs: curs.execute(sql) results = [dict((curs.description[i][0], value) for i, value in enumerate(row)) for row in curs.fetchall()] - - # In Python, empty sequences (strings, lists, tuples) are false - if not results: - return Response ("Could not find the target id: " + hmid, 404) - + + # In Python, empty sequences (strings, lists, tuples) are false + if not results: + return Response ("Could not find the target id: " + hmid, 404) + return json.dumps(results, indent=4, sort_keys=True, default=str) def uuidExists(self, hmid): From 95b4e7ce2f836196e58febaba8d0916011956d59 Mon Sep 17 00:00:00 2001 From: Bill Shirey Date: Wed, 9 Dec 2020 14:46:36 -0500 Subject: [PATCH 09/37] new ddl for uuid schema update for generating submission ids --- .gitignore | 1 + reload_from_neo4j/app.cfg.example | 11 +++++++++++ reload_from_neo4j/reload_from_neo.py | 4 ++++ sql/uuid.ddl | 28 +++++++++++++++++++++++++--- 4 files changed, 41 insertions(+), 3 deletions(-) create mode 100644 reload_from_neo4j/app.cfg.example create mode 100644 reload_from_neo4j/reload_from_neo.py diff --git a/.gitignore b/.gitignore index 53d8c4c..d7f5e8c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Ignore config files src/instance/app.cfg +app.cfg BUILD # mounted BUILD/VERSION file diff --git a/reload_from_neo4j/app.cfg.example b/reload_from_neo4j/app.cfg.example new file mode 100644 index 0000000..4a43cba --- /dev/null +++ b/reload_from_neo4j/app.cfg.example @@ -0,0 +1,11 @@ +# Globus App ID and secret +APP_CLIENT_ID = '' +APP_CLIENT_SECRET = '' + +# MySQL connection (default value used for docker dev environment) +# Point to remote MySQL for testing and production deployment +DB_HOST = 'hubmap-mysql' +DB_NAME = 'hm_uuid' +DB_USERNAME = 'root' +DB_PASSWORD = '123' + diff --git a/reload_from_neo4j/reload_from_neo.py b/reload_from_neo4j/reload_from_neo.py new file mode 100644 index 0000000..920176d --- /dev/null +++ b/reload_from_neo4j/reload_from_neo.py @@ -0,0 +1,4 @@ +#Standalone script to reload, from scratch, the UUID relational database +#This script reads all Nodes in a HuBMAP Neo4j database and stores all id +#information in the UUID database + diff --git a/sql/uuid.ddl b/sql/uuid.ddl index 9d9359b..ae80818 100644 --- a/sql/uuid.ddl +++ b/sql/uuid.ddl @@ -1,15 +1,37 @@ CREATE TABLE hm_uuids ( HMUUID VARCHAR(35) PRIMARY KEY, - DOI_SUFFIX VARCHAR(10) NULL UNIQUE, + HUBMAP_BASE_ID VARCHAR(10) NULL UNIQUE, ENTITY_TYPE VARCHAR(20) NOT NULL, PARENT_UUID VARCHAR(35) NULL, TIME_GENERATED TIMESTAMP NOT NULL, USER_ID VARCHAR(50) NOT NULL, USER_EMAIL VARCHAR(50) NULL, - HUBMAP_ID VARCHAR(170) NULL + SUBMISSION_ID VARCHAR(170) NULL, + DESCENDANT_COUNT INT NULL ); +CREATE TABLE hm_ancestors +( + ANCESTOR_UUID VARCHAR(35) NOT NULL, + DESCENDANT_UUID VARCHAR(35) NOT NULL +); + +CREATE TABLE hm_organs +( + HM_UUID VARCHAR(35) PRIMARY KEY, + ORGAN_CODE VARCHAR(8) +); + +CREATE TABLE hm_data_centers +( + DC_UUID VARCHAR(40) PRIMARY KEY, + DONOR_COUNT INT NOT NULL DEFAULT 0 +) + CREATE UNIQUE INDEX UUID_IDX ON hm_uuids (HMUUID); -CREATE UNIQUE INDEX DOI_IDX ON hm_uuids (DOI_SUFFIX); +CREATE UNIQUE INDEX HM_BASE_IDX ON hm_uuids (HM_BASE_IDX); CREATE UNIQUE INDEX HM_ID_IDX on hm_uuids (HUBMAP_ID); +CREATE INDEX ANC_UUID on hm_ancestors (ANCESTOR_UUID); +CREATE INDEX DEC_UUID on hm_ancestors (DESCENDANT_UUID); +CREATE UNIQUE INDEX ANC_DEC_UUID on hm_ancestors (ANCESTOR_UUID, DESCENDANT_UUID); From 4a7c5f493ec2bc23c0df5a82bf16a24f4ce04bcb Mon Sep 17 00:00:00 2001 From: "Zhou (Joe) Yuan" Date: Fri, 18 Dec 2020 09:51:04 -0500 Subject: [PATCH 10/37] Bump version for test-release workflow --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 9dbb0c0..943f9cb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.7.0 \ No newline at end of file +1.7.1 From f4e5d8663f85d195f2a74b7ee5ba964003a359b2 Mon Sep 17 00:00:00 2001 From: yuanzhou Date: Fri, 8 Jan 2021 11:01:27 -0500 Subject: [PATCH 11/37] Simplify BUILD VERSION path --- docker/docker-compose.dev.yml | 4 ++-- docker/docker-compose.localhost.yml | 4 ++-- docker/docker-compose.refactor.yml | 4 ++-- docker/uuid-api-docker.sh | 14 +++++++------- src/app.py | 4 ++-- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index d560e17..c6ef89a 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -10,8 +10,8 @@ services: restart: always volumes: # Mount the VERSION file and BUILD file - - "../VERSION:/usr/src/app/src/VERSION" - - "../BUILD:/usr/src/app/src/BUILD" + - "../VERSION:/usr/src/app/VERSION" + - "../BUILD:/usr/src/app/BUILD" # Mount the source code to container - "../src:/usr/src/app/src" \ No newline at end of file diff --git a/docker/docker-compose.localhost.yml b/docker/docker-compose.localhost.yml index e4a9e92..b332191 100644 --- a/docker/docker-compose.localhost.yml +++ b/docker/docker-compose.localhost.yml @@ -8,8 +8,8 @@ services: - HOST_UID=${HOST_UID:-1000} volumes: # Mount the VERSION file and BUILD file - - "../VERSION:/usr/src/app/src/VERSION" - - "../BUILD:/usr/src/app/src/BUILD" + - "../VERSION:/usr/src/app/VERSION" + - "../BUILD:/usr/src/app/BUILD" # Mount the source code to container - "../src:/usr/src/app/src" diff --git a/docker/docker-compose.refactor.yml b/docker/docker-compose.refactor.yml index ee5f21f..4fd3976 100644 --- a/docker/docker-compose.refactor.yml +++ b/docker/docker-compose.refactor.yml @@ -10,7 +10,7 @@ services: restart: always volumes: # Mount the VERSION file and BUILD file - - "../VERSION:/usr/src/app/src/VERSION" - - "../BUILD:/usr/src/app/src/BUILD" + - "../VERSION:/usr/src/app/VERSION" + - "../BUILD:/usr/src/app/BUILD" # Mount the source code to container - "../src:/usr/src/app/src" diff --git a/docker/uuid-api-docker.sh b/docker/uuid-api-docker.sh index 65dcf7e..7acc941 100755 --- a/docker/uuid-api-docker.sh +++ b/docker/uuid-api-docker.sh @@ -99,17 +99,17 @@ else # On test/stage/prod, copy the VERSION file and BUILD file to image if [[ "$1" != "localhost" && "$1" != "dev" && "$1" != "refactor" ]]; then # Delete old VERSION and BUILD files if found - if [ -f "uuid-api/src/VERSION" ]; then - rm -rf uuid-api/src/VERSION + if [ -f "uuid-api/VERSION" ]; then + rm -rf uuid-api/VERSION fi - if [ -f "uuid-api/src/BUILD" ]; then - rm -rf uuid-api/src/BUILD + if [ -f "uuid-api/BUILD" ]; then + rm -rf uuid-api/BUILD fi - # Copy over the one files - cp ../VERSION uuid-api/src - cp ../BUILD uuid-api/src + # Copy over the BUILD and VERSION files + cp ../VERSION uuid-api + cp ../BUILD uuid-api fi docker-compose -f docker-compose.yml -f docker-compose.$1.yml -p uuid-api build diff --git a/src/app.py b/src/app.py index dd04e8a..bc1c229 100644 --- a/src/app.py +++ b/src/app.py @@ -68,8 +68,8 @@ def status(): response_data = { # Use strip() to remove leading and trailing spaces, newlines, and tabs - 'version': (Path(__file__).parent / 'VERSION').read_text().strip(), - 'build': (Path(__file__).parent / 'BUILD').read_text().strip(), + 'version': (Path(__file__).absolute().parent.parent / 'VERSION').read_text().strip(), + 'build': (Path(__file__).absolute().parent.parent / 'BUILD').read_text().strip(), 'mysql_connection': False } From 4f56b6915a725ad0a7cc5cd8dcb19db0d3e089eb Mon Sep 17 00:00:00 2001 From: Bill Shirey Date: Fri, 8 Jan 2021 14:34:26 -0500 Subject: [PATCH 12/37] remove unneeded update method --- src/uuid_worker.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/uuid_worker.py b/src/uuid_worker.py index 7717bcf..f23481d 100644 --- a/src/uuid_worker.py +++ b/src/uuid_worker.py @@ -22,7 +22,7 @@ INSERT_SQL = "INSERT INTO hm_uuids (HM_UUID, HUBMAP_BASE_ID, ENTITY_TYPE, TIME_GENERATED, USER_ID, USER_EMAIL) VALUES (%s, %s, %s, %s, %s, %s)" INSERT_SQL_WITH_SUBMISSION_ID = "INSERT INTO hm_uuids (HM_UUID, HUBMAP_BASE_ID, ENTITY_TYPE, TIME_GENERATED, USER_ID, USER_EMAIL, SUBMISSION_ID) VALUES (%s, %s, %s, %s, %s, %s,%s)" INSERT_ANCESTOR_SQL = "INSERT INTO hm_ancestors (DESCENDANT_UUID, ANCESTOR_UUID) VALUES (%s, %s)" -UPDATE_SQL = "UPDATE hm_uuids set hubmap_id = %s where HMUUID = %s" +#UPDATE_SQL = "UPDATE hm_uuids set hubmap_id = %s where HMUUID = %s" HMID_ALPHA_CHARS=['B','C','D','F','G','H','J','K','L','M','N','P','Q','R','S','T','V','W','X','Z'] HMID_NUM_CHARS=['2','3','4','5','6','7','8','9'] @@ -206,6 +206,8 @@ def uuidGen(self): hexVal = hexVal.lower() return hexVal + + ''' remove not needed def updateUUIDs(self, uuids, display_ids): if len(uuids) != len(display_ids): raise Exception("The number of uuids must match the number of display ids") @@ -251,6 +253,7 @@ def updateUUIDs(self, uuids, display_ids): with closing(dbConn.cursor()) as curs: curs.executemany(UPDATE_SQL, updateVals) dbConn.commit() + ''' def __create_submission_ids(self, num_to_gen, parent_id, entity_type, organ_code = None, lab_code = None): parent_id = parent_id.strip().lower() From 956ceba6bf3d8e8d8838bd6a277c73a745f6717d Mon Sep 17 00:00:00 2001 From: yuanzhou Date: Fri, 8 Jan 2021 21:16:47 -0500 Subject: [PATCH 13/37] Use globus_groups instead of Provenance --- reload_from_neo4j/reload_from_neo.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/reload_from_neo4j/reload_from_neo.py b/reload_from_neo4j/reload_from_neo.py index 67f8cbf..ff4375b 100644 --- a/reload_from_neo4j/reload_from_neo.py +++ b/reload_from_neo4j/reload_from_neo.py @@ -15,7 +15,9 @@ import mysql from datetime import datetime from hubmap_commons.string_helper import isBlank -from hubmap_commons.provenance import Provenance + +# Deprecate the use of Provenance +#from hubmap_commons.provenance import Provenance UUID_DB_TABLE_NAMES = ['hm_uuids', 'hm_ancestors', 'hm_organs', 'hm_data_centers'] LABS_CYPHER = "match (l:Lab) return l.uuid as labid, l.next_identifier, l.submission_id as lab_code" @@ -318,7 +320,13 @@ def test(self, lab_id): if result is None: prov_helper = Provenance("a", "b", "c") try: - lab = prov_helper.get_group_by_identifier(check_id) + # Deprecate the use of Provenance, use the new globus_groups module - Zhou + #lab = prov_helper.get_group_by_identifier(check_id) + + # Get the globus groups info based on the groups json file in commons package + globus_groups_info = globus_groups.get_globus_groups_info() + groups_by_tmc_prefix_dict = globus_groups_info['by_tmc_prefix'] + lab = groups_by_tmc_prefix_dict[check_id] except ValueError: return Response("") if not 'tmc_prefix' in lab: From ba17ebd853b0f679e6242048175dcb0e6ca3d13d Mon Sep 17 00:00:00 2001 From: yuanzhou Date: Fri, 8 Jan 2021 21:24:11 -0500 Subject: [PATCH 14/37] Fix the use of Provenance in uuid_worker --- src/uuid_worker.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/uuid_worker.py b/src/uuid_worker.py index f23481d..d5d9384 100644 --- a/src/uuid_worker.py +++ b/src/uuid_worker.py @@ -11,7 +11,9 @@ # HuBMAP commons from hubmap_commons.string_helper import isBlank, listToCommaSeparated, padLeadingZeros from hubmap_commons.hm_auth import AuthHelper -from hubmap_commons.provenance import Provenance + +# Deprecate the use of Provenance +#from hubmap_commons.provenance import Provenance SUBMISSION_ID_ENTITY_TYPES = ['SAMPLE', 'DONOR'] ANCESTOR_REQUIRED_ENTITY_TYPES = ['SAMPLE', 'DONOR', 'DATASET'] @@ -84,7 +86,9 @@ def __init__(self, clientId, clientSecret, dbHost, dbName, dbUsername, dbPasswor self.dbPassword = dbPassword self.lock = threading.RLock() self.hmdb = DBConn(self.dbHost, self.dbUsername, self.dbPassword, self.dbName) - self.prov_helper = Provenance(clientId, clientSecret, None) + + # Deprecate the use of Provenance + #self.prov_helper = Provenance(clientId, clientSecret, None) def __resolve_lab_id(self, lab_id, user_id, user_email): if isBlank(lab_id): @@ -97,7 +101,13 @@ def __resolve_lab_id(self, lab_id, user_id, user_email): result = curs.fetchone() if result is None: try: - lab = self.prov_helper.get_group_by_identifier(check_id) + # Deprecate the use of Provenance + #lab = self.prov_helper.get_group_by_identifier(check_id) + + # Get the globus groups info based on the groups json file in commons package + globus_groups_info = globus_groups.get_globus_groups_info() + groups_by_tmc_prefix_dict = globus_groups_info['by_tmc_prefix'] + lab = groups_by_tmc_prefix_dict[check_id] except ValueError: return Response("A valid lab with specified id not found id:" + check_id, 400) From 0b04860a5f9930e8a8fdf129de76f75a20663517 Mon Sep 17 00:00:00 2001 From: yuanzhou Date: Fri, 8 Jan 2021 21:26:54 -0500 Subject: [PATCH 15/37] Add missing import globus_groups --- reload_from_neo4j/reload_from_neo.py | 3 +++ src/uuid_worker.py | 1 + 2 files changed, 4 insertions(+) diff --git a/reload_from_neo4j/reload_from_neo.py b/reload_from_neo4j/reload_from_neo.py index ff4375b..5f8d71a 100644 --- a/reload_from_neo4j/reload_from_neo.py +++ b/reload_from_neo4j/reload_from_neo.py @@ -14,7 +14,10 @@ from contextlib import closing import mysql from datetime import datetime + +# HuBMAP commons from hubmap_commons.string_helper import isBlank +from hubmap_commons import globus_groups # Deprecate the use of Provenance #from hubmap_commons.provenance import Provenance diff --git a/src/uuid_worker.py b/src/uuid_worker.py index d5d9384..327f00d 100644 --- a/src/uuid_worker.py +++ b/src/uuid_worker.py @@ -11,6 +11,7 @@ # HuBMAP commons from hubmap_commons.string_helper import isBlank, listToCommaSeparated, padLeadingZeros from hubmap_commons.hm_auth import AuthHelper +from hubmap_commons import globus_groups # Deprecate the use of Provenance #from hubmap_commons.provenance import Provenance From 0814017f67e138ff95e8b56b977e2473002b5275 Mon Sep 17 00:00:00 2001 From: yuanzhou Date: Fri, 8 Jan 2021 21:28:53 -0500 Subject: [PATCH 16/37] Convert tab to 4 spaces for consistent indentation --- src/hmdb.py | 130 ++--- src/uuid_worker.py | 1240 ++++++++++++++++++++++---------------------- 2 files changed, 685 insertions(+), 685 deletions(-) diff --git a/src/hmdb.py b/src/hmdb.py index e2e9b6e..b0c7d44 100644 --- a/src/hmdb.py +++ b/src/hmdb.py @@ -1,71 +1,71 @@ import mysql.connector #pip install mysql-connector-python from contextlib import closing from mysql.connector.errors import OperationalError -class DBConn: - def getDBConnection(self): - try: - #if self._db is None: - # self._openDBConnection(server=self.server, user=self.user, password=self.password, dbName=self.dbName) - cnx = mysql.connector.connect(pool_name="hm_uuid_db") - with closing(cnx.cursor()) as curs: - curs = cnx.cursor() - curs.execute("SELECT VERSION()") - results = curs.fetchone() - if not results: - raise Exception("Database connection test failed") - except OperationalError: - try: - if not self._openDBConnection(server=self.server, user=self.user, password=self.password, dbName=self.dbName): - raise Exception("Database reconnection test failed") - cnx = mysql.connector.connect(pool_name="hm_uuid_db") - except Exception as e: - raise e - #return self._db - return cnx +class DBConn: + def getDBConnection(self): + try: + #if self._db is None: + # self._openDBConnection(server=self.server, user=self.user, password=self.password, dbName=self.dbName) + cnx = mysql.connector.connect(pool_name="hm_uuid_db") + with closing(cnx.cursor()) as curs: + curs = cnx.cursor() + curs.execute("SELECT VERSION()") + results = curs.fetchone() + if not results: + raise Exception("Database connection test failed") + except OperationalError: + try: + if not self._openDBConnection(server=self.server, user=self.user, password=self.password, dbName=self.dbName): + raise Exception("Database reconnection test failed") + cnx = mysql.connector.connect(pool_name="hm_uuid_db") + except Exception as e: + raise e + #return self._db + return cnx - def _openDBConnection(self, server, user, password, dbName): - self._db = None - try: - #if self._dbpool is None: - #dbargs = { - # "host": server, - # "user": user, - # "password": password, - # "database": dbName - #} - #self._dbpool = mysql.connector.pooling.MySQLConnectionPool(pool_name = "hm-mysql-pool", pool_size = 10, **dbargs) - - cnx = mysql.connector.connect(pool_name ="hm_uuid_db", pool_size = 8, host=server, user=user, password=password, database=dbName) + def _openDBConnection(self, server, user, password, dbName): + self._db = None + try: + #if self._dbpool is None: + #dbargs = { + # "host": server, + # "user": user, + # "password": password, + # "database": dbName + #} + #self._dbpool = mysql.connector.pooling.MySQLConnectionPool(pool_name = "hm-mysql-pool", pool_size = 10, **dbargs) + + cnx = mysql.connector.connect(pool_name ="hm_uuid_db", pool_size = 8, host=server, user=user, password=password, database=dbName) - with closing(cnx.cursor()) as curs: - curs = cnx.cursor() - curs.execute("SELECT VERSION()") - results = curs.fetchone() - # Check if anything at all is returned - cnx.close() - if results: - return True - else: - return False - except Exception as e: - raise e - - return False + with closing(cnx.cursor()) as curs: + curs = cnx.cursor() + curs.execute("SELECT VERSION()") + results = curs.fetchone() + # Check if anything at all is returned + cnx.close() + if results: + return True + else: + return False + except Exception as e: + raise e + + return False - def __init__(self, dbHost, username, password, dbName): - try: - #dbConnected = self._openDBConnection(dbHost, username, password, dbName) - self._openDBConnection(dbHost, username, password, dbName) - self.server = dbHost - self.user = username - self.password = password - self.dbName = dbName - except Exception as e: - raise Exception("Error opening database connection for " + username + "@" + dbHost + " on " + username + "@" + dbName + "\n" + str(e)) - #if not dbConnected: - # raise Exception("Error connecting to " + dbName + " at " + username + "@" + dbHost) - - def __del__(self): - if not self._db is None: - self._db.close() - + def __init__(self, dbHost, username, password, dbName): + try: + #dbConnected = self._openDBConnection(dbHost, username, password, dbName) + self._openDBConnection(dbHost, username, password, dbName) + self.server = dbHost + self.user = username + self.password = password + self.dbName = dbName + except Exception as e: + raise Exception("Error opening database connection for " + username + "@" + dbHost + " on " + username + "@" + dbName + "\n" + str(e)) + #if not dbConnected: + # raise Exception("Error connecting to " + dbName + " at " + username + "@" + dbHost) + + def __del__(self): + if not self._db is None: + self._db.close() + diff --git a/src/uuid_worker.py b/src/uuid_worker.py index 327f00d..da0df3f 100644 --- a/src/uuid_worker.py +++ b/src/uuid_worker.py @@ -33,639 +33,639 @@ #UUID_SELECTS = "HMUUID as hmuuid, DOI_SUFFIX as doiSuffix, ENTITY_TYPE as type, PARENT_UUID as parentId, TIME_GENERATED as timeStamp, USER_ID as userId, HUBMAP_ID as hubmapId, USER_EMAIL as email" UUID_SELECTS = "HM_UUID as hm_uuid, HUBMAP_BASE_ID as hubmap_base_id, ENTITY_TYPE as type, TIME_GENERATED as time_generated, USER_ID as user_id, SUBMISSION_ID as submission_id, USER_EMAIL as email, GROUP_CONCAT(ancestor_uuid) as ancestor_ids" -def startsWithComponentPrefix(hmid): - tidl = hmid.strip().lower() - if tidl.startswith('test') or tidl.startswith('van') or tidl.startswith('ufl') or tidl.startswith('stan') or tidl.startswith('ucsd') or tidl.startswith('calt') or tidl.startswith('rtibd') or tidl.startswith('rtige') or tidl.startswith('rtinw') or tidl.startswith('rtist') or tidl.startswith('ttdct') or tidl.startswith('ttdhv') or tidl.startswith('ttdpd') or tidl.startswith('ttdst'): - return True - else: - return False +def startsWithComponentPrefix(hmid): + tidl = hmid.strip().lower() + if tidl.startswith('test') or tidl.startswith('van') or tidl.startswith('ufl') or tidl.startswith('stan') or tidl.startswith('ucsd') or tidl.startswith('calt') or tidl.startswith('rtibd') or tidl.startswith('rtige') or tidl.startswith('rtinw') or tidl.startswith('rtist') or tidl.startswith('ttdct') or tidl.startswith('ttdhv') or tidl.startswith('ttdpd') or tidl.startswith('ttdst'): + return True + else: + return False def isValidHMId(hmid): - if isBlank(hmid): return False - if startsWithComponentPrefix(hmid): - return True - tid = stripHMid(hmid) - l = len(tid) - if not (l == 10 or l == 32): return False - tid = tid.upper() - if l == 10: - if not set(tid[0:3]).issubset(HMID_NUM_CHARS): return False - if not set(tid[3:7]).issubset(HMID_ALPHA_CHARS): return False - if not set(tid[7:]).issubset(HMID_NUM_CHARS): return False - if l == 32: - if not set(tid).issubset(HEX_CHARS): return False - return True + if isBlank(hmid): return False + if startsWithComponentPrefix(hmid): + return True + tid = stripHMid(hmid) + l = len(tid) + if not (l == 10 or l == 32): return False + tid = tid.upper() + if l == 10: + if not set(tid[0:3]).issubset(HMID_NUM_CHARS): return False + if not set(tid[3:7]).issubset(HMID_ALPHA_CHARS): return False + if not set(tid[7:]).issubset(HMID_NUM_CHARS): return False + if l == 32: + if not set(tid).issubset(HEX_CHARS): return False + return True def stripHMid(hmid): - if isBlank(hmid): return hmid - thmid = hmid.strip(); - if thmid.lower().startswith('hbm'): thmid = thmid[3:] - if thmid.startswith(':'): thmid = thmid[1:] - return thmid.strip().replace('-', '').replace('.', '').replace(' ', '') + if isBlank(hmid): return hmid + thmid = hmid.strip(); + if thmid.lower().startswith('hbm'): thmid = thmid[3:] + if thmid.startswith(':'): thmid = thmid[1:] + return thmid.strip().replace('-', '').replace('.', '').replace(' ', '') class UUIDWorker: - authHelper = None - - def __init__(self, clientId, clientSecret, dbHost, dbName, dbUsername, dbPassword): - if clientId is None or clientSecret is None or isBlank(clientId) or isBlank(clientSecret): - raise Exception("Globus client id and secret are required in AuthHelper") - - if not AuthHelper.isInitialized(): - self.authHelper = AuthHelper.create(clientId=clientId, clientSecret=clientSecret) - else: - self.authHelper.instance() - - #Open the config file - self.logger = logging.getLogger('uuid.service') - - self.dbHost = dbHost - self.dbName = dbName - self.dbUsername = dbUsername - self.dbPassword = dbPassword - self.lock = threading.RLock() - self.hmdb = DBConn(self.dbHost, self.dbUsername, self.dbPassword, self.dbName) - - # Deprecate the use of Provenance - #self.prov_helper = Provenance(clientId, clientSecret, None) - - def __resolve_lab_id(self, lab_id, user_id, user_email): - if isBlank(lab_id): - return None - check_id = lab_id.strip().lower() - r_val = {} - with closing(self.hmdb.getDBConnection()) as dbConn: - with closing(dbConn.cursor()) as curs: - curs.execute("select hm_uuid, dc_code from hm_data_centers where hm_uuid = '" + check_id + "' or dc_uuid = '" + check_id + "'") - result = curs.fetchone() - if result is None: - try: - # Deprecate the use of Provenance - #lab = self.prov_helper.get_group_by_identifier(check_id) - - # Get the globus groups info based on the groups json file in commons package + authHelper = None + + def __init__(self, clientId, clientSecret, dbHost, dbName, dbUsername, dbPassword): + if clientId is None or clientSecret is None or isBlank(clientId) or isBlank(clientSecret): + raise Exception("Globus client id and secret are required in AuthHelper") + + if not AuthHelper.isInitialized(): + self.authHelper = AuthHelper.create(clientId=clientId, clientSecret=clientSecret) + else: + self.authHelper.instance() + + #Open the config file + self.logger = logging.getLogger('uuid.service') + + self.dbHost = dbHost + self.dbName = dbName + self.dbUsername = dbUsername + self.dbPassword = dbPassword + self.lock = threading.RLock() + self.hmdb = DBConn(self.dbHost, self.dbUsername, self.dbPassword, self.dbName) + + # Deprecate the use of Provenance + #self.prov_helper = Provenance(clientId, clientSecret, None) + + def __resolve_lab_id(self, lab_id, user_id, user_email): + if isBlank(lab_id): + return None + check_id = lab_id.strip().lower() + r_val = {} + with closing(self.hmdb.getDBConnection()) as dbConn: + with closing(dbConn.cursor()) as curs: + curs.execute("select hm_uuid, dc_code from hm_data_centers where hm_uuid = '" + check_id + "' or dc_uuid = '" + check_id + "'") + result = curs.fetchone() + if result is None: + try: + # Deprecate the use of Provenance + #lab = self.prov_helper.get_group_by_identifier(check_id) + + # Get the globus groups info based on the groups json file in commons package globus_groups_info = globus_groups.get_globus_groups_info() groups_by_tmc_prefix_dict = globus_groups_info['by_tmc_prefix'] lab = groups_by_tmc_prefix_dict[check_id] - except ValueError: - return Response("A valid lab with specified id not found id:" + check_id, 400) - - if not 'tmc_prefix' in lab: - return Response("Lab with specified id:" + check_id + " does not contain a tmc_prefix.", 400) - - uuid_json = self.newUUIDs([], "LAB", user_id, user_email, 1, gen_base_ids = False) - uuid_info = json.loads(uuid_json) - r_val['dc_code'] = lab['tmc_prefix'] - r_val['hm_uuid'] = uuid_info[0]['uuid'] - curs.execute("insert into hm_data_centers (HM_UUID, DC_UUID, DC_CODE) VALUES ('" + r_val['hm_uuid'] + "','" + check_id + "','" + r_val['dc_code'] + "')") - dbConn.commit() - else: - r_val['dc_code'] = result[1] - r_val['hm_uuid'] = result[0] - return r_val - - def uuidPost(self, req, nIds): - userInfo = self.authHelper.getUserInfoUsingRequest(req) - if isinstance(userInfo, Response): - return userInfo; - - if not 'sub' in userInfo: - return Response("Unable to get user id (sub) via introspection", 400) - - userId = userInfo['sub'] - userEmail = None - if 'email' in userInfo: - userEmail = userInfo['email'] - - if not req.is_json: - return(Response("Invalid input, json required.", 400)) - content = req.get_json() - if content is None or len(content) <= 0: - return(Response("Invalid input, uuid attributes required", 400)) - if not 'entity_type' in content or isBlank(content['entity_type']): - return(Response("entity_type is a required attribute", 400)) - - entityType = content['entity_type'].upper().strip() - organ_code = None - if 'organ_code' in content and not isBlank(content['organ_code']): - if not entityType == 'SAMPLE': - return(Response("Organ code " + content['organ_code'] + " found for entity type of " + entityType + ", but SAMPLE entity type required to specify organ.", 400)) - organ_code = content['organ_code'].strip().upper() - - - parentIds = None - if('parent_ids' in content): - parentIds = content['parent_ids'] - - if(entityType in ANCESTOR_REQUIRED_ENTITY_TYPES and parentIds is None): - return(Response("parentId is a required attribute for entities " + ", ".join(ANCESTOR_REQUIRED_ENTITY_TYPES), 400)) - - lab_code = None - - if not parentIds is None: - if(entityType in SUBMISSION_ID_ENTITY_TYPES): - n_parents = len(parentIds) - if n_parents != 1: - return(Response("Entity type " + entityType + " requires a single ancestor id, " + str(n_parents) + " provided.", 400)) - - if entityType == "DONOR": - ancestor_ids = [] - lab_info = self.__resolve_lab_id(parentIds[0], userId, userEmail) - if isinstance(lab_info, Response): - return lab_info - ancestor_ids.append(lab_info['hm_uuid']) - lab_code = lab_info['dc_code'] - elif parentIds is None: - ancestor_ids = [] - else: - ancestor_ids = parentIds - - for parentId in ancestor_ids: - if not self.uuid_exists(parentId): - return(Response("Parent id " + parentId + " does not exist", 400)) - - return self.newUUIDs(ancestor_ids, entityType, userId, userEmail, nIds, organ_code = organ_code, lab_code = lab_code) - - def uuidPut(self, req): - if not req.is_json: - return(Response("Invalid input, json required.", 400)) - content = req.get_json() - if content is None or len(content) <= 0: - return(Response("Invalid input, hubmap-uuids and display-ids arrays required", 400)) - if (not 'hubmap-uuids' in content) or content['hubmap-uuids'] is None or (not isinstance(content['hubmap-uuids'], list)) or (not len(content['hubmap-uuids']) > 0): - return(Response("hubmap-uuids is a required attribute", 400)) - if (not 'display-ids' in content) or content['display-ids'] is None or (not isinstance(content['display-ids'], list) or (not len(content['display-ids']) > 0)): - return(Response("display-ids is a required attribute", 400)) - - uuids = content['hubmap-uuids'] - disp_ids = content['display-ids'] - - if len(uuids) != len(disp_ids): - return(Response("The length of the diplay-ids and hubmap-uuids arrays must be the same length", 400)) - - return self.updateUUIDs(uuids, disp_ids) - - def newUUIDTest(self, parentIds, entityType, userId, userEmail): - return self.newUUIDs(parentIds, entityType, userId, userEmail) - - def uuidGen(self): - hexVal = "" - for _ in range(32): - hexVal = hexVal + secrets.choice(HEX_CHARS) - hexVal = hexVal.lower() - return hexVal - - - ''' remove not needed - def updateUUIDs(self, uuids, display_ids): - if len(uuids) != len(display_ids): - raise Exception("The number of uuids must match the number of display ids") - - idSet = set() - for uuid in uuids: - uuidt = uuid.lower().strip() - if uuidt in idSet: - raise Exception("During UUID update the uuid " + uuid + " is contained multiple times in the hubmap-uuids array") - idSet.add(uuidt) - - idSet = set() - for dispId in display_ids: - disIdT = dispId.lower().strip() - if disIdT in idSet: - raise Exception("During UUID update the display id " + dispId + " is contained multiple times in the diplay-ids array") - idSet.add(disIdT) - - excluded = self.__findExclusionsInDB("HMUUID", uuids) - if(excluded is not None and len(excluded) > 0): - raise Exception("UUIDs not contained in the id database " + listToCommaSeparated(excluded)) - - #only id records that have a null display id can be updated - sql = "select hmuuid from hm_uuids where hmuuid in (" + listToCommaSeparated(uuids, "'", True) + ") and hubmap_id is not null" - with closing(self.hmdb.getDBConnection()) as dbConn: - with closing(dbConn.cursor()) as curs: - curs.execute(sql) - nonNull = curs.fetchall() - if nonNull is not None and len(nonNull) > 0: - raise Exception("Some uuids do not have null display ids: " + listToCommaSeparated(nonNull)) - - #if there are any display ids that match what we're updating to send an error - dupes = self.__findDupsInDB("HUBMAP_ID", display_ids) - if(dupes is not None and len(dupes) > 0): - raise Exception("Display ID(s) are not unique " + listToCommaSeparated(dupes)) - - updateVals = [] - for uuid, dispid in zip(uuids, display_ids): - updateRow = (dispid, uuid) - updateVals.append(updateRow) - - with closing(self.hmdb.getDBConnection()) as dbConn: - with closing(dbConn.cursor()) as curs: - curs.executemany(UPDATE_SQL, updateVals) - dbConn.commit() - ''' - - def __create_submission_ids(self, num_to_gen, parent_id, entity_type, organ_code = None, lab_code = None): - parent_id = parent_id.strip().lower() - with closing(self.hmdb.getDBConnection()) as dbConn: - with closing(dbConn.cursor()) as curs: - curs.execute("select entity_type, descendant_count, submission_id from hm_uuids where hm_uuid = '" + parent_id + "'") - results = curs.fetchone() - anc_entity_type = results[0] - desc_count = results[1] - anc_submission_id = results[2] - #a donor - if entity_type == 'DONOR': - if not anc_entity_type == 'LAB': - return Response("An id can't be created for a DONOR because a DATA CENTER is required as the direct ancestor and an ancestor of type " + anc_entity_type + " found.", 400) - if isBlank(lab_code): - return Response("No data center code found data center with uuid:" + parent_id + ". Unable to generate an id for a DONOR.", 400) - r_val = [] - for _ in range(0, num_to_gen): - desc_count = desc_count + 1 - r_val.append(lab_code.strip().upper() + padLeadingZeros(desc_count, 4)) - curs.execute("update hm_uuids set descendant_count = " + str(desc_count) + " where hm_uuid = '" + parent_id + "'") - dbConn.commit() - return r_val - #an organ - elif entity_type == 'SAMPLE' and anc_entity_type == 'DONOR': - if isBlank(organ_code): - return Response("An id can't be created for a SAMPLE because the immediate ancestor is a DONOR and the SAMPLE was not supplied with an associated organ code (SAMPLE must be an organ to have a DONOR as a direct ancestor)", 400) - organ_code = organ_code.strip().upper() - curs.execute("select organ_count from hm_organs where donor_uuid = '" + parent_id + "' and organ_code = '" + organ_code + "'") - org_res = curs.fetchone() - if org_res is None: - curs.execute("insert into hm_organs (DONOR_UUID, ORGAN_CODE, ORGAN_COUNT) VALUE ('" + parent_id + "', '" + organ_code + "', 0)") - dbConn.commit() - org_count = 0 - else: - org_count = org_res[0] - if not organ_code in MULTIPLE_ALLOWED_ORGANS: - if org_count >= 1: - return Response("Cannot add another organ of type " + organ_code + " to DONOR " + parent_id + " exists already.", 400) - if num_to_gen > 1: - return Response("Cannot create multiple submission ids for organ of type " + organ_code + ". " + str(num_to_gen) + " requested.") - org_count = 1 - r_val = [anc_submission_id + "-" + organ_code] - else: - r_val = [] - for _ in range(0, num_to_gen): - org_count = org_count + 1 - r_val.append(anc_submission_id + "-" + organ_code + padLeadingZeros(org_count, 2)) - - curs.execute("update hm_organs set organ_count = " + str(org_count) + " where donor_uuid = '" + parent_id + "' and organ_code = '" + organ_code + "'") - dbConn.commit() - return r_val - - #error if remaining non-organ samples are not the descendants of a SAMPLE - elif entity_type == 'SAMPLE' and not anc_entity_type == 'SAMPLE': - return Response("Cannot create a submission id for a SAMPLE with a direct ancestor of " + anc_entity_type) - elif entity_type == 'SAMPLE': - r_val = [] - for _ in range(0, num_to_gen): - desc_count = desc_count + 1 - r_val.append(anc_submission_id + "-" + str(desc_count)) - curs.execute("update hm_uuids set descendant_count = " + str(desc_count) + " where hm_uuid = '" + parent_id + "'") - dbConn.commit() - return r_val - else: - return Response("Cannot create a submission id for an entity of type " + entity_type) - - #generate multiple ids, one for each display id in the displayIds array - def newUUIDs(self, parentIDs, entityType, userId, userEmail, nIds, organ_code = None, lab_code = None, gen_base_ids = True): - #if entityType == 'DONOR': - - returnIds = [] - now = time.strftime('%Y-%m-%d %H:%M:%S') - with self.lock: - #generate in batches - previousUUIDs = set() - previous_hubmap_ids = set() - - gen_submission_ids = False - if entityType in SUBMISSION_ID_ENTITY_TYPES: - gen_submission_ids = True - - for i in range(0, nIds, MAX_GEN_IDS): - insertVals = [] - insertParents = [] - numToGen = min(MAX_GEN_IDS, nIds - i) - #generate uuids - uuids = self.__nUniqueIds(numToGen, self.uuidGen, "HM_UUID", previousGeneratedIds=previousUUIDs) - if gen_base_ids: - hubmap_base_ids = self.__nUniqueIds(numToGen, self.hmidGen, "HUBMAP_BASE_ID", previousGeneratedIds=previous_hubmap_ids) - else: - hubmap_base_ids = [None] * numToGen - - count_increase_q = None - submission_ids = None - if gen_submission_ids: - submission_ids = self.__create_submission_ids(numToGen, parentIDs[0], entityType, organ_code = organ_code, lab_code = lab_code) - if isinstance(submission_ids, Response): - return submission_ids - - for n in range(0, numToGen): - insUuid = uuids[n] - previousUUIDs.add(insUuid) - thisId = {"uuid":insUuid} - - if gen_base_ids: - ins_hubmap_base_id = hubmap_base_ids[n] - previous_hubmap_ids.add(ins_hubmap_base_id) - ins_display_hubmap_id = self.__display_hm_id(ins_hubmap_base_id) - thisId["hubmap_base_id"] = ins_hubmap_base_id - thisId["hubmap_id"] = ins_display_hubmap_id - else: - ins_hubmap_base_id = None - - if gen_submission_ids: - thisId["submission_id"] = submission_ids[n] - insRow = (insUuid, ins_hubmap_base_id, entityType, now, userId, userEmail, submission_ids[n]) - else: - insRow = (insUuid, ins_hubmap_base_id, entityType, now, userId, userEmail) - - returnIds.append(thisId) - insertVals.append(insRow) - - for parentId in parentIDs: - parRow = (insUuid, parentId) - insertParents.append(parRow) - - with closing(self.hmdb.getDBConnection()) as dbConn: - with closing(dbConn.cursor()) as curs: - if gen_submission_ids: - curs.executemany(INSERT_SQL_WITH_SUBMISSION_ID, insertVals) - curs.execute(count_increase_q) - else: - curs.executemany(INSERT_SQL, insertVals) - curs.executemany(INSERT_ANCESTOR_SQL, insertParents) - dbConn.commit() - - return json.dumps(returnIds) - - def nUniqueIds(self, nIds, idGenMethod, dbColumn): - return self.__nUniqueIds(nIds, idGenMethod, dbColumn) - - #generate unique ids - #generates ids with provided id generation method and checks them against existing ids in the DB - #this method MUST BE CALLED FROM WITHIN a self.lock block - def __nUniqueIds(self, nIds, idGenMethod, dbColumn, previousGeneratedIds=set(), iteration=1): - ids = set() - lclPreviousIds = copy.deepcopy(previousGeneratedIds) - for _ in range(nIds): - newId = idGenMethod() - count = 1 - while (newId in ids or newId in lclPreviousIds) and count < 100: - newId = idGenMethod() - count = count + 1 - if count == 100: - raise Exception("Unable to generate an initial unique id for " + dbColumn + " after 100 attempts.") - ids.add(newId) - lclPreviousIds.add(newId) - dupes = self.__findDupsInDB(dbColumn, ids) - if dupes is not None and len(dupes) > 0: - n_iter = iteration + 1 - if n_iter > 100: - raise Exception("Unable to generate unique id(s) for " + dbColumn + " after 100 attempts.") - replacements = self.__nUniqueIds(len(dupes), idGenMethod, dbColumn, previousGeneratedIds=lclPreviousIds, iteration=n_iter) - for val in dupes: - ids.remove(val[0]) - for val in replacements: - ids.add(val) - return list(ids) - - def __findDupsInDB(self, dbColumn, idSet): - sql = "select " + dbColumn + " from hm_uuids where " + dbColumn + " IN(" + listToCommaSeparated(idSet, "'", True) + ")" - with closing(self.hmdb.getDBConnection()) as dbConn: - with closing(dbConn.cursor()) as curs: - curs.execute(sql) - dupes = curs.fetchall() - - return dupes - - - - #which items in idSet are not in the database - def __findExclusionsInDB(self, dbColumn, idSet): - - sql = "select " + dbColumn + " from ( " - first = True - for ex_id in idSet: - if first: first = False - else: sql = sql + " UNION ALL " - sql = sql + "(select '" + ex_id + "' as " + dbColumn + ")" - sql = sql + ") as list left join hm_uuids using (" + dbColumn + ") where hm_uuids." + dbColumn + " is null" - with closing(self.hmdb.getDBConnection()) as dbConn: - with closing(dbConn.cursor()) as curs: - curs.execute(sql) - excluded = curs.fetchall() - - return excluded - - def __display_hm_id(self, hm_base_id): - hubmap_id = 'HBM' + hm_base_id[0:3] + '.' + hm_base_id[3:7] + '.' + hm_base_id[7:] - return hubmap_id - - def hmidGen(self): - nums1 = '' - nums2 = '' - alphs = '' - for _ in range(3): - nums1 = nums1 + secrets.choice(HMID_NUM_CHARS) #[random.randint(0,len(DOI_NUM_CHARS)-1)] - for _ in range(3): - nums2 = nums2 + secrets.choice(HMID_NUM_CHARS) #[random.randint(0,len(DOI_NUM_CHARS)-1)] - for _ in range(4): - alphs = alphs + secrets.choice(HMID_ALPHA_CHARS) #[random.randint(0,len(DOI_ALPHA_CHARS)-1)] - - val = nums1 + alphs + nums2 - return(val) - - def getIdExists(self, hmid): - if not isValidHMId(hmid): - return Response("Invalid HuBMAP Id", 400) - tid = stripHMid(hmid) - if startsWithComponentPrefix(hmid): - return self.submission_id_exists(hmid.strip()) - elif len(tid) == 10: - return self.base_id_exists(tid.upper()) - elif len(tid) == 32: - return self.uuid_exists(tid.lower()) - else: - return Response("Invalid HuBMAP Id (or empty or bad length)", 400) - - - #convert csv list of ancestor ids to a list - #convert hubmap base id to a hubmap id (display version) - def _convert_result_id_array(self, results, hmid): - if isinstance(results, list): - asize = len(results) - if asize == 0: - record = None - elif asize == 1: - record = results[0] - if not 'ancestor_ids' in record: - return record - ancestor_ids = record['ancestor_ids'] - if ancestor_ids is None or ancestor_ids.strip() == '': - record.pop('ancestor_ids', '') - elif isinstance(ancestor_ids, str): - record['ancestor_ids'] = ancestor_ids.split(',') - if len(record['ancestor_ids']) == 1: - record['ancestor_id'] = record['ancestor_ids'][0] - else: - raise Exception("Unknown ancestor type for id:" + hmid) - - if 'hubmap_base_id' in record: - if not record['hubmap_base_id'].strip() == '': - record['hubmap_id'] = self.__display_hm_id(record['hubmap_base_id']) - record.pop('hubmap_base_id', '') - else: - raise Exception("Multiple results exist for id:" + hmid) - - return record - - def getIdInfo(self, hmid): - if not isValidHMId(hmid): - return Response(hmid + " is not a valid id format", 400) - tidl = hmid.strip().lower() - tid = stripHMid(hmid) - if startsWithComponentPrefix(hmid): - sql = "select " + UUID_SELECTS + " from hm_uuids inner join hm_ancestors on hm_ancestors.descendant_uuid = hm_uuids.hm_uuid where lower(submission_id) ='" + tidl + "'" - elif len(tid) == 10: - sql = "select " + UUID_SELECTS + " from hm_uuids inner join hm_ancestors on hm_ancestors.descendant_uuid = hm_uuids.hm_uuid where hubmap_base_id ='" + tid + "'" - elif len(tid) == 32: - sql = "select " + UUID_SELECTS + " from hm_uuids inner join hm_ancestors on hm_ancestors.descendant_uuid = hm_uuids.hm_uuid where hm_uuid ='" + tid + "'" - else: - return Response("Invalid id (empty or bad length)", 400) - with closing(self.hmdb.getDBConnection()) as dbConn: - with closing(dbConn.cursor()) as curs: - curs.execute(sql) - results = [dict((curs.description[i][0], value) for i, value in enumerate(row)) for row in curs.fetchall()] - - # In Python, empty sequences (strings, lists, tuples) are false - if results is None or not results: - return Response ("Could not find the target id: " + hmid, 404) - if isinstance(results, list) and (len(results) == 0): - return Response ("Could not find the target id: " + hmid, 404) - if not 'hm_uuid' in results[0]: - return Response ("Could not find the target id: " + hmid, 404) - if results[0]['hm_uuid'] is None: - return Response ("Could not find the target id: " + hmid, 404) - - rdict = self._convert_result_id_array(results, hmid) - return json.dumps(rdict, indent=4, sort_keys=True, default=str) - - def uuid_exists(self, hmid): - with closing(self.hmdb.getDBConnection()) as dbConn: - with closing(dbConn.cursor()) as curs: - curs.execute("select count(*) from hm_uuids where hm_uuid = '" + hmid + "'") - res = curs.fetchone() - - if(res is None or len(res) == 0): return False - if(res[0] == 1): return True - if(res[0] == 0): return False - raise Exception("Multiple uuids found matching " + hmid) - -# def uuidsExist(self, uuids): -# uuid_list = '(' -# first = True -# comma = '' -# for uuid in uuids: -# uuid_list = uuid_list + comma + "'" + uuid.strip() + "'" -# if first: -# first = False -# comma = ',' -# uuid_list = uuid_list + ')' + except ValueError: + return Response("A valid lab with specified id not found id:" + check_id, 400) + + if not 'tmc_prefix' in lab: + return Response("Lab with specified id:" + check_id + " does not contain a tmc_prefix.", 400) + + uuid_json = self.newUUIDs([], "LAB", user_id, user_email, 1, gen_base_ids = False) + uuid_info = json.loads(uuid_json) + r_val['dc_code'] = lab['tmc_prefix'] + r_val['hm_uuid'] = uuid_info[0]['uuid'] + curs.execute("insert into hm_data_centers (HM_UUID, DC_UUID, DC_CODE) VALUES ('" + r_val['hm_uuid'] + "','" + check_id + "','" + r_val['dc_code'] + "')") + dbConn.commit() + else: + r_val['dc_code'] = result[1] + r_val['hm_uuid'] = result[0] + return r_val + + def uuidPost(self, req, nIds): + userInfo = self.authHelper.getUserInfoUsingRequest(req) + if isinstance(userInfo, Response): + return userInfo; + + if not 'sub' in userInfo: + return Response("Unable to get user id (sub) via introspection", 400) + + userId = userInfo['sub'] + userEmail = None + if 'email' in userInfo: + userEmail = userInfo['email'] + + if not req.is_json: + return(Response("Invalid input, json required.", 400)) + content = req.get_json() + if content is None or len(content) <= 0: + return(Response("Invalid input, uuid attributes required", 400)) + if not 'entity_type' in content or isBlank(content['entity_type']): + return(Response("entity_type is a required attribute", 400)) + + entityType = content['entity_type'].upper().strip() + organ_code = None + if 'organ_code' in content and not isBlank(content['organ_code']): + if not entityType == 'SAMPLE': + return(Response("Organ code " + content['organ_code'] + " found for entity type of " + entityType + ", but SAMPLE entity type required to specify organ.", 400)) + organ_code = content['organ_code'].strip().upper() + + + parentIds = None + if('parent_ids' in content): + parentIds = content['parent_ids'] + + if(entityType in ANCESTOR_REQUIRED_ENTITY_TYPES and parentIds is None): + return(Response("parentId is a required attribute for entities " + ", ".join(ANCESTOR_REQUIRED_ENTITY_TYPES), 400)) + + lab_code = None + + if not parentIds is None: + if(entityType in SUBMISSION_ID_ENTITY_TYPES): + n_parents = len(parentIds) + if n_parents != 1: + return(Response("Entity type " + entityType + " requires a single ancestor id, " + str(n_parents) + " provided.", 400)) + + if entityType == "DONOR": + ancestor_ids = [] + lab_info = self.__resolve_lab_id(parentIds[0], userId, userEmail) + if isinstance(lab_info, Response): + return lab_info + ancestor_ids.append(lab_info['hm_uuid']) + lab_code = lab_info['dc_code'] + elif parentIds is None: + ancestor_ids = [] + else: + ancestor_ids = parentIds + + for parentId in ancestor_ids: + if not self.uuid_exists(parentId): + return(Response("Parent id " + parentId + " does not exist", 400)) + + return self.newUUIDs(ancestor_ids, entityType, userId, userEmail, nIds, organ_code = organ_code, lab_code = lab_code) + + def uuidPut(self, req): + if not req.is_json: + return(Response("Invalid input, json required.", 400)) + content = req.get_json() + if content is None or len(content) <= 0: + return(Response("Invalid input, hubmap-uuids and display-ids arrays required", 400)) + if (not 'hubmap-uuids' in content) or content['hubmap-uuids'] is None or (not isinstance(content['hubmap-uuids'], list)) or (not len(content['hubmap-uuids']) > 0): + return(Response("hubmap-uuids is a required attribute", 400)) + if (not 'display-ids' in content) or content['display-ids'] is None or (not isinstance(content['display-ids'], list) or (not len(content['display-ids']) > 0)): + return(Response("display-ids is a required attribute", 400)) + + uuids = content['hubmap-uuids'] + disp_ids = content['display-ids'] + + if len(uuids) != len(disp_ids): + return(Response("The length of the diplay-ids and hubmap-uuids arrays must be the same length", 400)) + + return self.updateUUIDs(uuids, disp_ids) + + def newUUIDTest(self, parentIds, entityType, userId, userEmail): + return self.newUUIDs(parentIds, entityType, userId, userEmail) + + def uuidGen(self): + hexVal = "" + for _ in range(32): + hexVal = hexVal + secrets.choice(HEX_CHARS) + hexVal = hexVal.lower() + return hexVal + + + ''' remove not needed + def updateUUIDs(self, uuids, display_ids): + if len(uuids) != len(display_ids): + raise Exception("The number of uuids must match the number of display ids") + + idSet = set() + for uuid in uuids: + uuidt = uuid.lower().strip() + if uuidt in idSet: + raise Exception("During UUID update the uuid " + uuid + " is contained multiple times in the hubmap-uuids array") + idSet.add(uuidt) + + idSet = set() + for dispId in display_ids: + disIdT = dispId.lower().strip() + if disIdT in idSet: + raise Exception("During UUID update the display id " + dispId + " is contained multiple times in the diplay-ids array") + idSet.add(disIdT) + + excluded = self.__findExclusionsInDB("HMUUID", uuids) + if(excluded is not None and len(excluded) > 0): + raise Exception("UUIDs not contained in the id database " + listToCommaSeparated(excluded)) + + #only id records that have a null display id can be updated + sql = "select hmuuid from hm_uuids where hmuuid in (" + listToCommaSeparated(uuids, "'", True) + ") and hubmap_id is not null" + with closing(self.hmdb.getDBConnection()) as dbConn: + with closing(dbConn.cursor()) as curs: + curs.execute(sql) + nonNull = curs.fetchall() + if nonNull is not None and len(nonNull) > 0: + raise Exception("Some uuids do not have null display ids: " + listToCommaSeparated(nonNull)) + + #if there are any display ids that match what we're updating to send an error + dupes = self.__findDupsInDB("HUBMAP_ID", display_ids) + if(dupes is not None and len(dupes) > 0): + raise Exception("Display ID(s) are not unique " + listToCommaSeparated(dupes)) + + updateVals = [] + for uuid, dispid in zip(uuids, display_ids): + updateRow = (dispid, uuid) + updateVals.append(updateRow) + + with closing(self.hmdb.getDBConnection()) as dbConn: + with closing(dbConn.cursor()) as curs: + curs.executemany(UPDATE_SQL, updateVals) + dbConn.commit() + ''' + + def __create_submission_ids(self, num_to_gen, parent_id, entity_type, organ_code = None, lab_code = None): + parent_id = parent_id.strip().lower() + with closing(self.hmdb.getDBConnection()) as dbConn: + with closing(dbConn.cursor()) as curs: + curs.execute("select entity_type, descendant_count, submission_id from hm_uuids where hm_uuid = '" + parent_id + "'") + results = curs.fetchone() + anc_entity_type = results[0] + desc_count = results[1] + anc_submission_id = results[2] + #a donor + if entity_type == 'DONOR': + if not anc_entity_type == 'LAB': + return Response("An id can't be created for a DONOR because a DATA CENTER is required as the direct ancestor and an ancestor of type " + anc_entity_type + " found.", 400) + if isBlank(lab_code): + return Response("No data center code found data center with uuid:" + parent_id + ". Unable to generate an id for a DONOR.", 400) + r_val = [] + for _ in range(0, num_to_gen): + desc_count = desc_count + 1 + r_val.append(lab_code.strip().upper() + padLeadingZeros(desc_count, 4)) + curs.execute("update hm_uuids set descendant_count = " + str(desc_count) + " where hm_uuid = '" + parent_id + "'") + dbConn.commit() + return r_val + #an organ + elif entity_type == 'SAMPLE' and anc_entity_type == 'DONOR': + if isBlank(organ_code): + return Response("An id can't be created for a SAMPLE because the immediate ancestor is a DONOR and the SAMPLE was not supplied with an associated organ code (SAMPLE must be an organ to have a DONOR as a direct ancestor)", 400) + organ_code = organ_code.strip().upper() + curs.execute("select organ_count from hm_organs where donor_uuid = '" + parent_id + "' and organ_code = '" + organ_code + "'") + org_res = curs.fetchone() + if org_res is None: + curs.execute("insert into hm_organs (DONOR_UUID, ORGAN_CODE, ORGAN_COUNT) VALUE ('" + parent_id + "', '" + organ_code + "', 0)") + dbConn.commit() + org_count = 0 + else: + org_count = org_res[0] + if not organ_code in MULTIPLE_ALLOWED_ORGANS: + if org_count >= 1: + return Response("Cannot add another organ of type " + organ_code + " to DONOR " + parent_id + " exists already.", 400) + if num_to_gen > 1: + return Response("Cannot create multiple submission ids for organ of type " + organ_code + ". " + str(num_to_gen) + " requested.") + org_count = 1 + r_val = [anc_submission_id + "-" + organ_code] + else: + r_val = [] + for _ in range(0, num_to_gen): + org_count = org_count + 1 + r_val.append(anc_submission_id + "-" + organ_code + padLeadingZeros(org_count, 2)) + + curs.execute("update hm_organs set organ_count = " + str(org_count) + " where donor_uuid = '" + parent_id + "' and organ_code = '" + organ_code + "'") + dbConn.commit() + return r_val + + #error if remaining non-organ samples are not the descendants of a SAMPLE + elif entity_type == 'SAMPLE' and not anc_entity_type == 'SAMPLE': + return Response("Cannot create a submission id for a SAMPLE with a direct ancestor of " + anc_entity_type) + elif entity_type == 'SAMPLE': + r_val = [] + for _ in range(0, num_to_gen): + desc_count = desc_count + 1 + r_val.append(anc_submission_id + "-" + str(desc_count)) + curs.execute("update hm_uuids set descendant_count = " + str(desc_count) + " where hm_uuid = '" + parent_id + "'") + dbConn.commit() + return r_val + else: + return Response("Cannot create a submission id for an entity of type " + entity_type) + + #generate multiple ids, one for each display id in the displayIds array + def newUUIDs(self, parentIDs, entityType, userId, userEmail, nIds, organ_code = None, lab_code = None, gen_base_ids = True): + #if entityType == 'DONOR': + + returnIds = [] + now = time.strftime('%Y-%m-%d %H:%M:%S') + with self.lock: + #generate in batches + previousUUIDs = set() + previous_hubmap_ids = set() + + gen_submission_ids = False + if entityType in SUBMISSION_ID_ENTITY_TYPES: + gen_submission_ids = True + + for i in range(0, nIds, MAX_GEN_IDS): + insertVals = [] + insertParents = [] + numToGen = min(MAX_GEN_IDS, nIds - i) + #generate uuids + uuids = self.__nUniqueIds(numToGen, self.uuidGen, "HM_UUID", previousGeneratedIds=previousUUIDs) + if gen_base_ids: + hubmap_base_ids = self.__nUniqueIds(numToGen, self.hmidGen, "HUBMAP_BASE_ID", previousGeneratedIds=previous_hubmap_ids) + else: + hubmap_base_ids = [None] * numToGen + + count_increase_q = None + submission_ids = None + if gen_submission_ids: + submission_ids = self.__create_submission_ids(numToGen, parentIDs[0], entityType, organ_code = organ_code, lab_code = lab_code) + if isinstance(submission_ids, Response): + return submission_ids + + for n in range(0, numToGen): + insUuid = uuids[n] + previousUUIDs.add(insUuid) + thisId = {"uuid":insUuid} + + if gen_base_ids: + ins_hubmap_base_id = hubmap_base_ids[n] + previous_hubmap_ids.add(ins_hubmap_base_id) + ins_display_hubmap_id = self.__display_hm_id(ins_hubmap_base_id) + thisId["hubmap_base_id"] = ins_hubmap_base_id + thisId["hubmap_id"] = ins_display_hubmap_id + else: + ins_hubmap_base_id = None + + if gen_submission_ids: + thisId["submission_id"] = submission_ids[n] + insRow = (insUuid, ins_hubmap_base_id, entityType, now, userId, userEmail, submission_ids[n]) + else: + insRow = (insUuid, ins_hubmap_base_id, entityType, now, userId, userEmail) + + returnIds.append(thisId) + insertVals.append(insRow) + + for parentId in parentIDs: + parRow = (insUuid, parentId) + insertParents.append(parRow) + + with closing(self.hmdb.getDBConnection()) as dbConn: + with closing(dbConn.cursor()) as curs: + if gen_submission_ids: + curs.executemany(INSERT_SQL_WITH_SUBMISSION_ID, insertVals) + curs.execute(count_increase_q) + else: + curs.executemany(INSERT_SQL, insertVals) + curs.executemany(INSERT_ANCESTOR_SQL, insertParents) + dbConn.commit() + + return json.dumps(returnIds) + + def nUniqueIds(self, nIds, idGenMethod, dbColumn): + return self.__nUniqueIds(nIds, idGenMethod, dbColumn) + + #generate unique ids + #generates ids with provided id generation method and checks them against existing ids in the DB + #this method MUST BE CALLED FROM WITHIN a self.lock block + def __nUniqueIds(self, nIds, idGenMethod, dbColumn, previousGeneratedIds=set(), iteration=1): + ids = set() + lclPreviousIds = copy.deepcopy(previousGeneratedIds) + for _ in range(nIds): + newId = idGenMethod() + count = 1 + while (newId in ids or newId in lclPreviousIds) and count < 100: + newId = idGenMethod() + count = count + 1 + if count == 100: + raise Exception("Unable to generate an initial unique id for " + dbColumn + " after 100 attempts.") + ids.add(newId) + lclPreviousIds.add(newId) + dupes = self.__findDupsInDB(dbColumn, ids) + if dupes is not None and len(dupes) > 0: + n_iter = iteration + 1 + if n_iter > 100: + raise Exception("Unable to generate unique id(s) for " + dbColumn + " after 100 attempts.") + replacements = self.__nUniqueIds(len(dupes), idGenMethod, dbColumn, previousGeneratedIds=lclPreviousIds, iteration=n_iter) + for val in dupes: + ids.remove(val[0]) + for val in replacements: + ids.add(val) + return list(ids) + + def __findDupsInDB(self, dbColumn, idSet): + sql = "select " + dbColumn + " from hm_uuids where " + dbColumn + " IN(" + listToCommaSeparated(idSet, "'", True) + ")" + with closing(self.hmdb.getDBConnection()) as dbConn: + with closing(dbConn.cursor()) as curs: + curs.execute(sql) + dupes = curs.fetchall() + + return dupes + + + + #which items in idSet are not in the database + def __findExclusionsInDB(self, dbColumn, idSet): + + sql = "select " + dbColumn + " from ( " + first = True + for ex_id in idSet: + if first: first = False + else: sql = sql + " UNION ALL " + sql = sql + "(select '" + ex_id + "' as " + dbColumn + ")" + sql = sql + ") as list left join hm_uuids using (" + dbColumn + ") where hm_uuids." + dbColumn + " is null" + with closing(self.hmdb.getDBConnection()) as dbConn: + with closing(dbConn.cursor()) as curs: + curs.execute(sql) + excluded = curs.fetchall() + + return excluded + + def __display_hm_id(self, hm_base_id): + hubmap_id = 'HBM' + hm_base_id[0:3] + '.' + hm_base_id[3:7] + '.' + hm_base_id[7:] + return hubmap_id + + def hmidGen(self): + nums1 = '' + nums2 = '' + alphs = '' + for _ in range(3): + nums1 = nums1 + secrets.choice(HMID_NUM_CHARS) #[random.randint(0,len(DOI_NUM_CHARS)-1)] + for _ in range(3): + nums2 = nums2 + secrets.choice(HMID_NUM_CHARS) #[random.randint(0,len(DOI_NUM_CHARS)-1)] + for _ in range(4): + alphs = alphs + secrets.choice(HMID_ALPHA_CHARS) #[random.randint(0,len(DOI_ALPHA_CHARS)-1)] + + val = nums1 + alphs + nums2 + return(val) + + def getIdExists(self, hmid): + if not isValidHMId(hmid): + return Response("Invalid HuBMAP Id", 400) + tid = stripHMid(hmid) + if startsWithComponentPrefix(hmid): + return self.submission_id_exists(hmid.strip()) + elif len(tid) == 10: + return self.base_id_exists(tid.upper()) + elif len(tid) == 32: + return self.uuid_exists(tid.lower()) + else: + return Response("Invalid HuBMAP Id (or empty or bad length)", 400) + + + #convert csv list of ancestor ids to a list + #convert hubmap base id to a hubmap id (display version) + def _convert_result_id_array(self, results, hmid): + if isinstance(results, list): + asize = len(results) + if asize == 0: + record = None + elif asize == 1: + record = results[0] + if not 'ancestor_ids' in record: + return record + ancestor_ids = record['ancestor_ids'] + if ancestor_ids is None or ancestor_ids.strip() == '': + record.pop('ancestor_ids', '') + elif isinstance(ancestor_ids, str): + record['ancestor_ids'] = ancestor_ids.split(',') + if len(record['ancestor_ids']) == 1: + record['ancestor_id'] = record['ancestor_ids'][0] + else: + raise Exception("Unknown ancestor type for id:" + hmid) + + if 'hubmap_base_id' in record: + if not record['hubmap_base_id'].strip() == '': + record['hubmap_id'] = self.__display_hm_id(record['hubmap_base_id']) + record.pop('hubmap_base_id', '') + else: + raise Exception("Multiple results exist for id:" + hmid) + + return record + + def getIdInfo(self, hmid): + if not isValidHMId(hmid): + return Response(hmid + " is not a valid id format", 400) + tidl = hmid.strip().lower() + tid = stripHMid(hmid) + if startsWithComponentPrefix(hmid): + sql = "select " + UUID_SELECTS + " from hm_uuids inner join hm_ancestors on hm_ancestors.descendant_uuid = hm_uuids.hm_uuid where lower(submission_id) ='" + tidl + "'" + elif len(tid) == 10: + sql = "select " + UUID_SELECTS + " from hm_uuids inner join hm_ancestors on hm_ancestors.descendant_uuid = hm_uuids.hm_uuid where hubmap_base_id ='" + tid + "'" + elif len(tid) == 32: + sql = "select " + UUID_SELECTS + " from hm_uuids inner join hm_ancestors on hm_ancestors.descendant_uuid = hm_uuids.hm_uuid where hm_uuid ='" + tid + "'" + else: + return Response("Invalid id (empty or bad length)", 400) + with closing(self.hmdb.getDBConnection()) as dbConn: + with closing(dbConn.cursor()) as curs: + curs.execute(sql) + results = [dict((curs.description[i][0], value) for i, value in enumerate(row)) for row in curs.fetchall()] + + # In Python, empty sequences (strings, lists, tuples) are false + if results is None or not results: + return Response ("Could not find the target id: " + hmid, 404) + if isinstance(results, list) and (len(results) == 0): + return Response ("Could not find the target id: " + hmid, 404) + if not 'hm_uuid' in results[0]: + return Response ("Could not find the target id: " + hmid, 404) + if results[0]['hm_uuid'] is None: + return Response ("Could not find the target id: " + hmid, 404) + + rdict = self._convert_result_id_array(results, hmid) + return json.dumps(rdict, indent=4, sort_keys=True, default=str) + + def uuid_exists(self, hmid): + with closing(self.hmdb.getDBConnection()) as dbConn: + with closing(dbConn.cursor()) as curs: + curs.execute("select count(*) from hm_uuids where hm_uuid = '" + hmid + "'") + res = curs.fetchone() + + if(res is None or len(res) == 0): return False + if(res[0] == 1): return True + if(res[0] == 0): return False + raise Exception("Multiple uuids found matching " + hmid) + +# def uuidsExist(self, uuids): +# uuid_list = '(' +# first = True +# comma = '' +# for uuid in uuids: +# uuid_list = uuid_list + comma + "'" + uuid.strip() + "'" +# if first: +# first = False +# comma = ',' +# uuid_list = uuid_list + ')' # -# with closing(self.hmdb.getDBConnection()) as dbConn: -# with closing(dbConn.cursor()) as curs: -# curs.execute("select count(*) from hm_uuids where hm_uuid in " + uuid_list) -# res = curs.fetchone() +# with closing(self.hmdb.getDBConnection()) as dbConn: +# with closing(dbConn.cursor()) as curs: +# curs.execute("select count(*) from hm_uuids where hm_uuid in " + uuid_list) +# res = curs.fetchone() # -# if(res is None or len(res) == 0): return False -# count = res[0] -# if count != len(uuids): return False -# return True - - def base_id_exists(self, base_id): - with closing(self.hmdb.getDBConnection()) as dbConn: - with closing(dbConn.cursor()) as curs: - curs.execute("select count(*) from hm_uuids where hubmap_base_id = '" + base_id + "'") - res = curs.fetchone() - if(res is None or len(res) == 0): return False - if(res[0] == 1): return True - if(res[0] == 0): return False - raise Exception("Multiple hubmap base ids found matching " + base_id) - - def submission_id_exists(self, hmid): - with closing(self.hmdb.getDBConnection()) as dbConn: - with closing(dbConn.cursor()) as curs: - curs.execute("select count(*) from hm_uuids where submission_id = '" + hmid + "'") - res = curs.fetchone() - if(res is None or len(res) == 0): return False - if(res[0] == 1): return True - if(res[0] == 0): return False - raise Exception("Multiple HuBMAP IDs found matching " + hmid) - - def testConnection(self): - try: - res = None - with closing(self.hmdb.getDBConnection()) as dbConn: - with closing(dbConn.cursor()) as curs: - curs.execute("select 'ANYTHING'") - res = curs.fetchone() - - if(res is None or len(res) == 0): return False - if(res[0] == 'ANYTHING'): - return True - else: - return False - except Exception as e: - self.logger.error(e, exc_info=True) - return False - -''' - def newUUID(self, parentID, entityType, userId, userEmail, submissionId=None): - uuid = self.uuidGen() #uuid.uuid4().hex - hubmap_id = self.hmidGen() - - with self.lock: - count = 0 - while(self.uuidExists(uuid) and count < 100): - uuid = self.uuidGen() #uuid.uuid4().hex - count = count + 1 - if count == 100: - raise Exception("Unable to generate a unique uuid after 100 attempts") - - count = 0; - while(self.doiExists(hubmap_id) and count < 100): - hubmap_id = self.hmidGen() - count = count + 1 - if count == 100: - raise Exception("Unable to generate a unique hubmap id after 100 attempts") - now = time.strftime('%Y-%m-%d %H:%M:%S') - sql = "INSERT INTO hm_uuids (HMUUID, DOI_SUFFIX, ENTITY_TYPE, PARENT_UUID, TIME_GENERATED, USER_ID, USER_EMAIL, HUBMAP_ID) VALUES (%s, %s, %s, %s, %s, %s,%s, %s)" - vals = (uuid, hubmap_id, entityType, parentID, now, userId, userEmail, submissionId) - with closing(self.hmdb.getDBConnection()) as dbConn: - with closing(dbConn.cursor()) as curs: - curs.execute(sql, vals) - dbConn.commit() - - disp_hubmap_id = self.__displayDoi(hubmap_id) - if submissionId is None: - rVal = { - "displayDoi": disp_hubmap_id, - "doi": hubmap_id, - "uuid": uuid - } - else: - rVal = { - "displayDoi": disp_hubmap_id, - "doi": hubmap_id, - "uuid": uuid, - "hubmapId": submissionId - } - return rVal +# if(res is None or len(res) == 0): return False +# count = res[0] +# if count != len(uuids): return False +# return True + + def base_id_exists(self, base_id): + with closing(self.hmdb.getDBConnection()) as dbConn: + with closing(dbConn.cursor()) as curs: + curs.execute("select count(*) from hm_uuids where hubmap_base_id = '" + base_id + "'") + res = curs.fetchone() + if(res is None or len(res) == 0): return False + if(res[0] == 1): return True + if(res[0] == 0): return False + raise Exception("Multiple hubmap base ids found matching " + base_id) + + def submission_id_exists(self, hmid): + with closing(self.hmdb.getDBConnection()) as dbConn: + with closing(dbConn.cursor()) as curs: + curs.execute("select count(*) from hm_uuids where submission_id = '" + hmid + "'") + res = curs.fetchone() + if(res is None or len(res) == 0): return False + if(res[0] == 1): return True + if(res[0] == 0): return False + raise Exception("Multiple HuBMAP IDs found matching " + hmid) + + def testConnection(self): + try: + res = None + with closing(self.hmdb.getDBConnection()) as dbConn: + with closing(dbConn.cursor()) as curs: + curs.execute("select 'ANYTHING'") + res = curs.fetchone() + + if(res is None or len(res) == 0): return False + if(res[0] == 'ANYTHING'): + return True + else: + return False + except Exception as e: + self.logger.error(e, exc_info=True) + return False + +''' + def newUUID(self, parentID, entityType, userId, userEmail, submissionId=None): + uuid = self.uuidGen() #uuid.uuid4().hex + hubmap_id = self.hmidGen() + + with self.lock: + count = 0 + while(self.uuidExists(uuid) and count < 100): + uuid = self.uuidGen() #uuid.uuid4().hex + count = count + 1 + if count == 100: + raise Exception("Unable to generate a unique uuid after 100 attempts") + + count = 0; + while(self.doiExists(hubmap_id) and count < 100): + hubmap_id = self.hmidGen() + count = count + 1 + if count == 100: + raise Exception("Unable to generate a unique hubmap id after 100 attempts") + now = time.strftime('%Y-%m-%d %H:%M:%S') + sql = "INSERT INTO hm_uuids (HMUUID, DOI_SUFFIX, ENTITY_TYPE, PARENT_UUID, TIME_GENERATED, USER_ID, USER_EMAIL, HUBMAP_ID) VALUES (%s, %s, %s, %s, %s, %s,%s, %s)" + vals = (uuid, hubmap_id, entityType, parentID, now, userId, userEmail, submissionId) + with closing(self.hmdb.getDBConnection()) as dbConn: + with closing(dbConn.cursor()) as curs: + curs.execute(sql, vals) + dbConn.commit() + + disp_hubmap_id = self.__displayDoi(hubmap_id) + if submissionId is None: + rVal = { + "displayDoi": disp_hubmap_id, + "doi": hubmap_id, + "uuid": uuid + } + else: + rVal = { + "displayDoi": disp_hubmap_id, + "doi": hubmap_id, + "uuid": uuid, + "hubmapId": submissionId + } + return rVal ''' From f1d106868f7289bbfe5d7a78bd3bd04d4e404315 Mon Sep 17 00:00:00 2001 From: yuanzhou Date: Fri, 8 Jan 2021 21:49:52 -0500 Subject: [PATCH 17/37] Add MIME type to response --- src/app.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/app.py b/src/app.py index 1736512..4e6fdd1 100644 --- a/src/app.py +++ b/src/app.py @@ -140,8 +140,11 @@ def get_hmuuid(hmuuid): global logger try: if request.method == "GET": + # The info is a pretty print json string info = worker.getIdInfo(hmuuid) - return info + + # Add the json MIME header in response + return Response(response=info, mimetype="application/json") else: return Response ("Invalid request use GET to retrieve UUID information", 500) except Exception as e: From 0d42ffac70a27e7116c1b7ee9435f294acfb2b67 Mon Sep 17 00:00:00 2001 From: yuanzhou Date: Sun, 10 Jan 2021 23:05:06 -0500 Subject: [PATCH 18/37] Update cryptography>=3.2 --- src/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/requirements.txt b/src/requirements.txt index 60aa9f3..5b1824d 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -3,7 +3,7 @@ certifi==2019.9.11 cffi==1.12.3 chardet==3.0.4 Click==7.0 -cryptography==2.7 +cryptography>=3.2 Flask==1.1.1 globus-sdk==1.8.0 idna==2.8 From cceaadaff961f98df49a0d9b5dc7c72a227dbc36 Mon Sep 17 00:00:00 2001 From: Bill Shirey Date: Mon, 11 Jan 2021 12:04:38 -0500 Subject: [PATCH 19/37] add requirement for package cachetools --- src/requirements.txt | 3 ++- src/requirements_dev.txt | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/requirements.txt b/src/requirements.txt index 5b1824d..15dd662 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,4 +1,5 @@ asn1crypto==0.24.0 +cachetools==4.1.0 certifi==2019.9.11 cffi==1.12.3 chardet==3.0.4 @@ -21,4 +22,4 @@ urllib3==1.25.6 Werkzeug==0.16.0 # The branch name of commons to be used during image build # Default is master branch specified in docker-compose.yml if not set -git+git://github.com/hubmapconsortium/commons.git@${COMMONS_BRANCH}#egg=hubmap-commons \ No newline at end of file +git+git://github.com/hubmapconsortium/commons.git@${COMMONS_BRANCH}#egg=hubmap-commons diff --git a/src/requirements_dev.txt b/src/requirements_dev.txt index d2e8ce3..8190b2e 100644 --- a/src/requirements_dev.txt +++ b/src/requirements_dev.txt @@ -1,4 +1,5 @@ asn1crypto==0.24.0 +cachetools==4.1.0 certifi==2019.9.11 cffi==1.12.3 chardet==3.0.4 From 942a0d2fab178cd77ec1b0e62b30c5ce979cbdad Mon Sep 17 00:00:00 2001 From: Bill Shirey Date: Mon, 11 Jan 2021 12:47:53 -0500 Subject: [PATCH 20/37] ckeck for an html Response object returned from getidInfo --- src/app.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/app.py b/src/app.py index 4e6fdd1..42e692a 100644 --- a/src/app.py +++ b/src/app.py @@ -143,6 +143,10 @@ def get_hmuuid(hmuuid): # The info is a pretty print json string info = worker.getIdInfo(hmuuid) + # if getIdInfo returns a Response, it is sending an error back + if isinstance(info, Response): + return info + # Add the json MIME header in response return Response(response=info, mimetype="application/json") else: From 3ae87ac0c41cc948c8c9ab1043a4ed6558172d7a Mon Sep 17 00:00:00 2001 From: Bill Shirey Date: Mon, 11 Jan 2021 14:15:51 -0500 Subject: [PATCH 21/37] Fix bug when data provider group id not found in yaml file --- src/uuid_worker.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/uuid_worker.py b/src/uuid_worker.py index da0df3f..c557e55 100644 --- a/src/uuid_worker.py +++ b/src/uuid_worker.py @@ -108,7 +108,10 @@ def __resolve_lab_id(self, lab_id, user_id, user_email): # Get the globus groups info based on the groups json file in commons package globus_groups_info = globus_groups.get_globus_groups_info() groups_by_tmc_prefix_dict = globus_groups_info['by_tmc_prefix'] - lab = groups_by_tmc_prefix_dict[check_id] + if not check_id in groups_by_tmc_prefix_dict: + lab = {} + else: + lab = groups_by_tmc_prefix_dict[check_id] except ValueError: return Response("A valid lab with specified id not found id:" + check_id, 400) From 8871c15c2a7ebb7471c44d1dbbb525747f5bf6bb Mon Sep 17 00:00:00 2001 From: "Zhou (Joe) Yuan" Date: Wed, 13 Jan 2021 12:56:24 -0500 Subject: [PATCH 22/37] Fix typo in code comment --- src/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app.py b/src/app.py index 42e692a..fbbfc2a 100644 --- a/src/app.py +++ b/src/app.py @@ -90,7 +90,7 @@ def status(): have a single ancestor, not multiple). For DATASETs at least one ancestor UUID is required, but multiple can be specified. (A DATASET can be derived from multiple SAMPLEs or DATASETs.) - organ_id- required only in the case where an id is being generated for a SAMPLE that + organ_code- required only in the case where an id is being generated for a SAMPLE that has a DONOR as a direct ancestor. Must be one of the codes from: https://github.com/hubmapconsortium/search-api/blob/test-release/src/search-schema/data/definitions/enums/organ_types.yaml From 5828b5d56e1b112da0bd0cf954a2560a2ee906eb Mon Sep 17 00:00:00 2001 From: Bill Shirey Date: Sat, 30 Jan 2021 14:41:19 -0500 Subject: [PATCH 23/37] add support for FILE uuids --- reload_from_neo4j/reload_from_neo.py | 6 +- sql/uuid.ddl | 20 +++-- src/app.py | 63 ++++++------- src/uuid_worker.py | 128 +++++++++++++++------------ 4 files changed, 114 insertions(+), 103 deletions(-) diff --git a/reload_from_neo4j/reload_from_neo.py b/reload_from_neo4j/reload_from_neo.py index 5f8d71a..6afa2c1 100644 --- a/reload_from_neo4j/reload_from_neo.py +++ b/reload_from_neo4j/reload_from_neo.py @@ -24,9 +24,9 @@ UUID_DB_TABLE_NAMES = ['hm_uuids', 'hm_ancestors', 'hm_organs', 'hm_data_centers'] LABS_CYPHER = "match (l:Lab) return l.uuid as labid, l.next_identifier, l.submission_id as lab_code" -DONORS_CYPHER = "match (d:Donor {entity_class:'Donor'})<-[:ACTIVITY_OUTPUT]-(a)-[:ACTIVITY_INPUT]-(lab:Lab) return d.uuid, d.submission_id, d.doi_suffix_id, d.hubmap_id, lab.uuid, d.created_by_user_email, d.created_timestamp, d.created_by_user_displayname, d.created_by_user_sub" -SAMPLES_CYPHER = "match (s:Sample {entity_class:'Sample'})<-[:ACTIVITY_OUTPUT]-(a)-[:ACTIVITY_INPUT]-(anc:Entity) return s.uuid, s.submission_id, s.doi_suffix_id, s.hubmap_id, anc.uuid, s.created_by_user_email, s.created_timestamp, s.created_by_user_displayname, s.created_by_user_sub, s.organ, s.specimen_type" -DATASETS_CYPHER = "match (ds:Dataset {entity_class:'Dataset'})<-[:ACTIVITY_OUTPUT]-(a)-[:ACTIVITY_INPUT]-(anc:Entity) return collect(anc.uuid) as ancestor_ids, ds.uuid, ds.doi_suffix_id, ds.hubmap_id, ds.created_by_user_email, ds.created_timestamp, ds.created_by_user_displayname, ds.created_by_user_sub" +DONORS_CYPHER = "match (d:Donor)<-[:ACTIVITY_OUTPUT]-(a)-[:ACTIVITY_INPUT]-(lab:Lab) return d.uuid, d.submission_id, d.doi_suffix_id, d.hubmap_id, lab.uuid, d.created_by_user_email, d.created_timestamp, d.created_by_user_displayname, d.created_by_user_sub" +SAMPLES_CYPHER = "match (s:Sample)<-[:ACTIVITY_OUTPUT]-(a)-[:ACTIVITY_INPUT]-(anc:Entity) return s.uuid, s.submission_id, s.doi_suffix_id, s.hubmap_id, anc.uuid, s.created_by_user_email, s.created_timestamp, s.created_by_user_displayname, s.created_by_user_sub, s.organ, s.specimen_type" +DATASETS_CYPHER = "match (ds:Dataset)<-[:ACTIVITY_OUTPUT]-(a)-[:ACTIVITY_INPUT]-(anc:Entity) return collect(anc.uuid) as ancestor_ids, ds.uuid, ds.doi_suffix_id, ds.hubmap_id, ds.created_by_user_email, ds.created_timestamp, ds.created_by_user_displayname, ds.created_by_user_sub" ACTIVITIES_CYPHER = "match (a:Activity) return a.uuid, a.doi_suffix_id, a.hubmap_id, a.created_by_user_email, a.created_timestamp, a.created_by_user_displayname, a.created_by_user_sub" COLLECTIONS_CYPHER = "match (c:Collection)<-[:IN_COLLECTION]-(ds:Dataset) return c.uuid, c.doi_suffix_id, c.hubmap_id, c.created_by_user_email, c.created_timestamp, c.created_by_user_displayname, c.created_by_user_sub, collect(ds.created_by_user_email) as ds_created_by_user_email, collect(ds.created_timestamp) as ds_created_timestamp, collect(ds.created_by_user_displayname) as ds_created_by_user_displayname, collect(ds.created_by_user_sub) as ds_created_by_user_sub" diff --git a/sql/uuid.ddl b/sql/uuid.ddl index 8e0ea8f..361e2f9 100644 --- a/sql/uuid.ddl +++ b/sql/uuid.ddl @@ -1,6 +1,6 @@ CREATE TABLE hm_uuids ( - HM_UUID VARCHAR(35) PRIMARY KEY, + HM_UUID CHAR(32) PRIMARY KEY, HUBMAP_BASE_ID VARCHAR(10) NULL UNIQUE, ENTITY_TYPE VARCHAR(20) NOT NULL, TIME_GENERATED TIMESTAMP NOT NULL, @@ -12,28 +12,36 @@ CREATE TABLE hm_uuids CREATE TABLE hm_ancestors ( - ANCESTOR_UUID VARCHAR(35) NOT NULL, - DESCENDANT_UUID VARCHAR(35) NOT NULL + ANCESTOR_UUID CHAR(32) NOT NULL, + DESCENDANT_UUID CHAR(32) NOT NULL ); CREATE TABLE hm_organs ( - DONOR_UUID VARCHAR(35) NOT NULL, + DONOR_UUID CHAR(32) NOT NULL, ORGAN_CODE VARCHAR(8) NOT NULL, ORGAN_COUNT INT NOT NULL DEFAULT 0 ); CREATE TABLE hm_data_centers ( - HM_UUID VARCHAR(35) PRIMARY KEY, + HM_UUID CHAR(32) PRIMARY KEY, DC_UUID VARCHAR(40) NOT NULL UNIQUE, DC_CODE VARCHAR(8) NOT NULL UNIQUE ); -CREATE UNIQUE INDEX UUID_IDX ON hm_uuids (HM_UUID); +CREATE TABLE hm_files +( + HM_UUID CHAR(32) PRIMARY KEY, + PATH VARCHAR(65000) NULL, + CHECKSUM VARCHAR(50) NULL, + SIZE BIGINT +); + CREATE UNIQUE INDEX HM_BASE_IDX ON hm_uuids (HUBMAP_BASE_ID); CREATE UNIQUE INDEX HM_ID_IDX on hm_uuids (SUBMISSION_ID); CREATE INDEX ANC_UUID on hm_ancestors (ANCESTOR_UUID); CREATE INDEX DEC_UUID on hm_ancestors (DESCENDANT_UUID); +CREATE INDEX DC_UUID_IDX on hm_data_centers (HM_UUID, DC_UUID, DC_CODE); CREATE UNIQUE INDEX ANC_DEC_UUID on hm_ancestors (ANCESTOR_UUID, DESCENDANT_UUID); CREATE INDEX ORG_CODE ON hm_organs (ORGAN_CODE); diff --git a/src/app.py b/src/app.py index fbbfc2a..45acad5 100644 --- a/src/app.py +++ b/src/app.py @@ -90,22 +90,32 @@ def status(): have a single ancestor, not multiple). For DATASETs at least one ancestor UUID is required, but multiple can be specified. (A DATASET can be derived from multiple SAMPLEs or DATASETs.) - organ_code- required only in the case where an id is being generated for a SAMPLE that + organ_code- required only in the case where an id is being generated for a SAMPLE that has a DONOR as a direct ancestor. Must be one of the codes from: https://github.com/hubmapconsortium/search-api/blob/test-release/src/search-schema/data/definitions/enums/organ_types.yaml + file_info- required only if the entity type is FILE. A list/array of information about each + file that requires an id to be generated for it. The size of this array is required + to match the optional URL argument, entity_count (or be 1 in the case where this argument + is defaulted to 1). + Each file info element should contain: + path- required: the path to the file in storage. For the purposes of the + UUID system this can be a full path or relative, but it is + recommended that a relative path be used. + checksum- optional: An MD5 checksum/hash of the file + size- optional: The size of the file as an integer + + Query (in url) argument entity_count optional, the number of ids to generate. If omitted, defaults to 1 - For REMOVED: generateDOI- optional, defaults to false submission_ids- optional, an array of ids to associate and store with the generated ids. If provide, the length of this array must match the entity_count argument - curl example: curl -d '{"entityType":"BILL-TEST","generateDOI":"true","hubmap-ids":["Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"] }' -H "Content-Type: application/json" -H "Authorization: Bearer AgX07PVbz9Wb6WDK3QvP9p23j2vWV7bYnMGBvaQxQQGEY5MjJEIwC8x3vwqYmzwEzPw93qWYeGpEkKu4nOkPgs7VKQ" -X POST http://localhost:5000/hmuuid?entity_count=7 ''' @app.route('/hmuuid', methods=["POST"]) @secured(groups="HuBMAP-read") @@ -156,39 +166,6 @@ def get_hmuuid(hmuuid): logger.error(e, exc_info=True) return(Response("Unexpected error: " + eMsg, 500)) -''' -Update id records to include display ids - -PUT arguments in json - hubmap-uuids- an array of uuids to update - display-ids- an array of display ids to update for the associated (same order) - records of the uuids passed in the hubmap-uuids array. - - Only id records that currently DO NOT have an associated display-id can be updated. - - curl example: curl -d '{"hubmap-uuids":["32ae6d81bc9df2d62a550c62c02df39a", "3a2bdd52946a14081d448261c7b52bb2"], "display-ids"["lab-id-1","lab-id-2"] }' -H "Content-Type: application/json" -H "Authorization: Bearer AgX07PVbz9Wb6WDK3QvP9p23j2vWV7bYnMGBvaQxQQGEY5MjJEIwC8x3vwqYmzwEzPw93qWYeGpEkKu4nOkPgs7VKQ" -X PUT http://localhost:5000/hmuuid -''' -''' -@app.route('/hmuuid', methods=["PUT"]) -@secured(groups="HuBMAP-read") -def update_hmuuid(): - global worker - global logger - try: - if request.method == "PUT": - rjson = worker.uuidPut(request) - if isinstance(rjson, Response): - return rjson - - return Response(rjson, 200, {'Content-Type': "application/json"}) - else: - return Response("Invalid request. Use PUT to update a UUID", 500) - except Exception as e: - eMsg = str(e) - logger.error(e, exc_info=True) - return(Response("Unexpected error: " + eMsg, 500)) -''' - @app.route('/hmuuid//exists', methods=["GET"]) @secured(groups="HuBMAP-read") def is_hmuuid(hmuuid): @@ -207,7 +184,19 @@ def is_hmuuid(hmuuid): logger.error(e, exc_info=True) return(Response("Unexpected error: " + eMsg, 500)) - +@app.route('/file-id/', methods=["GET"]) +@secured(groups="HuBMAP-read") +def get_file_id(file_uuid): + global worker + global logger + try: + file_id_info = worker.getFileIdInfo(file_uuid) + if isinstance(file_id_info, Response): return file_id_info + return Response(response=file_id_info, mimetype="application/json") + except Exception as e: + eMsg = str(e) + logger.error(e, exc_info=True) + return(Response("Unexpected error: " + eMsg, 500)) if __name__ == "__main__": try: diff --git a/src/uuid_worker.py b/src/uuid_worker.py index c557e55..5c4f3fa 100644 --- a/src/uuid_worker.py +++ b/src/uuid_worker.py @@ -16,8 +16,9 @@ # Deprecate the use of Provenance #from hubmap_commons.provenance import Provenance +HUBMAP_ID_ENTITY_TYPES = ['ACTIVITY', 'SAMPLE', 'DONOR', 'DATASET', 'COLLECTION'] SUBMISSION_ID_ENTITY_TYPES = ['SAMPLE', 'DONOR'] -ANCESTOR_REQUIRED_ENTITY_TYPES = ['SAMPLE', 'DONOR', 'DATASET'] +ANCESTOR_REQUIRED_ENTITY_TYPES = ['SAMPLE', 'DONOR', 'DATASET', 'FILE'] MULTIPLE_ALLOWED_ORGANS = ['LY', 'SK'] @@ -25,6 +26,8 @@ INSERT_SQL = "INSERT INTO hm_uuids (HM_UUID, HUBMAP_BASE_ID, ENTITY_TYPE, TIME_GENERATED, USER_ID, USER_EMAIL) VALUES (%s, %s, %s, %s, %s, %s)" INSERT_SQL_WITH_SUBMISSION_ID = "INSERT INTO hm_uuids (HM_UUID, HUBMAP_BASE_ID, ENTITY_TYPE, TIME_GENERATED, USER_ID, USER_EMAIL, SUBMISSION_ID) VALUES (%s, %s, %s, %s, %s, %s,%s)" INSERT_ANCESTOR_SQL = "INSERT INTO hm_ancestors (DESCENDANT_UUID, ANCESTOR_UUID) VALUES (%s, %s)" +INSERT_FILE_INFO_SQL = "INSERT INTO hm_files (HM_UUID, PATH, CHECKSUM, SIZE) VALUES (%s, %s, %s, %s)" +#INSERT_FILE_INFO_SQL = "INSERT INTO hm_files (HM_UUID, PATH, CHECKSUM) VALUES (%s, %s, %s)" #UPDATE_SQL = "UPDATE hm_uuids set hubmap_id = %s where HMUUID = %s" HMID_ALPHA_CHARS=['B','C','D','F','G','H','J','K','L','M','N','P','Q','R','S','T','V','W','X','Z'] @@ -33,7 +36,7 @@ #UUID_SELECTS = "HMUUID as hmuuid, DOI_SUFFIX as doiSuffix, ENTITY_TYPE as type, PARENT_UUID as parentId, TIME_GENERATED as timeStamp, USER_ID as userId, HUBMAP_ID as hubmapId, USER_EMAIL as email" UUID_SELECTS = "HM_UUID as hm_uuid, HUBMAP_BASE_ID as hubmap_base_id, ENTITY_TYPE as type, TIME_GENERATED as time_generated, USER_ID as user_id, SUBMISSION_ID as submission_id, USER_EMAIL as email, GROUP_CONCAT(ancestor_uuid) as ancestor_ids" -def startsWithComponentPrefix(hmid): +def startsWithComponentPrefix(hmid): tidl = hmid.strip().lower() if tidl.startswith('test') or tidl.startswith('van') or tidl.startswith('ufl') or tidl.startswith('stan') or tidl.startswith('ucsd') or tidl.startswith('calt') or tidl.startswith('rtibd') or tidl.startswith('rtige') or tidl.startswith('rtinw') or tidl.startswith('rtist') or tidl.startswith('ttdct') or tidl.startswith('ttdhv') or tidl.startswith('ttdpd') or tidl.startswith('ttdst'): return True @@ -184,13 +187,27 @@ def uuidPost(self, req, nIds): ancestor_ids = [] else: ancestor_ids = parentIds - + + file_info = None + if entityType == "FILE": + if not 'file_info' in content: + return(Response("Entity type of FILE requires a file_info array provide in the request body.", 400)) + file_info = content['file_info'] + if not isinstance(file_info, list): + return(Response("file_info attribute must be a list", 400)) + if not len(file_info) == nIds: + return(Response("number file_info list must contain the same number of entries as ids being generated " + nIds)) + for fi in file_info: + if not 'path' in fi or isBlank(fi['path']): + return(Response("The 'path' attribute is required for each file_info entry", 400)) + for parentId in ancestor_ids: if not self.uuid_exists(parentId): return(Response("Parent id " + parentId + " does not exist", 400)) - return self.newUUIDs(ancestor_ids, entityType, userId, userEmail, nIds, organ_code = organ_code, lab_code = lab_code) + return self.newUUIDs(ancestor_ids, entityType, userId, userEmail, nIds, organ_code = organ_code, lab_code = lab_code, file_info_array = file_info) + ''' def uuidPut(self, req): if not req.is_json: return(Response("Invalid input, json required.", 400)) @@ -209,6 +226,7 @@ def uuidPut(self, req): return(Response("The length of the diplay-ids and hubmap-uuids arrays must be the same length", 400)) return self.updateUUIDs(uuids, disp_ids) + ''' def newUUIDTest(self, parentIds, entityType, userId, userEmail): return self.newUUIDs(parentIds, entityType, userId, userEmail) @@ -220,55 +238,6 @@ def uuidGen(self): hexVal = hexVal.lower() return hexVal - - ''' remove not needed - def updateUUIDs(self, uuids, display_ids): - if len(uuids) != len(display_ids): - raise Exception("The number of uuids must match the number of display ids") - - idSet = set() - for uuid in uuids: - uuidt = uuid.lower().strip() - if uuidt in idSet: - raise Exception("During UUID update the uuid " + uuid + " is contained multiple times in the hubmap-uuids array") - idSet.add(uuidt) - - idSet = set() - for dispId in display_ids: - disIdT = dispId.lower().strip() - if disIdT in idSet: - raise Exception("During UUID update the display id " + dispId + " is contained multiple times in the diplay-ids array") - idSet.add(disIdT) - - excluded = self.__findExclusionsInDB("HMUUID", uuids) - if(excluded is not None and len(excluded) > 0): - raise Exception("UUIDs not contained in the id database " + listToCommaSeparated(excluded)) - - #only id records that have a null display id can be updated - sql = "select hmuuid from hm_uuids where hmuuid in (" + listToCommaSeparated(uuids, "'", True) + ") and hubmap_id is not null" - with closing(self.hmdb.getDBConnection()) as dbConn: - with closing(dbConn.cursor()) as curs: - curs.execute(sql) - nonNull = curs.fetchall() - if nonNull is not None and len(nonNull) > 0: - raise Exception("Some uuids do not have null display ids: " + listToCommaSeparated(nonNull)) - - #if there are any display ids that match what we're updating to send an error - dupes = self.__findDupsInDB("HUBMAP_ID", display_ids) - if(dupes is not None and len(dupes) > 0): - raise Exception("Display ID(s) are not unique " + listToCommaSeparated(dupes)) - - updateVals = [] - for uuid, dispid in zip(uuids, display_ids): - updateRow = (dispid, uuid) - updateVals.append(updateRow) - - with closing(self.hmdb.getDBConnection()) as dbConn: - with closing(dbConn.cursor()) as curs: - curs.executemany(UPDATE_SQL, updateVals) - dbConn.commit() - ''' - def __create_submission_ids(self, num_to_gen, parent_id, entity_type, organ_code = None, lab_code = None): parent_id = parent_id.strip().lower() with closing(self.hmdb.getDBConnection()) as dbConn: @@ -336,11 +305,15 @@ def __create_submission_ids(self, num_to_gen, parent_id, entity_type, organ_code return Response("Cannot create a submission id for an entity of type " + entity_type) #generate multiple ids, one for each display id in the displayIds array - def newUUIDs(self, parentIDs, entityType, userId, userEmail, nIds, organ_code = None, lab_code = None, gen_base_ids = True): + + def newUUIDs(self, parentIDs, entityType, userId, userEmail, nIds, organ_code = None, lab_code = None, file_info_array = None): #if entityType == 'DONOR': - + gen_base_ids = entityType in HUBMAP_ID_ENTITY_TYPES returnIds = [] now = time.strftime('%Y-%m-%d %H:%M:%S') + store_file_info = False + if entityType == 'FILE': + store_file_info = True with self.lock: #generate in batches previousUUIDs = set() @@ -352,6 +325,7 @@ def newUUIDs(self, parentIDs, entityType, userId, userEmail, nIds, organ_code = for i in range(0, nIds, MAX_GEN_IDS): insertVals = [] + file_info_insert_vals = [] insertParents = [] numToGen = min(MAX_GEN_IDS, nIds - i) #generate uuids @@ -387,7 +361,20 @@ def newUUIDs(self, parentIDs, entityType, userId, userEmail, nIds, organ_code = insRow = (insUuid, ins_hubmap_base_id, entityType, now, userId, userEmail, submission_ids[n]) else: insRow = (insUuid, ins_hubmap_base_id, entityType, now, userId, userEmail) - + + if store_file_info: + file_path = file_info_array[i]['path'] + file_checksum = None + file_size = None + if 'checksum' in file_info_array[i]: + file_checksum = file_info_array[i]['checksum'] + if 'size' in file_info_array[i]: + file_size = file_info_array[i]['size'] + file_info_ins_row = (insUuid, file_path, file_checksum, file_size) + #file_info_ins_row = (insUuid, file_path, file_checksum) + file_info_insert_vals.append(file_info_ins_row) + thisId['file_path'] = file_path + returnIds.append(thisId) insertVals.append(insRow) @@ -402,6 +389,9 @@ def newUUIDs(self, parentIDs, entityType, userId, userEmail, nIds, organ_code = curs.execute(count_increase_q) else: curs.executemany(INSERT_SQL, insertVals) + if store_file_info: + curs.executemany(INSERT_FILE_INFO_SQL, file_info_insert_vals) + curs.executemany(INSERT_ANCESTOR_SQL, insertParents) dbConn.commit() @@ -558,7 +548,31 @@ def getIdInfo(self, hmid): rdict = self._convert_result_id_array(results, hmid) return json.dumps(rdict, indent=4, sort_keys=True, default=str) - + + def getFileIdInfo(self, fid): + check_id = fid.strip() + if isBlank(check_id) or len(check_id) != 32: + return Response("Invalid file id format. 32 digit hex only.", 400) + sql = "select hm_uuid, path, checksum, size, ancestor_uuid from hm_files inner join hm_ancestors on hm_ancestors.descendant_uuid = hm_files.hm_uuid where hm_uuid = '" + check_id + "'" + with closing(self.hmdb.getDBConnection()) as dbConn: + with closing(dbConn.cursor()) as curs: + curs.execute(sql) + results = [dict((curs.description[i][0], value) for i, value in enumerate(row)) for row in curs.fetchall()] + + if results is None or not results: + return Response ("Could not find the target id: " + fid, 404) + if isinstance(results, list) and (len(results) == 0): + return Response ("Could not find the target id: " + fid, 404) + if not 'hm_uuid' in results[0]: + return Response ("Could not find the target id: " + fid, 404) + if results[0]['hm_uuid'] is None: + return Response ("Could not find the target id: " + fid, 404) + + rdict = self._convert_result_id_array(results, check_id) + if 'checksum' in rdict and rdict['checksum'] is None: rdict.pop('checksum') + if 'size' in rdict and rdict['size'] is None: rdict.pop('size') + return json.dumps(rdict, indent=4, sort_keys=True, default=str) + def uuid_exists(self, hmid): with closing(self.hmdb.getDBConnection()) as dbConn: with closing(dbConn.cursor()) as curs: From e6e676053999381e9277b5bbea82efa3f3050163 Mon Sep 17 00:00:00 2001 From: Bill Shirey Date: Sat, 30 Jan 2021 22:16:55 -0500 Subject: [PATCH 24/37] added BASE_DIR attribute to files --- sql/uuid.ddl | 3 ++- src/app.py | 3 +++ src/uuid_worker.py | 31 +++++++++++++++++++------------ 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/sql/uuid.ddl b/sql/uuid.ddl index 361e2f9..1ddbb52 100644 --- a/sql/uuid.ddl +++ b/sql/uuid.ddl @@ -35,7 +35,8 @@ CREATE TABLE hm_files HM_UUID CHAR(32) PRIMARY KEY, PATH VARCHAR(65000) NULL, CHECKSUM VARCHAR(50) NULL, - SIZE BIGINT + SIZE BIGINT, + BASE_DIR VARCHAR(50) NOT NULL ); CREATE UNIQUE INDEX HM_BASE_IDX ON hm_uuids (HUBMAP_BASE_ID); diff --git a/src/app.py b/src/app.py index 45acad5..da12207 100644 --- a/src/app.py +++ b/src/app.py @@ -101,6 +101,9 @@ def status(): path- required: the path to the file in storage. For the purposes of the UUID system this can be a full path or relative, but it is recommended that a relative path be used. + base_dir- required: a specifier for the base directory where the file is stored + valid values are: DATA_UPLOAD or INGEST_PORTAL_UPLOAD + checksum- optional: An MD5 checksum/hash of the file size- optional: The size of the file as an integer diff --git a/src/uuid_worker.py b/src/uuid_worker.py index 5c4f3fa..efbaac8 100644 --- a/src/uuid_worker.py +++ b/src/uuid_worker.py @@ -16,6 +16,8 @@ # Deprecate the use of Provenance #from hubmap_commons.provenance import Provenance +BASE_DIR_TYPES = ['DATA_UPLOAD', 'INGEST_PORTAL_UPLOAD'] + HUBMAP_ID_ENTITY_TYPES = ['ACTIVITY', 'SAMPLE', 'DONOR', 'DATASET', 'COLLECTION'] SUBMISSION_ID_ENTITY_TYPES = ['SAMPLE', 'DONOR'] ANCESTOR_REQUIRED_ENTITY_TYPES = ['SAMPLE', 'DONOR', 'DATASET', 'FILE'] @@ -26,7 +28,7 @@ INSERT_SQL = "INSERT INTO hm_uuids (HM_UUID, HUBMAP_BASE_ID, ENTITY_TYPE, TIME_GENERATED, USER_ID, USER_EMAIL) VALUES (%s, %s, %s, %s, %s, %s)" INSERT_SQL_WITH_SUBMISSION_ID = "INSERT INTO hm_uuids (HM_UUID, HUBMAP_BASE_ID, ENTITY_TYPE, TIME_GENERATED, USER_ID, USER_EMAIL, SUBMISSION_ID) VALUES (%s, %s, %s, %s, %s, %s,%s)" INSERT_ANCESTOR_SQL = "INSERT INTO hm_ancestors (DESCENDANT_UUID, ANCESTOR_UUID) VALUES (%s, %s)" -INSERT_FILE_INFO_SQL = "INSERT INTO hm_files (HM_UUID, PATH, CHECKSUM, SIZE) VALUES (%s, %s, %s, %s)" +INSERT_FILE_INFO_SQL = "INSERT INTO hm_files (HM_UUID, PATH, CHECKSUM, SIZE, BASE_DIR) VALUES (%s, %s, %s, %s, %s)" #INSERT_FILE_INFO_SQL = "INSERT INTO hm_files (HM_UUID, PATH, CHECKSUM) VALUES (%s, %s, %s)" #UPDATE_SQL = "UPDATE hm_uuids set hubmap_id = %s where HMUUID = %s" @@ -196,16 +198,20 @@ def uuidPost(self, req, nIds): if not isinstance(file_info, list): return(Response("file_info attribute must be a list", 400)) if not len(file_info) == nIds: - return(Response("number file_info list must contain the same number of entries as ids being generated " + nIds)) + return(Response("number file_info list must contain the same number of entries as ids being generated " + str(nIds))) for fi in file_info: if not 'path' in fi or isBlank(fi['path']): return(Response("The 'path' attribute is required for each file_info entry", 400)) - + if not 'base_dir' in fi or isBlank(fi['base_dir']): + return(Response("the 'base_dir' attribute is required for each file_info entity. Valid values are " + " ".join(BASE_DIR_TYPES), 400)) + base_dir = fi['base_dir'].strip().upper() + if not base_dir in BASE_DIR_TYPES: + return(Response("valid base_dir values are " + " ".join(BASE_DIR_TYPES))) for parentId in ancestor_ids: if not self.uuid_exists(parentId): return(Response("Parent id " + parentId + " does not exist", 400)) - return self.newUUIDs(ancestor_ids, entityType, userId, userEmail, nIds, organ_code = organ_code, lab_code = lab_code, file_info_array = file_info) + return self.newUUIDs(ancestor_ids, entityType, userId, userEmail, nIds, organ_code = organ_code, lab_code = lab_code, file_info_array = file_info, base_dir_type = base_dir) ''' def uuidPut(self, req): @@ -306,7 +312,7 @@ def __create_submission_ids(self, num_to_gen, parent_id, entity_type, organ_code #generate multiple ids, one for each display id in the displayIds array - def newUUIDs(self, parentIDs, entityType, userId, userEmail, nIds, organ_code = None, lab_code = None, file_info_array = None): + def newUUIDs(self, parentIDs, entityType, userId, userEmail, nIds, organ_code = None, lab_code = None, file_info_array = None, base_dir_type = None): #if entityType == 'DONOR': gen_base_ids = entityType in HUBMAP_ID_ENTITY_TYPES returnIds = [] @@ -363,14 +369,15 @@ def newUUIDs(self, parentIDs, entityType, userId, userEmail, nIds, organ_code = insRow = (insUuid, ins_hubmap_base_id, entityType, now, userId, userEmail) if store_file_info: - file_path = file_info_array[i]['path'] + info_idx = i + n + file_path = file_info_array[info_idx]['path'] file_checksum = None file_size = None - if 'checksum' in file_info_array[i]: - file_checksum = file_info_array[i]['checksum'] - if 'size' in file_info_array[i]: - file_size = file_info_array[i]['size'] - file_info_ins_row = (insUuid, file_path, file_checksum, file_size) + if 'checksum' in file_info_array[info_idx]: + file_checksum = file_info_array[info_idx]['checksum'] + if 'size' in file_info_array[info_idx]: + file_size = file_info_array[info_idx]['size'] + file_info_ins_row = (insUuid, file_path, file_checksum, file_size, base_dir_type) #file_info_ins_row = (insUuid, file_path, file_checksum) file_info_insert_vals.append(file_info_ins_row) thisId['file_path'] = file_path @@ -553,7 +560,7 @@ def getFileIdInfo(self, fid): check_id = fid.strip() if isBlank(check_id) or len(check_id) != 32: return Response("Invalid file id format. 32 digit hex only.", 400) - sql = "select hm_uuid, path, checksum, size, ancestor_uuid from hm_files inner join hm_ancestors on hm_ancestors.descendant_uuid = hm_files.hm_uuid where hm_uuid = '" + check_id + "'" + sql = "select hm_uuid, path, checksum, size, base_dir, ancestor_uuid from hm_files inner join hm_ancestors on hm_ancestors.descendant_uuid = hm_files.hm_uuid where hm_uuid = '" + check_id + "'" with closing(self.hmdb.getDBConnection()) as dbConn: with closing(dbConn.cursor()) as curs: curs.execute(sql) From e6a94f1785e71ee1a4e28ea60a9299fdd46573fb Mon Sep 17 00:00:00 2001 From: Bill Shirey Date: Mon, 1 Feb 2021 10:44:04 -0500 Subject: [PATCH 25/37] fix base_dir reference before assignment bug --- src/uuid_worker.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/uuid_worker.py b/src/uuid_worker.py index efbaac8..6add218 100644 --- a/src/uuid_worker.py +++ b/src/uuid_worker.py @@ -191,6 +191,7 @@ def uuidPost(self, req, nIds): ancestor_ids = parentIds file_info = None + base_dir = None if entityType == "FILE": if not 'file_info' in content: return(Response("Entity type of FILE requires a file_info array provide in the request body.", 400)) From 9038180cfdfc3afc75a77bfa5e98df7d0cfcbe0c Mon Sep 17 00:00:00 2001 From: Bill Shirey Date: Tue, 9 Feb 2021 14:23:38 -0500 Subject: [PATCH 26/37] added utility to convert old files uploaded via ingest portal and functionality to us tag in file path --- reload_from_neo4j/fix_sample_file_uploads.py | 133 +++++++++++++++++++ reload_from_neo4j/reload.properties.example | 8 ++ src/app.py | 4 + src/uuid_worker.py | 2 + 4 files changed, 147 insertions(+) create mode 100644 reload_from_neo4j/fix_sample_file_uploads.py diff --git a/reload_from_neo4j/fix_sample_file_uploads.py b/reload_from_neo4j/fix_sample_file_uploads.py new file mode 100644 index 0000000..7587fde --- /dev/null +++ b/reload_from_neo4j/fix_sample_file_uploads.py @@ -0,0 +1,133 @@ +import traceback +from py2neo import Graph +from hubmap_commons.exceptions import ErrorMessage +from hubmap_commons.properties import PropHelper +from hubmap_commons import file_helper +from uuid_worker import UUIDWorker +import sys +import os +import json +import hashlib +import requests +from shutil import copyfile + +SAMPLES_TO_FIX_Q = "match (s:Sample) where s.portal_metadata_upload_files is not null and not s.portal_metadata_upload_files = '[]' return s.uuid, s.portal_metadata_upload_files" + +class FixFileUploads: + + def __init__(self, property_file_name): + self.props = PropHelper(property_file_name, required_props = ['neo4j.server', 'neo4j.username', 'neo4j.password', 'db.host', 'db.name', 'db.username', 'db.password', 'user.mapping', 'client.id', 'client.secret', 'old.ingest.upload.dir', 'new.ingest.upload.dir', 'uuid.api.url', 'user.nexus.token']) + self.graph = Graph(self.props.get('neo4j.server'), auth=(self.props.get('neo4j.username'), self.props.get('neo4j.password'))) + self.uuid_wrker = UUIDWorker(self.props.get('client.id'), self.props.get('client.secret'), self.props.get('db.host'), self.props.get('db.name'), self.props.get('db.username'), self.props.get('db.password')) + self.old_ingest_upload_dir = file_helper.ensureTrailingSlash(self.props.get('old.ingest.upload.dir')) + self.new_ingest_upload_dir = file_helper.ensureTrailingSlash(self.props.get('new.ingest.upload.dir')) + self.uuid_api_url = file_helper.ensureTrailingSlashURL(self.props.get('uuid.api.url')) + "hmuuid" + self.user_token = self.props.get('user.nexus.token') + + def fix_sample_file_uploads(self): + samples = self.graph.run(SAMPLES_TO_FIX_Q).data() + tofix = [] + for sample in samples: + uuid = sample['s.uuid'] + file_json = sample['s.portal_metadata_upload_files'].replace("'", '"') + file_info = json.loads(file_json) + changes = {"uuid":uuid, "changes":[]} + tofix.append(changes) + for fi in file_info: + change_info = {} + file_path = fi['filepath'] + dirs = file_path.split('/') + n_dirs = len(dirs) + old_rel_dir = dirs[n_dirs-3] + os.sep + dirs[n_dirs-2] + + change_info['copy_from'] = self.old_ingest_upload_dir + old_rel_dir + os.sep + dirs[n_dirs-1] + if not os.path.exists(change_info['copy_from']): + raise Exception(f'File {change_info["copy_from"]} attached to entity with uuid:{uuid} does not exist') + change_info['file_uuid'] = self.__gen_file_uuid(change_info['copy_from'], uuid + os.sep + '' + os.sep + dirs[n_dirs-1], uuid) + new_rel_dir = uuid + os.sep + change_info['file_uuid'] + new_rel_path = new_rel_dir + os.sep + dirs[n_dirs-1] + change_info['copy_to'] = self.new_ingest_upload_dir + new_rel_path + change_info['file_path'] = new_rel_path + change_info['file_name'] = dirs[n_dirs-1] + if not os.path.exists(self.new_ingest_upload_dir + new_rel_dir): + os.makedirs(self.new_ingest_upload_dir + new_rel_dir) + description = "" + if 'description' in fi: + change_info['description'] = fi['description'] + changes['changes'].append(change_info) + + for fix_rcd in tofix: + uuid = fix_rcd['uuid'] + new_files_recd = [] + for change in fix_rcd['changes']: + print(f'copy {change["copy_from"]} to {change["copy_to"]}') + file_rcd = {} + file_rcd['filename'] = change['file_name'] + file_rcd['file_uuid'] = change['file_uuid'] + if 'description' in change: + file_rcd['description'] = change['description'] + new_files_recd.append(file_rcd) + copyfile(change["copy_from"], change["copy_to"]) + + #update the record + update_cql = "match (s:Sample {uuid:'" + uuid + "'}) set s.metadata_files = '" + json.dumps(new_files_recd) + "' return s.uuid" + self.graph.run(update_cql) + #print(uuid + ":" + json.dumps(new_files_recd)) + #if file_path + #print(f"{uuid}:{file_json}") + + def __gen_file_uuid(self, file_current_path, file_rel_path, parent_entity_uuid): + checksum = hashlib.md5(open(file_current_path, 'rb').read()).hexdigest() + filesize = os.path.getsize(file_current_path) + headers = {'Authorization': 'Bearer ' + self.user_token, 'Content-Type': 'application/json'} + data = {} + data['entity_type'] = 'FILE' + data['parent_ids'] = [parent_entity_uuid] + file_info= {} + file_info['path'] = file_rel_path + file_info['checksum'] = checksum + file_info['base_dir'] = 'INGEST_PORTAL_UPLOAD' + file_info['size'] = filesize + data['file_info'] = [file_info] + response = requests.post(self.uuid_api_url, json = data, headers = headers, verify = False) + if response is None or response.status_code != 200: + raise Exception(f"Unable to generate uuid for file {file_rel_path}") + + rsjs = response.json() + file_uuid = rsjs[0]['uuid'] + return file_uuid + + + def tfix(self): + samples = self.graph.run("match (s:Sample) where not s.metadata_files is null return s.uuid, s.metadata_files").data() + for sample in samples: + uuid = sample['s.uuid'] + file_json = sample['s.metadata_files'].replace("'", '"') + file_info_arry = json.loads(file_json) + replace_arry = [] + for file_info in file_info_arry: + if 'file_path' in file_info: + f_path = file_info['file_path'] + s_path = f_path.split('/') + file_info['filename'] = s_path[3] + print(file_info['filename']) + file_info.pop('file_path') + replace_arry.append(file_info) + update_cql = "match (s:Sample {uuid:'" + uuid + "'}) set s.metadata_files = '" + json.dumps(replace_arry) + "' return s.uuid" + self.graph.run(update_cql) + +try: + fixer = FixFileUploads(os.path.dirname(os.path.realpath(__file__)) + "/reload.properties") + fixer.fix_sample_file_uploads() + +except ErrorMessage as em: + print(em.get_message()) + exit(1) +except Exception as e: + exc_type, exc_value, exc_traceback = sys.exc_info() + eMsg = str(e) + print("ERROR Occurred: " + eMsg) + traceback.print_tb(exc_traceback) + exit(1) + + \ No newline at end of file diff --git a/reload_from_neo4j/reload.properties.example b/reload_from_neo4j/reload.properties.example index c50b3a7..1b445a0 100644 --- a/reload_from_neo4j/reload.properties.example +++ b/reload_from_neo4j/reload.properties.example @@ -8,6 +8,14 @@ db.password=123 #a user id is missing in neo4j user.mapping={"someone@pitt.edu":"43e53b4a-7853-33e4-99f3-bce224a0e312","someone@ufl.edu":"5eab9484-6bec-486d-88cc-7492620a3d6c","someone@stanford.edu":"6ce3090a22-11c8-3544-cc53-b91f8d257362"} +#the location of the files previously upload via the ingest portal +old.ingest.upload.dir=/Users/bill/projects/hubmap/ingest-uploads-testing/uploads +#location where the previously uploaded files should be copied +new.ingest.upload.dir=/Users/bill/projects/hubmap/file_uploads + +uuid.api.url=https://uuid-api.refactor.hubmapconsortium.org/ + + # Point to the Neo4j database that will be used # to load the uuid database neo4j.server=bolt://hubmap-neo4j:7687 diff --git a/src/app.py b/src/app.py index da12207..1206aa5 100644 --- a/src/app.py +++ b/src/app.py @@ -101,6 +101,10 @@ def status(): path- required: the path to the file in storage. For the purposes of the UUID system this can be a full path or relative, but it is recommended that a relative path be used. + The path attribute can contain an optional "" tag, which + will be replaced by the generated file uuid before being stored. + This is useful in the case where the path to the file will include + the file uuid, such as for files uploaded via the ingest portal. base_dir- required: a specifier for the base directory where the file is stored valid values are: DATA_UPLOAD or INGEST_PORTAL_UPLOAD diff --git a/src/uuid_worker.py b/src/uuid_worker.py index 6add218..9510d46 100644 --- a/src/uuid_worker.py +++ b/src/uuid_worker.py @@ -372,6 +372,8 @@ def newUUIDs(self, parentIDs, entityType, userId, userEmail, nIds, organ_code = if store_file_info: info_idx = i + n file_path = file_info_array[info_idx]['path'] + #replace any tags in the file path with the generated uuid + file_path = file_path.replace('', insUuid) file_checksum = None file_size = None if 'checksum' in file_info_array[info_idx]: From 6a103dbfb3f482a20a44975f0a272fcd7573056f Mon Sep 17 00:00:00 2001 From: Bill Shirey Date: Tue, 16 Feb 2021 09:04:00 -0500 Subject: [PATCH 27/37] added blood (BD) and bone marrow (BM) as organs allowed to be registered multiple times --- src/uuid_worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/uuid_worker.py b/src/uuid_worker.py index 9510d46..50cf870 100644 --- a/src/uuid_worker.py +++ b/src/uuid_worker.py @@ -22,7 +22,7 @@ SUBMISSION_ID_ENTITY_TYPES = ['SAMPLE', 'DONOR'] ANCESTOR_REQUIRED_ENTITY_TYPES = ['SAMPLE', 'DONOR', 'DATASET', 'FILE'] -MULTIPLE_ALLOWED_ORGANS = ['LY', 'SK'] +MULTIPLE_ALLOWED_ORGANS = ['LY', 'SK', 'BD', 'BM'] MAX_GEN_IDS = 200 INSERT_SQL = "INSERT INTO hm_uuids (HM_UUID, HUBMAP_BASE_ID, ENTITY_TYPE, TIME_GENERATED, USER_ID, USER_EMAIL) VALUES (%s, %s, %s, %s, %s, %s)" From 1595ae708283aec1099d24638a6172fbe810811c Mon Sep 17 00:00:00 2001 From: Bill Shirey Date: Tue, 16 Feb 2021 09:25:42 -0500 Subject: [PATCH 28/37] cypher to fix codes for serialized lymph nodes --- reload_from_neo4j/fix-organs.cypher | 1 + 1 file changed, 1 insertion(+) create mode 100644 reload_from_neo4j/fix-organs.cypher diff --git a/reload_from_neo4j/fix-organs.cypher b/reload_from_neo4j/fix-organs.cypher new file mode 100644 index 0000000..625413b --- /dev/null +++ b/reload_from_neo4j/fix-organs.cypher @@ -0,0 +1 @@ +match (s:Sample) where s.organ starts with 'LY' set s.organ = 'LY' \ No newline at end of file From 28452bc4d0f1db25af63623b84eea283cbdeaad4 Mon Sep 17 00:00:00 2001 From: Bill Shirey Date: Tue, 16 Feb 2021 22:37:17 -0500 Subject: [PATCH 29/37] set the descsendant entity counts --- reload_from_neo4j/reload_from_neo.py | 40 ++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/reload_from_neo4j/reload_from_neo.py b/reload_from_neo4j/reload_from_neo.py index 6afa2c1..96beed5 100644 --- a/reload_from_neo4j/reload_from_neo.py +++ b/reload_from_neo4j/reload_from_neo.py @@ -67,7 +67,8 @@ def clear_tables(self): def load_lab_info(self): str_now = datetime.now().strftime('%Y-%m-%d %H:%M:%S') - labs = self.graph.run(LABS_CYPHER).data() + labs_r = self.graph.run(LABS_CYPHER) + labs = labs_r.data() with closing(self.uuid_db.getDBConnection()) as db_conn: with closing(db_conn.cursor()) as curs: count = 0 @@ -310,6 +311,39 @@ def convert_ancestor_array(self, results): return record + def fix_descendant_counts(self): + #set donor count per lab + donors = self.graph.run(DONORS_CYPHER).data() + lab_counts = {} + for donor in donors: + submission_id = donor['d.submission_id'] + lab_id = donor['lab.uuid'] + if not lab_id in lab_counts: lab_counts[lab_id] = 0 + n = int(submission_id[len(submission_id)-4:]) + if n > lab_counts[lab_id]: lab_counts[lab_id] = n + + samples = self.graph.run("match(sp:Sample)-[:ACTIVITY_INPUT]->(a)-[:ACTIVITY_OUTPUT]->(sc:Sample) return sp.uuid, sc.submission_id").data() + sample_counts = {} + for sample in samples: + submission_id = sample['sc.submission_id'] + parent_id = sample['sp.uuid'] + n = int(submission_id[submission_id.rfind('-')+1:]) + if not parent_id in sample_counts: sample_counts[parent_id] = 0 + if n > sample_counts[parent_id]: sample_counts[parent_id] = n + + with closing(self.uuid_db.getDBConnection()) as dbConn: + with closing(dbConn.cursor()) as curs: + for lab_uuid in lab_counts.keys(): + sql = f"update hm_uuids join hm_data_centers on hm_uuids.hm_uuid = hm_data_centers.hm_uuid set hm_uuids.descendant_count = {lab_counts[lab_uuid]} where hm_data_centers.dc_uuid = '{lab_uuid}';" + curs.execute(sql) + for sample_uuid in sample_counts.keys(): + sql = f"update hm_uuids set descendant_count = {sample_counts[sample_uuid]} where hm_uuid = '{sample_uuid}'" + curs.execute(sql) + dbConn.commit() + + + + def test(self, lab_id): if isBlank(lab_id): @@ -353,8 +387,10 @@ def test(self, lab_id): reloader.load_activity_info() print("\nLoading collections.", end='') reloader.load_collection_info() - print("\nReplacing temp UUIDs") + print("\nReplacing temp UUIDs", end='') reloader.replace_temp_uuids() + print("\nFixing descendant counts") + reloader.fix_descendant_counts() #print(reloader.test('2cf25858-ed44-11e8-991d-0e368f3075e8Z')) except ErrorMessage as em: From 28980839d5c56bb1537c6dac18fd6cabf3223ecf Mon Sep 17 00:00:00 2001 From: "Zhou (Joe) Yuan" Date: Tue, 2 Mar 2021 22:08:53 -0500 Subject: [PATCH 30/37] Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5c7d6d9..4bb98c2 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ -# uuid-api for HuBMAP +# HuBMAP UUID API The uuid-api service is a restful web service used to create and query UUIDs used across HuBMAP. Three types of IDs are supported: * HuBMAP UUID: Standard randomly generated 128 bit UUIDs represented as 32 hexadecimal digits. These are generated by the service. * HuBMAP DOI prefix: A randomly generated unique id that can be used to construct a HuBMAP DOI in the format ###.XXXX.###. These are optionally generated by the service. * HuBMAP Display Id: An id specified when generating a UUID and stored by the service to associate user defined ids with UUIDs. -## Development and deployment environments +## Docker development and deployment environments We have the following 5 development and deployment environments: @@ -89,4 +89,4 @@ You can also stop the running container and remove it by: ### Updating API Documentation -The documentation for the API calls is hosted on SmartAPI. Modifying the `entity-api-spec.yaml` file and commititng the changes to github should update the API shown on SmartAPI. SmartAPI allows users to register API documents. The documentation is associated with this github account: api-developers@hubmapconsortium.org. Please contact Chuck Borromeo (chb69@pitt.edu) if you want to register a new API on SmartAPI. +The documentation for the API calls is hosted on SmartAPI. Modifying the `entity-api-spec.yaml` file and commititng the changes to github should update the API shown on SmartAPI. SmartAPI allows users to register API documents. The documentation is associated with this github account: api-developers@hubmapconsortium.org. From ea93e24b422336a8e826fafef1da393da70aa6fa Mon Sep 17 00:00:00 2001 From: "Zhou (Joe) Yuan" Date: Fri, 5 Mar 2021 18:04:16 -0500 Subject: [PATCH 31/37] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 943f9cb..227cea2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.7.1 +2.0.0 From b4ad496c11932363c00cf699a9d1ae3c25fa29d2 Mon Sep 17 00:00:00 2001 From: yuanzhou Date: Fri, 5 Mar 2021 21:17:49 -0500 Subject: [PATCH 32/37] Update dependencies --- src/requirements.txt | 28 ++++++---------------------- src/requirements_dev.txt | 22 ++++++---------------- 2 files changed, 12 insertions(+), 38 deletions(-) diff --git a/src/requirements.txt b/src/requirements.txt index 15dd662..7c42673 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,25 +1,9 @@ -asn1crypto==0.24.0 -cachetools==4.1.0 -certifi==2019.9.11 -cffi==1.12.3 -chardet==3.0.4 -Click==7.0 -cryptography>=3.2 -Flask==1.1.1 -globus-sdk==1.8.0 -idna==2.8 -itsdangerous==1.1.0 -Jinja2==2.10.1 -MarkupSafe==1.1.1 -mysql-connector-python==8.0.16 -property==2.2 -protobuf==3.9.2 -pycparser==2.19 -PyJWT==1.7.1 -requests==2.22.0 -six==1.12.0 -urllib3==1.25.6 -Werkzeug==0.16.0 +requests>=2.25.1 +Flask>=1.1.2 +mysql_connector_repackaged>=0.3.1 +py2neo>=2021.0.1 +secrets>=1.0.2 + # The branch name of commons to be used during image build # Default is master branch specified in docker-compose.yml if not set git+git://github.com/hubmapconsortium/commons.git@${COMMONS_BRANCH}#egg=hubmap-commons diff --git a/src/requirements_dev.txt b/src/requirements_dev.txt index 8190b2e..56e55d5 100644 --- a/src/requirements_dev.txt +++ b/src/requirements_dev.txt @@ -4,21 +4,11 @@ certifi==2019.9.11 cffi==1.12.3 chardet==3.0.4 Click==7.0 -cryptography==2.7 -Flask==1.1.1 -globus-sdk==1.8.0 -idna==2.8 -itsdangerous==1.1.0 -Jinja2==2.10.1 -MarkupSafe==1.1.1 -mysql-connector-python==8.0.16 -property==2.2 -protobuf==3.9.2 -pycparser==2.19 -PyJWT==1.7.1 -requests==2.22.0 -six==1.12.0 -urllib3==1.25.6 -Werkzeug==0.16.0 +requests>=2.25.1 +Flask>=1.1.2 +mysql_connector_repackaged>=0.3.1 +py2neo>=2021.0.1 +secrets>=1.0.2 +# The commons to be used for local development file:../../commons#egg=hubmap-commons --no-index From 9042dd2b58879e912948deb2256f44386855e011 Mon Sep 17 00:00:00 2001 From: yuanzhou Date: Fri, 5 Mar 2021 21:56:24 -0500 Subject: [PATCH 33/37] Remove secrects dependency install --- src/requirements.txt | 1 - src/requirements_dev.txt | 1 - 2 files changed, 2 deletions(-) diff --git a/src/requirements.txt b/src/requirements.txt index 7c42673..366a222 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -2,7 +2,6 @@ requests>=2.25.1 Flask>=1.1.2 mysql_connector_repackaged>=0.3.1 py2neo>=2021.0.1 -secrets>=1.0.2 # The branch name of commons to be used during image build # Default is master branch specified in docker-compose.yml if not set diff --git a/src/requirements_dev.txt b/src/requirements_dev.txt index 56e55d5..b7da000 100644 --- a/src/requirements_dev.txt +++ b/src/requirements_dev.txt @@ -8,7 +8,6 @@ requests>=2.25.1 Flask>=1.1.2 mysql_connector_repackaged>=0.3.1 py2neo>=2021.0.1 -secrets>=1.0.2 # The commons to be used for local development file:../../commons#egg=hubmap-commons --no-index From 9490a0efa4cbce0e1f3ef6ac2066d2e90ab8b76e Mon Sep 17 00:00:00 2001 From: yuanzhou Date: Fri, 5 Mar 2021 23:18:44 -0500 Subject: [PATCH 34/37] Mysql dependency change --- src/requirements.txt | 2 +- src/requirements_dev.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/requirements.txt b/src/requirements.txt index 366a222..97427f5 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,6 +1,6 @@ requests>=2.25.1 Flask>=1.1.2 -mysql_connector_repackaged>=0.3.1 +mysql-connector-python>=8.0.16 py2neo>=2021.0.1 # The branch name of commons to be used during image build diff --git a/src/requirements_dev.txt b/src/requirements_dev.txt index b7da000..8fd6bf6 100644 --- a/src/requirements_dev.txt +++ b/src/requirements_dev.txt @@ -6,7 +6,7 @@ chardet==3.0.4 Click==7.0 requests>=2.25.1 Flask>=1.1.2 -mysql_connector_repackaged>=0.3.1 +mysql-connector-python>=8.0.16 py2neo>=2021.0.1 # The commons to be used for local development From a66bf021c605912e0722270885eac0ab7c3ef62e Mon Sep 17 00:00:00 2001 From: yuanzhou Date: Sat, 6 Mar 2021 12:03:33 -0500 Subject: [PATCH 35/37] Fix dependency issue - stick with mysql-connector-python==8.0.16 --- src/requirements.txt | 5 ++++- src/requirements_dev.txt | 11 ++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/requirements.txt b/src/requirements.txt index 97427f5..e892193 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,6 +1,9 @@ requests>=2.25.1 Flask>=1.1.2 -mysql-connector-python>=8.0.16 + +# Higher version would cause issue +mysql-connector-python==8.0.16 + py2neo>=2021.0.1 # The branch name of commons to be used during image build diff --git a/src/requirements_dev.txt b/src/requirements_dev.txt index 8fd6bf6..3d9813c 100644 --- a/src/requirements_dev.txt +++ b/src/requirements_dev.txt @@ -1,12 +1,9 @@ -asn1crypto==0.24.0 -cachetools==4.1.0 -certifi==2019.9.11 -cffi==1.12.3 -chardet==3.0.4 -Click==7.0 requests>=2.25.1 Flask>=1.1.2 -mysql-connector-python>=8.0.16 + +# Higher version would cause issue +mysql-connector-python==8.0.16 + py2neo>=2021.0.1 # The commons to be used for local development From 88e624018a96e08f5a241c5acca575ee09e7a46e Mon Sep 17 00:00:00 2001 From: yuanzhou Date: Mon, 8 Mar 2021 13:45:51 -0500 Subject: [PATCH 36/37] Remove requirements_dev.txt and use agreed versions for requests and PyYAML --- src/requirements.txt | 5 +++-- src/requirements_dev.txt | 10 ---------- 2 files changed, 3 insertions(+), 12 deletions(-) delete mode 100644 src/requirements_dev.txt diff --git a/src/requirements.txt b/src/requirements.txt index e892193..0b37170 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,10 +1,11 @@ -requests>=2.25.1 Flask>=1.1.2 +py2neo>=2021.0.1 # Higher version would cause issue mysql-connector-python==8.0.16 -py2neo>=2021.0.1 +# It's an agreement with other collaborators to use the beblow version for requests +requests>=2.22.0 # The branch name of commons to be used during image build # Default is master branch specified in docker-compose.yml if not set diff --git a/src/requirements_dev.txt b/src/requirements_dev.txt deleted file mode 100644 index 3d9813c..0000000 --- a/src/requirements_dev.txt +++ /dev/null @@ -1,10 +0,0 @@ -requests>=2.25.1 -Flask>=1.1.2 - -# Higher version would cause issue -mysql-connector-python==8.0.16 - -py2neo>=2021.0.1 - -# The commons to be used for local development -file:../../commons#egg=hubmap-commons --no-index From d67341b4a6cc42a7984f97efc4335121115a553d Mon Sep 17 00:00:00 2001 From: "Zhou (Joe) Yuan" Date: Sat, 13 Mar 2021 21:41:30 -0500 Subject: [PATCH 37/37] Use exact dependency versions These versions have been tested on TEST --- src/requirements.txt | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/requirements.txt b/src/requirements.txt index 0b37170..c785a48 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,11 +1,14 @@ -Flask>=1.1.2 -py2neo>=2021.0.1 +Flask==1.1.2 -# Higher version would cause issue +# Used by reload_from_neo4j +py2neo==2021.0.1 + +# Higher version (even 8.0.23) would cause issue because +# we still use RDS with older 5.6.mysql_aurora mysql-connector-python==8.0.16 -# It's an agreement with other collaborators to use the beblow version for requests -requests>=2.22.0 +# The commons package requires requests>=2.22.0 +requests==2.25.1 # The branch name of commons to be used during image build # Default is master branch specified in docker-compose.yml if not set