From 9b3059f9e73f857e4ae7c0ef5cf7fa3a81129c4a Mon Sep 17 00:00:00 2001 From: sarayourfriend <24264157+sarayourfriend@users.noreply.github.com> Date: Thu, 15 Dec 2022 15:21:30 +1100 Subject: [PATCH] Replace grequests with asyncio solution (#1027) * Draft pass at asyncio replacement for grequests * Use async_to_sync to ensure Django safety * Update unit tests for asyncio validate_images * Add back 2s timeout and do not follow redirects * Use better test assertion Co-authored-by: Madison Swain-Bowden * Add return type annotations and clean up dead code * Fix suggested code typo * Add type annotations and switch to comprehension Co-authored-by: Madison Swain-Bowden --- api/Pipfile | 3 +- api/Pipfile.lock | 464 ++++++++++++++++++-- api/catalog/api/utils/validate_images.py | 59 +-- api/test/dead_link_filter_test.py | 41 +- api/test/unit/utils/validate_images_test.py | 57 +-- 5 files changed, 488 insertions(+), 136 deletions(-) diff --git a/api/Pipfile b/api/Pipfile index 6b3438526..24c5aa17e 100644 --- a/api/Pipfile +++ b/api/Pipfile @@ -18,6 +18,7 @@ pytest-raises = "~=0.11" remote-pdb = "~=2.1" sphinx = "~=5.2" sphinx-autobuild = "~=2021.3" +pook = "~=1.0" [packages] aws-requests-auth = "~=0.4" @@ -40,7 +41,6 @@ drf-yasg = "~=1.21" elasticsearch-dsl = "~=7.4" future = "~=0.18" gevent = "~=22.10" -grequests = "~=0.6" gunicorn = "~=20.1" hvac = "~=1.0" ipaddress = "~=1.0" @@ -56,6 +56,7 @@ redlock-py = "~=1.0" requests-oauthlib = "~=1.3" sentry-sdk = "~=1.11" wsgi-basic-auth = "~=1.1" +aiohttp = "~=3.8" [requires] python_version = "3.10" diff --git a/api/Pipfile.lock b/api/Pipfile.lock index 4a042639b..9eb73c0f0 100644 --- a/api/Pipfile.lock +++ b/api/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "63f9c181a0ffea5cea44dcebec6e231613e6942fc6266479c5ccbf592893fffc" + "sha256": "9cf1b1dd4f7b74b6c711f7508836782b3259903eea10669c5ab8144ab36eee4e" }, "pipfile-spec": 6, "requires": { @@ -16,6 +16,107 @@ ] }, "default": { + "aiohttp": { + "hashes": [ + "sha256:02f9a2c72fc95d59b881cf38a4b2be9381b9527f9d328771e90f72ac76f31ad8", + "sha256:059a91e88f2c00fe40aed9031b3606c3f311414f86a90d696dd982e7aec48142", + "sha256:05a3c31c6d7cd08c149e50dc7aa2568317f5844acd745621983380597f027a18", + "sha256:08c78317e950e0762c2983f4dd58dc5e6c9ff75c8a0efeae299d363d439c8e34", + "sha256:09e28f572b21642128ef31f4e8372adb6888846f32fecb288c8b0457597ba61a", + "sha256:0d2c6d8c6872df4a6ec37d2ede71eff62395b9e337b4e18efd2177de883a5033", + "sha256:16c121ba0b1ec2b44b73e3a8a171c4f999b33929cd2397124a8c7fcfc8cd9e06", + "sha256:1d90043c1882067f1bd26196d5d2db9aa6d268def3293ed5fb317e13c9413ea4", + "sha256:1e56b9cafcd6531bab5d9b2e890bb4937f4165109fe98e2b98ef0dcfcb06ee9d", + "sha256:20acae4f268317bb975671e375493dbdbc67cddb5f6c71eebdb85b34444ac46b", + "sha256:21b30885a63c3f4ff5b77a5d6caf008b037cb521a5f33eab445dc566f6d092cc", + "sha256:21d69797eb951f155026651f7e9362877334508d39c2fc37bd04ff55b2007091", + "sha256:256deb4b29fe5e47893fa32e1de2d73c3afe7407738bd3c63829874661d4822d", + "sha256:25892c92bee6d9449ffac82c2fe257f3a6f297792cdb18ad784737d61e7a9a85", + "sha256:2ca9af5f8f5812d475c5259393f52d712f6d5f0d7fdad9acdb1107dd9e3cb7eb", + "sha256:2d252771fc85e0cf8da0b823157962d70639e63cb9b578b1dec9868dd1f4f937", + "sha256:2dea10edfa1a54098703cb7acaa665c07b4e7568472a47f4e64e6319d3821ccf", + "sha256:2df5f139233060578d8c2c975128fb231a89ca0a462b35d4b5fcf7c501ebdbe1", + "sha256:2feebbb6074cdbd1ac276dbd737b40e890a1361b3cc30b74ac2f5e24aab41f7b", + "sha256:309aa21c1d54b8ef0723181d430347d7452daaff93e8e2363db8e75c72c2fb2d", + "sha256:3828fb41b7203176b82fe5d699e0d845435f2374750a44b480ea6b930f6be269", + "sha256:398701865e7a9565d49189f6c90868efaca21be65c725fc87fc305906be915da", + "sha256:43046a319664a04b146f81b40e1545d4c8ac7b7dd04c47e40bf09f65f2437346", + "sha256:437399385f2abcd634865705bdc180c8314124b98299d54fe1d4c8990f2f9494", + "sha256:45d88b016c849d74ebc6f2b6e8bc17cabf26e7e40c0661ddd8fae4c00f015697", + "sha256:47841407cc89a4b80b0c52276f3cc8138bbbfba4b179ee3acbd7d77ae33f7ac4", + "sha256:4a4fbc769ea9b6bd97f4ad0b430a6807f92f0e5eb020f1e42ece59f3ecfc4585", + "sha256:4ab94426ddb1ecc6a0b601d832d5d9d421820989b8caa929114811369673235c", + "sha256:4b0f30372cef3fdc262f33d06e7b411cd59058ce9174ef159ad938c4a34a89da", + "sha256:4e3a23ec214e95c9fe85a58470b660efe6534b83e6cbe38b3ed52b053d7cb6ad", + "sha256:512bd5ab136b8dc0ffe3fdf2dfb0c4b4f49c8577f6cae55dca862cd37a4564e2", + "sha256:527b3b87b24844ea7865284aabfab08eb0faf599b385b03c2aa91fc6edd6e4b6", + "sha256:54d107c89a3ebcd13228278d68f1436d3f33f2dd2af5415e3feaeb1156e1a62c", + "sha256:5835f258ca9f7c455493a57ee707b76d2d9634d84d5d7f62e77be984ea80b849", + "sha256:598adde339d2cf7d67beaccda3f2ce7c57b3b412702f29c946708f69cf8222aa", + "sha256:599418aaaf88a6d02a8c515e656f6faf3d10618d3dd95866eb4436520096c84b", + "sha256:5bf651afd22d5f0c4be16cf39d0482ea494f5c88f03e75e5fef3a85177fecdeb", + "sha256:5c59fcd80b9049b49acd29bd3598cada4afc8d8d69bd4160cd613246912535d7", + "sha256:653acc3880459f82a65e27bd6526e47ddf19e643457d36a2250b85b41a564715", + "sha256:66bd5f950344fb2b3dbdd421aaa4e84f4411a1a13fca3aeb2bcbe667f80c9f76", + "sha256:6f3553510abdbec67c043ca85727396ceed1272eef029b050677046d3387be8d", + "sha256:7018ecc5fe97027214556afbc7c502fbd718d0740e87eb1217b17efd05b3d276", + "sha256:713d22cd9643ba9025d33c4af43943c7a1eb8547729228de18d3e02e278472b6", + "sha256:73a4131962e6d91109bca6536416aa067cf6c4efb871975df734f8d2fd821b37", + "sha256:75880ed07be39beff1881d81e4a907cafb802f306efd6d2d15f2b3c69935f6fb", + "sha256:75e14eac916f024305db517e00a9252714fce0abcb10ad327fb6dcdc0d060f1d", + "sha256:8135fa153a20d82ffb64f70a1b5c2738684afa197839b34cc3e3c72fa88d302c", + "sha256:84b14f36e85295fe69c6b9789b51a0903b774046d5f7df538176516c3e422446", + "sha256:86fc24e58ecb32aee09f864cb11bb91bc4c1086615001647dbfc4dc8c32f4008", + "sha256:87f44875f2804bc0511a69ce44a9595d5944837a62caecc8490bbdb0e18b1342", + "sha256:88c70ed9da9963d5496d38320160e8eb7e5f1886f9290475a881db12f351ab5d", + "sha256:88e5be56c231981428f4f506c68b6a46fa25c4123a2e86d156c58a8369d31ab7", + "sha256:89d2e02167fa95172c017732ed7725bc8523c598757f08d13c5acca308e1a061", + "sha256:8d6aaa4e7155afaf994d7924eb290abbe81a6905b303d8cb61310a2aba1c68ba", + "sha256:92a2964319d359f494f16011e23434f6f8ef0434acd3cf154a6b7bec511e2fb7", + "sha256:96372fc29471646b9b106ee918c8eeb4cca423fcbf9a34daa1b93767a88a2290", + "sha256:978b046ca728073070e9abc074b6299ebf3501e8dee5e26efacb13cec2b2dea0", + "sha256:9c7149272fb5834fc186328e2c1fa01dda3e1fa940ce18fded6d412e8f2cf76d", + "sha256:a0239da9fbafd9ff82fd67c16704a7d1bccf0d107a300e790587ad05547681c8", + "sha256:ad5383a67514e8e76906a06741febd9126fc7c7ff0f599d6fcce3e82b80d026f", + "sha256:ad61a9639792fd790523ba072c0555cd6be5a0baf03a49a5dd8cfcf20d56df48", + "sha256:b29bfd650ed8e148f9c515474a6ef0ba1090b7a8faeee26b74a8ff3b33617502", + "sha256:b97decbb3372d4b69e4d4c8117f44632551c692bb1361b356a02b97b69e18a62", + "sha256:ba71c9b4dcbb16212f334126cc3d8beb6af377f6703d9dc2d9fb3874fd667ee9", + "sha256:c37c5cce780349d4d51739ae682dec63573847a2a8dcb44381b174c3d9c8d403", + "sha256:c971bf3786b5fad82ce5ad570dc6ee420f5b12527157929e830f51c55dc8af77", + "sha256:d1fde0f44029e02d02d3993ad55ce93ead9bb9b15c6b7ccd580f90bd7e3de476", + "sha256:d24b8bb40d5c61ef2d9b6a8f4528c2f17f1c5d2d31fed62ec860f6006142e83e", + "sha256:d5ba88df9aa5e2f806650fcbeedbe4f6e8736e92fc0e73b0400538fd25a4dd96", + "sha256:d6f76310355e9fae637c3162936e9504b4767d5c52ca268331e2756e54fd4ca5", + "sha256:d737fc67b9a970f3234754974531dc9afeea11c70791dcb7db53b0cf81b79784", + "sha256:da22885266bbfb3f78218dc40205fed2671909fbd0720aedba39b4515c038091", + "sha256:da37dcfbf4b7f45d80ee386a5f81122501ec75672f475da34784196690762f4b", + "sha256:db19d60d846283ee275d0416e2a23493f4e6b6028825b51290ac05afc87a6f97", + "sha256:db4c979b0b3e0fa7e9e69ecd11b2b3174c6963cebadeecfb7ad24532ffcdd11a", + "sha256:e164e0a98e92d06da343d17d4e9c4da4654f4a4588a20d6c73548a29f176abe2", + "sha256:e168a7560b7c61342ae0412997b069753f27ac4862ec7867eff74f0fe4ea2ad9", + "sha256:e381581b37db1db7597b62a2e6b8b57c3deec95d93b6d6407c5b61ddc98aca6d", + "sha256:e65bc19919c910127c06759a63747ebe14f386cda573d95bcc62b427ca1afc73", + "sha256:e7b8813be97cab8cb52b1375f41f8e6804f6507fe4660152e8ca5c48f0436017", + "sha256:e8a78079d9a39ca9ca99a8b0ac2fdc0c4d25fc80c8a8a82e5c8211509c523363", + "sha256:ebf909ea0a3fc9596e40d55d8000702a85e27fd578ff41a5500f68f20fd32e6c", + "sha256:ec40170327d4a404b0d91855d41bfe1fe4b699222b2b93e3d833a27330a87a6d", + "sha256:f178d2aadf0166be4df834c4953da2d7eef24719e8aec9a65289483eeea9d618", + "sha256:f88df3a83cf9df566f171adba39d5bd52814ac0b94778d2448652fc77f9eb491", + "sha256:f973157ffeab5459eefe7b97a804987876dd0a55570b8fa56b4e1954bf11329b", + "sha256:ff25f48fc8e623d95eca0670b8cc1469a83783c924a602e0fbd47363bb54aaca" + ], + "index": "pypi", + "version": "==3.8.3" + }, + "aiosignal": { + "hashes": [ + "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc", + "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17" + ], + "markers": "python_version >= '3.7'", + "version": "==1.3.1" + }, "asgiref": { "hashes": [ "sha256:1d2880b792ae8757289136f1db2b7b99100ce959b2aa57fd69dab783d05afac4", @@ -32,6 +133,14 @@ "markers": "python_version >= '3.6'", "version": "==4.0.2" }, + "attrs": { + "hashes": [ + "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6", + "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c" + ], + "markers": "python_version >= '3.5'", + "version": "==22.1.0" + }, "aws-requests-auth": { "hashes": [ "sha256:33593372018b960a31dbbe236f89421678b885c35f0b6a7abfae35bb77e069b2", @@ -42,26 +151,26 @@ }, "boto3": { "hashes": [ - "sha256:a4cf7fe649b0202f2f2307eb982ca215edcf9c87b6e65f5b3405f66c289603f2", - "sha256:f16c3aef1432c5083a9e1c36ac08b70b9072e87205e970f8d7520b10b964858f" + "sha256:2e5e80daae3873185b046d1fabc13676aea519e891faf4f27ca71d287bc26039", + "sha256:4fb4a0ce2679e5dc1719441192b45687654201d5de76224f2c376b689c9ed4aa" ], "index": "pypi", - "version": "==1.26.22" + "version": "==1.26.29" }, "botocore": { "hashes": [ - "sha256:03ead47c52c5caee4c69498a655444569e9d86c0e15e3884ce8c3eca01b4f658", - "sha256:0932b22d8737b11037adf7e734f9b90425b575d0757e4c1a035e99f382955221" + "sha256:97a6d059e688ff9caa7c0a4e30cc58fa27be8bf3347578ce7c62fb808380fb55", + "sha256:dca2daf108aae6c847d8ec99b7e918b46ae81713bf70b2199ab94627faf935a1" ], "markers": "python_version >= '3.7'", - "version": "==1.29.27" + "version": "==1.29.29" }, "certifi": { "hashes": [ "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3", "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18" ], - "index": "pypi", + "markers": "python_version >= '3.6'", "version": "==2022.12.7" }, "cffi": { @@ -189,11 +298,11 @@ }, "deepdiff": { "hashes": [ - "sha256:3fe134dde5b3922ff8c51fc1e95a972e659c853797231b836a5ccf15532fd516", - "sha256:8ba27c185f9197b78c316ce7bb0c743d25d14f7cdb8ec3b340437dbc93dcbff2" + "sha256:d04d997a68bf8bea01f8a97395877314ef5c2131d8f57bba2295f3adda725282", + "sha256:dea62316741f86c1d8e946f47c4c21386788457c898a495a5e6b0ccdcd76d9b6" ], "index": "pypi", - "version": "==6.2.1" + "version": "==6.2.2" }, "defusedxml": { "hashes": [ @@ -213,11 +322,11 @@ }, "django": { "hashes": [ - "sha256:678bbfc8604eb246ed54e2063f0765f13b321a50526bdc8cb1f943eda7fa31f1", - "sha256:6b1de6886cae14c7c44d188f580f8ba8da05750f544c80ae5ad43375ab293cd5" + "sha256:0b223bfa55511f950ff741983d408d78d772351284c75e9f77d2b830b6b4d148", + "sha256:d38a4e108d2386cb9637da66a82dc8d0733caede4c83c4afdbda78af4214211b" ], "index": "pypi", - "version": "==4.1.3" + "version": "==4.1.4" }, "django-braces": { "hashes": [ @@ -326,7 +435,7 @@ "sha256:840adeb45a5ec9102a83f3cf481aae83a3775b75d6dd83a7310b04e44a5d0308", "sha256:f511ea92e96db09b0e96b0de5fbbb7aa5c3740b0c571a364a2c3a1cc7ec06203" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_full_version < '4.0.0'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'", "version": "==7.17.8" }, "elasticsearch-dsl": { @@ -337,6 +446,86 @@ "index": "pypi", "version": "==7.4.0" }, + "frozenlist": { + "hashes": [ + "sha256:008a054b75d77c995ea26629ab3a0c0d7281341f2fa7e1e85fa6153ae29ae99c", + "sha256:02c9ac843e3390826a265e331105efeab489ffaf4dd86384595ee8ce6d35ae7f", + "sha256:034a5c08d36649591be1cbb10e09da9f531034acfe29275fc5454a3b101ce41a", + "sha256:05cdb16d09a0832eedf770cb7bd1fe57d8cf4eaf5aced29c4e41e3f20b30a784", + "sha256:0693c609e9742c66ba4870bcee1ad5ff35462d5ffec18710b4ac89337ff16e27", + "sha256:0771aed7f596c7d73444c847a1c16288937ef988dc04fb9f7be4b2aa91db609d", + "sha256:0af2e7c87d35b38732e810befb9d797a99279cbb85374d42ea61c1e9d23094b3", + "sha256:14143ae966a6229350021384870458e4777d1eae4c28d1a7aa47f24d030e6678", + "sha256:180c00c66bde6146a860cbb81b54ee0df350d2daf13ca85b275123bbf85de18a", + "sha256:1841e200fdafc3d51f974d9d377c079a0694a8f06de2e67b48150328d66d5483", + "sha256:23d16d9f477bb55b6154654e0e74557040575d9d19fe78a161bd33d7d76808e8", + "sha256:2b07ae0c1edaa0a36339ec6cce700f51b14a3fc6545fdd32930d2c83917332cf", + "sha256:2c926450857408e42f0bbc295e84395722ce74bae69a3b2aa2a65fe22cb14b99", + "sha256:2e24900aa13212e75e5b366cb9065e78bbf3893d4baab6052d1aca10d46d944c", + "sha256:303e04d422e9b911a09ad499b0368dc551e8c3cd15293c99160c7f1f07b59a48", + "sha256:352bd4c8c72d508778cf05ab491f6ef36149f4d0cb3c56b1b4302852255d05d5", + "sha256:3843f84a6c465a36559161e6c59dce2f2ac10943040c2fd021cfb70d58c4ad56", + "sha256:394c9c242113bfb4b9aa36e2b80a05ffa163a30691c7b5a29eba82e937895d5e", + "sha256:3bbdf44855ed8f0fbcd102ef05ec3012d6a4fd7c7562403f76ce6a52aeffb2b1", + "sha256:40de71985e9042ca00b7953c4f41eabc3dc514a2d1ff534027f091bc74416401", + "sha256:41fe21dc74ad3a779c3d73a2786bdf622ea81234bdd4faf90b8b03cad0c2c0b4", + "sha256:47df36a9fe24054b950bbc2db630d508cca3aa27ed0566c0baf661225e52c18e", + "sha256:4ea42116ceb6bb16dbb7d526e242cb6747b08b7710d9782aa3d6732bd8d27649", + "sha256:58bcc55721e8a90b88332d6cd441261ebb22342e238296bb330968952fbb3a6a", + "sha256:5c11e43016b9024240212d2a65043b70ed8dfd3b52678a1271972702d990ac6d", + "sha256:5cf820485f1b4c91e0417ea0afd41ce5cf5965011b3c22c400f6d144296ccbc0", + "sha256:5d8860749e813a6f65bad8285a0520607c9500caa23fea6ee407e63debcdbef6", + "sha256:6327eb8e419f7d9c38f333cde41b9ae348bec26d840927332f17e887a8dcb70d", + "sha256:65a5e4d3aa679610ac6e3569e865425b23b372277f89b5ef06cf2cdaf1ebf22b", + "sha256:66080ec69883597e4d026f2f71a231a1ee9887835902dbe6b6467d5a89216cf6", + "sha256:783263a4eaad7c49983fe4b2e7b53fa9770c136c270d2d4bbb6d2192bf4d9caf", + "sha256:7f44e24fa70f6fbc74aeec3e971f60a14dde85da364aa87f15d1be94ae75aeef", + "sha256:7fdfc24dcfce5b48109867c13b4cb15e4660e7bd7661741a391f821f23dfdca7", + "sha256:810860bb4bdce7557bc0febb84bbd88198b9dbc2022d8eebe5b3590b2ad6c842", + "sha256:841ea19b43d438a80b4de62ac6ab21cfe6827bb8a9dc62b896acc88eaf9cecba", + "sha256:84610c1502b2461255b4c9b7d5e9c48052601a8957cd0aea6ec7a7a1e1fb9420", + "sha256:899c5e1928eec13fd6f6d8dc51be23f0d09c5281e40d9cf4273d188d9feeaf9b", + "sha256:8bae29d60768bfa8fb92244b74502b18fae55a80eac13c88eb0b496d4268fd2d", + "sha256:8df3de3a9ab8325f94f646609a66cbeeede263910c5c0de0101079ad541af332", + "sha256:8fa3c6e3305aa1146b59a09b32b2e04074945ffcfb2f0931836d103a2c38f936", + "sha256:924620eef691990dfb56dc4709f280f40baee568c794b5c1885800c3ecc69816", + "sha256:9309869032abb23d196cb4e4db574232abe8b8be1339026f489eeb34a4acfd91", + "sha256:9545a33965d0d377b0bc823dcabf26980e77f1b6a7caa368a365a9497fb09420", + "sha256:9ac5995f2b408017b0be26d4a1d7c61bce106ff3d9e3324374d66b5964325448", + "sha256:9bbbcedd75acdfecf2159663b87f1bb5cfc80e7cd99f7ddd9d66eb98b14a8411", + "sha256:a4ae8135b11652b08a8baf07631d3ebfe65a4c87909dbef5fa0cdde440444ee4", + "sha256:a6394d7dadd3cfe3f4b3b186e54d5d8504d44f2d58dcc89d693698e8b7132b32", + "sha256:a97b4fe50b5890d36300820abd305694cb865ddb7885049587a5678215782a6b", + "sha256:ae4dc05c465a08a866b7a1baf360747078b362e6a6dbeb0c57f234db0ef88ae0", + "sha256:b1c63e8d377d039ac769cd0926558bb7068a1f7abb0f003e3717ee003ad85530", + "sha256:b1e2c1185858d7e10ff045c496bbf90ae752c28b365fef2c09cf0fa309291669", + "sha256:b4395e2f8d83fbe0c627b2b696acce67868793d7d9750e90e39592b3626691b7", + "sha256:b756072364347cb6aa5b60f9bc18e94b2f79632de3b0190253ad770c5df17db1", + "sha256:ba64dc2b3b7b158c6660d49cdb1d872d1d0bf4e42043ad8d5006099479a194e5", + "sha256:bed331fe18f58d844d39ceb398b77d6ac0b010d571cba8267c2e7165806b00ce", + "sha256:c188512b43542b1e91cadc3c6c915a82a5eb95929134faf7fd109f14f9892ce4", + "sha256:c21b9aa40e08e4f63a2f92ff3748e6b6c84d717d033c7b3438dd3123ee18f70e", + "sha256:ca713d4af15bae6e5d79b15c10c8522859a9a89d3b361a50b817c98c2fb402a2", + "sha256:cd4210baef299717db0a600d7a3cac81d46ef0e007f88c9335db79f8979c0d3d", + "sha256:cfe33efc9cb900a4c46f91a5ceba26d6df370ffddd9ca386eb1d4f0ad97b9ea9", + "sha256:d5cd3ab21acbdb414bb6c31958d7b06b85eeb40f66463c264a9b343a4e238642", + "sha256:dfbac4c2dfcc082fcf8d942d1e49b6aa0766c19d3358bd86e2000bf0fa4a9cf0", + "sha256:e235688f42b36be2b6b06fc37ac2126a73b75fb8d6bc66dd632aa35286238703", + "sha256:eb82dbba47a8318e75f679690190c10a5e1f447fbf9df41cbc4c3afd726d88cb", + "sha256:ebb86518203e12e96af765ee89034a1dbb0c3c65052d1b0c19bbbd6af8a145e1", + "sha256:ee78feb9d293c323b59a6f2dd441b63339a30edf35abcb51187d2fc26e696d13", + "sha256:eedab4c310c0299961ac285591acd53dc6723a1ebd90a57207c71f6e0c2153ab", + "sha256:efa568b885bca461f7c7b9e032655c0c143d305bf01c30caf6db2854a4532b38", + "sha256:efce6ae830831ab6a22b9b4091d411698145cb9b8fc869e1397ccf4b4b6455cb", + "sha256:f163d2fd041c630fed01bc48d28c3ed4a3b003c00acd396900e11ee5316b56bb", + "sha256:f20380df709d91525e4bee04746ba612a4df0972c1b8f8e1e8af997e678c7b81", + "sha256:f30f1928162e189091cf4d9da2eac617bfe78ef907a761614ff577ef4edfb3c8", + "sha256:f470c92737afa7d4c3aacc001e335062d582053d4dbe73cda126f2d7031068dd", + "sha256:ff8bf625fe85e119553b5383ba0fb6aa3d0ec2ae980295aaefa552374926b3f4" + ], + "markers": "python_version >= '3.7'", + "version": "==1.3.3" + }, "future": { "hashes": [ "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d" @@ -468,14 +657,6 @@ "markers": "platform_python_implementation == 'CPython'", "version": "==2.0.1" }, - "grequests": { - "hashes": [ - "sha256:6eff964416021bb1dee4182a56cc2b551bfa42d37820e6aee6f2efa00d43a061", - "sha256:7dec890c6668e6755a1ea968565535867956639301268394d24df67b478df666" - ], - "index": "pypi", - "version": "==0.6.0" - }, "gunicorn": { "hashes": [ "sha256:9dcc4547dbb1cb284accfb15ab5667a0e5d1881cc443e0677b4882a4067a807e", @@ -599,6 +780,86 @@ "markers": "python_version >= '3.7'", "version": "==2.1.1" }, + "multidict": { + "hashes": [ + "sha256:018c8e3be7f161a12b3e41741b6721f9baeb2210f4ab25a6359b7d76c1017dce", + "sha256:01b456046a05ff7cceefb0e1d2a9d32f05efcb1c7e0d152446304e11557639ce", + "sha256:114a4ab3e5cfbc56c4b6697686ecb92376c7e8c56893ef20547921552f8bdf57", + "sha256:12e0d396faa6dc55ff5379eee54d1df3b508243ff15bfc8295a6ec7a4483a335", + "sha256:190626ced82d4cc567a09e7346340d380154a493bac6905e0095d8158cdf1e38", + "sha256:1f5d5129a937af4e3c4a1d6c139f4051b7d17d43276cefdd8d442a7031f7eef2", + "sha256:21e1ce0b187c4e93112304dcde2aa18922fdbe8fb4f13d8aa72a5657bce0563a", + "sha256:24e8d513bfcaadc1f8b0ebece3ff50961951c54b07d5a775008a882966102418", + "sha256:2523a29006c034687eccd3ee70093a697129a3ffe8732535d3b2df6a4ecc279d", + "sha256:26fbbe17f8a7211b623502d2bf41022a51da3025142401417c765bf9a56fed4c", + "sha256:2b66d61966b12e6bba500e5cbb2c721a35e119c30ee02495c5629bd0e91eea30", + "sha256:2cf5d19e12eff855aa198259c0b02fd3f5d07e1291fbd20279c37b3b0e6c9852", + "sha256:2cfda34b7cb99eacada2072e0f69c0ad3285cb6f8e480b11f2b6d6c1c6f92718", + "sha256:3541882266247c7cd3dba78d6ef28dbe704774df60c9e4231edaa4493522e614", + "sha256:36df958b15639e40472adaa4f0c2c7828fe680f894a6b48c4ce229f59a6a798b", + "sha256:38d394814b39be1c36ac709006d39d50d72a884f9551acd9c8cc1ffae3fc8c4e", + "sha256:4159fc1ec9ede8ab93382e0d6ba9b1b3d23c72da39a834db7a116986605c7ab4", + "sha256:445c0851a1cbc1f2ec3b40bc22f9c4a235edb3c9a0906122a9df6ea8d51f886c", + "sha256:47defc0218682281a52fb1f6346ebb8b68b17538163a89ea24dfe4da37a8a9a3", + "sha256:4cc5c8cd205a9810d16a5cd428cd81bac554ad1477cb87f4ad722b10992e794d", + "sha256:4ccf55f28066b4f08666764a957c2b7c241c7547b0921d69c7ceab5f74fe1a45", + "sha256:4fb3fe591956d8841882c463f934c9f7485cfd5f763a08c0d467b513dc18ef89", + "sha256:526f8397fc124674b8f39748680a0ff673bd6a715fecb4866716d36e380f015f", + "sha256:578bfcb16f4b8675ef71b960c00f174b0426e0eeb796bab6737389d8288eb827", + "sha256:5b51969503709415a35754954c2763f536a70b8bf7360322b2edb0c0a44391f6", + "sha256:5e58ec0375803526d395f6f7e730ecc45d06e15f68f7b9cdbf644a2918324e51", + "sha256:62db44727d0befea68e8ad2881bb87a9cfb6b87d45dd78609009627167f37b69", + "sha256:67090b17a0a5be5704fd109f231ee73cefb1b3802d41288d6378b5df46ae89ba", + "sha256:6cd14e61f0da2a2cfb9fe05bfced2a1ed7063ce46a7a8cd473be4973de9a7f91", + "sha256:70740c2bc9ab1c99f7cdcb104f27d16c63860c56d51c5bf0ef82fc1d892a2131", + "sha256:73009ea04205966d47e16d98686ac5c438af23a1bb30b48a2c5da3423ec9ce37", + "sha256:791458a1f7d1b4ab3bd9e93e0dcd1d59ef7ee9aa051dcd1ea030e62e49b923fd", + "sha256:7f9511e48bde6b995825e8d35e434fc96296cf07a25f4aae24ff9162be7eaa46", + "sha256:81c3d597591b0940e04949e4e4f79359b2d2e542a686ba0da5e25de33fec13e0", + "sha256:8230a39bae6c2e8a09e4da6bace5064693b00590a4a213e38f9a9366da10e7dd", + "sha256:8b92a9f3ab904397a33b193000dc4de7318ea175c4c460a1e154c415f9008e3d", + "sha256:94cbe5535ef150546b8321aebea22862a3284da51e7b55f6f95b7d73e96d90ee", + "sha256:960ce1b790952916e682093788696ef7e33ac6a97482f9b983abdc293091b531", + "sha256:99341ca1f1db9e7f47914cb2461305665a662383765ced6f843712564766956d", + "sha256:9aac6881454a750554ed4b280a839dcf9e2133a9d12ab4d417d673fb102289b7", + "sha256:9d359b0a962e052b713647ac1f13eabf2263167b149ed1e27d5c579f5c8c7d2c", + "sha256:9dbab2a7e9c073bc9538824a01f5ed689194db7f55f2b8102766873e906a6c1a", + "sha256:a27b029caa3b555a4f3da54bc1e718eb55fcf1a11fda8bf0132147b476cf4c08", + "sha256:a8b817d4ed68fd568ec5e45dd75ddf30cc72a47a6b41b74d5bb211374c296f5e", + "sha256:ad7d66422b9cc51125509229693d27e18c08f2dea3ac9de408d821932b1b3759", + "sha256:b46e79a9f4db53897d17bc64a39d1c7c2be3e3d4f8dba6d6730a2b13ddf0f986", + "sha256:baa96a3418e27d723064854143b2f414a422c84cc87285a71558722049bebc5a", + "sha256:beeca903e4270b4afcd114f371a9602240dc143f9e944edfea00f8d4ad56c40d", + "sha256:c2a1168e5aa7c72499fb03c850e0f03f624fa4a5c8d2e215c518d0a73872eb64", + "sha256:c5790cc603456b6dcf8a9a4765f666895a6afddc88b3d3ba7b53dea2b6e23116", + "sha256:cb4a08f0aaaa869f189ffea0e17b86ad0237b51116d494da15ef7991ee6ad2d7", + "sha256:cd5771e8ea325f85cbb361ddbdeb9ae424a68e5dfb6eea786afdcd22e68a7d5d", + "sha256:ce8e51774eb03844588d3c279adb94efcd0edeccd2f97516623292445bcc01f9", + "sha256:d09daf5c6ce7fc6ed444c9339bbde5ea84e2534d1ca1cd37b60f365c77f00dea", + "sha256:d0e798b072cf2aab9daceb43d97c9c527a0c7593e67a7846ad4cc6051de1e303", + "sha256:d325d61cac602976a5d47b19eaa7d04e3daf4efce2164c630219885087234102", + "sha256:d408172519049e36fb6d29672f060dc8461fc7174eba9883c7026041ef9bfb38", + "sha256:d52442e7c951e4c9ee591d6047706e66923d248d83958bbf99b8b19515fffaef", + "sha256:dc4cfef5d899f5f1a15f3d2ac49f71107a01a5a2745b4dd53fa0cede1419385a", + "sha256:df7b4cee3ff31b3335aba602f8d70dbc641e5b7164b1e9565570c9d3c536a438", + "sha256:e068dfeadbce63072b2d8096486713d04db4946aad0a0f849bd4fc300799d0d3", + "sha256:e07c24018986fb00d6e7eafca8fcd6e05095649e17fcf0e33a592caaa62a78b9", + "sha256:e0bce9f7c30e7e3a9e683f670314c0144e8d34be6b7019e40604763bd278d84f", + "sha256:e1925f78a543b94c3d46274c66a366fee8a263747060220ed0188e5f3eeea1c0", + "sha256:e322c94596054352f5a02771eec71563c018b15699b961aba14d6dd943367022", + "sha256:e4a095e18847c12ec20e55326ab8782d9c2d599400a3a2f174fab4796875d0e2", + "sha256:e5a811aab1b4aea0b4be669363c19847a8c547510f0e18fb632956369fdbdf67", + "sha256:eddf604a3de2ace3d9a4e4d491be7562a1ac095a0a1c95a9ec5781ef0273ef11", + "sha256:ee9b1cae9a6c5d023e5a150f6f6b9dbb3c3bbc7887d6ee07d4c0ecb49a473734", + "sha256:f1650ea41c408755da5eed52ac6ccbc8938ccc3e698d81e6f6a1be02ff2a0945", + "sha256:f2c0957b3e8c66c10d27272709a5299ab3670a0f187c9428f3b90d267119aedb", + "sha256:f76109387e1ec8d8e2137c94c437b89fe002f29e0881aae8ae45529bdff92000", + "sha256:f8a728511c977df6f3d8af388fcb157e49f11db4a6637dd60131b8b6e40b0253", + "sha256:fb6c3dc3d65014d2c782f5acf0b3ba14e639c6c33d3ed8932ead76b9080b3544" + ], + "markers": "python_version >= '3.7'", + "version": "==6.0.3" + }, "oauthlib": { "hashes": [ "sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca", @@ -743,7 +1004,7 @@ "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", "version": "==2.8.2" }, "python-decouple": { @@ -804,7 +1065,7 @@ "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983", "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349" ], - "markers": "python_version >= '3.7' and python_full_version < '4.0.0'", + "markers": "python_version >= '3.7' and python_version < '4'", "version": "==2.28.1" }, "requests-oauthlib": { @@ -860,7 +1121,7 @@ "sha256:f01da5790e95815eb5a8a138508c01c758e5f5bc0ce4286c4f7028b8dd7ac3d0", "sha256:f34019dced51047d6f70cb9383b2ae2853b7fc4dce65129a5acd49f4f9256646" ], - "markers": "python_version < '3.11' and platform_python_implementation == 'CPython'", + "markers": "platform_python_implementation == 'CPython' and python_version < '3.11'", "version": "==0.2.7" }, "s3transfer": { @@ -892,7 +1153,7 @@ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", "version": "==1.16.0" }, "sqlparse": { @@ -939,7 +1200,7 @@ "sha256:73aae30359291c14fa3b956f8b5ca31960e420c28c1bec002547fb04928cf89b", "sha256:b64ef5141be559cfade448f044fa45c2260351edcb6a8ef6b7e00c7dcef0c323" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", "version": "==1.8.7" }, "wrapt": { @@ -1020,6 +1281,86 @@ "index": "pypi", "version": "==1.1.0" }, + "yarl": { + "hashes": [ + "sha256:009a028127e0a1755c38b03244c0bea9d5565630db9c4cf9572496e947137a87", + "sha256:0414fd91ce0b763d4eadb4456795b307a71524dbacd015c657bb2a39db2eab89", + "sha256:0978f29222e649c351b173da2b9b4665ad1feb8d1daa9d971eb90df08702668a", + "sha256:0ef8fb25e52663a1c85d608f6dd72e19bd390e2ecaf29c17fb08f730226e3a08", + "sha256:10b08293cda921157f1e7c2790999d903b3fd28cd5c208cf8826b3b508026996", + "sha256:1684a9bd9077e922300ecd48003ddae7a7474e0412bea38d4631443a91d61077", + "sha256:1b372aad2b5f81db66ee7ec085cbad72c4da660d994e8e590c997e9b01e44901", + "sha256:1e21fb44e1eff06dd6ef971d4bdc611807d6bd3691223d9c01a18cec3677939e", + "sha256:2305517e332a862ef75be8fad3606ea10108662bc6fe08509d5ca99503ac2aee", + "sha256:24ad1d10c9db1953291f56b5fe76203977f1ed05f82d09ec97acb623a7976574", + "sha256:272b4f1599f1b621bf2aabe4e5b54f39a933971f4e7c9aa311d6d7dc06965165", + "sha256:2a1fca9588f360036242f379bfea2b8b44cae2721859b1c56d033adfd5893634", + "sha256:2b4fa2606adf392051d990c3b3877d768771adc3faf2e117b9de7eb977741229", + "sha256:3150078118f62371375e1e69b13b48288e44f6691c1069340081c3fd12c94d5b", + "sha256:326dd1d3caf910cd26a26ccbfb84c03b608ba32499b5d6eeb09252c920bcbe4f", + "sha256:34c09b43bd538bf6c4b891ecce94b6fa4f1f10663a8d4ca589a079a5018f6ed7", + "sha256:388a45dc77198b2460eac0aca1efd6a7c09e976ee768b0d5109173e521a19daf", + "sha256:3adeef150d528ded2a8e734ebf9ae2e658f4c49bf413f5f157a470e17a4a2e89", + "sha256:3edac5d74bb3209c418805bda77f973117836e1de7c000e9755e572c1f7850d0", + "sha256:3f6b4aca43b602ba0f1459de647af954769919c4714706be36af670a5f44c9c1", + "sha256:3fc056e35fa6fba63248d93ff6e672c096f95f7836938241ebc8260e062832fe", + "sha256:418857f837347e8aaef682679f41e36c24250097f9e2f315d39bae3a99a34cbf", + "sha256:42430ff511571940d51e75cf42f1e4dbdded477e71c1b7a17f4da76c1da8ea76", + "sha256:44ceac0450e648de86da8e42674f9b7077d763ea80c8ceb9d1c3e41f0f0a9951", + "sha256:47d49ac96156f0928f002e2424299b2c91d9db73e08c4cd6742923a086f1c863", + "sha256:48dd18adcf98ea9cd721a25313aef49d70d413a999d7d89df44f469edfb38a06", + "sha256:49d43402c6e3013ad0978602bf6bf5328535c48d192304b91b97a3c6790b1562", + "sha256:4d04acba75c72e6eb90745447d69f84e6c9056390f7a9724605ca9c56b4afcc6", + "sha256:57a7c87927a468e5a1dc60c17caf9597161d66457a34273ab1760219953f7f4c", + "sha256:58a3c13d1c3005dbbac5c9f0d3210b60220a65a999b1833aa46bd6677c69b08e", + "sha256:5df5e3d04101c1e5c3b1d69710b0574171cc02fddc4b23d1b2813e75f35a30b1", + "sha256:63243b21c6e28ec2375f932a10ce7eda65139b5b854c0f6b82ed945ba526bff3", + "sha256:64dd68a92cab699a233641f5929a40f02a4ede8c009068ca8aa1fe87b8c20ae3", + "sha256:6604711362f2dbf7160df21c416f81fac0de6dbcf0b5445a2ef25478ecc4c778", + "sha256:6c4fcfa71e2c6a3cb568cf81aadc12768b9995323186a10827beccf5fa23d4f8", + "sha256:6d88056a04860a98341a0cf53e950e3ac9f4e51d1b6f61a53b0609df342cc8b2", + "sha256:705227dccbe96ab02c7cb2c43e1228e2826e7ead880bb19ec94ef279e9555b5b", + "sha256:728be34f70a190566d20aa13dc1f01dc44b6aa74580e10a3fb159691bc76909d", + "sha256:74dece2bfc60f0f70907c34b857ee98f2c6dd0f75185db133770cd67300d505f", + "sha256:75c16b2a900b3536dfc7014905a128a2bea8fb01f9ee26d2d7d8db0a08e7cb2c", + "sha256:77e913b846a6b9c5f767b14dc1e759e5aff05502fe73079f6f4176359d832581", + "sha256:7a66c506ec67eb3159eea5096acd05f5e788ceec7b96087d30c7d2865a243918", + "sha256:8c46d3d89902c393a1d1e243ac847e0442d0196bbd81aecc94fcebbc2fd5857c", + "sha256:93202666046d9edadfe9f2e7bf5e0782ea0d497b6d63da322e541665d65a044e", + "sha256:97209cc91189b48e7cfe777237c04af8e7cc51eb369004e061809bcdf4e55220", + "sha256:a48f4f7fea9a51098b02209d90297ac324241bf37ff6be6d2b0149ab2bd51b37", + "sha256:a783cd344113cb88c5ff7ca32f1f16532a6f2142185147822187913eb989f739", + "sha256:ae0eec05ab49e91a78700761777f284c2df119376e391db42c38ab46fd662b77", + "sha256:ae4d7ff1049f36accde9e1ef7301912a751e5bae0a9d142459646114c70ecba6", + "sha256:b05df9ea7496df11b710081bd90ecc3a3db6adb4fee36f6a411e7bc91a18aa42", + "sha256:baf211dcad448a87a0d9047dc8282d7de59473ade7d7fdf22150b1d23859f946", + "sha256:bb81f753c815f6b8e2ddd2eef3c855cf7da193b82396ac013c661aaa6cc6b0a5", + "sha256:bcd7bb1e5c45274af9a1dd7494d3c52b2be5e6bd8d7e49c612705fd45420b12d", + "sha256:bf071f797aec5b96abfc735ab97da9fd8f8768b43ce2abd85356a3127909d146", + "sha256:c15163b6125db87c8f53c98baa5e785782078fbd2dbeaa04c6141935eb6dab7a", + "sha256:cb6d48d80a41f68de41212f3dfd1a9d9898d7841c8f7ce6696cf2fd9cb57ef83", + "sha256:ceff9722e0df2e0a9e8a79c610842004fa54e5b309fe6d218e47cd52f791d7ef", + "sha256:cfa2bbca929aa742b5084fd4663dd4b87c191c844326fcb21c3afd2d11497f80", + "sha256:d617c241c8c3ad5c4e78a08429fa49e4b04bedfc507b34b4d8dceb83b4af3588", + "sha256:d881d152ae0007809c2c02e22aa534e702f12071e6b285e90945aa3c376463c5", + "sha256:da65c3f263729e47351261351b8679c6429151ef9649bba08ef2528ff2c423b2", + "sha256:de986979bbd87272fe557e0a8fcb66fd40ae2ddfe28a8b1ce4eae22681728fef", + "sha256:df60a94d332158b444301c7f569659c926168e4d4aad2cfbf4bce0e8fb8be826", + "sha256:dfef7350ee369197106805e193d420b75467b6cceac646ea5ed3049fcc950a05", + "sha256:e59399dda559688461762800d7fb34d9e8a6a7444fd76ec33220a926c8be1516", + "sha256:e6f3515aafe0209dd17fb9bdd3b4e892963370b3de781f53e1746a521fb39fc0", + "sha256:e7fd20d6576c10306dea2d6a5765f46f0ac5d6f53436217913e952d19237efc4", + "sha256:ebb78745273e51b9832ef90c0898501006670d6e059f2cdb0e999494eb1450c2", + "sha256:efff27bd8cbe1f9bd127e7894942ccc20c857aa8b5a0327874f30201e5ce83d0", + "sha256:f37db05c6051eff17bc832914fe46869f8849de5b92dc4a3466cd63095d23dfd", + "sha256:f8ca8ad414c85bbc50f49c0a106f951613dfa5f948ab69c10ce9b128d368baf8", + "sha256:fb742dcdd5eec9f26b61224c23baea46c9055cf16f62475e11b9b15dfd5c117b", + "sha256:fc77086ce244453e074e445104f0ecb27530d6fd3a46698e33f6c38951d5a0f1", + "sha256:ff205b58dc2929191f68162633d5e10e8044398d7a45265f90a0f1d51f85f72c" + ], + "markers": "python_version >= '3.7'", + "version": "==1.8.2" + }, "zope.event": { "hashes": [ "sha256:2666401939cdaa5f4e0c08cf7f20c9b21423b95e88f4675b1443973bdb080c42", @@ -1129,7 +1470,7 @@ "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3", "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18" ], - "index": "pypi", + "markers": "python_version >= '3.6'", "version": "==2022.12.7" }, "cfgv": { @@ -1212,11 +1553,11 @@ }, "fakeredis": { "hashes": [ - "sha256:0be420a79fabda234963a2730c4ce609a6d44a598e8dd253ce97785bef944285", - "sha256:2b02370118535893d832bcd3c099ef282de3f13b29ae3922432e2225794ec334" + "sha256:054ec109ec2cb4b1f0c6ed8d53603b07504c971bf8865dc6459eb5598e02ef1b", + "sha256:2d0d9163ce9ea7e3eae44c65d2e4ed168a8a3827a1d3ba430ca6f31ab0e762ff" ], "index": "pypi", - "version": "==1.10.0" + "version": "==1.10.1" }, "filelock": { "hashes": [ @@ -1226,13 +1567,20 @@ "markers": "python_version >= '3.7'", "version": "==3.8.2" }, + "furl": { + "hashes": [ + "sha256:5a6188fe2666c484a12159c18be97a1977a71d632ef5bb867ef15f54af39cc4e", + "sha256:9ab425062c4217f9802508e45feb4a83e54324273ac4b202f1850363309666c0" + ], + "version": "==2.1.3" + }, "furo": { "hashes": [ - "sha256:559ee17999c0f52728481dcf6b1b0cf8c9743e68c5e3a18cb45a7992747869a9", - "sha256:d4238145629c623609c2deb5384f8d036e2a1ee2a101d64b67b4348112470dbd" + "sha256:7cb76c12a25ef65db85ab0743df907573d03027a33631f17d267e598ebb191f7", + "sha256:d8008f8efbe7587a97ba533c8b2df1f9c21ee9b3e5cad0d27f61193d38b1a986" ], "index": "pypi", - "version": "==2022.9.29" + "version": "==2022.12.7" }, "identify": { "hashes": [ @@ -1310,7 +1658,7 @@ "sha256:1e525177574c23ae0f55cd62382632a083a0339928f0ca846a975a4da9851cec", "sha256:780a22d517cdc857d9714a80d8349c546945063f20853ea32ba7f85bc643ec7d" ], - "markers": "python_version >= '3.7' and python_version < '4.0'", + "markers": "python_version >= '3.7' and python_version < '4'", "version": "==0.1.2" }, "lazy-object-proxy": { @@ -1443,7 +1791,7 @@ "sha256:34fbd14b7501abe25e64d7b4624a9db02cde1a578d285b3da6f34b290cdf0b3a", "sha256:7cf27585dd7970b7257cefe48e1a3a10d4e34421831bdb472d96967433bc27bd" ], - "markers": "python_version >= '3.7' and python_version < '4.0'", + "markers": "python_version >= '3.7' and python_version < '4'", "version": "==0.3.4" }, "openapi-spec-validator": { @@ -1454,6 +1802,13 @@ "index": "pypi", "version": "==0.5.1" }, + "orderedmultidict": { + "hashes": [ + "sha256:04070bbb5e87291cc9bfa51df413677faf2141c73c61d2a5f7b26bea3cd882ad", + "sha256:43c839a17ee3cdd62234c47deca1a8508a3f2ca1d0678a3bf791c87cf84adbf3" + ], + "version": "==1.0.1" + }, "packaging": { "hashes": [ "sha256:2198ec20bd4c017b8f9717e00f0c8714076fc2fd93816750ab48e2c41de2cfd3", @@ -1475,7 +1830,7 @@ "sha256:5c869d315be50776cc8a993f3af43e0c60dc01506b399643f919034ebf4cdcab", "sha256:cdd7b1f9d7d5c8b8d3315dbf5a86b2596053ae845f056f57d97c0eefff84da14" ], - "markers": "python_version >= '3.7' and python_version < '4.0'", + "markers": "python_version >= '3.7' and python_version < '4'", "version": "==0.4.3" }, "pexpect": { @@ -1509,6 +1864,15 @@ "markers": "python_version >= '3.6'", "version": "==1.0.0" }, + "pook": { + "hashes": [ + "sha256:2e16d231ec9fe071c14cad7fe41261f65b401f6cb30935a169cf6fc229bd0a1d", + "sha256:cd3cbfe280d544e672f41a5b9482883841ba247f865858b57fd59f729e37616a", + "sha256:f28112db062d17db245b351c80f2bb5bf1e56ebfa93d3d75cc44f500c15c40eb" + ], + "index": "pypi", + "version": "==1.0.2" + }, "pre-commit": { "hashes": [ "sha256:51a5ba7c480ae8072ecdb6933df22d2f812dc897d5fe848778116129a681aac7", @@ -1612,7 +1976,7 @@ "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", "version": "==2.8.2" }, "pytz": { @@ -1689,7 +2053,7 @@ "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983", "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349" ], - "markers": "python_version >= '3.7' and python_full_version < '4.0.0'", + "markers": "python_version >= '3.7' and python_version < '4'", "version": "==2.28.1" }, "setuptools": { @@ -1705,7 +2069,7 @@ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", "version": "==1.16.0" }, "snowballstemmer": { @@ -1814,7 +2178,7 @@ "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", "version": "==0.10.2" }, "tomli": { @@ -1839,16 +2203,16 @@ "sha256:d3a2f5999215a3a06a4fc218026cd84c61b8b2b40ac5296a6db1f1451ef04c1e", "sha256:e5f923aa6a47e133d1cf87d60700889d7eae68988704e20c75fb2d65677a8e4b" ], - "markers": "python_version >= '3.7'", + "markers": "python_version > '2.7'", "version": "==6.2" }, "traitlets": { "hashes": [ - "sha256:61832ea7b7f910f5745e27e9bb269a181fd15af76027d99560299209d5b17c94", - "sha256:bd0fca5c890a09bf66b33cce67ca14156b080429bc39c7ef26b075a4bd4f9fc3" + "sha256:57ba2ba951632eeab9388fa45f342a5402060a5cc9f0bb942f760fafb6641581", + "sha256:fde8f62c05204ead43c2c1b9389cfc85befa7f54acb5da28529d671175bb4108" ], "markers": "python_version >= '3.7'", - "version": "==5.7.0" + "version": "==5.7.1" }, "typing-extensions": { "hashes": [ @@ -1880,6 +2244,14 @@ "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83" ], "version": "==0.2.5" + }, + "xmltodict": { + "hashes": [ + "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56", + "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852" + ], + "markers": "python_version >= '3.4'", + "version": "==0.13.0" } } } diff --git a/api/catalog/api/utils/validate_images.py b/api/catalog/api/utils/validate_images.py index b888fa286..11de88ce4 100644 --- a/api/catalog/api/utils/validate_images.py +++ b/api/catalog/api/utils/validate_images.py @@ -1,11 +1,14 @@ +import asyncio import logging import time from django.conf import settings +import aiohttp import django_redis -import grequests +from asgiref.sync import async_to_sync from decouple import config +from elasticsearch_dsl.response import Hit from catalog.api.utils.dead_link_mask import get_query_mask, save_query_mask @@ -28,7 +31,29 @@ def _get_expiry(status, default): return config(f"LINK_VALIDATION_CACHE_EXPIRY__{status}", default=default, cast=int) -def validate_images(query_hash, start_slice, results, image_urls): +async def _head(url: str, session: aiohttp.ClientSession) -> tuple[str, int]: + try: + async with session.head(url, timeout=2, allow_redirects=False) as response: + return url, response.status + except aiohttp.ClientError as exception: + _log_validation_failure(exception) + return url, -1 + + +# https://stackoverflow.com/q/55259755 +@async_to_sync +async def _make_head_requests(urls: list[str]) -> list[tuple[str, int]]: + tasks = [] + async with aiohttp.ClientSession(headers=HEADERS) as session: + tasks = [asyncio.ensure_future(_head(url, session)) for url in urls] + responses = asyncio.gather(*tasks) + await responses + return responses.result() + + +def validate_images( + query_hash: str, start_slice: int, results: list[Hit], image_urls: list[str] +) -> None: """ Make sure images exist before we display them. Treat redirects as broken links since 99% of the time the redirect leads to a generic "not found" @@ -57,25 +82,10 @@ def validate_images(query_hash, start_slice, results, image_urls): to_verify[url] = idx logger.debug(f"len(to_verify)={len(to_verify)}") - reqs = ( - grequests.head( - url, headers=HEADERS, allow_redirects=False, timeout=2, verify=False - ) - for url in to_verify.keys() - ) - verified = grequests.map(reqs, exception_handler=_validation_failure) + verified = _make_head_requests(to_verify.keys()) + # Cache newly verified image statuses. - to_cache = {} - # relies on the consistenct of the order returned by `dict::keys()` which - # is safe as of Python 3 dot something - for idx, url in enumerate(to_verify.keys()): - cache_key = CACHE_PREFIX + url - if verified[idx]: - status = verified[idx].status_code - # Response didn't arrive in time. Try again later. - else: - status = -1 - to_cache[cache_key] = status + to_cache = {CACHE_PREFIX + url: status for url, status in verified} pipe = redis.pipeline() if len(to_cache) > 0: @@ -98,10 +108,7 @@ def validate_images(query_hash, start_slice, results, image_urls): # Merge newly verified results with cached statuses for idx, url in enumerate(to_verify): cache_idx = to_verify[url] - if verified[idx] is not None: - cached_statuses[cache_idx] = verified[idx].status_code - else: - cached_statuses[cache_idx] = -1 + cached_statuses[cache_idx] = verified[idx][1] # Create a new dead link mask new_mask = [1] * len(results) @@ -145,6 +152,6 @@ def validate_images(query_hash, start_slice, results, image_urls): ) -def _validation_failure(request, exception): - logger = parent_logger.getChild("_validation_failure") +def _log_validation_failure(exception): + logger = parent_logger.getChild("_log_validation_failure") logger.warning(f"Failed to validate image! Reason: {exception}") diff --git a/api/test/dead_link_filter_test.py b/api/test/dead_link_filter_test.py index ba03924fb..af4d4cb95 100644 --- a/api/test/dead_link_filter_test.py +++ b/api/test/dead_link_filter_test.py @@ -1,5 +1,5 @@ from test.constants import API_URL -from unittest.mock import MagicMock, patch +from unittest.mock import patch from uuid import uuid4 from django.conf import settings @@ -48,44 +48,39 @@ def get_empty_cached_statuses(_, image_urls): ) -def _patch_grequests(): - def grequests_map(reqs, *_, **__): - """ - Patch for ``grequests.map`` used by ``validate_images`` to filter - and remove dead links - """ +_MAKE_HEAD_REQUESTS_MODULE_PATH = ( + "catalog.api.utils.validate_images._make_head_requests" +) + + +def _patch_make_head_requests(): + def _make_head_requests(urls): responses = [] - for idx in range(len(list(reqs))): - mocked_res = MagicMock() - mocked_res.status_code = 200 if idx % 10 != 0 else 404 - responses.append(mocked_res) + for idx, url in enumerate(urls): + status_code = 200 if idx % 10 != 0 else 404 + responses.append((url, status_code)) return responses - return patch("grequests.map", side_effect=grequests_map) + return patch(_MAKE_HEAD_REQUESTS_MODULE_PATH, side_effect=_make_head_requests) def patch_link_validation_dead_for_count(count): total_res_count = 0 - def grequests_map(reqs, *_, **__): + def _make_head_requests(urls): nonlocal total_res_count - """ - Patch for ``grequests.map`` used by ``validate_images`` to filter - and remove dead links - """ responses = [] - for idx in range(len(list(reqs))): + for idx, url in enumerate(urls): total_res_count += 1 - mocked_res = MagicMock() - mocked_res.status_code = 404 if total_res_count <= count else 200 - responses.append(mocked_res) + status_code = 404 if total_res_count <= count else 200 + responses.append((url, status_code)) return responses - return patch("grequests.map", side_effect=grequests_map) + return patch(_MAKE_HEAD_REQUESTS_MODULE_PATH, side_effect=_make_head_requests) @pytest.mark.django_db -@_patch_grequests() +@_patch_make_head_requests() def test_dead_link_filtering(mocked_map, client): path = "/v1/images/" query_params = {"q": "*", "page_size": 20} diff --git a/api/test/unit/utils/validate_images_test.py b/api/test/unit/utils/validate_images_test.py index 2a7889505..9987709b1 100644 --- a/api/test/unit/utils/validate_images_test.py +++ b/api/test/unit/utils/validate_images_test.py @@ -1,10 +1,9 @@ -from collections.abc import Callable -from dataclasses import dataclass +from unittest import mock +import aiohttp +import pook import pytest from fakeredis import FakeRedis -from grequests import AsyncRequest -from requests import Response from catalog.api.utils.validate_images import HEADERS, validate_images @@ -22,48 +21,26 @@ def get_redis_connection(*args, **kwargs): fake_redis.client().close() -@dataclass -class GRequestsFixture: - requests: list[AsyncRequest] - response_factory: Callable[ - [AsyncRequest], Response - ] = lambda x: GRequestsFixture._default_response_factory(x) - - @staticmethod - def _default_response_factory(req: AsyncRequest) -> Response: - res = Response() - res.url = req.url - res.status_code = 200 - return res - - -@pytest.fixture(autouse=True) -def grequests(monkeypatch) -> GRequestsFixture: - fixture = GRequestsFixture([]) - - def map_reqs(reqs, **kwargs): - nonlocal fixture - fixture.requests += list(reqs) - responses = [fixture.response_factory(r) for r in fixture.requests] - return responses - - monkeypatch.setattr("grequests.map", map_reqs) - - return fixture - - -def test_sends_user_agent(grequests): +@mock.patch.object(aiohttp, "ClientSession", wraps=aiohttp.ClientSession) +@pook.on +def test_sends_user_agent(wrapped_client_session: mock.AsyncMock): query_hash = "test_sends_user_agent" results = [object() for _ in range(40)] - image_urls = [f"http://example.org/{i}" for i in range(len(results))] + image_urls = [f"https://example.org/{i}" for i in range(len(results))] start_slice = 0 + head_mock = ( + pook.head(pook.regex(r"https://example.org/\d")) + .times(len(results)) + .reply(200) + .mock + ) + validate_images(query_hash, start_slice, results, image_urls) - requested_urls = [r.url for r in grequests.requests] + assert head_mock.calls == len(results) + requested_urls = [req.rawurl for req in head_mock.matches] for url in image_urls: assert url in requested_urls - assert len(grequests.requests) > 0 - for r in grequests.requests: - assert r.kwargs["headers"] == HEADERS + wrapped_client_session.assert_called_once_with(headers=HEADERS)