From 65e315af67caf472803f6b6c85a52ae07bbd71fa Mon Sep 17 00:00:00 2001 From: palewire Date: Wed, 23 Feb 2022 11:55:17 -0800 Subject: [PATCH 1/6] Add KY scraper --- Pipfile | 1 + Pipfile.lock | 463 +++++++++++++++++++++++++++++--------------- setup.py | 1 + warn/scrapers/ky.py | 132 +++++++++++++ 4 files changed, 438 insertions(+), 159 deletions(-) create mode 100644 warn/scrapers/ky.py diff --git a/Pipfile b/Pipfile index 3457d1b1..03052ba3 100644 --- a/Pipfile +++ b/Pipfile @@ -31,6 +31,7 @@ openpyxl = "*" pdfplumber = "*" tenacity = "*" click = "*" +xlrd = "*" [requires] python_version = "3.8" diff --git a/Pipfile.lock b/Pipfile.lock index 9167fe2f..97c8bfdb 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "97e198b5a311117a20e31347ebce62931573b0af1e7c95f55bcf8bc9c56ec3b7" + "sha256": "50d67d7c9271b38b8db74dc2a9ea20398b113642c7f280cbc15b825719675a5b" }, "pipfile-spec": 6, "requires": { @@ -103,19 +103,19 @@ }, "charset-normalizer": { "hashes": [ - "sha256:2842d8f5e82a1f6aa437380934d5e1cd4fcf2003b06fed6940769c164a480a45", - "sha256:98398a9d69ee80548c762ba991a4728bfc3836768ed226b3945908d1a688371c" + "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597", + "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df" ], "markers": "python_version >= '3'", - "version": "==2.0.11" + "version": "==2.0.12" }, "click": { "hashes": [ - "sha256:353f466495adaeb40b6b5f592f9f91cb22372351c84caeb068132442a4518ef3", - "sha256:410e932b050f5eed773c4cda94de75971c89cdb3155a72a0831139a79e5ecb5b" + "sha256:6a7a62563bbfabfda3a38f3023a1db4a35978c0abd76f6c9605ecd6554d6d9b1", + "sha256:8458d7b1287c5fb128c90e23381cf99dcde74beaf6c7ff6384ce84d6fe090adb" ], "index": "pypi", - "version": "==8.0.3" + "version": "==8.0.4" }, "cryptography": { "hashes": [ @@ -167,6 +167,14 @@ "markers": "python_version >= '3'", "version": "==3.3" }, + "importlib-metadata": { + "hashes": [ + "sha256:175f4ee440a0317f6e8d81b7f8d4869f93316170a65ad2b007d2929186c8052c", + "sha256:e0bc84ff355328a4adfc5240c4f211e0ab386f80aa640d1b11f0618a1d282094" + ], + "markers": "python_version < '3.8'", + "version": "==4.11.1" + }, "openpyxl": { "hashes": [ "sha256:40f568b9829bf9e446acfffce30250ac1fa39035124d55fc024025c41481c90f", @@ -270,6 +278,14 @@ "index": "pypi", "version": "==8.0.1" }, + "typing-extensions": { + "hashes": [ + "sha256:1a9462dcc3347a79b1f1c0271fbe79e844580bb598bafa1ed208b94da3cdcd42", + "sha256:21c85e0fe4b9a155d0799430b0ad741cdce7e359660ccbd8b530613e8df88ce2" + ], + "markers": "python_version < '3.8'", + "version": "==4.1.1" + }, "urllib3": { "hashes": [ "sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed", @@ -291,6 +307,22 @@ "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923" ], "version": "==0.5.1" + }, + "xlrd": { + "hashes": [ + "sha256:6a33ee89877bd9abc1158129f6e94be74e2679636b8a205b43b85206c3f0bbdd", + "sha256:f72f148f54442c6b056bf931dbc34f986fd0c3b0b6b5a58d013c9aef274d0c88" + ], + "index": "pypi", + "version": "==2.0.1" + }, + "zipp": { + "hashes": [ + "sha256:9f50f446828eb9d45b267433fd3e9da8d801f614129124863f9c51ebceafb87d", + "sha256:b47250dd24f92b7dd6a0a8fc5244da14608f3ca90a5efcd37a3b1642fac9a375" + ], + "markers": "python_version >= '3.7'", + "version": "==3.7.0" } }, "develop": { @@ -361,6 +393,61 @@ ], "version": "==2021.10.8" }, + "cffi": { + "hashes": [ + "sha256:00c878c90cb53ccfaae6b8bc18ad05d2036553e6d9d1d9dbcf323bbe83854ca3", + "sha256:0104fb5ae2391d46a4cb082abdd5c69ea4eab79d8d44eaaf79f1b1fd806ee4c2", + "sha256:06c48159c1abed75c2e721b1715c379fa3200c7784271b3c46df01383b593636", + "sha256:0808014eb713677ec1292301ea4c81ad277b6cdf2fdd90fd540af98c0b101d20", + "sha256:10dffb601ccfb65262a27233ac273d552ddc4d8ae1bf93b21c94b8511bffe728", + "sha256:14cd121ea63ecdae71efa69c15c5543a4b5fbcd0bbe2aad864baca0063cecf27", + "sha256:17771976e82e9f94976180f76468546834d22a7cc404b17c22df2a2c81db0c66", + "sha256:181dee03b1170ff1969489acf1c26533710231c58f95534e3edac87fff06c443", + "sha256:23cfe892bd5dd8941608f93348c0737e369e51c100d03718f108bf1add7bd6d0", + "sha256:263cc3d821c4ab2213cbe8cd8b355a7f72a8324577dc865ef98487c1aeee2bc7", + "sha256:2756c88cbb94231c7a147402476be2c4df2f6078099a6f4a480d239a8817ae39", + "sha256:27c219baf94952ae9d50ec19651a687b826792055353d07648a5695413e0c605", + "sha256:2a23af14f408d53d5e6cd4e3d9a24ff9e05906ad574822a10563efcef137979a", + "sha256:31fb708d9d7c3f49a60f04cf5b119aeefe5644daba1cd2a0fe389b674fd1de37", + "sha256:3415c89f9204ee60cd09b235810be700e993e343a408693e80ce7f6a40108029", + "sha256:3773c4d81e6e818df2efbc7dd77325ca0dcb688116050fb2b3011218eda36139", + "sha256:3b96a311ac60a3f6be21d2572e46ce67f09abcf4d09344c49274eb9e0bf345fc", + "sha256:3f7d084648d77af029acb79a0ff49a0ad7e9d09057a9bf46596dac9514dc07df", + "sha256:41d45de54cd277a7878919867c0f08b0cf817605e4eb94093e7516505d3c8d14", + "sha256:4238e6dab5d6a8ba812de994bbb0a79bddbdf80994e4ce802b6f6f3142fcc880", + "sha256:45db3a33139e9c8f7c09234b5784a5e33d31fd6907800b316decad50af323ff2", + "sha256:45e8636704eacc432a206ac7345a5d3d2c62d95a507ec70d62f23cd91770482a", + "sha256:4958391dbd6249d7ad855b9ca88fae690783a6be9e86df65865058ed81fc860e", + "sha256:4a306fa632e8f0928956a41fa8e1d6243c71e7eb59ffbd165fc0b41e316b2474", + "sha256:57e9ac9ccc3101fac9d6014fba037473e4358ef4e89f8e181f8951a2c0162024", + "sha256:59888172256cac5629e60e72e86598027aca6bf01fa2465bdb676d37636573e8", + "sha256:5e069f72d497312b24fcc02073d70cb989045d1c91cbd53979366077959933e0", + "sha256:64d4ec9f448dfe041705426000cc13e34e6e5bb13736e9fd62e34a0b0c41566e", + "sha256:6dc2737a3674b3e344847c8686cf29e500584ccad76204efea14f451d4cc669a", + "sha256:74fdfdbfdc48d3f47148976f49fab3251e550a8720bebc99bf1483f5bfb5db3e", + "sha256:75e4024375654472cc27e91cbe9eaa08567f7fbdf822638be2814ce059f58032", + "sha256:786902fb9ba7433aae840e0ed609f45c7bcd4e225ebb9c753aa39725bb3e6ad6", + "sha256:8b6c2ea03845c9f501ed1313e78de148cd3f6cad741a75d43a29b43da27f2e1e", + "sha256:91d77d2a782be4274da750752bb1650a97bfd8f291022b379bb8e01c66b4e96b", + "sha256:91ec59c33514b7c7559a6acda53bbfe1b283949c34fe7440bcf917f96ac0723e", + "sha256:920f0d66a896c2d99f0adbb391f990a84091179542c205fa53ce5787aff87954", + "sha256:a5263e363c27b653a90078143adb3d076c1a748ec9ecc78ea2fb916f9b861962", + "sha256:abb9a20a72ac4e0fdb50dae135ba5e77880518e742077ced47eb1499e29a443c", + "sha256:c2051981a968d7de9dd2d7b87bcb9c939c74a34626a6e2f8181455dd49ed69e4", + "sha256:c21c9e3896c23007803a875460fb786118f0cdd4434359577ea25eb556e34c55", + "sha256:c2502a1a03b6312837279c8c1bd3ebedf6c12c4228ddbad40912d671ccc8a962", + "sha256:d4d692a89c5cf08a8557fdeb329b82e7bf609aadfaed6c0d79f5a449a3c7c023", + "sha256:da5db4e883f1ce37f55c667e5c0de439df76ac4cb55964655906306918e7363c", + "sha256:e7022a66d9b55e93e1a845d8c9eba2a1bebd4966cd8bfc25d9cd07d515b33fa6", + "sha256:ef1f279350da2c586a69d32fc8733092fd32cc8ac95139a00377841f59a3f8d8", + "sha256:f54a64f8b0c8ff0b64d18aa76675262e1700f3995182267998c31ae974fbc382", + "sha256:f5c7150ad32ba43a07c4479f40241756145a1f03b43480e058cfd862bf5041c7", + "sha256:f6f824dc3bce0edab5f427efcfb1d63ee75b6fcb7282900ccaf925be84efb0fc", + "sha256:fd8a250edc26254fe5b33be00402e6d287f562b6a5b2152dec302fa15bb3e997", + "sha256:ffaa5c925128e29efbde7301d8ecaf35c8c60ffbcd6a1ffd3a552177c8e5e796" + ], + "version": "==1.15.0" + }, "cfgv": { "hashes": [ "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426", @@ -371,19 +458,19 @@ }, "charset-normalizer": { "hashes": [ - "sha256:2842d8f5e82a1f6aa437380934d5e1cd4fcf2003b06fed6940769c164a480a45", - "sha256:98398a9d69ee80548c762ba991a4728bfc3836768ed226b3945908d1a688371c" + "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597", + "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df" ], "markers": "python_version >= '3'", - "version": "==2.0.11" + "version": "==2.0.12" }, "click": { "hashes": [ - "sha256:353f466495adaeb40b6b5f592f9f91cb22372351c84caeb068132442a4518ef3", - "sha256:410e932b050f5eed773c4cda94de75971c89cdb3155a72a0831139a79e5ecb5b" + "sha256:6a7a62563bbfabfda3a38f3023a1db4a35978c0abd76f6c9605ecd6554d6d9b1", + "sha256:8458d7b1287c5fb128c90e23381cf99dcde74beaf6c7ff6384ce84d6fe090adb" ], "index": "pypi", - "version": "==8.0.3" + "version": "==8.0.4" }, "colorama": { "hashes": [ @@ -395,50 +482,76 @@ }, "coverage": { "hashes": [ - "sha256:1245ab82e8554fa88c4b2ab1e098ae051faac5af829efdcf2ce6b34dccd5567c", - "sha256:1bc6d709939ff262fd1432f03f080c5042dc6508b6e0d3d20e61dd045456a1a0", - "sha256:25e73d4c81efa8ea3785274a2f7f3bfbbeccb6fcba2a0bdd3be9223371c37554", - "sha256:276b13cc085474e482566c477c25ed66a097b44c6e77132f3304ac0b039f83eb", - "sha256:2aed4761809640f02e44e16b8b32c1a5dee5e80ea30a0ff0912158bde9c501f2", - "sha256:2dd70a167843b4b4b2630c0c56f1b586fe965b4f8ac5da05b6690344fd065c6b", - "sha256:352c68e233409c31048a3725c446a9e48bbff36e39db92774d4f2380d630d8f8", - "sha256:3f2b05757c92ad96b33dbf8e8ec8d4ccb9af6ae3c9e9bd141c7cc44d20c6bcba", - "sha256:448d7bde7ceb6c69e08474c2ddbc5b4cd13c9e4aa4a717467f716b5fc938a734", - "sha256:463e52616ea687fd323888e86bf25e864a3cc6335a043fad6bbb037dbf49bbe2", - "sha256:482fb42eea6164894ff82abbcf33d526362de5d1a7ed25af7ecbdddd28fc124f", - "sha256:56c4a409381ddd7bbff134e9756077860d4e8a583d310a6f38a2315b9ce301d0", - "sha256:56d296cbc8254a7dffdd7bcc2eb70be5a233aae7c01856d2d936f5ac4e8ac1f1", - "sha256:5e15d424b8153756b7c903bde6d4610be0c3daca3986173c18dd5c1a1625e4cd", - "sha256:618eeba986cea7f621d8607ee378ecc8c2504b98b3fdc4952b30fe3578304687", - "sha256:61d47a897c1e91f33f177c21de897267b38fbb45f2cd8e22a710bcef1df09ac1", - "sha256:621f6ea7260ea2ffdaec64fe5cb521669984f567b66f62f81445221d4754df4c", - "sha256:6a5cdc3adb4f8bb8d8f5e64c2e9e282bc12980ef055ec6da59db562ee9bdfefa", - "sha256:6c3f6158b02ac403868eea390930ae64e9a9a2a5bbfafefbb920d29258d9f2f8", - "sha256:704f89b87c4f4737da2860695a18c852b78ec7279b24eedacab10b29067d3a38", - "sha256:72128176fea72012063200b7b395ed8a57849282b207321124d7ff14e26988e8", - "sha256:78fbb2be068a13a5d99dce9e1e7d168db880870f7bc73f876152130575bd6167", - "sha256:7bff3a98f63b47464480de1b5bdd80c8fade0ba2832c9381253c9b74c4153c27", - "sha256:84f2436d6742c01136dd940ee158bfc7cf5ced3da7e4c949662b8703b5cd8145", - "sha256:9976fb0a5709988778ac9bc44f3d50fccd989987876dfd7716dee28beed0a9fa", - "sha256:9ad0a117b8dc2061ce9461ea4c1b4799e55edceb236522c5b8f958ce9ed8fa9a", - "sha256:9e3dd806f34de38d4c01416344e98eab2437ac450b3ae39c62a0ede2f8b5e4ed", - "sha256:9eb494070aa060ceba6e4bbf44c1bc5fa97bfb883a0d9b0c9049415f9e944793", - "sha256:9fde6b90889522c220dd56a670102ceef24955d994ff7af2cb786b4ba8fe11e4", - "sha256:9fff3ff052922cb99f9e52f63f985d4f7a54f6b94287463bc66b7cdf3eb41217", - "sha256:a06c358f4aed05fa1099c39decc8022261bb07dfadc127c08cfbd1391b09689e", - "sha256:a4f923b9ab265136e57cc14794a15b9dcea07a9c578609cd5dbbfff28a0d15e6", - "sha256:c5b81fb37db76ebea79aa963b76d96ff854e7662921ce742293463635a87a78d", - "sha256:d5ed164af5c9078596cfc40b078c3b337911190d3faeac830c3f1274f26b8320", - "sha256:d651fde74a4d3122e5562705824507e2f5b2d3d57557f1916c4b27635f8fbe3f", - "sha256:de73fca6fb403dd72d4da517cfc49fcf791f74eee697d3219f6be29adf5af6ce", - "sha256:e647a0be741edbb529a72644e999acb09f2ad60465f80757da183528941ff975", - "sha256:e92c7a5f7d62edff50f60a045dc9542bf939758c95b2fcd686175dd10ce0ed10", - "sha256:eeffd96882d8c06d31b65dddcf51db7c612547babc1c4c5db6a011abe9798525", - "sha256:f5a4551dfd09c3bd12fca8144d47fe7745275adf3229b7223c2f9e29a975ebda", - "sha256:fac0bcc5b7e8169bffa87f0dcc24435446d329cbc2b5486d155c2e0f3b493ae1" + "sha256:03e2a7826086b91ef345ff18742ee9fc47a6839ccd517061ef8fa1976e652ce9", + "sha256:07e6db90cd9686c767dcc593dff16c8c09f9814f5e9c51034066cad3373b914d", + "sha256:18d520c6860515a771708937d2f78f63cc47ab3b80cb78e86573b0a760161faf", + "sha256:1ebf730d2381158ecf3dfd4453fbca0613e16eaa547b4170e2450c9707665ce7", + "sha256:21b7745788866028adeb1e0eca3bf1101109e2dc58456cb49d2d9b99a8c516e6", + "sha256:26e2deacd414fc2f97dd9f7676ee3eaecd299ca751412d89f40bc01557a6b1b4", + "sha256:2c6dbb42f3ad25760010c45191e9757e7dce981cbfb90e42feef301d71540059", + "sha256:2fea046bfb455510e05be95e879f0e768d45c10c11509e20e06d8fcaa31d9e39", + "sha256:34626a7eee2a3da12af0507780bb51eb52dca0e1751fd1471d0810539cefb536", + "sha256:37d1141ad6b2466a7b53a22e08fe76994c2d35a5b6b469590424a9953155afac", + "sha256:46191097ebc381fbf89bdce207a6c107ac4ec0890d8d20f3360345ff5976155c", + "sha256:4dd8bafa458b5c7d061540f1ee9f18025a68e2d8471b3e858a9dad47c8d41903", + "sha256:4e21876082ed887baed0146fe222f861b5815455ada3b33b890f4105d806128d", + "sha256:58303469e9a272b4abdb9e302a780072c0633cdcc0165db7eec0f9e32f901e05", + "sha256:5ca5aeb4344b30d0bec47481536b8ba1181d50dbe783b0e4ad03c95dc1296684", + "sha256:68353fe7cdf91f109fc7d474461b46e7f1f14e533e911a2a2cbb8b0fc8613cf1", + "sha256:6f89d05e028d274ce4fa1a86887b071ae1755082ef94a6740238cd7a8178804f", + "sha256:7a15dc0a14008f1da3d1ebd44bdda3e357dbabdf5a0b5034d38fcde0b5c234b7", + "sha256:8bdde1177f2311ee552f47ae6e5aa7750c0e3291ca6b75f71f7ffe1f1dab3dca", + "sha256:8ce257cac556cb03be4a248d92ed36904a59a4a5ff55a994e92214cde15c5bad", + "sha256:8cf5cfcb1521dc3255d845d9dca3ff204b3229401994ef8d1984b32746bb45ca", + "sha256:8fbbdc8d55990eac1b0919ca69eb5a988a802b854488c34b8f37f3e2025fa90d", + "sha256:9548f10d8be799551eb3a9c74bbf2b4934ddb330e08a73320123c07f95cc2d92", + "sha256:96f8a1cb43ca1422f36492bebe63312d396491a9165ed3b9231e778d43a7fca4", + "sha256:9b27d894748475fa858f9597c0ee1d4829f44683f3813633aaf94b19cb5453cf", + "sha256:9baff2a45ae1f17c8078452e9e5962e518eab705e50a0aa8083733ea7d45f3a6", + "sha256:a2a8b8bcc399edb4347a5ca8b9b87e7524c0967b335fbb08a83c8421489ddee1", + "sha256:acf53bc2cf7282ab9b8ba346746afe703474004d9e566ad164c91a7a59f188a4", + "sha256:b0be84e5a6209858a1d3e8d1806c46214e867ce1b0fd32e4ea03f4bd8b2e3359", + "sha256:b31651d018b23ec463e95cf10070d0b2c548aa950a03d0b559eaa11c7e5a6fa3", + "sha256:b78e5afb39941572209f71866aa0b206c12f0109835aa0d601e41552f9b3e620", + "sha256:c76aeef1b95aff3905fb2ae2d96e319caca5b76fa41d3470b19d4e4a3a313512", + "sha256:dd035edafefee4d573140a76fdc785dc38829fe5a455c4bb12bac8c20cfc3d69", + "sha256:dd6fe30bd519694b356cbfcaca9bd5c1737cddd20778c6a581ae20dc8c04def2", + "sha256:e5f4e1edcf57ce94e5475fe09e5afa3e3145081318e5fd1a43a6b4539a97e518", + "sha256:ec6bc7fe73a938933d4178c9b23c4e0568e43e220aef9472c4f6044bfc6dd0f0", + "sha256:f1555ea6d6da108e1999b2463ea1003fe03f29213e459145e70edbaf3e004aaa", + "sha256:f5fa5803f47e095d7ad8443d28b01d48c0359484fec1b9d8606d0e3282084bc4", + "sha256:f7331dbf301b7289013175087636bbaf5b2405e57259dd2c42fdcc9fcc47325e", + "sha256:f9987b0354b06d4df0f4d3e0ec1ae76d7ce7cbca9a2f98c25041eb79eec766f1", + "sha256:fd9e830e9d8d89b20ab1e5af09b32d33e1a08ef4c4e14411e559556fd788e6b2" ], "index": "pypi", - "version": "==6.3.1" + "version": "==6.3.2" + }, + "cryptography": { + "hashes": [ + "sha256:0a817b961b46894c5ca8a66b599c745b9a3d9f822725221f0e0fe49dc043a3a3", + "sha256:2d87cdcb378d3cfed944dac30596da1968f88fb96d7fc34fdae30a99054b2e31", + "sha256:30ee1eb3ebe1644d1c3f183d115a8c04e4e603ed6ce8e394ed39eea4a98469ac", + "sha256:391432971a66cfaf94b21c24ab465a4cc3e8bf4a939c1ca5c3e3a6e0abebdbcf", + "sha256:39bdf8e70eee6b1c7b289ec6e5d84d49a6bfa11f8b8646b5b3dfe41219153316", + "sha256:4caa4b893d8fad33cf1964d3e51842cd78ba87401ab1d2e44556826df849a8ca", + "sha256:53e5c1dc3d7a953de055d77bef2ff607ceef7a2aac0353b5d630ab67f7423638", + "sha256:596f3cd67e1b950bc372c33f1a28a0692080625592ea6392987dba7f09f17a94", + "sha256:5d59a9d55027a8b88fd9fd2826c4392bd487d74bf628bb9d39beecc62a644c12", + "sha256:6c0c021f35b421ebf5976abf2daacc47e235f8b6082d3396a2fe3ccd537ab173", + "sha256:73bc2d3f2444bcfeac67dd130ff2ea598ea5f20b40e36d19821b4df8c9c5037b", + "sha256:74d6c7e80609c0f4c2434b97b80c7f8fdfaa072ca4baab7e239a15d6d70ed73a", + "sha256:7be0eec337359c155df191d6ae00a5e8bbb63933883f4f5dffc439dac5348c3f", + "sha256:94ae132f0e40fe48f310bba63f477f14a43116f05ddb69d6fa31e93f05848ae2", + "sha256:bb5829d027ff82aa872d76158919045a7c1e91fbf241aec32cb07956e9ebd3c9", + "sha256:ca238ceb7ba0bdf6ce88c1b74a87bffcee5afbfa1e41e173b1ceb095b39add46", + "sha256:ca28641954f767f9822c24e927ad894d45d5a1e501767599647259cbf030b903", + "sha256:e0344c14c9cb89e76eb6a060e67980c9e35b3f36691e15e1b7a9e58a0a6c6dc3", + "sha256:ebc15b1c22e55c4d5566e3ca4db8689470a0ca2babef8e3a9ee057a8b82ce4b1", + "sha256:ec63da4e7e4a5f924b90af42eddf20b698a70e58d86a72d943857c4c6045b3ee" + ], + "markers": "python_version >= '3.6'", + "version": "==36.0.1" }, "distlib": { "hashes": [ @@ -457,11 +570,11 @@ }, "filelock": { "hashes": [ - "sha256:38b4f4c989f9d06d44524df1b24bd19e167d851f19b50bf3e3559952dddc5b80", - "sha256:cf0fc6a2f8d26bd900f19bf33915ca70ba4dd8c56903eeb14e1e7a2fd7590146" + "sha256:9cd540a9352e432c7246a48fe4e8712b10acb1df2ad1f30e8c070b82ae1fed85", + "sha256:f8314284bfffbdcfa0ff3d7992b023d4c628ced6feb957351d4c48d059f56bc0" ], "markers": "python_version >= '3.7'", - "version": "==3.4.2" + "version": "==3.6.0" }, "flake8": { "hashes": [ @@ -489,11 +602,11 @@ }, "identify": { "hashes": [ - "sha256:97e839c1779f07011b84c92af183e1883d9745d532d83412cca1ca76d3808c1c", - "sha256:a55bdd671b6063eb837af938c250ec00bba6e610454265133b0d2db7ae718d0f" + "sha256:2986942d3974c8f2e5019a190523b0b0e2a07cb8e89bf236727fb4b26f27f8fd", + "sha256:fd906823ed1db23c7a48f9b176a1d71cb8abede1e21ebe614bac7bdd688d9213" ], "markers": "python_version >= '3.7'", - "version": "==2.4.8" + "version": "==2.4.11" }, "idna": { "hashes": [ @@ -513,11 +626,11 @@ }, "importlib-metadata": { "hashes": [ - "sha256:899e2a40a8c4a1aec681feef45733de8a6c58f3f6a0dbed2eb6574b4387a77b6", - "sha256:951f0d8a5b7260e9db5e41d429285b5f451e928479f19d80818878527d36e95e" + "sha256:175f4ee440a0317f6e8d81b7f8d4869f93316170a65ad2b007d2929186c8052c", + "sha256:e0bc84ff355328a4adfc5240c4f211e0ab386f80aa640d1b11f0618a1d282094" ], - "markers": "python_version >= '3.7'", - "version": "==4.10.1" + "markers": "python_version < '3.8'", + "version": "==4.11.1" }, "iniconfig": { "hashes": [ @@ -526,6 +639,14 @@ ], "version": "==1.1.1" }, + "jeepney": { + "hashes": [ + "sha256:1b5a0ea5c0e7b166b2f5895b91a08c14de8915afda4407fb5022a195224958ac", + "sha256:fa9e232dfa0c498bd0b8a3a73b8d8a31978304dcef0515adc859d4e096f96f4f" + ], + "markers": "sys_platform == 'linux'", + "version": "==0.7.1" + }, "jellyfish": { "hashes": [ "sha256:5104e45a2b804b48a46a92a5e6d6e86830fe60ae83b1da32c867402c8f4c2094" @@ -564,78 +685,49 @@ }, "markupsafe": { "hashes": [ - "sha256:01a9b8ea66f1658938f65b93a85ebe8bc016e6769611be228d797c9d998dd298", - "sha256:023cb26ec21ece8dc3907c0e8320058b2e0cb3c55cf9564da612bc325bed5e64", - "sha256:0446679737af14f45767963a1a9ef7620189912317d095f2d9ffa183a4d25d2b", - "sha256:04635854b943835a6ea959e948d19dcd311762c5c0c6e1f0e16ee57022669194", - "sha256:0717a7390a68be14b8c793ba258e075c6f4ca819f15edfc2a3a027c823718567", - "sha256:0955295dd5eec6cb6cc2fe1698f4c6d84af2e92de33fbcac4111913cd100a6ff", - "sha256:0d4b31cc67ab36e3392bbf3862cfbadac3db12bdd8b02a2731f509ed5b829724", - "sha256:10f82115e21dc0dfec9ab5c0223652f7197feb168c940f3ef61563fc2d6beb74", - "sha256:168cd0a3642de83558a5153c8bd34f175a9a6e7f6dc6384b9655d2697312a646", - "sha256:1d609f577dc6e1aa17d746f8bd3c31aa4d258f4070d61b2aa5c4166c1539de35", - "sha256:1f2ade76b9903f39aa442b4aadd2177decb66525062db244b35d71d0ee8599b6", - "sha256:20dca64a3ef2d6e4d5d615a3fd418ad3bde77a47ec8a23d984a12b5b4c74491a", - "sha256:2a7d351cbd8cfeb19ca00de495e224dea7e7d919659c2841bbb7f420ad03e2d6", - "sha256:2d7d807855b419fc2ed3e631034685db6079889a1f01d5d9dac950f764da3dad", - "sha256:2ef54abee730b502252bcdf31b10dacb0a416229b72c18b19e24a4509f273d26", - "sha256:36bc903cbb393720fad60fc28c10de6acf10dc6cc883f3e24ee4012371399a38", - "sha256:37205cac2a79194e3750b0af2a5720d95f786a55ce7df90c3af697bfa100eaac", - "sha256:3c112550557578c26af18a1ccc9e090bfe03832ae994343cfdacd287db6a6ae7", - "sha256:3dd007d54ee88b46be476e293f48c85048603f5f516008bee124ddd891398ed6", - "sha256:4296f2b1ce8c86a6aea78613c34bb1a672ea0e3de9c6ba08a960efe0b0a09047", - "sha256:47ab1e7b91c098ab893b828deafa1203de86d0bc6ab587b160f78fe6c4011f75", - "sha256:49e3ceeabbfb9d66c3aef5af3a60cc43b85c33df25ce03d0031a608b0a8b2e3f", - "sha256:4dc8f9fb58f7364b63fd9f85013b780ef83c11857ae79f2feda41e270468dd9b", - "sha256:4efca8f86c54b22348a5467704e3fec767b2db12fc39c6d963168ab1d3fc9135", - "sha256:53edb4da6925ad13c07b6d26c2a852bd81e364f95301c66e930ab2aef5b5ddd8", - "sha256:5855f8438a7d1d458206a2466bf82b0f104a3724bf96a1c781ab731e4201731a", - "sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a", - "sha256:5b6d930f030f8ed98e3e6c98ffa0652bdb82601e7a016ec2ab5d7ff23baa78d1", - "sha256:5bb28c636d87e840583ee3adeb78172efc47c8b26127267f54a9c0ec251d41a9", - "sha256:60bf42e36abfaf9aff1f50f52644b336d4f0a3fd6d8a60ca0d054ac9f713a864", - "sha256:611d1ad9a4288cf3e3c16014564df047fe08410e628f89805e475368bd304914", - "sha256:6300b8454aa6930a24b9618fbb54b5a68135092bc666f7b06901f897fa5c2fee", - "sha256:63f3268ba69ace99cab4e3e3b5840b03340efed0948ab8f78d2fd87ee5442a4f", - "sha256:6557b31b5e2c9ddf0de32a691f2312a32f77cd7681d8af66c2692efdbef84c18", - "sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8", - "sha256:6a7fae0dd14cf60ad5ff42baa2e95727c3d81ded453457771d02b7d2b3f9c0c2", - "sha256:6c4ca60fa24e85fe25b912b01e62cb969d69a23a5d5867682dd3e80b5b02581d", - "sha256:6fcf051089389abe060c9cd7caa212c707e58153afa2c649f00346ce6d260f1b", - "sha256:7d91275b0245b1da4d4cfa07e0faedd5b0812efc15b702576d103293e252af1b", - "sha256:89c687013cb1cd489a0f0ac24febe8c7a666e6e221b783e53ac50ebf68e45d86", - "sha256:8d206346619592c6200148b01a2142798c989edcb9c896f9ac9722a99d4e77e6", - "sha256:905fec760bd2fa1388bb5b489ee8ee5f7291d692638ea5f67982d968366bef9f", - "sha256:97383d78eb34da7e1fa37dd273c20ad4320929af65d156e35a5e2d89566d9dfb", - "sha256:984d76483eb32f1bcb536dc27e4ad56bba4baa70be32fa87152832cdd9db0833", - "sha256:99df47edb6bda1249d3e80fdabb1dab8c08ef3975f69aed437cb69d0a5de1e28", - "sha256:9f02365d4e99430a12647f09b6cc8bab61a6564363f313126f775eb4f6ef798e", - "sha256:a30e67a65b53ea0a5e62fe23682cfe22712e01f453b95233b25502f7c61cb415", - "sha256:ab3ef638ace319fa26553db0624c4699e31a28bb2a835c5faca8f8acf6a5a902", - "sha256:aca6377c0cb8a8253e493c6b451565ac77e98c2951c45f913e0b52facdcff83f", - "sha256:add36cb2dbb8b736611303cd3bfcee00afd96471b09cda130da3581cbdc56a6d", - "sha256:b2f4bf27480f5e5e8ce285a8c8fd176c0b03e93dcc6646477d4630e83440c6a9", - "sha256:b7f2d075102dc8c794cbde1947378051c4e5180d52d276987b8d28a3bd58c17d", - "sha256:baa1a4e8f868845af802979fcdbf0bb11f94f1cb7ced4c4b8a351bb60d108145", - "sha256:be98f628055368795d818ebf93da628541e10b75b41c559fdf36d104c5787066", - "sha256:bf5d821ffabf0ef3533c39c518f3357b171a1651c1ff6827325e4489b0e46c3c", - "sha256:c47adbc92fc1bb2b3274c4b3a43ae0e4573d9fbff4f54cd484555edbf030baf1", - "sha256:cdfba22ea2f0029c9261a4bd07e830a8da012291fbe44dc794e488b6c9bb353a", - "sha256:d6c7ebd4e944c85e2c3421e612a7057a2f48d478d79e61800d81468a8d842207", - "sha256:d7f9850398e85aba693bb640262d3611788b1f29a79f0c93c565694658f4071f", - "sha256:d8446c54dc28c01e5a2dbac5a25f071f6653e6e40f3a8818e8b45d790fe6ef53", - "sha256:deb993cacb280823246a026e3b2d81c493c53de6acfd5e6bfe31ab3402bb37dd", - "sha256:e0f138900af21926a02425cf736db95be9f4af72ba1bb21453432a07f6082134", - "sha256:e9936f0b261d4df76ad22f8fee3ae83b60d7c3e871292cd42f40b81b70afae85", - "sha256:f0567c4dc99f264f49fe27da5f735f414c4e7e7dd850cfd8e69f0862d7c74ea9", - "sha256:f5653a225f31e113b152e56f154ccbe59eeb1c7487b39b9d9f9cdb58e6c79dc5", - "sha256:f826e31d18b516f653fe296d967d700fddad5901ae07c622bb3705955e1faa94", - "sha256:f8ba0e8349a38d3001fae7eadded3f6606f0da5d748ee53cc1dab1d6527b9509", - "sha256:f9081981fe268bd86831e5c75f7de206ef275defcb82bc70740ae6dc507aee51", - "sha256:fa130dd50c57d53368c9d59395cb5526eda596d3ffe36666cd81a44d56e48872" + "sha256:023af8c54fe63530545f70dd2a2a7eed18d07a9a77b94e8bf1e2ff7f252db9a3", + "sha256:09c86c9643cceb1d87ca08cdc30160d1b7ab49a8a21564868921959bd16441b8", + "sha256:142119fb14a1ef6d758912b25c4e803c3ff66920635c44078666fe7cc3f8f759", + "sha256:1d1fb9b2eec3c9714dd936860850300b51dbaa37404209c8d4cb66547884b7ed", + "sha256:204730fd5fe2fe3b1e9ccadb2bd18ba8712b111dcabce185af0b3b5285a7c989", + "sha256:24c3be29abb6b34052fd26fc7a8e0a49b1ee9d282e3665e8ad09a0a68faee5b3", + "sha256:290b02bab3c9e216da57c1d11d2ba73a9f73a614bbdcc027d299a60cdfabb11a", + "sha256:3028252424c72b2602a323f70fbf50aa80a5d3aa616ea6add4ba21ae9cc9da4c", + "sha256:30c653fde75a6e5eb814d2a0a89378f83d1d3f502ab710904ee585c38888816c", + "sha256:3cace1837bc84e63b3fd2dfce37f08f8c18aeb81ef5cf6bb9b51f625cb4e6cd8", + "sha256:4056f752015dfa9828dce3140dbadd543b555afb3252507348c493def166d454", + "sha256:454ffc1cbb75227d15667c09f164a0099159da0c1f3d2636aa648f12675491ad", + "sha256:598b65d74615c021423bd45c2bc5e9b59539c875a9bdb7e5f2a6b92dfcfc268d", + "sha256:599941da468f2cf22bf90a84f6e2a65524e87be2fce844f96f2dd9a6c9d1e635", + "sha256:5ddea4c352a488b5e1069069f2f501006b1a4362cb906bee9a193ef1245a7a61", + "sha256:62c0285e91414f5c8f621a17b69fc0088394ccdaa961ef469e833dbff64bd5ea", + "sha256:679cbb78914ab212c49c67ba2c7396dc599a8479de51b9a87b174700abd9ea49", + "sha256:6e104c0c2b4cd765b4e83909cde7ec61a1e313f8a75775897db321450e928cce", + "sha256:736895a020e31b428b3382a7887bfea96102c529530299f426bf2e636aacec9e", + "sha256:75bb36f134883fdbe13d8e63b8675f5f12b80bb6627f7714c7d6c5becf22719f", + "sha256:7d2f5d97fcbd004c03df8d8fe2b973fe2b14e7bfeb2cfa012eaa8759ce9a762f", + "sha256:80beaf63ddfbc64a0452b841d8036ca0611e049650e20afcb882f5d3c266d65f", + "sha256:84ad5e29bf8bab3ad70fd707d3c05524862bddc54dc040982b0dbcff36481de7", + "sha256:8da5924cb1f9064589767b0f3fc39d03e3d0fb5aa29e0cb21d43106519bd624a", + "sha256:961eb86e5be7d0973789f30ebcf6caab60b844203f4396ece27310295a6082c7", + "sha256:96de1932237abe0a13ba68b63e94113678c379dca45afa040a17b6e1ad7ed076", + "sha256:a0a0abef2ca47b33fb615b491ce31b055ef2430de52c5b3fb19a4042dbc5cadb", + "sha256:b2a5a856019d2833c56a3dcac1b80fe795c95f401818ea963594b345929dffa7", + "sha256:b8811d48078d1cf2a6863dafb896e68406c5f513048451cd2ded0473133473c7", + "sha256:c532d5ab79be0199fa2658e24a02fce8542df196e60665dd322409a03db6a52c", + "sha256:d3b64c65328cb4cd252c94f83e66e3d7acf8891e60ebf588d7b493a55a1dbf26", + "sha256:d4e702eea4a2903441f2735799d217f4ac1b55f7d8ad96ab7d4e25417cb0827c", + "sha256:d5653619b3eb5cbd35bfba3c12d575db2a74d15e0e1c08bf1db788069d410ce8", + "sha256:d66624f04de4af8bbf1c7f21cc06649c1c69a7f84109179add573ce35e46d448", + "sha256:e67ec74fada3841b8c5f4c4f197bea916025cb9aa3fe5abf7d52b655d042f956", + "sha256:e6f7f3f41faffaea6596da86ecc2389672fa949bd035251eab26dc6697451d05", + "sha256:f02cf7221d5cd915d7fa58ab64f7ee6dd0f6cddbb48683debf5d04ae9b1c2cc1", + "sha256:f0eddfcabd6936558ec020130f932d479930581171368fd728efcfb6ef0dd357", + "sha256:fabbe18087c3d33c5824cb145ffca52eccd053061df1d79d4b66dafa5ad2a5ea", + "sha256:fc3150f85e2dbcf99e65238c842d1cfe69d3e7649b19864c1cc043213d9cd730" ], - "markers": "python_version >= '3.6'", - "version": "==2.0.1" + "markers": "python_version >= '3.7'", + "version": "==2.1.0" }, "mccabe": { "hashes": [ @@ -760,11 +852,11 @@ }, "myst-parser": { "hashes": [ - "sha256:617a90ceda2162ebf81cd13ad17d879bd4f49e7fb5c4f177bb905272555a2268", - "sha256:a6473b9735c8c74959b49b36550725464f4aecc4481340c9a5f9153829191f83" + "sha256:555ec2950aba5ae5dac5c162c7e9a43ad4a7291cfac644d8f5f84da8efa6f356", + "sha256:d412347a5cacb77ebc03d7f7ffef050cd61957d46f234313d350e84e24972260" ], "index": "pypi", - "version": "==0.16.1" + "version": "==0.17.0" }, "nodeenv": { "hashes": [ @@ -797,11 +889,11 @@ }, "platformdirs": { "hashes": [ - "sha256:1d7385c7db91728b83efd0ca99a5afb296cab9d0ed8313a45ed8ba17967ecfca", - "sha256:440633ddfebcc36264232365d7840a970e75e1018d15b4327d11f91909045fda" + "sha256:7535e70dfa32e84d4b34996ea99c5e432fa29a708d0f4e394bbcb2a8faa4f16d", + "sha256:bcae7cab893c2d310a711b70b24efb93334febe65f8de776ee320b517471e227" ], "markers": "python_version >= '3.7'", - "version": "==2.4.1" + "version": "==2.5.1" }, "pluggy": { "hashes": [ @@ -842,6 +934,13 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==2.8.0" }, + "pycparser": { + "hashes": [ + "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9", + "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206" + ], + "version": "==2.21" + }, "pydocstyle": { "hashes": [ "sha256:1d41b7c459ba0ee6c345f2eb9ae827cab14a7533a88c5c6f7e94923f72df92dc", @@ -876,11 +975,11 @@ }, "pytest": { "hashes": [ - "sha256:42901e6bd4bd4a0e533358a86e848427a49005a3256f657c5c8f8dd35ef137a9", - "sha256:dad48ffda394e5ad9aa3b7d7ddf339ed502e5e365b1350e0af65f4a602344b11" + "sha256:9ce3ff477af913ecf6321fe337b93a2c0dcf2a0a1439c43f5452112c1e4280db", + "sha256:e30905a0c131d3d94b89624a1cc5afec3e0ba2fbdb151867d8e0ebd49850f171" ], "index": "pypi", - "version": "==7.0.0" + "version": "==7.0.1" }, "pytest-vcr": { "hashes": [ @@ -967,6 +1066,22 @@ "markers": "python_version >= '3.7'", "version": "==2.0.0" }, + "secretstorage": { + "hashes": [ + "sha256:422d82c36172d88d6a0ed5afdec956514b189ddbfb72fefab0c8a1cee4eaf71f", + "sha256:fd666c51a6bf200643495a04abb261f83229dcb6fd8472ec393df7ffc8b6f195" + ], + "markers": "sys_platform == 'linux'", + "version": "==3.3.1" + }, + "setuptools": { + "hashes": [ + "sha256:2347b2b432c891a863acadca2da9ac101eae6169b1d3dfee2ec605ecd50dbfe5", + "sha256:e4f30b9f84e5ab3decf945113119649fec09c1fc3507c6ebffec75646c56e62b" + ], + "markers": "python_version >= '3.7'", + "version": "==60.9.3" + }, "setuptools-scm": { "hashes": [ "sha256:6833ac65c6ed9711a4d5d2266f8024cfa07c533a0e55f4c12f6eff280a5a9e30", @@ -1072,11 +1187,11 @@ }, "tomli": { "hashes": [ - "sha256:b5bde28da1fed24b9bd1d4d2b8cba62300bfb4ec9a6187a957e8ddb9434c5224", - "sha256:c292c34f58502a1eb2bbb9f5bbc9a5ebc37bee10ffb8c2d6bbdfa8eb13cc14e1" + "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", + "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f" ], "markers": "python_version >= '3.7'", - "version": "==2.0.0" + "version": "==2.0.1" }, "tornado": { "hashes": [ @@ -1122,7 +1237,7 @@ "sha256:fa2ba70284fa42c2a5ecb35e322e68823288a4251f9ba9cc77be04ae15eada68", "sha256:fba85b6cd9c39be262fcd23865652920832b61583de2a2ca907dbd8e8a8c81e5" ], - "markers": "python_version >= '3.5'", + "markers": "python_version > '2.7'", "version": "==6.1" }, "tqdm": { @@ -1141,13 +1256,43 @@ "index": "pypi", "version": "==3.8.0" }, + "typed-ast": { + "hashes": [ + "sha256:0eb77764ea470f14fcbb89d51bc6bbf5e7623446ac4ed06cbd9ca9495b62e36e", + "sha256:1098df9a0592dd4c8c0ccfc2e98931278a6c6c53cb3a3e2cf7e9ee3b06153344", + "sha256:183b183b7771a508395d2cbffd6db67d6ad52958a5fdc99f450d954003900266", + "sha256:18fe320f354d6f9ad3147859b6e16649a0781425268c4dde596093177660e71a", + "sha256:26a432dc219c6b6f38be20a958cbe1abffcc5492821d7e27f08606ef99e0dffd", + "sha256:294a6903a4d087db805a7656989f613371915fc45c8cc0ddc5c5a0a8ad9bea4d", + "sha256:31d8c6b2df19a777bc8826770b872a45a1f30cfefcfd729491baa5237faae837", + "sha256:33b4a19ddc9fc551ebabca9765d54d04600c4a50eda13893dadf67ed81d9a098", + "sha256:42c47c3b43fe3a39ddf8de1d40dbbfca60ac8530a36c9b198ea5b9efac75c09e", + "sha256:525a2d4088e70a9f75b08b3f87a51acc9cde640e19cc523c7e41aa355564ae27", + "sha256:58ae097a325e9bb7a684572d20eb3e1809802c5c9ec7108e85da1eb6c1a3331b", + "sha256:676d051b1da67a852c0447621fdd11c4e104827417bf216092ec3e286f7da596", + "sha256:74cac86cc586db8dfda0ce65d8bcd2bf17b58668dfcc3652762f3ef0e6677e76", + "sha256:8c08d6625bb258179b6e512f55ad20f9dfef019bbfbe3095247401e053a3ea30", + "sha256:90904d889ab8e81a956f2c0935a523cc4e077c7847a836abee832f868d5c26a4", + "sha256:963a0ccc9a4188524e6e6d39b12c9ca24cc2d45a71cfdd04a26d883c922b4b78", + "sha256:bbebc31bf11762b63bf61aaae232becb41c5bf6b3461b80a4df7e791fabb3aca", + "sha256:bc2542e83ac8399752bc16e0b35e038bdb659ba237f4222616b4e83fb9654985", + "sha256:c29dd9a3a9d259c9fa19d19738d021632d673f6ed9b35a739f48e5f807f264fb", + "sha256:c7407cfcad702f0b6c0e0f3e7ab876cd1d2c13b14ce770e412c0c4b9728a0f88", + "sha256:da0a98d458010bf4fe535f2d1e367a2e2060e105978873c04c04212fb20543f7", + "sha256:df05aa5b241e2e8045f5f4367a9f6187b09c4cdf8578bb219861c4e27c443db5", + "sha256:f290617f74a610849bd8f5514e34ae3d09eafd521dceaa6cf68b3f4414266d4e", + "sha256:f30ddd110634c2d7534b2d4e0e22967e88366b0d356b24de87419cc4410c41b7" + ], + "markers": "python_version < '3.8'", + "version": "==1.5.2" + }, "types-requests": { "hashes": [ - "sha256:8ec9f5f84adc6f579f53943312c28a84e87dc70201b54f7c4fbc7d22ecfa8a3e", - "sha256:c2f4e4754d07ca0a88fd8a89bbc6c8a9f90fb441f9c9b572fd5c484f04817486" + "sha256:506279bad570c7b4b19ac1f22e50146538befbe0c133b2cea66a9b04a533a859", + "sha256:6a7ed24b21780af4a5b5e24c310b2cd885fb612df5fd95584d03d87e5f2a195a" ], "index": "pypi", - "version": "==2.27.8" + "version": "==2.27.11" }, "types-urllib3": { "hashes": [ @@ -1158,11 +1303,11 @@ }, "typing-extensions": { "hashes": [ - "sha256:4ca091dea149f945ec56afb48dae714f21e8692ef22a395223bcd328961b6a0e", - "sha256:7f001e5ac290a0c0401508864c7ec868be4e701886d5b573a9528ed3973d9d3b" + "sha256:1a9462dcc3347a79b1f1c0271fbe79e844580bb598bafa1ed208b94da3cdcd42", + "sha256:21c85e0fe4b9a155d0799430b0ad741cdce7e359660ccbd8b530613e8df88ce2" ], - "markers": "python_version < '3.10'", - "version": "==4.0.1" + "markers": "python_version < '3.8'", + "version": "==4.1.1" }, "urllib3": { "hashes": [ @@ -1189,11 +1334,11 @@ }, "virtualenv": { "hashes": [ - "sha256:339f16c4a86b44240ba7223d0f93a7887c3ca04b5f9c8129da7958447d079b09", - "sha256:d8458cf8d59d0ea495ad9b34c2599487f8a7772d796f9910858376d1600dd2dd" + "sha256:45e1d053cad4cd453181ae877c4ffc053546ae99e7dd049b9ff1d9be7491abf7", + "sha256:e0621bcbf4160e4e1030f05065c8834b4e93f4fcc223255db2a823440aca9c14" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==20.13.0" + "version": "==20.13.1" }, "webencodings": { "hashes": [ diff --git a/setup.py b/setup.py index e90b8fdd..61453b81 100644 --- a/setup.py +++ b/setup.py @@ -131,6 +131,7 @@ def run(self): "pdfplumber", "requests", "openpyxl", + "xlrd", "tenacity", ], license="Apache 2.0 license", diff --git a/warn/scrapers/ky.py b/warn/scrapers/ky.py new file mode 100644 index 00000000..35a0f8a7 --- /dev/null +++ b/warn/scrapers/ky.py @@ -0,0 +1,132 @@ +import typing +from pathlib import Path + +import xlrd +from openpyxl import load_workbook + +from .. import utils +from ..cache import Cache + +__authors__ = [ + "palewire", +] +__tags__ = [ + "excel", +] + + +def scrape( + data_dir: Path = utils.WARN_DATA_DIR, + cache_dir: Path = utils.WARN_CACHE_DIR, +) -> Path: + """ + Scrape data from Kentucky. + + Arguments: + output_dir -- the Path were the result will be saved + + Keyword arguments: + data_dir -- the Path were the result will be saved (default WARN_DATA_DIR) + cache_dir -- the Path where results can be cached (default WARN_CACHE_DIR) + + Returns: the Path where the file is written + """ + # Get the latest workbook + cache = Cache(cache_dir) + latest_url = "https://kcc.ky.gov/WARN%20notices/WARN%20NOTICES%202022/WARN%20Notice%20Report%2001262022.xls" + latest_path = cache.download("ky/latest.xls", latest_url) + + # Open it up + latest_workbook = xlrd.open_workbook(latest_path) + + # Loop through the worksheets + dict_list = [] + for sheet in latest_workbook.sheet_names(): + # Get the data as a list of lists + sheet_obj = latest_workbook.sheet_by_name(sheet) + sheet_list = parse_xls(sheet_obj) + + # Convert to list of dicts + headers = sheet_list[0] + for row in sheet_list[1:]: + d = {} + for x, value in enumerate(row): + d[headers[x]] = value + dict_list.append(d) + + # Same thing for the archived file + archive_url = "https://kcc.ky.gov/WARN%20notices/2017%20WARN%20docs/Kentucky%20%20WARN%20Report%20-Tracking%20Form%20%281998-2016%29.xlsx" + archive_path = cache.download("ky/archive.xlsx", archive_url) + + # Open it up + archive_workbook = load_workbook(filename=archive_path) + + # Loop through the worksheets + for sheet in archive_workbook.worksheets: + # Get the data as a list of lists + sheet_list = parse_xlsx(sheet) + + # Convert to list of dicts + headers = sheet_list[0] + for row in sheet_list[1:]: + d = {} + for i, value in enumerate(row): + d[headers[i]] = value + dict_list.append(d) + + # Write out the results + data_path = data_dir / "ky.csv" + all_headers = list({value for dic in dict_list for value in dic.keys()}) + utils.write_dict_rows_to_csv( + data_path, all_headers, dict_list, extrasaction="ignore" + ) + + # Pass it out + return data_path + + +def parse_xls(worksheet) -> typing.List[typing.List]: + """Parse the Excel xls file at the provided path. + + Args: + worksheet: An xlrd worksheet ready to parse + + Returns: List of dicts ready to write. + """ + row_list = [] + for i in range(0, worksheet.nrows): + row = worksheet.row_values(i) + row_list.append(row) + return row_list + + +def parse_xlsx(worksheet) -> typing.List[typing.List]: + """Parse the Excel xlsx file at the provided path. + + Args: + worksheet: An openpyxl worksheet ready to parse + + Returns: List of values ready to write. + """ + # Convert the sheet to a list of lists + row_list = [] + for r in worksheet.rows: + # Parse cells + cell_list = [cell.value for cell in r] + + # Skip empty rows + try: + # A list with only empty cells will throw an error + next(c for c in cell_list if c) + except StopIteration: + continue + + # Add to the master list + row_list.append(cell_list) + + # Pass it back + return row_list + + +if __name__ == "__main__": + scrape() From 17372fcdf6e09d8f38e632512c51f51ffa61455d Mon Sep 17 00:00:00 2001 From: palewire Date: Wed, 23 Feb 2022 11:55:36 -0800 Subject: [PATCH 2/6] Docs --- docs/sources.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/sources.md b/docs/sources.md index 678a64ed..7a43818d 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -1,6 +1,6 @@ # Sources -There are currently scrapers for 34 of America's 56 states and territories. +There are currently scrapers for 35 of America's 56 states and territories. | State | Docs | Authors | Tags | | :---- | :--: | :------ | :--- | @@ -17,6 +17,7 @@ There are currently scrapers for 34 of America's 56 states and territories. |[Indiana](https://github.com/biglocalnews/warn-scraper/blob/main/warn/scrapers/in.py)|[📃](scrapers/in.md)|[Dilcia19](https://github.com/Dilcia19), [zstumgoren](https://github.com/zstumgoren)|html| |[Iowa](https://github.com/biglocalnews/warn-scraper/blob/main/warn/scrapers/ia.py)|[📃](scrapers/ia.md)|[Dilcia19](https://github.com/Dilcia19), [palewire](https://github.com/palewire), [shallotly](https://github.com/shallotly), [zstumgoren](https://github.com/zstumgoren)|excel, html| |[Kansas](https://github.com/biglocalnews/warn-scraper/blob/main/warn/scrapers/ks.py)|[📃](scrapers/ks.md)|[Dilcia19](https://github.com/Dilcia19), [zstumgoren](https://github.com/zstumgoren)|jobcenter| +|[Kentucky](https://github.com/biglocalnews/warn-scraper/blob/main/warn/scrapers/ky.py)||[palewire](https://github.com/palewire)|excel| |[Louisiana](https://github.com/biglocalnews/warn-scraper/blob/main/warn/scrapers/la.py)||[chriszs](https://github.com/chriszs)|html, pdf| |[Maine](https://github.com/biglocalnews/warn-scraper/blob/main/warn/scrapers/me.py)|[📃](scrapers/me.md)|[zstumgoren](https://github.com/zstumgoren)|jobcenter| |[Maryland](https://github.com/biglocalnews/warn-scraper/blob/main/warn/scrapers/md.py)|[📃](scrapers/md.md)|[Dilcia19](https://github.com/Dilcia19), [shallotly](https://github.com/shallotly), [zstumgoren](https://github.com/zstumgoren)|html| @@ -42,12 +43,11 @@ There are currently scrapers for 34 of America's 56 states and territories. ## To do -These 22 areas need a scraper: +These 21 areas need a scraper: - Arkansas - Colorado - Hawaii -- Kentucky - Massachusetts - Michigan - Minnesota From d942883241edafc7cb093c7e8909dbfc23ac1c55 Mon Sep 17 00:00:00 2001 From: palewire Date: Wed, 23 Feb 2022 12:00:32 -0800 Subject: [PATCH 3/6] Try this with 3.9 --- .github/workflows/continuous-deployment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/continuous-deployment.yml b/.github/workflows/continuous-deployment.yml index 3f2c7e80..3f06af58 100644 --- a/.github/workflows/continuous-deployment.yml +++ b/.github/workflows/continuous-deployment.yml @@ -38,7 +38,7 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: '3.8' + python-version: '3.9' cache: 'pipenv' - run: pipenv install --dev --pre From 2b8084724fa0ab6a69d7d689549f9bfae761198b Mon Sep 17 00:00:00 2001 From: palewire Date: Wed, 23 Feb 2022 17:29:14 -0800 Subject: [PATCH 4/6] Added date handling --- warn/scrapers/ky.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/warn/scrapers/ky.py b/warn/scrapers/ky.py index 35a0f8a7..823caae3 100644 --- a/warn/scrapers/ky.py +++ b/warn/scrapers/ky.py @@ -44,7 +44,7 @@ def scrape( for sheet in latest_workbook.sheet_names(): # Get the data as a list of lists sheet_obj = latest_workbook.sheet_by_name(sheet) - sheet_list = parse_xls(sheet_obj) + sheet_list = parse_xls(sheet_obj, latest_workbook.datemode) # Convert to list of dicts headers = sheet_list[0] @@ -85,7 +85,7 @@ def scrape( return data_path -def parse_xls(worksheet) -> typing.List[typing.List]: +def parse_xls(worksheet, datemode) -> typing.List[typing.List]: """Parse the Excel xls file at the provided path. Args: @@ -94,8 +94,17 @@ def parse_xls(worksheet) -> typing.List[typing.List]: Returns: List of dicts ready to write. """ row_list = [] - for i in range(0, worksheet.nrows): - row = worksheet.row_values(i) + num_cols = worksheet.ncols + for row_idx in range(0, worksheet.nrows): + row = [] + for col_idx in range(0, num_cols): + cell_obj = worksheet.cell(row_idx, col_idx) + cell_type = worksheet.cell_type(row_idx, col_idx) + if cell_type == xlrd.XL_CELL_DATE: + value = xlrd.xldate.xldate_as_datetime(cell_obj.value, datemode) + else: + value = cell_obj.value + row.append(value) row_list.append(row) return row_list From b43cd6a0ec4c67fa1d0b942e0cd301f53b62933d Mon Sep 17 00:00:00 2001 From: palewire Date: Wed, 23 Feb 2022 17:34:55 -0800 Subject: [PATCH 5/6] Added typing_extensions --- Pipfile | 1 + Pipfile.lock | 60 +++------------------------------------------------- 2 files changed, 4 insertions(+), 57 deletions(-) diff --git a/Pipfile b/Pipfile index 03052ba3..82c75587 100644 --- a/Pipfile +++ b/Pipfile @@ -22,6 +22,7 @@ pre-commit = "*" sphinxcontrib-napoleon = "*" types-requests = "*" mypy = "*" +typing-extensions = "*" [packages] bs4 = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 97c8bfdb..375cf5b4 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "50d67d7c9271b38b8db74dc2a9ea20398b113642c7f280cbc15b825719675a5b" + "sha256": "31a3a9aed21c7f0f40ca01aba8b6cacc9e4b5a0904fa60aa6ed779a14e94faae" }, "pipfile-spec": 6, "requires": { @@ -167,14 +167,6 @@ "markers": "python_version >= '3'", "version": "==3.3" }, - "importlib-metadata": { - "hashes": [ - "sha256:175f4ee440a0317f6e8d81b7f8d4869f93316170a65ad2b007d2929186c8052c", - "sha256:e0bc84ff355328a4adfc5240c4f211e0ab386f80aa640d1b11f0618a1d282094" - ], - "markers": "python_version < '3.8'", - "version": "==4.11.1" - }, "openpyxl": { "hashes": [ "sha256:40f568b9829bf9e446acfffce30250ac1fa39035124d55fc024025c41481c90f", @@ -278,14 +270,6 @@ "index": "pypi", "version": "==8.0.1" }, - "typing-extensions": { - "hashes": [ - "sha256:1a9462dcc3347a79b1f1c0271fbe79e844580bb598bafa1ed208b94da3cdcd42", - "sha256:21c85e0fe4b9a155d0799430b0ad741cdce7e359660ccbd8b530613e8df88ce2" - ], - "markers": "python_version < '3.8'", - "version": "==4.1.1" - }, "urllib3": { "hashes": [ "sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed", @@ -315,14 +299,6 @@ ], "index": "pypi", "version": "==2.0.1" - }, - "zipp": { - "hashes": [ - "sha256:9f50f446828eb9d45b267433fd3e9da8d801f614129124863f9c51ebceafb87d", - "sha256:b47250dd24f92b7dd6a0a8fc5244da14608f3ca90a5efcd37a3b1642fac9a375" - ], - "markers": "python_version >= '3.7'", - "version": "==3.7.0" } }, "develop": { @@ -629,7 +605,7 @@ "sha256:175f4ee440a0317f6e8d81b7f8d4869f93316170a65ad2b007d2929186c8052c", "sha256:e0bc84ff355328a4adfc5240c4f211e0ab386f80aa640d1b11f0618a1d282094" ], - "markers": "python_version < '3.8'", + "markers": "python_version >= '3.7'", "version": "==4.11.1" }, "iniconfig": { @@ -1256,36 +1232,6 @@ "index": "pypi", "version": "==3.8.0" }, - "typed-ast": { - "hashes": [ - "sha256:0eb77764ea470f14fcbb89d51bc6bbf5e7623446ac4ed06cbd9ca9495b62e36e", - "sha256:1098df9a0592dd4c8c0ccfc2e98931278a6c6c53cb3a3e2cf7e9ee3b06153344", - "sha256:183b183b7771a508395d2cbffd6db67d6ad52958a5fdc99f450d954003900266", - "sha256:18fe320f354d6f9ad3147859b6e16649a0781425268c4dde596093177660e71a", - "sha256:26a432dc219c6b6f38be20a958cbe1abffcc5492821d7e27f08606ef99e0dffd", - "sha256:294a6903a4d087db805a7656989f613371915fc45c8cc0ddc5c5a0a8ad9bea4d", - "sha256:31d8c6b2df19a777bc8826770b872a45a1f30cfefcfd729491baa5237faae837", - "sha256:33b4a19ddc9fc551ebabca9765d54d04600c4a50eda13893dadf67ed81d9a098", - "sha256:42c47c3b43fe3a39ddf8de1d40dbbfca60ac8530a36c9b198ea5b9efac75c09e", - "sha256:525a2d4088e70a9f75b08b3f87a51acc9cde640e19cc523c7e41aa355564ae27", - "sha256:58ae097a325e9bb7a684572d20eb3e1809802c5c9ec7108e85da1eb6c1a3331b", - "sha256:676d051b1da67a852c0447621fdd11c4e104827417bf216092ec3e286f7da596", - "sha256:74cac86cc586db8dfda0ce65d8bcd2bf17b58668dfcc3652762f3ef0e6677e76", - "sha256:8c08d6625bb258179b6e512f55ad20f9dfef019bbfbe3095247401e053a3ea30", - "sha256:90904d889ab8e81a956f2c0935a523cc4e077c7847a836abee832f868d5c26a4", - "sha256:963a0ccc9a4188524e6e6d39b12c9ca24cc2d45a71cfdd04a26d883c922b4b78", - "sha256:bbebc31bf11762b63bf61aaae232becb41c5bf6b3461b80a4df7e791fabb3aca", - "sha256:bc2542e83ac8399752bc16e0b35e038bdb659ba237f4222616b4e83fb9654985", - "sha256:c29dd9a3a9d259c9fa19d19738d021632d673f6ed9b35a739f48e5f807f264fb", - "sha256:c7407cfcad702f0b6c0e0f3e7ab876cd1d2c13b14ce770e412c0c4b9728a0f88", - "sha256:da0a98d458010bf4fe535f2d1e367a2e2060e105978873c04c04212fb20543f7", - "sha256:df05aa5b241e2e8045f5f4367a9f6187b09c4cdf8578bb219861c4e27c443db5", - "sha256:f290617f74a610849bd8f5514e34ae3d09eafd521dceaa6cf68b3f4414266d4e", - "sha256:f30ddd110634c2d7534b2d4e0e22967e88366b0d356b24de87419cc4410c41b7" - ], - "markers": "python_version < '3.8'", - "version": "==1.5.2" - }, "types-requests": { "hashes": [ "sha256:506279bad570c7b4b19ac1f22e50146538befbe0c133b2cea66a9b04a533a859", @@ -1306,7 +1252,7 @@ "sha256:1a9462dcc3347a79b1f1c0271fbe79e844580bb598bafa1ed208b94da3cdcd42", "sha256:21c85e0fe4b9a155d0799430b0ad741cdce7e359660ccbd8b530613e8df88ce2" ], - "markers": "python_version < '3.8'", + "index": "pypi", "version": "==4.1.1" }, "urllib3": { From ac9a11235fb5ce50fba6e9ce3f501a8d818b687d Mon Sep 17 00:00:00 2001 From: palewire Date: Wed, 23 Feb 2022 17:37:13 -0800 Subject: [PATCH 6/6] Smooth out yml stuff --- .github/workflows/continuous-deployment.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/continuous-deployment.yml b/.github/workflows/continuous-deployment.yml index 3f06af58..8d7dda8f 100644 --- a/.github/workflows/continuous-deployment.yml +++ b/.github/workflows/continuous-deployment.yml @@ -20,7 +20,7 @@ jobs: python-version: '3.8' cache: 'pipenv' - - run: pipenv install --dev --pre + - run: pipenv install --dev - id: lint name: Lint @@ -38,10 +38,10 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: '3.9' + python-version: '3.8' cache: 'pipenv' - - run: pipenv install --dev --pre + - run: pipenv install --dev - id: mypy name: Mypy @@ -62,7 +62,7 @@ jobs: python-version: '3.8' cache: 'pipenv' - - run: pipenv install --dev --pre + - run: pipenv install --dev - id: tally name: Tally sources @@ -98,7 +98,7 @@ jobs: python-version: ${{ matrix.python }} cache: 'pipenv' - - run: pipenv install --dev --pre + - run: pipenv install --dev - id: run name: Run tests @@ -121,10 +121,10 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: '3.7' + python-version: '3.8' cache: 'pipenv' - - run: pipenv install --dev --pre + - run: pipenv install --dev - id: build name: Build release @@ -151,7 +151,7 @@ jobs: - name: Set up Python 3.9 uses: actions/setup-python@v1 with: - python-version: 3.9 + python-version: '3.8' - id: fetch name: Fetch artifact @@ -181,7 +181,7 @@ jobs: steps: - uses: actions/setup-python@v2 with: - python-version: '3.9' + python-version: '3.8' - id: fetch name: Fetch artifact