From 2e1c6480cc2d371946969ce4480bf2f4681a24d5 Mon Sep 17 00:00:00 2001 From: Wey Gu Date: Wed, 22 Mar 2023 22:08:04 +0800 Subject: [PATCH] feat: integration end to end test --- .gitignore | 15 +- HACKING.md | 23 ++ README.md | 16 +- examples/ng_ai_from_ngql_udf.ipynb | 2 +- examples/run_ng_ai_api.py | 2 +- examples/spark_engine.ipynb | 22 +- ng_ai/engines.py | 8 +- pdm.lock | 389 +++++++++++++++++- pyproject.toml | 83 +++- tests/integration/README.md | 6 + .../notebooks/spark_engine_e2e.ipynb | 188 +++++++++ tests/integration/setup/docker-compose.yaml | 47 ++- tests/integration/spark_engine_cases/algo.py | 23 ++ .../spark_engine_cases/query_reader.py | 24 ++ .../spark_engine_cases/scan_reader.py | 20 + .../integration/spark_engine_cases/writer.py | 39 ++ tests/integration/test_e2e_spark_engine.py | 191 +++++++++ 17 files changed, 1054 insertions(+), 44 deletions(-) create mode 100644 HACKING.md create mode 100644 tests/integration/README.md create mode 100644 tests/integration/notebooks/spark_engine_e2e.ipynb create mode 100644 tests/integration/spark_engine_cases/algo.py create mode 100644 tests/integration/spark_engine_cases/query_reader.py create mode 100644 tests/integration/spark_engine_cases/scan_reader.py create mode 100644 tests/integration/spark_engine_cases/writer.py diff --git a/.gitignore b/.gitignore index 204d745..ceef717 100644 --- a/.gitignore +++ b/.gitignore @@ -137,4 +137,17 @@ requirements.txt # ide -.vscode \ No newline at end of file +.vscode + +# integration + +.bash_history +.jupyter +.ipython +.local +data +logs +download +run +*.gz +*.whl diff --git a/HACKING.md b/HACKING.md new file mode 100644 index 0000000..21b6a4c --- /dev/null +++ b/HACKING.md @@ -0,0 +1,23 @@ +## Hacking + +```bash +pip3 install pdm +# build and install ng_ai +pdm install +# run unit tests +pdm run test +# run integration tests +pdm run int-test +# lint +pdm run lint +# format +pdm run format + +# integration tests environment setup +pdm run dockerup +pdm run dockerdown +pdm run dockerstatus + +# integration tests environment teardown +pdm run teardown +``` \ No newline at end of file diff --git a/README.md b/README.md index c094b11..52f9f04 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -NebulaGraph Data Intelligence Suite(ng_ai) +NebulaGraph AI Suite(ng_ai)

NebulaGraph AI Suite with 4 line code to run Graph Algo on NebulaGraph @@ -136,7 +136,7 @@ ng_ai is an unified abstraction layer for different engines, the current impleme Engine└───────────────────────────────────────────────────┘ │ ┌────────────────────────────────────────────────────┬──────────┐ └──┤ │ │ - │ NebulaGraph Data Intelligence Suite(ngai) │ ngai-api │◀─┐ + │ NebulaGraph AI Suite(ngai) │ ngai-api │◀─┐ │ │ │ │ │ └──────────┤ │ │ ┌────────┐ ┌──────┐ ┌────────┐ ┌─────┐ │ │ @@ -196,17 +196,7 @@ ng_ai is an unified abstraction layer for different engines, the current impleme ## Contributing -```bash -pip3 install pdm -# build and install ng_ai -pdm install -# run tests -pdm run test -# lint -pdm run lint -# format -pdm run format -``` +See HACKING.md for details. ## License diff --git a/examples/ng_ai_from_ngql_udf.ipynb b/examples/ng_ai_from_ngql_udf.ipynb index ca36229..cc4c2d4 100644 --- a/examples/ng_ai_from_ngql_udf.ipynb +++ b/examples/ng_ai_from_ngql_udf.ipynb @@ -79,7 +79,7 @@ "source": [ "from ng_ai import ng_ai_api_app as app\n", "\n", - "app.run(port=9999, host='0.0.0.0')" + "app.run(port=9999, host=\"0.0.0.0\")" ] }, { diff --git a/examples/run_ng_ai_api.py b/examples/run_ng_ai_api.py index 5cf02b4..2949f2d 100644 --- a/examples/run_ng_ai_api.py +++ b/examples/run_ng_ai_api.py @@ -1,3 +1,3 @@ from ng_ai import ng_ai_api_app as app -app.run(port=9999, host='0.0.0.0') +app.run(port=9999, host="0.0.0.0") diff --git a/examples/spark_engine.ipynb b/examples/spark_engine.ipynb index bf2b065..aeaa934 100644 --- a/examples/spark_engine.ipynb +++ b/examples/spark_engine.ipynb @@ -20,20 +20,22 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "5b4e4143", "metadata": {}, "source": [ - "## Data Intelligence Suite Spark Engine Examples\n", + "## AI Suite Spark Engine Examples\n", "### read data with spark engine, scan mode" ] }, { + "attachments": {}, "cell_type": "markdown", "id": "f17abcf8", "metadata": {}, "source": [ - "In this example, we are leveraging the Spark Engine of NebulaGraph DI Suite, with the Storage Scan mode.\n", + "In this example, we are leveraging the Spark Engine of NebulaGraph AI Suite, with the Storage Scan mode.\n", "\n", "#### Step 1, get dataframe by scanning the Graph\n", "\n", @@ -50,7 +52,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "\r", + "\r\n", "[Stage 0:> (0 + 1) / 1]" ] }, @@ -72,13 +74,14 @@ "name": "stderr", "output_type": "stream", "text": [ - "\r", + "\r\n", " \r" ] } ], "source": [ "from ng_ai import NebulaReader\n", + "\n", "# read data with spark engine, scan mode\n", "reader = NebulaReader(engine=\"spark\")\n", "reader.scan(edge=\"follow\", props=\"degree\")\n", @@ -246,6 +249,7 @@ ], "source": [ "from ng_ai import NebulaReader\n", + "\n", "# read data with spark engine, query mode\n", "reader = NebulaReader(engine=\"spark\")\n", "query = \"\"\"\n", @@ -253,7 +257,7 @@ " RETURN e LIMIT 100000\n", "\"\"\"\n", "reader.query(query=query, edge=\"follow\", props=\"degree\")\n", - "df = reader.read() # this will take some time\n", + "df = reader.read() # this will take some time\n", "df.show(2)" ] }, @@ -517,12 +521,12 @@ "from ng_ai.config import NebulaGraphConfig\n", "\n", "config = NebulaGraphConfig()\n", - "writer = NebulaWriter(data=df_result, sink=\"nebulagraph_vertex\", config=config, engine=\"spark\")\n", + "writer = NebulaWriter(\n", + " data=df_result, sink=\"nebulagraph_vertex\", config=config, engine=\"spark\"\n", + ")\n", "\n", "# map column louvain into property cluster_id\n", - "properties = {\n", - " \"lpa\": \"cluster_id\"\n", - "}\n", + "properties = {\"lpa\": \"cluster_id\"}\n", "\n", "writer.set_options(\n", " tag=\"label_propagation\",\n", diff --git a/ng_ai/engines.py b/ng_ai/engines.py index 8799bda..67d8c6e 100644 --- a/ng_ai/engines.py +++ b/ng_ai/engines.py @@ -38,13 +38,13 @@ def __init__(self, config): self.shuffle_partitions = DEFAULT_SHUFFLE_PARTITIONS self.executor_memory = DEFAULT_EXECUTOR_MEMORY self.driver_memory = DEFAULT_DRIVER_MEMORY - self.encode_vertex_id = ENCODE_VERTEX_ID + self.encode_vid = ENCODE_VERTEX_ID self.parse_config() from pyspark.sql import SparkSession self.spark = ( - SparkSession.builder.appName("NebulaGraph Data Intelligence") + SparkSession.builder.appName("NebulaGraph AI") .config("spark.sql.shuffle.partitions", self.shuffle_partitions) .config("spark.executor.memory", self.executor_memory) .config("spark.driver.memory", self.driver_memory) @@ -76,8 +76,8 @@ def parse_config(self): if self.config.driver_memory is not None: self.driver_memory = self.config.driver_memory - if self.config.encode_vertex_id is not None: - self.encode_vertex_id = self.config.encode_vertex_id + if self.config.encode_vid is not None: + self.encode_vid = self.config.encode_vid def prepare(self): self.java_import = self._get_java_import() diff --git a/pdm.lock b/pdm.lock index 1c98446..7f996e0 100644 --- a/pdm.lock +++ b/pdm.lock @@ -1,12 +1,22 @@ # This file is @generated by PDM. # It is not intended for manual editing. +[[package]] +name = "appnope" +version = "0.1.3" +summary = "Disable App Nap on macOS >= 10.9" + [[package]] name = "attrs" version = "22.2.0" requires_python = ">=3.6" summary = "Classes Without Boilerplate" +[[package]] +name = "backcall" +version = "0.2.0" +summary = "Specifications for callback functions passed in to an API" + [[package]] name = "black" version = "23.1.0" @@ -23,6 +33,30 @@ dependencies = [ "typing-extensions>=3.10.0.0; python_version < \"3.10\"", ] +[[package]] +name = "black" +version = "23.1.0" +extras = ["jupyter"] +requires_python = ">=3.7" +summary = "The uncompromising code formatter." +dependencies = [ + "black==23.1.0", + "ipython>=7.8.0", + "tokenize-rt>=3.2.0", +] + +[[package]] +name = "certifi" +version = "2022.12.7" +requires_python = ">=3.6" +summary = "Python package for providing Mozilla's CA Bundle." + +[[package]] +name = "charset-normalizer" +version = "3.1.0" +requires_python = ">=3.7.0" +summary = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." + [[package]] name = "chispa" version = "0.9.2" @@ -45,6 +79,12 @@ version = "0.4.6" requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" summary = "Cross-platform colored terminal text." +[[package]] +name = "decorator" +version = "5.1.1" +requires_python = ">=3.5" +summary = "Decorators for Humans" + [[package]] name = "exceptiongroup" version = "1.1.1" @@ -76,6 +116,27 @@ dependencies = [ "itsdangerous>=2.0", ] +[[package]] +name = "future" +version = "0.18.3" +requires_python = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +summary = "Clean single-source support for Python 3 and 2" + +[[package]] +name = "httplib2" +version = "0.22.0" +requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +summary = "A comprehensive HTTP client library." +dependencies = [ + "pyparsing!=3.0.0,!=3.0.1,!=3.0.2,!=3.0.3,<4,>=2.4.2; python_version > \"3.0\"", +] + +[[package]] +name = "idna" +version = "3.4" +requires_python = ">=3.5" +summary = "Internationalized Domain Names in Applications (IDNA)" + [[package]] name = "importlib-metadata" version = "4.2.0" @@ -92,6 +153,26 @@ version = "2.0.0" requires_python = ">=3.7" summary = "brain-dead simple config-ini parsing" +[[package]] +name = "ipython" +version = "7.34.0" +requires_python = ">=3.7" +summary = "IPython: Productive Interactive Computing" +dependencies = [ + "appnope; sys_platform == \"darwin\"", + "backcall", + "colorama; sys_platform == \"win32\"", + "decorator", + "jedi>=0.16", + "matplotlib-inline", + "pexpect>4.3; sys_platform != \"win32\"", + "pickleshare", + "prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0", + "pygments", + "setuptools>=18.5", + "traitlets>=4.2", +] + [[package]] name = "isort" version = "5.11.5" @@ -104,6 +185,15 @@ version = "2.1.2" requires_python = ">=3.7" summary = "Safely pass data to untrusted environments and back." +[[package]] +name = "jedi" +version = "0.18.2" +requires_python = ">=3.6" +summary = "An autocompletion tool for Python that can be used for text editors." +dependencies = [ + "parso<0.9.0,>=0.8.0", +] + [[package]] name = "jinja2" version = "3.1.2" @@ -119,6 +209,15 @@ version = "2.1.2" requires_python = ">=3.7" summary = "Safely add untrusted strings to HTML/XML markup." +[[package]] +name = "matplotlib-inline" +version = "0.1.6" +requires_python = ">=3.5" +summary = "Inline Matplotlib backend for Jupyter" +dependencies = [ + "traitlets", +] + [[package]] name = "mccabe" version = "0.7.0" @@ -131,6 +230,17 @@ version = "1.0.0" requires_python = ">=3.5" summary = "Type system extensions for programs checked with the mypy type checker." +[[package]] +name = "nebula3-python" +version = "3.4.0" +summary = "Python client for NebulaGraph V3.4" +dependencies = [ + "future>=0.18.0", + "httplib2>=0.20.0", + "pytz>=2021.1", + "six>=1.16.0", +] + [[package]] name = "networkx" version = "2.6.3" @@ -143,12 +253,31 @@ version = "23.0" requires_python = ">=3.7" summary = "Core utilities for Python packages" +[[package]] +name = "parso" +version = "0.8.3" +requires_python = ">=3.6" +summary = "A Python Parser" + [[package]] name = "pathspec" version = "0.11.1" requires_python = ">=3.7" summary = "Utility library for gitignore style pattern matching of file paths." +[[package]] +name = "pexpect" +version = "4.8.0" +summary = "Pexpect allows easy control of interactive console applications." +dependencies = [ + "ptyprocess>=0.5", +] + +[[package]] +name = "pickleshare" +version = "0.7.5" +summary = "Tiny 'shelve'-like database with concurrency support" + [[package]] name = "platformdirs" version = "3.1.1" @@ -167,6 +296,20 @@ dependencies = [ "importlib-metadata>=0.12; python_version < \"3.8\"", ] +[[package]] +name = "prompt-toolkit" +version = "3.0.38" +requires_python = ">=3.7.0" +summary = "Library for building powerful interactive command lines in Python" +dependencies = [ + "wcwidth", +] + +[[package]] +name = "ptyprocess" +version = "0.7.0" +summary = "Run a subprocess in a pseudo terminal" + [[package]] name = "py4j" version = "0.10.9.5" @@ -184,6 +327,18 @@ version = "2.5.0" requires_python = ">=3.6" summary = "passive checker of Python programs" +[[package]] +name = "pygments" +version = "2.14.0" +requires_python = ">=3.6" +summary = "Pygments is a syntax highlighting package written in Python." + +[[package]] +name = "pyparsing" +version = "3.0.9" +requires_python = ">=3.6.8" +summary = "pyparsing module - Classes and methods to define and execute parsing grammars" + [[package]] name = "pyspark" version = "3.3.2" @@ -209,12 +364,53 @@ dependencies = [ "tomli>=1.0.0; python_version < \"3.11\"", ] +[[package]] +name = "pytz" +version = "2022.7.1" +summary = "World timezone definitions, modern and historical" + +[[package]] +name = "requests" +version = "2.28.2" +requires_python = ">=3.7, <4" +summary = "Python HTTP for Humans." +dependencies = [ + "certifi>=2017.4.17", + "charset-normalizer<4,>=2", + "idna<4,>=2.5", + "urllib3<1.27,>=1.21.1", +] + +[[package]] +name = "setuptools" +version = "67.6.0" +requires_python = ">=3.7" +summary = "Easily download, build, install, upgrade, and uninstall Python packages" + +[[package]] +name = "six" +version = "1.16.0" +requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +summary = "Python 2 and 3 compatibility utilities" + +[[package]] +name = "tokenize-rt" +version = "5.0.0" +requires_python = ">=3.7" +summary = "A wrapper around the stdlib `tokenize` which roundtrips." + [[package]] name = "tomli" version = "2.0.1" requires_python = ">=3.7" summary = "A lil' TOML parser" +[[package]] +name = "traitlets" +version = "5.9.0" +requires_python = ">=3.7" +summary = "Traitlets Python configuration system" + [[package]] name = "typed-ast" version = "1.5.4" @@ -227,6 +423,17 @@ version = "4.5.0" requires_python = ">=3.7" summary = "Backported and Experimental Type Hints for Python 3.7+" +[[package]] +name = "urllib3" +version = "1.26.15" +requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +summary = "HTTP library with thread-safe connection pooling, file post, and more." + +[[package]] +name = "wcwidth" +version = "0.2.6" +summary = "Measures the displayed width of unicode strings in a terminal" + [[package]] name = "werkzeug" version = "2.2.3" @@ -244,13 +451,21 @@ summary = "Backport of pathlib-compatible object wrapper for zip files" [metadata] lock_version = "4.1" -content_hash = "sha256:744e451e5442eda41d35ca976c8ea011eb0f1aae04f1c23af3847063d95f7b9b" +content_hash = "sha256:259dbbfabfc660f911efea986e129584b563c4b23576fe7b8670bdd8a237646b" [metadata.files] +"appnope 0.1.3" = [ + {url = "https://files.pythonhosted.org/packages/41/4a/381783f26df413dde4c70c734163d88ca0550a1361cb74a1c68f47550619/appnope-0.1.3-py2.py3-none-any.whl", hash = "sha256:265a455292d0bd8a72453494fa24df5a11eb18373a60c7c0430889f22548605e"}, + {url = "https://files.pythonhosted.org/packages/6a/cd/355842c0db33192ac0fc822e2dcae835669ef317fe56c795fb55fcddb26f/appnope-0.1.3.tar.gz", hash = "sha256:02bd91c4de869fbb1e1c50aafc4098827a7a54ab2f39d9dcba6c9547ed920e24"}, +] "attrs 22.2.0" = [ {url = "https://files.pythonhosted.org/packages/21/31/3f468da74c7de4fcf9b25591e682856389b3400b4b62f201e65f15ea3e07/attrs-22.2.0.tar.gz", hash = "sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99"}, {url = "https://files.pythonhosted.org/packages/fb/6e/6f83bf616d2becdf333a1640f1d463fef3150e2e926b7010cb0f81c95e88/attrs-22.2.0-py3-none-any.whl", hash = "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836"}, ] +"backcall 0.2.0" = [ + {url = "https://files.pythonhosted.org/packages/4c/1c/ff6546b6c12603d8dd1070aa3c3d273ad4c07f5771689a7b69a550e8c951/backcall-0.2.0-py2.py3-none-any.whl", hash = "sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255"}, + {url = "https://files.pythonhosted.org/packages/a2/40/764a663805d84deee23043e1426a9175567db89c8b3287b5c2ad9f71aa93/backcall-0.2.0.tar.gz", hash = "sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e"}, +] "black 23.1.0" = [ {url = "https://files.pythonhosted.org/packages/01/8a/065d0a59c1ebe13186b12a2fa3965a41fc1588828709995e2630004d216e/black-23.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8178318cb74f98bc571eef19068f6ab5613b3e59d4f47771582f04e175570ed8"}, {url = "https://files.pythonhosted.org/packages/15/11/533355217b1cc4a6df3263048060c1527f733d4720e158de2085293112bb/black-23.1.0.tar.gz", hash = "sha256:b0bd97bea8903f5a2ba7219257a44e3f1f9d00073d6cc1add68f0beec69692ac"}, @@ -278,6 +493,87 @@ content_hash = "sha256:744e451e5442eda41d35ca976c8ea011eb0f1aae04f1c23af3847063d {url = "https://files.pythonhosted.org/packages/e6/0a/9a5fca4a2ca07d4dbc3b00445c9353f05ea182b000f68c9ad6ba1da87a47/black-23.1.0-cp38-cp38-macosx_10_16_universal2.whl", hash = "sha256:0052dba51dec07ed029ed61b18183942043e00008ec65d5028814afaab9a22fd"}, {url = "https://files.pythonhosted.org/packages/f1/89/ccc28cb74a66c094b609295b009b5e0350c10b75661d2450eeed2f60ce37/black-23.1.0-cp311-cp311-macosx_10_16_x86_64.whl", hash = "sha256:382998821f58e5c8238d3166c492139573325287820963d2f7de4d518bd76958"}, ] +"certifi 2022.12.7" = [ + {url = "https://files.pythonhosted.org/packages/37/f7/2b1b0ec44fdc30a3d31dfebe52226be9ddc40cd6c0f34ffc8923ba423b69/certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, + {url = "https://files.pythonhosted.org/packages/71/4c/3db2b8021bd6f2f0ceb0e088d6b2d49147671f25832fb17970e9b583d742/certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"}, +] +"charset-normalizer 3.1.0" = [ + {url = "https://files.pythonhosted.org/packages/00/47/f14533da238134f5067fb1d951eb03d5c4be895d6afb11c7ebd07d111acb/charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28"}, + {url = "https://files.pythonhosted.org/packages/01/c7/0407de35b70525dba2a58a2724a525cf882ee76c3d2171d834463c5d2881/charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb"}, + {url = "https://files.pythonhosted.org/packages/05/f3/86b5fcb5c8fe8b4231362918a7c4d8f549c56561c5fdb495a3c5b41c6862/charset_normalizer-3.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8"}, + {url = "https://files.pythonhosted.org/packages/07/6b/98d41a0221991a806e88c95bfeecf8935fbf465b02eb4b469770d572183a/charset_normalizer-3.1.0-cp37-cp37m-win32.whl", hash = "sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974"}, + {url = "https://files.pythonhosted.org/packages/0a/67/8d3d162ec6641911879651cdef670c3c6136782b711d7f8e82e2fffe06e0/charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017"}, + {url = "https://files.pythonhosted.org/packages/12/12/c5c39f5a149cd6788d2e40cea5618bae37380e2754fcdf53dc9e01bdd33a/charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41"}, + {url = "https://files.pythonhosted.org/packages/12/68/4812f9b05ac0a2b7619ac3dd7d7e3fc52c12006b84617021c615fc2fcf42/charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e"}, + {url = "https://files.pythonhosted.org/packages/13/b7/21729a6d512246aa0bb872b90aea0d9fcd1b293762cdb1d1d33c01140074/charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326"}, + {url = "https://files.pythonhosted.org/packages/16/58/19fd2f62e6ff44ba0db0cd44b584790555e2cde09293149f4409d654811b/charset_normalizer-3.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6"}, + {url = "https://files.pythonhosted.org/packages/18/36/7ae10a3dd7f9117b61180671f8d1e4802080cca88ad40aaabd3dad8bab0e/charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62"}, + {url = "https://files.pythonhosted.org/packages/1c/9b/de2adc43345623da8e7c958719528a42b6d87d2601017ce1187d43b8a2d7/charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230"}, + {url = "https://files.pythonhosted.org/packages/1f/be/c6c76cf8fcf6918922223203c83ba8192eff1c6a709e8cfec7f5ca3e7d2d/charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854"}, + {url = "https://files.pythonhosted.org/packages/21/16/1b0d8fdcb81bbf180976af4f867ce0f2244d303ab10d452fde361dec3b5c/charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be"}, + {url = "https://files.pythonhosted.org/packages/23/13/cf5d7bb5bc95f120df64d6c470581189df51d7f011560b2a06a395b7a120/charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137"}, + {url = "https://files.pythonhosted.org/packages/26/20/83e1804a62b25891c4e770c94d9fd80233bbb3f2a51c4fadee7a196e5a5b/charset_normalizer-3.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59"}, + {url = "https://files.pythonhosted.org/packages/2c/2f/ec805104098085728b7cb610deede7195c6fa59f51942422f02cc427b6f6/charset_normalizer-3.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649"}, + {url = "https://files.pythonhosted.org/packages/2e/25/3eab2b38fef9ae59f7b4e9c1e62eb50609d911867e5acabace95fe25c0b1/charset_normalizer-3.1.0-cp310-cp310-win32.whl", hash = "sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448"}, + {url = "https://files.pythonhosted.org/packages/31/8b/81c3515a69d06b501fcce69506af57a7a19bd9f42cabd1a667b1b40f2c55/charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b"}, + {url = "https://files.pythonhosted.org/packages/33/10/c87ba15f779f8251ae55fa147631339cd91e7af51c3c133d2687c6e41800/charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11"}, + {url = "https://files.pythonhosted.org/packages/33/97/9967fb2d364a9da38557e4af323abcd58cc05bdd8f77e9fd5ae4882772cc/charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706"}, + {url = "https://files.pythonhosted.org/packages/45/3d/fa2683f5604f99fba5098a7313e5d4846baaecbee754faf115907f21a85f/charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f"}, + {url = "https://files.pythonhosted.org/packages/4e/11/f7077d78b18aca8ea3186a706c0221aa2bc34c442a3d3bdf3ad401a29052/charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0"}, + {url = "https://files.pythonhosted.org/packages/4f/18/92866f050f7114ba38aba4f4a69f83cc2a25dc2e5a8af4b44fd1bfd6d528/charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d"}, + {url = "https://files.pythonhosted.org/packages/4f/7c/af43743567a7da2a069b4f9fa31874c3c02b963cd1fb84fe1e7568a567e6/charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7"}, + {url = "https://files.pythonhosted.org/packages/4f/a2/9031ba4a008e11a21d7b7aa41751290d2f2035a2f14ecb6e589771a17c47/charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b"}, + {url = "https://files.pythonhosted.org/packages/56/24/5f2dedcf3d0673931b6200c410832ae44b376848bc899dbf1fa6c91c4ebe/charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324"}, + {url = "https://files.pythonhosted.org/packages/5d/2b/4d8c80400c04ae3c8dbc847de092e282b5c7b17f8f9505d68bb3e5815c71/charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb"}, + {url = "https://files.pythonhosted.org/packages/61/e3/ad9ae58b28482d1069eba1edec2be87701f5dd6fd6024a665020d66677a0/charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d"}, + {url = "https://files.pythonhosted.org/packages/67/30/dbab1fe5ab2ce5d3d517ad9936170d896e9687f3860a092519f1fe359812/charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1"}, + {url = "https://files.pythonhosted.org/packages/67/df/660e9665ace7ad711e275194a86cb757fb4d4e513fae5ff3d39573db4984/charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60"}, + {url = "https://files.pythonhosted.org/packages/68/77/af702eba147ba963b27eb00832cef6b8c4cb9fcf7404a476993876434b93/charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd"}, + {url = "https://files.pythonhosted.org/packages/69/22/66351781e668158feef71c5e3b059a79ecc9efc3ef84a45888b0f3a933d5/charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f"}, + {url = "https://files.pythonhosted.org/packages/6d/59/59a3f4d8a59ee270da77f9e954a0e284c9d6884d39ec69d696d9aa5ff2f2/charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0"}, + {url = "https://files.pythonhosted.org/packages/72/90/667a6bc6abe42fc10adf4cd2c1e1c399d78e653dbac4c8018350843d4ab7/charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0"}, + {url = "https://files.pythonhosted.org/packages/74/5f/361202de730532028458b729781b8435f320e31a622c27f30e25eec80513/charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0"}, + {url = "https://files.pythonhosted.org/packages/74/f1/d0b8385b574f7e086fb6709e104b696707bd3742d54a6caf0cebbb7e975b/charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce"}, + {url = "https://files.pythonhosted.org/packages/76/ad/516fed8ffaf02e7a01cd6f6e9d101a6dec64d4db53bec89d30802bf30a96/charset_normalizer-3.1.0-cp38-cp38-win32.whl", hash = "sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0"}, + {url = "https://files.pythonhosted.org/packages/82/b9/51b66a647be8685dee75b7807e0f750edf5c1e4f29bc562ad285c501e3c7/charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c"}, + {url = "https://files.pythonhosted.org/packages/84/23/f60cda6c70ae922ad78368982f06e7fef258fba833212f26275fe4727dc4/charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84"}, + {url = "https://files.pythonhosted.org/packages/85/e8/18d408d8fe29a56012c10d6b15960940b83f06620e9d7481581cdc6d9901/charset_normalizer-3.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df"}, + {url = "https://files.pythonhosted.org/packages/94/70/23981e7bf098efbc4037e7c66d28a10e950d9296c08c6dea8ef290f9c79e/charset_normalizer-3.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e"}, + {url = "https://files.pythonhosted.org/packages/9a/f1/ff81439aa09070fee64173e6ca6ce1342f2b1cca997bcaae89e443812684/charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f"}, + {url = "https://files.pythonhosted.org/packages/9e/62/a1e0a8f8830c92014602c8a88a1a20b8a68d636378077381f671e6e1cec9/charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a"}, + {url = "https://files.pythonhosted.org/packages/a2/6c/5167f08da5298f383036c33cb749ab5b3405fd07853edc8314c6882c01b8/charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e"}, + {url = "https://files.pythonhosted.org/packages/a4/03/355281b62c26712a50c6a9dd75339d8cdd58488fd7bf2556ba1320ebd315/charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d"}, + {url = "https://files.pythonhosted.org/packages/a9/83/138d2624fdbcb62b7e14715eb721d44347e41a1b4c16544661e940793f49/charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f"}, + {url = "https://files.pythonhosted.org/packages/ac/7f/62d5dff4e9cb993e4b0d4ea78a74cc84d7d92120879529e0ce0965765936/charset_normalizer-3.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f"}, + {url = "https://files.pythonhosted.org/packages/ac/c5/990bc41a98b7fa2677c665737fdf278bb74ad4b199c56b6b564b3d4cbfc5/charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c"}, + {url = "https://files.pythonhosted.org/packages/ad/83/994bfca99e29f1bab66b9248e739360ee70b5aae0a5ee488cd776501edbc/charset_normalizer-3.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755"}, + {url = "https://files.pythonhosted.org/packages/b0/55/d8ef4c8c7d2a8b3a16e7d9b03c59475c2ee96a0e0c90b14c99faaac0ee3b/charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8"}, + {url = "https://files.pythonhosted.org/packages/bb/dc/58fdef3ab85e8e7953a8b89ef1d2c06938b8ad88d9617f22967e1a90e6b8/charset_normalizer-3.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c"}, + {url = "https://files.pythonhosted.org/packages/bc/08/7e7c97399806366ca515a049c3a1e4b644a6a2048bed16e5e67bfaafd0aa/charset_normalizer-3.1.0-cp311-cp311-win32.whl", hash = "sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909"}, + {url = "https://files.pythonhosted.org/packages/bc/92/ac692a303e53cdc8852ce72b1ac364b493ca5c9206a5c8db5b30a7f3019c/charset_normalizer-3.1.0-cp39-cp39-win32.whl", hash = "sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1"}, + {url = "https://files.pythonhosted.org/packages/c2/35/dfb4032f5712747d3dcfdd19d0768f6d8f60910ae24ed066ecbf442be013/charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203"}, + {url = "https://files.pythonhosted.org/packages/c6/ab/43ea052756b2f2dcb6a131897811c0e2704b0288f090336217d3346cd682/charset_normalizer-3.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1"}, + {url = "https://files.pythonhosted.org/packages/c9/8c/a76dd9f2c8803eb147e1e715727f5c3ba0ef39adaadf66a7b3698c113180/charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5"}, + {url = "https://files.pythonhosted.org/packages/cc/f6/21a66e524658bd1dd7b89ac9d1ee8f7823f2d9701a2fbc458ab9ede53c63/charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795"}, + {url = "https://files.pythonhosted.org/packages/d1/ff/51fe7e6446415f143b159740c727850172bc35622b2a06dde3354bdebaf3/charset_normalizer-3.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b"}, + {url = "https://files.pythonhosted.org/packages/d5/92/86c0f0e66e897f6818c46dadef328a5b345d061688f9960fc6ca1fd03dbe/charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31"}, + {url = "https://files.pythonhosted.org/packages/d7/4c/37ad75674e8c6bc22ab01bef673d2d6e46ee44203498c9a26aa23959afe5/charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1"}, + {url = "https://files.pythonhosted.org/packages/d8/ca/a7ff600781bf1e5f702ba26bb82f2ba1d3a873a3f8ad73cc44c79dfaefa9/charset_normalizer-3.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14"}, + {url = "https://files.pythonhosted.org/packages/dd/39/6276cf5a395ffd39b77dadf0e2fcbfca8dbfe48c56ada250c40086055143/charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9"}, + {url = "https://files.pythonhosted.org/packages/e1/7c/398600268fc98b7e007f5a716bd60903fff1ecff75e45f5700212df5cd76/charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19"}, + {url = "https://files.pythonhosted.org/packages/e1/b4/53678b2a14e0496fc167fe9b9e726ad33d670cfd2011031aa5caeee6b784/charset_normalizer-3.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373"}, + {url = "https://files.pythonhosted.org/packages/e5/aa/9d2d60d6a566423da96c15cd11cbb88a70f9aff9a4db096094ee19179cab/charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac"}, + {url = "https://files.pythonhosted.org/packages/e6/98/a3f65f57651da1cecaed91d6f75291995d56c97442fa2a43d2a421139adf/charset_normalizer-3.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23"}, + {url = "https://files.pythonhosted.org/packages/ea/38/d31c7906c4be13060c1a5034087966774ef33ab57ff2eee76d71265173c3/charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531"}, + {url = "https://files.pythonhosted.org/packages/ef/81/14b3b8f01ddaddad6cdec97f2f599aa2fa466bd5ee9af99b08b7713ccd29/charset_normalizer-3.1.0-py3-none-any.whl", hash = "sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d"}, + {url = "https://files.pythonhosted.org/packages/f2/b7/e21e16c98575616f4ce09dc766dbccdac0ca119c176b184d46105e971a84/charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab"}, + {url = "https://files.pythonhosted.org/packages/f2/d7/6ee92c11eda3f3c9cac1e059901092bfdf07388be7d2e60ac627527eee62/charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1"}, + {url = "https://files.pythonhosted.org/packages/f4/0a/8c03913ed1eca9d831db0c28759edb6ce87af22bb55dbc005a52525a75b6/charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a"}, + {url = "https://files.pythonhosted.org/packages/f6/0f/de1c4030fd669e6719277043e3b0f152a83c118dd1020cf85b51d443d04a/charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e"}, + {url = "https://files.pythonhosted.org/packages/f8/ed/500609cb2457b002242b090c814549997424d72690ef3058cfdfca91f68b/charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b"}, + {url = "https://files.pythonhosted.org/packages/fa/8e/2e5c742c3082bce3eea2ddd5b331d08050cda458bc362d71c48e07a44719/charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6"}, + {url = "https://files.pythonhosted.org/packages/ff/d7/8d757f8bd45be079d76309248845a04f09619a7b17d6dfc8c9ff6433cac2/charset-normalizer-3.1.0.tar.gz", hash = "sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5"}, +] "chispa 0.9.2" = [ {url = "https://files.pythonhosted.org/packages/03/8a/060df449d0a53a9c8480ccadfbcbba000a009d6e357f183a88adbe23f458/chispa-0.9.2.tar.gz", hash = "sha256:621ad2e64fd27e7372c7b90ab2d5ad1f8dd69b737a3421ba5b6f84b113a18b84"}, {url = "https://files.pythonhosted.org/packages/51/b3/c7af5ae69e59b545ed6fcfe7f606e32a0efad36ba21a15b393ae69015ffd/chispa-0.9.2-py3-none-any.whl", hash = "sha256:c6eae922f5c3ccd08f4dc3707202291bb249e68e319d0641795d92d80cfb1cad"}, @@ -290,6 +586,10 @@ content_hash = "sha256:744e451e5442eda41d35ca976c8ea011eb0f1aae04f1c23af3847063d {url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +"decorator 5.1.1" = [ + {url = "https://files.pythonhosted.org/packages/66/0c/8d907af351aa16b42caae42f9d6aa37b900c67308052d10fdce809f8d952/decorator-5.1.1.tar.gz", hash = "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330"}, + {url = "https://files.pythonhosted.org/packages/d5/50/83c593b07763e1161326b3b8c6686f0f4b0f24d5526546bee538c89837d6/decorator-5.1.1-py3-none-any.whl", hash = "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186"}, +] "exceptiongroup 1.1.1" = [ {url = "https://files.pythonhosted.org/packages/61/97/17ed81b7a8d24d8f69b62c0db37abbd8c0042d4b3fc429c73dab986e7483/exceptiongroup-1.1.1-py3-none-any.whl", hash = "sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e"}, {url = "https://files.pythonhosted.org/packages/cc/38/57f14ddc8e8baeddd8993a36fe57ce7b4ba174c35048b9a6d270bb01e833/exceptiongroup-1.1.1.tar.gz", hash = "sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785"}, @@ -302,6 +602,17 @@ content_hash = "sha256:744e451e5442eda41d35ca976c8ea011eb0f1aae04f1c23af3847063d {url = "https://files.pythonhosted.org/packages/95/9c/a3542594ce4973786236a1b7b702b8ca81dbf40ea270f0f96284f0c27348/Flask-2.2.3-py3-none-any.whl", hash = "sha256:c0bec9477df1cb867e5a67c9e1ab758de9cb4a3e52dd70681f59fa40a62b3f2d"}, {url = "https://files.pythonhosted.org/packages/e8/5c/ff9047989bd995b1098d14b03013f160225db2282925b517bb4a967752ee/Flask-2.2.3.tar.gz", hash = "sha256:7eb373984bf1c770023fce9db164ed0c3353cd0b53f130f4693da0ca756a2e6d"}, ] +"future 0.18.3" = [ + {url = "https://files.pythonhosted.org/packages/8f/2e/cf6accf7415237d6faeeebdc7832023c90e0282aa16fd3263db0eb4715ec/future-0.18.3.tar.gz", hash = "sha256:34a17436ed1e96697a86f9de3d15a3b0be01d8bc8de9c1dffd59fb8234ed5307"}, +] +"httplib2 0.22.0" = [ + {url = "https://files.pythonhosted.org/packages/3d/ad/2371116b22d616c194aa25ec410c9c6c37f23599dcd590502b74db197584/httplib2-0.22.0.tar.gz", hash = "sha256:d7a10bc5ef5ab08322488bde8c726eeee5c8618723fdb399597ec58f3d82df81"}, + {url = "https://files.pythonhosted.org/packages/a8/6c/d2fbdaaa5959339d53ba38e94c123e4e84b8fbc4b84beb0e70d7c1608486/httplib2-0.22.0-py3-none-any.whl", hash = "sha256:14ae0a53c1ba8f3d37e9e27cf37eabb0fb9980f435ba405d546948b009dd64dc"}, +] +"idna 3.4" = [ + {url = "https://files.pythonhosted.org/packages/8b/e1/43beb3d38dba6cb420cefa297822eac205a277ab43e5ba5d5c46faf96438/idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, + {url = "https://files.pythonhosted.org/packages/fc/34/3030de6f1370931b9dbb4dad48f6ab1015ab1d32447850b9fc94e60097be/idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, +] "importlib-metadata 4.2.0" = [ {url = "https://files.pythonhosted.org/packages/22/51/52442c59db26637681148c21f8984eed58c9db67053a0a4783a047010c98/importlib_metadata-4.2.0-py3-none-any.whl", hash = "sha256:057e92c15bc8d9e8109738a48db0ccb31b4d9d5cfbee5a8670879a30be66304b"}, {url = "https://files.pythonhosted.org/packages/c7/7c/126a8686399ebe256b5e4343ea80b6f2ee91549969da2eef0bb2891b8d24/importlib_metadata-4.2.0.tar.gz", hash = "sha256:b7e52a1f8dec14a75ea73e0891f3060099ca1d8e6a462a4dff11c3e119ea1b31"}, @@ -310,6 +621,10 @@ content_hash = "sha256:744e451e5442eda41d35ca976c8ea011eb0f1aae04f1c23af3847063d {url = "https://files.pythonhosted.org/packages/d7/4b/cbd8e699e64a6f16ca3a8220661b5f83792b3017d0f79807cb8708d33913/iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, {url = "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, ] +"ipython 7.34.0" = [ + {url = "https://files.pythonhosted.org/packages/7c/6a/1f1365f4bf9fcb349fcaa5b61edfcefa721aa13ff37c5631296b12fab8e5/ipython-7.34.0-py3-none-any.whl", hash = "sha256:c175d2440a1caff76116eb719d40538fbb316e214eda85c5515c303aacbfb23e"}, + {url = "https://files.pythonhosted.org/packages/db/6c/3fcf0b8ee46656796099ac4b7b72497af5f090da3e43fd305f2a24c73915/ipython-7.34.0.tar.gz", hash = "sha256:af3bdb46aa292bce5615b1b2ebc76c2080c5f77f54bda2ec72461317273e7cd6"}, +] "isort 5.11.5" = [ {url = "https://files.pythonhosted.org/packages/5f/f6/c55db45970fbd14de6ab72082f1b8a143c3a69aa031c1e0dd4b9ecc8d496/isort-5.11.5-py3-none-any.whl", hash = "sha256:ba1d72fb2595a01c7895a5128f9585a5cc4b6d395f1c8d514989b9a7eb2a8746"}, {url = "https://files.pythonhosted.org/packages/67/63/18cc5c2f9084d3f91ce704f2b5c8e17bedd777244e7732c21a31992b0a78/isort-5.11.5.tar.gz", hash = "sha256:6be1f76a507cb2ecf16c7cf14a37e41609ca082330be4e3436a18ef74add55db"}, @@ -318,6 +633,10 @@ content_hash = "sha256:744e451e5442eda41d35ca976c8ea011eb0f1aae04f1c23af3847063d {url = "https://files.pythonhosted.org/packages/68/5f/447e04e828f47465eeab35b5d408b7ebaaaee207f48b7136c5a7267a30ae/itsdangerous-2.1.2-py3-none-any.whl", hash = "sha256:2c2349112351b88699d8d4b6b075022c0808887cb7ad10069318a8b0bc88db44"}, {url = "https://files.pythonhosted.org/packages/7f/a1/d3fb83e7a61fa0c0d3d08ad0a94ddbeff3731c05212617dff3a94e097f08/itsdangerous-2.1.2.tar.gz", hash = "sha256:5dbbc68b317e5e42f327f9021763545dc3fc3bfe22e6deb96aaf1fc38874156a"}, ] +"jedi 0.18.2" = [ + {url = "https://files.pythonhosted.org/packages/15/02/afd43c5066de05f6b3188f3aa74136a3289e6c30e7a45f351546cab0928c/jedi-0.18.2.tar.gz", hash = "sha256:bae794c30d07f6d910d32a7048af09b5a39ed740918da923c6b780790ebac612"}, + {url = "https://files.pythonhosted.org/packages/6d/60/4acda63286ef6023515eb914543ba36496b8929cb7af49ecce63afde09c6/jedi-0.18.2-py2.py3-none-any.whl", hash = "sha256:203c1fd9d969ab8f2119ec0a3342e0b49910045abe6af0a3ae83a5764d54639e"}, +] "jinja2 3.1.2" = [ {url = "https://files.pythonhosted.org/packages/7a/ff/75c28576a1d900e87eb6335b063fab47a8ef3c8b4d88524c4bf78f670cce/Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, {url = "https://files.pythonhosted.org/packages/bc/c3/f068337a370801f372f2f8f6bad74a5c140f6fda3d9de154052708dd3c65/Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, @@ -374,6 +693,10 @@ content_hash = "sha256:744e451e5442eda41d35ca976c8ea011eb0f1aae04f1c23af3847063d {url = "https://files.pythonhosted.org/packages/ea/60/2400ba59cf2465fa136487ee7299f52121a9d04b2cf8539ad43ad10e70e8/MarkupSafe-2.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:e55e40ff0cc8cc5c07996915ad367fa47da6b3fc091fdadca7f5403239c5fec3"}, {url = "https://files.pythonhosted.org/packages/f9/aa/ebcd114deab08f892b1d70badda4436dbad1747f9e5b72cffb3de4c7129d/MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7313ce6a199651c4ed9d7e4cfb4aa56fe923b1adf9af3b420ee14e6d9a73df65"}, ] +"matplotlib-inline 0.1.6" = [ + {url = "https://files.pythonhosted.org/packages/d9/50/3af8c0362f26108e54d58c7f38784a3bdae6b9a450bab48ee8482d737f44/matplotlib-inline-0.1.6.tar.gz", hash = "sha256:f887e5f10ba98e8d2b150ddcf4702c1e5f8b3a20005eb0f74bfdbd360ee6f304"}, + {url = "https://files.pythonhosted.org/packages/f2/51/c34d7a1d528efaae3d8ddb18ef45a41f284eacf9e514523b191b7d0872cc/matplotlib_inline-0.1.6-py3-none-any.whl", hash = "sha256:f1f41aab5328aa5aaea9b16d083b128102f8712542f819fe7e6a420ff581b311"}, +] "mccabe 0.7.0" = [ {url = "https://files.pythonhosted.org/packages/27/1a/1f68f9ba0c207934b35b86a8ca3aad8395a3d6dd7921c0686e23853ff5a9/mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, {url = "https://files.pythonhosted.org/packages/e7/ff/0ffefdcac38932a54d2b5eed4e0ba8a408f215002cd178ad1df0f2806ff8/mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, @@ -382,6 +705,10 @@ content_hash = "sha256:744e451e5442eda41d35ca976c8ea011eb0f1aae04f1c23af3847063d {url = "https://files.pythonhosted.org/packages/2a/e2/5d3f6ada4297caebe1a2add3b126fe800c96f56dbe5d1988a2cbe0b267aa/mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, {url = "https://files.pythonhosted.org/packages/98/a4/1ab47638b92648243faf97a5aeb6ea83059cc3624972ab6b8d2316078d3f/mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, ] +"nebula3-python 3.4.0" = [ + {url = "https://files.pythonhosted.org/packages/9e/2f/3f942deb4a5da23c0cabeac954ec6a224ee6e21d0f8e35845164dce65b79/nebula3-python-3.4.0.tar.gz", hash = "sha256:47bd8b1b4bb2c2f0e5122bc147926cb50578a66841acf6a743cae4d0362c9eaa"}, + {url = "https://files.pythonhosted.org/packages/ac/bc/23396a5d394801ec8a1cfb2eed6643ded878d77c9242fd097c4a4a5d3489/nebula3_python-3.4.0-py3-none-any.whl", hash = "sha256:d9d94c6a41712875e6ec866907de0789057f860e64f547f87d9f199439759dd6"}, +] "networkx 2.6.3" = [ {url = "https://files.pythonhosted.org/packages/97/ae/7497bc5e1c84af95e585e3f98585c9f06c627fac6340984c4243053e8f44/networkx-2.6.3.tar.gz", hash = "sha256:c0946ed31d71f1b732b5aaa6da5a0388a345019af232ce2f49c766e2d6795c51"}, {url = "https://files.pythonhosted.org/packages/e9/93/aa6613aa70d6eb4868e667068b5a11feca9645498fd31b954b6c4bb82fa5/networkx-2.6.3-py3-none-any.whl", hash = "sha256:80b6b89c77d1dfb64a4c7854981b60aeea6360ac02c6d4e4913319e0a313abef"}, @@ -390,10 +717,22 @@ content_hash = "sha256:744e451e5442eda41d35ca976c8ea011eb0f1aae04f1c23af3847063d {url = "https://files.pythonhosted.org/packages/47/d5/aca8ff6f49aa5565df1c826e7bf5e85a6df852ee063600c1efa5b932968c/packaging-23.0.tar.gz", hash = "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97"}, {url = "https://files.pythonhosted.org/packages/ed/35/a31aed2993e398f6b09a790a181a7927eb14610ee8bbf02dc14d31677f1c/packaging-23.0-py3-none-any.whl", hash = "sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2"}, ] +"parso 0.8.3" = [ + {url = "https://files.pythonhosted.org/packages/05/63/8011bd08a4111858f79d2b09aad86638490d62fbf881c44e434a6dfca87b/parso-0.8.3-py2.py3-none-any.whl", hash = "sha256:c001d4636cd3aecdaf33cbb40aebb59b094be2a74c556778ef5576c175e19e75"}, + {url = "https://files.pythonhosted.org/packages/a2/0e/41f0cca4b85a6ea74d66d2226a7cda8e41206a624f5b330b958ef48e2e52/parso-0.8.3.tar.gz", hash = "sha256:8c07be290bb59f03588915921e29e8a50002acaf2cdc5fa0e0114f91709fafa0"}, +] "pathspec 0.11.1" = [ {url = "https://files.pythonhosted.org/packages/95/60/d93628975242cc515ab2b8f5b2fc831d8be2eff32f5a1be4776d49305d13/pathspec-0.11.1.tar.gz", hash = "sha256:2798de800fa92780e33acca925945e9a19a133b715067cf165b8866c15a31687"}, {url = "https://files.pythonhosted.org/packages/be/c8/551a803a6ebb174ec1c124e68b449b98a0961f0b737def601e3c1fbb4cfd/pathspec-0.11.1-py3-none-any.whl", hash = "sha256:d8af70af76652554bd134c22b3e8a1cc46ed7d91edcdd721ef1a0c51a84a5293"}, ] +"pexpect 4.8.0" = [ + {url = "https://files.pythonhosted.org/packages/39/7b/88dbb785881c28a102619d46423cb853b46dbccc70d3ac362d99773a78ce/pexpect-4.8.0-py2.py3-none-any.whl", hash = "sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937"}, + {url = "https://files.pythonhosted.org/packages/e5/9b/ff402e0e930e70467a7178abb7c128709a30dfb22d8777c043e501bc1b10/pexpect-4.8.0.tar.gz", hash = "sha256:fc65a43959d153d0114afe13997d439c22823a27cefceb5ff35c2178c6784c0c"}, +] +"pickleshare 0.7.5" = [ + {url = "https://files.pythonhosted.org/packages/9a/41/220f49aaea88bc6fa6cba8d05ecf24676326156c23b991e80b3f2fc24c77/pickleshare-0.7.5-py2.py3-none-any.whl", hash = "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56"}, + {url = "https://files.pythonhosted.org/packages/d8/b6/df3c1c9b616e9c0edbc4fbab6ddd09df9535849c64ba51fcb6531c32d4d8/pickleshare-0.7.5.tar.gz", hash = "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca"}, +] "platformdirs 3.1.1" = [ {url = "https://files.pythonhosted.org/packages/79/c4/f98a05535344f79699bbd494e56ac9efc986b7a253fe9f4dba7414a7f505/platformdirs-3.1.1.tar.gz", hash = "sha256:024996549ee88ec1a9aa99ff7f8fc819bb59e2c3477b410d90a16d32d6e707aa"}, {url = "https://files.pythonhosted.org/packages/7b/e1/593f693096c50411a2bf9571f66bc3be9d0f79a4a50e95aab581458b0e3c/platformdirs-3.1.1-py3-none-any.whl", hash = "sha256:e5986afb596e4bb5bde29a79ac9061aa955b94fca2399b7aaac4090860920dd8"}, @@ -402,6 +741,14 @@ content_hash = "sha256:744e451e5442eda41d35ca976c8ea011eb0f1aae04f1c23af3847063d {url = "https://files.pythonhosted.org/packages/9e/01/f38e2ff29715251cf25532b9082a1589ab7e4f571ced434f98d0139336dc/pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, {url = "https://files.pythonhosted.org/packages/a1/16/db2d7de3474b6e37cbb9c008965ee63835bba517e22cdb8c35b5116b5ce1/pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] +"prompt-toolkit 3.0.38" = [ + {url = "https://files.pythonhosted.org/packages/4b/bb/75cdcd356f57d17b295aba121494c2333d26bfff1a837e6199b8b83c415a/prompt_toolkit-3.0.38.tar.gz", hash = "sha256:23ac5d50538a9a38c8bde05fecb47d0b403ecd0662857a86f886f798563d5b9b"}, + {url = "https://files.pythonhosted.org/packages/87/3f/1f5a0ff475ae6481f4b0d45d4d911824d3218b94ee2a97a8cb84e5569836/prompt_toolkit-3.0.38-py3-none-any.whl", hash = "sha256:45ea77a2f7c60418850331366c81cf6b5b9cf4c7fd34616f733c5427e6abbb1f"}, +] +"ptyprocess 0.7.0" = [ + {url = "https://files.pythonhosted.org/packages/20/e5/16ff212c1e452235a90aeb09066144d0c5a6a8c0834397e03f5224495c4e/ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220"}, + {url = "https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35"}, +] "py4j 0.10.9.5" = [ {url = "https://files.pythonhosted.org/packages/86/ec/60880978512d5569ca4bf32b3b4d7776a528ecf4bca4523936c98c92a3c8/py4j-0.10.9.5-py2.py3-none-any.whl", hash = "sha256:52d171a6a2b031d8a5d1de6efe451cf4f5baff1a2819aabc3741c8406539ba04"}, {url = "https://files.pythonhosted.org/packages/ce/1f/b00295b6da3bd2f050912b9f71fdb436ae8f1601cf161365937d8553e24b/py4j-0.10.9.5.tar.gz", hash = "sha256:276a4a3c5a2154df1860ef3303a927460e02e97b047dc0a47c1c3fb8cce34db6"}, @@ -414,6 +761,14 @@ content_hash = "sha256:744e451e5442eda41d35ca976c8ea011eb0f1aae04f1c23af3847063d {url = "https://files.pythonhosted.org/packages/07/92/f0cb5381f752e89a598dd2850941e7f570ac3cb8ea4a344854de486db152/pyflakes-2.5.0.tar.gz", hash = "sha256:491feb020dca48ccc562a8c0cbe8df07ee13078df59813b83959cbdada312ea3"}, {url = "https://files.pythonhosted.org/packages/dc/13/63178f59f74e53acc2165aee4b002619a3cfa7eeaeac989a9eb41edf364e/pyflakes-2.5.0-py2.py3-none-any.whl", hash = "sha256:4579f67d887f804e67edb544428f264b7b24f435b263c4614f384135cea553d2"}, ] +"pygments 2.14.0" = [ + {url = "https://files.pythonhosted.org/packages/0b/42/d9d95cc461f098f204cd20c85642ae40fbff81f74c300341b8d0e0df14e0/Pygments-2.14.0-py3-none-any.whl", hash = "sha256:fa7bd7bd2771287c0de303af8bfdfc731f51bd2c6a47ab69d117138893b82717"}, + {url = "https://files.pythonhosted.org/packages/da/6a/c427c06913204e24de28de5300d3f0e809933f376e0b7df95194b2bb3f71/Pygments-2.14.0.tar.gz", hash = "sha256:b3ed06a9e8ac9a9aae5a6f5dbe78a8a58655d17b43b93c078f094ddc476ae297"}, +] +"pyparsing 3.0.9" = [ + {url = "https://files.pythonhosted.org/packages/6c/10/a7d0fa5baea8fe7b50f448ab742f26f52b80bfca85ac2be9d35cdd9a3246/pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, + {url = "https://files.pythonhosted.org/packages/71/22/207523d16464c40a0310d2d4d8926daffa00ac1f5b1576170a32db749636/pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, +] "pyspark 3.3.2" = [ {url = "https://files.pythonhosted.org/packages/6d/08/87b404b8b3255d46caf0ecdccf871d501a2b58da9b844d3f9710ce9d4d53/pyspark-3.3.2.tar.gz", hash = "sha256:0dfd5db4300c1f6cc9c16d8dbdfb82d881b4b172984da71344ede1a9d4893da8"}, ] @@ -421,10 +776,34 @@ content_hash = "sha256:744e451e5442eda41d35ca976c8ea011eb0f1aae04f1c23af3847063d {url = "https://files.pythonhosted.org/packages/b2/68/5321b5793bd506961bd40bdbdd0674e7de4fb873ee7cab33dd27283ad513/pytest-7.2.2-py3-none-any.whl", hash = "sha256:130328f552dcfac0b1cec75c12e3f005619dc5f874f0a06e8ff7263f0ee6225e"}, {url = "https://files.pythonhosted.org/packages/b9/29/311895d9cd3f003dd58e8fdea36dd895ba2da5c0c90601836f7de79f76fe/pytest-7.2.2.tar.gz", hash = "sha256:c99ab0c73aceb050f68929bc93af19ab6db0558791c6a0715723abe9d0ade9d4"}, ] +"pytz 2022.7.1" = [ + {url = "https://files.pythonhosted.org/packages/03/3e/dc5c793b62c60d0ca0b7e58f1fdd84d5aaa9f8df23e7589b39cc9ce20a03/pytz-2022.7.1.tar.gz", hash = "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0"}, + {url = "https://files.pythonhosted.org/packages/2e/09/fbd3c46dce130958ee8e0090f910f1fe39e502cc5ba0aadca1e8a2b932e5/pytz-2022.7.1-py2.py3-none-any.whl", hash = "sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a"}, +] +"requests 2.28.2" = [ + {url = "https://files.pythonhosted.org/packages/9d/ee/391076f5937f0a8cdf5e53b701ffc91753e87b07d66bae4a09aa671897bf/requests-2.28.2.tar.gz", hash = "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf"}, + {url = "https://files.pythonhosted.org/packages/d2/f4/274d1dbe96b41cf4e0efb70cbced278ffd61b5c7bb70338b62af94ccb25b/requests-2.28.2-py3-none-any.whl", hash = "sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa"}, +] +"setuptools 67.6.0" = [ + {url = "https://files.pythonhosted.org/packages/25/f3/d68c20919bc774c6cb127f1762f2f2f999d700a58198556e883dd3700e58/setuptools-67.6.0.tar.gz", hash = "sha256:2ee892cd5f29f3373097f5a814697e397cf3ce313616df0af11231e2ad118077"}, + {url = "https://files.pythonhosted.org/packages/c3/9e/8a7ba2c9984a060daa6c6f9fff4d576b7ace3936239d6b771541eab972ed/setuptools-67.6.0-py3-none-any.whl", hash = "sha256:b78aaa36f6b90a074c1fa651168723acbf45d14cb1196b6f02c0fd07f17623b2"}, +] +"six 1.16.0" = [ + {url = "https://files.pythonhosted.org/packages/71/39/171f1c67cd00715f190ba0b100d606d440a28c93c7714febeca8b79af85e/six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, + {url = "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, +] +"tokenize-rt 5.0.0" = [ + {url = "https://files.pythonhosted.org/packages/40/01/fb40ea8c465f680bf7aa3f5bee39c62ba8b7f52c38048c27aa95aff4f779/tokenize_rt-5.0.0.tar.gz", hash = "sha256:3160bc0c3e8491312d0485171dea861fc160a240f5f5766b72a1165408d10740"}, + {url = "https://files.pythonhosted.org/packages/8d/12/4c7495f25b4c9131706f3aaffb185d4de32c02a6ee49d875e929c5b7c919/tokenize_rt-5.0.0-py2.py3-none-any.whl", hash = "sha256:c67772c662c6b3dc65edf66808577968fb10badfc2042e3027196bed4daf9e5a"}, +] "tomli 2.0.1" = [ {url = "https://files.pythonhosted.org/packages/97/75/10a9ebee3fd790d20926a90a2547f0bf78f371b2f13aa822c759680ca7b9/tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, {url = "https://files.pythonhosted.org/packages/c0/3f/d7af728f075fb08564c5949a9c95e44352e23dee646869fa104a3b2060a3/tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] +"traitlets 5.9.0" = [ + {url = "https://files.pythonhosted.org/packages/39/c3/205e88f02959712b62008502952707313640369144a7fded4cbc61f48321/traitlets-5.9.0.tar.gz", hash = "sha256:f6cde21a9c68cf756af02035f72d5a723bf607e862e7be33ece505abf4a3bad9"}, + {url = "https://files.pythonhosted.org/packages/77/75/c28e9ef7abec2b7e9ff35aea3e0be6c1aceaf7873c26c95ae1f0d594de71/traitlets-5.9.0-py3-none-any.whl", hash = "sha256:9e6ec080259b9a5940c797d58b613b5e31441c2257b87c2e795c5228ae80d2d8"}, +] "typed-ast 1.5.4" = [ {url = "https://files.pythonhosted.org/packages/04/93/482d12fd3334b53ec4087e658ab161ab23affcf8b052166b4cf972ca673b/typed_ast-1.5.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6778e1b2f81dfc7bc58e4b259363b83d2e509a65198e85d5700dfae4c6c8ff1c"}, {url = "https://files.pythonhosted.org/packages/07/d2/d55702e8deba2c80282fea0df53130790d8f398648be589750954c2dcce4/typed_ast-1.5.4.tar.gz", hash = "sha256:39e21ceb7388e4bb37f4c679d72707ed46c2fbf2a5609b8b8ebc4b067d977df2"}, @@ -455,6 +834,14 @@ content_hash = "sha256:744e451e5442eda41d35ca976c8ea011eb0f1aae04f1c23af3847063d {url = "https://files.pythonhosted.org/packages/31/25/5abcd82372d3d4a3932e1fa8c3dbf9efac10cc7c0d16e78467460571b404/typing_extensions-4.5.0-py3-none-any.whl", hash = "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4"}, {url = "https://files.pythonhosted.org/packages/d3/20/06270dac7316220643c32ae61694e451c98f8caf4c8eab3aa80a2bedf0df/typing_extensions-4.5.0.tar.gz", hash = "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb"}, ] +"urllib3 1.26.15" = [ + {url = "https://files.pythonhosted.org/packages/21/79/6372d8c0d0641b4072889f3ff84f279b738cd8595b64c8e0496d4e848122/urllib3-1.26.15.tar.gz", hash = "sha256:8a388717b9476f934a21484e8c8e61875ab60644d29b9b39e11e4b9dc1c6b305"}, + {url = "https://files.pythonhosted.org/packages/7b/f5/890a0baca17a61c1f92f72b81d3c31523c99bec609e60c292ea55b387ae8/urllib3-1.26.15-py2.py3-none-any.whl", hash = "sha256:aa751d169e23c7479ce47a0cb0da579e3ede798f994f5816a74e4f4500dcea42"}, +] +"wcwidth 0.2.6" = [ + {url = "https://files.pythonhosted.org/packages/20/f4/c0584a25144ce20bfcf1aecd041768b8c762c1eb0aa77502a3f0baa83f11/wcwidth-0.2.6-py2.py3-none-any.whl", hash = "sha256:795b138f6875577cd91bba52baf9e445cd5118fd32723b460e30a0af30ea230e"}, + {url = "https://files.pythonhosted.org/packages/5e/5f/1e4bd82a9cc1f17b2c2361a2d876d4c38973a997003ba5eb400e8a932b6c/wcwidth-0.2.6.tar.gz", hash = "sha256:a5220780a404dbe3353789870978e472cfe477761f06ee55077256e509b156d0"}, +] "werkzeug 2.2.3" = [ {url = "https://files.pythonhosted.org/packages/02/3c/baaebf3235c87d61d6593467056d5a8fba7c75ac838b8d100a5e64eba7a0/Werkzeug-2.2.3.tar.gz", hash = "sha256:2e1ccc9417d4da358b9de6f174e3ac094391ea1d4fbef2d667865d819dfd0afe"}, {url = "https://files.pythonhosted.org/packages/f6/f8/9da63c1617ae2a1dec2fbf6412f3a0cfe9d4ce029eccbda6e1e4258ca45f/Werkzeug-2.2.3-py3-none-any.whl", hash = "sha256:56433961bc1f12533306c624f3be5e744389ac61d722175d543e1751285da612"}, diff --git a/pyproject.toml b/pyproject.toml index f614e44..662db2f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ ng_ai-api = {call = "ng_ai.ng_ai_api.__main__:run"} [tool.pdm.scripts.test] shell = """ pdm update -dG dev - pytest + pytest tests/unit/ """ [tool.pdm.scripts.lint] @@ -21,7 +21,70 @@ shell = """ shell = """ pdm update -dG lint isort ng_ai tests - black --line-length 84 ng_ai tests + black --line-length 84 ng_ai tests examples +""" + +[tool.pdm.scripts.dockerup] +shell = """ + echo "Downloading jar packages for NebulaGraph Algorithm" + mkdir -p tests/integration/setup/download + mkdir -p tests/integration/setup/udf + ALGO_VERSION="3.1.0" + SPARK_C_VERSION="3.4.0" + wget -O tests/integration/setup/download/nebula-algo.jar https://repo1.maven.org/maven2/com/vesoft/nebula-algorithm/$ALGO_VERSION/nebula-algorithm-$ALGO_VERSION.jar + # wget -O tests/integration/setup/download/nebula-spark-connector.jar https://repo1.maven.org/maven2/com/vesoft/nebula-spark-connector/$SPARK_C_VERSION/nebula-spark-connector-$SPARK_C_VERSION.jar + wget -O tests/integration/setup/udf/ng_ai.so https://github.com/wey-gu/nebulagraph-ai/releases/download/0.2.9/ng_ai-ubuntu-2004-nebulagraph-nightly-2023.03.13.so + chmod +x tests/integration/setup/udf/ng_ai.so + echo "Starting NebulaGraph" + + docker-compose -f 'tests/integration/setup/docker-compose.yaml' up -d --remove-orphans + echo "Waiting for NebulaGraph to start..." + sleep 10 + for i in {1..50}; do docker-compose -f 'tests/integration/setup/docker-compose.yaml' ps | grep "unhealthy\\|starting" | wc -l | grep -q 0 && break; echo Waiting for another 5 sec; sleep 5; done + echo "NebulaGraph is up and running, removing console container" + docker-compose -f 'tests/integration/setup/docker-compose.yaml' stop console + docker-compose -f 'tests/integration/setup/docker-compose.yaml' rm -f console + docker-compose -f 'tests/integration/setup/docker-compose.yaml' ps +""" + +[tool.pdm.scripts.dockerstatus] +shell = """ + docker-compose -f 'tests/integration/setup/docker-compose.yaml' ps +""" + +[tool.pdm.scripts.dockerdown] +shell = """ + docker-compose -f 'tests/integration/setup/docker-compose.yaml' down +""" + +[tool.pdm.scripts.teardown] +shell = """ + docker-compose -f 'tests/integration/setup/docker-compose.yaml' down + rm -fr tests/integration/setup/download + rm -fr tests/integration/setup/udf +""" + +[tool.pdm.scripts.int-test] +shell = """ + pdm update -dG dev + pdm run dockerup + + mkdir -p tests/integration/setup/build/ + + echo "Build and Install ng_ai to jupyter_spark, master_spark container" + pdm build --dest tests/integration/setup/build/ + + docker exec -it master_spark sh -c 'pip3 uninstall ng_ai -y' + docker exec -it master_spark sh -c 'pip3 install /root/build/*.whl' + docker restart master_spark + docker exec -it jupyter_spark sh -c 'pip3 uninstall ng_ai -y' + docker exec -it jupyter_spark sh -c 'pip3 install /root/build/*.whl' + docker restart jupyter_spark + sleep 7 + echo "Run integration tests" + + pytest tests/integration/ + pdm run dockerdown """ [tool.black] @@ -47,21 +110,17 @@ atomic = true skip_glob = ["*/setup.py"] filter_files = true -[tool.pdm.dev-dependencies] -dev = [ - "pytest>=7.2.2", -] - [project] name = "ng_ai" -version = "0.2.9.2" -description = "NebulaGraph Data Intelligence Suite" +version = "0.2.9.5" +description = "NebulaGraph AI Suite" authors = [ {name = "Wey Gu", email = "weyl.gu@gmail.com"}, ] dependencies = [ "flask>=2.0.3", "networkx>=2.5.1", + "nebula3-python", ] # pyspark for unit test cannot work with python 3.10 requires-python = ">=3.7,<3.10" @@ -73,16 +132,18 @@ requires = ["pdm-pep517>=1.0"] build-backend = "pdm.pep517.api" [project.optional-dependencies] +# pyspark 2.4.8 doesn't work with python 3.8+, so we use 3.2.3 spark = ["pyspark>=3.2.3"] networkx = ["networkx>=2.5.1"] all = ["ng_ai[spark,networkx]"] lint = [ "flake8>=4.0.1", - "black>=23.1.0", + "black[jupyter]>=23.1.0", "isort>=5.11.5", ] test = [ "pytest>=7.2.2", - "chispa" + "chispa", + "requests", ] dev = ["ng_ai[all,lint,test]"] diff --git a/tests/integration/README.md b/tests/integration/README.md new file mode 100644 index 0000000..16cf295 --- /dev/null +++ b/tests/integration/README.md @@ -0,0 +1,6 @@ + +## Run in verbose mode + +```bash +pytest -s tests/integration/ +``` diff --git a/tests/integration/notebooks/spark_engine_e2e.ipynb b/tests/integration/notebooks/spark_engine_e2e.ipynb new file mode 100644 index 0000000..8cda942 --- /dev/null +++ b/tests/integration/notebooks/spark_engine_e2e.ipynb @@ -0,0 +1,188 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "8a6e2678", + "metadata": {}, + "source": [ + "## Test Reader Scan Mode" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dc946910", + "metadata": {}, + "outputs": [], + "source": [ + "from ng_ai import NebulaReader\n", + "from ng_ai import NebulaGraphConfig\n", + "\n", + "config_dict = {\n", + " \"graphd_hosts\": \"graphd:9669\",\n", + " \"metad_hosts\": \"metad0:9559\",\n", + " \"user\": \"root\",\n", + " \"password\": \"nebula\",\n", + " \"space\": \"basketballplayer\",\n", + "}\n", + "config = NebulaGraphConfig(**config_dict)\n", + "reader = NebulaReader(engine=\"spark\", config=config)\n", + "reader.scan(edge=\"follow\", props=\"degree\")\n", + "df = reader.read()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "acf8ea34", + "metadata": {}, + "outputs": [], + "source": [ + "df.show(2)" + ] + }, + { + "cell_type": "markdown", + "id": "170f8c38", + "metadata": {}, + "source": [ + "## Test Algo Pagerank" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "03975505", + "metadata": {}, + "outputs": [], + "source": [ + "pr_result = df.algo.pagerank(reset_prob=0.15, max_iter=10)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2b157bcb", + "metadata": {}, + "outputs": [], + "source": [ + "pr_result.show(5)" + ] + }, + { + "cell_type": "markdown", + "id": "6051622d", + "metadata": {}, + "source": [ + "## Test Reader Query Mode" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a05a12ad", + "metadata": {}, + "outputs": [], + "source": [ + "from ng_ai import NebulaReader\n", + "from ng_ai.config import NebulaGraphConfig\n", + "\n", + "# read data with spark engine, query mode\n", + "config_dict = {\n", + " \"graphd_hosts\": \"graphd:9669\",\n", + " \"metad_hosts\": \"metad0:9559\",\n", + " \"user\": \"root\",\n", + " \"password\": \"nebula\",\n", + " \"space\": \"basketballplayer\",\n", + "}\n", + "config = NebulaGraphConfig(**config_dict)\n", + "reader = NebulaReader(engine=\"spark\", config=config)\n", + "query = \"\"\"\n", + " MATCH ()-[e:follow]->()\n", + " RETURN e LIMIT 100000\n", + "\"\"\"\n", + "reader.query(query=query, edge=\"follow\", props=\"degree\")\n", + "df = reader.read() # this will take some time" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bfc30c38", + "metadata": {}, + "outputs": [], + "source": [ + "df.show(2)" + ] + }, + { + "cell_type": "markdown", + "id": "311b49c4", + "metadata": {}, + "source": [ + "## Test Writer insert mode in sink: `nebulagraph_vertex`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3d25ad22", + "metadata": {}, + "outputs": [], + "source": [ + "from ng_ai import NebulaWriter\n", + "from ng_ai.config import NebulaGraphConfig\n", + "\n", + "\n", + "df_result = df.algo.label_propagation()\n", + "\n", + "config_dict = {\n", + " \"graphd_hosts\": \"graphd:9669\",\n", + " \"metad_hosts\": \"metad0:9559\",\n", + " \"user\": \"root\",\n", + " \"password\": \"nebula\",\n", + " \"space\": \"basketballplayer\",\n", + "}\n", + "config = NebulaGraphConfig(**config_dict)\n", + "\n", + "writer = NebulaWriter(\n", + " data=df_result, sink=\"nebulagraph_vertex\", config=config, engine=\"spark\"\n", + ")\n", + "\n", + "# map column louvain into property cluster_id\n", + "properties = {\"lpa\": \"cluster_id\"}\n", + "\n", + "writer.set_options(\n", + " tag=\"label_propagation\",\n", + " vid_field=\"_id\",\n", + " properties=properties,\n", + " batch_size=256,\n", + " write_mode=\"insert\",\n", + ")\n", + "# write back to NebulaGraph\n", + "writer.write()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/integration/setup/docker-compose.yaml b/tests/integration/setup/docker-compose.yaml index 661a8fa..32f77ab 100644 --- a/tests/integration/setup/docker-compose.yaml +++ b/tests/integration/setup/docker-compose.yaml @@ -106,11 +106,11 @@ services: - ngai-net restart: on-failure - juptyer_spark: + jupyter_spark: labels: - "com.vesoft.scope=ngai" image: 'weygu/pyspark-notebook-nebulagraph:2.4.5-hadoop2.7' - container_name: juptyer_spark + container_name: jupyter_spark restart: always ports: - '28888:8888' @@ -127,6 +127,27 @@ services: environment: - PYSPARK_PYTHON=python3 + master_spark: + networks: + - ngai-net + volumes: + - ./:/root + image: 'bde2020/spark-master:2.4.5-hadoop2.7' + container_name: master_spark + ports: + - 37077:7077 + - 36066:6066 + - 38080:8080 + healthcheck: + test: ["CMD", "netstat", "-plunt", "| grep 8080"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 20s + environment: + - PYSPARK_PYTHON=python3 + - INIT_DAEMON_STEP=setup_spark + ngai_graphd: labels: - "com.vesoft.scope=ngai" @@ -165,6 +186,26 @@ services: cap_add: - SYS_PTRACE + console: + image: vesoft/nebula-console:v3 + container_name: ngai_console + entrypoint: "" + command: + - sh + - -c + - | + for i in `seq 1 180`;do + var=`nebula-console -addr graphd -port 9669 -u root -p nebula -e 'ADD HOSTS "storaged0":9779'`; + if [[ $$? == 0 ]];then + break; + fi; + sleep 1; + echo "retry to add hosts."; + done && tail -f /dev/null; + depends_on: + - graphd + networks: + - ngai-net + networks: ngai-net: - external: true diff --git a/tests/integration/spark_engine_cases/algo.py b/tests/integration/spark_engine_cases/algo.py new file mode 100644 index 0000000..7fd9e0a --- /dev/null +++ b/tests/integration/spark_engine_cases/algo.py @@ -0,0 +1,23 @@ +from ng_ai import NebulaGraphConfig, NebulaReader + +config_dict = { + "graphd_hosts": "graphd:9669", + "metad_hosts": "metad0:9559", + "user": "root", + "password": "nebula", + "space": "basketballplayer", +} +config = NebulaGraphConfig(**config_dict) +reader = NebulaReader(engine="spark", config=config) +reader.scan(edge="follow", props="degree") +df = reader.read() + +pr_result = df.algo.pagerank(reset_prob=0.15, max_iter=10) + +pr_result.show(5) + +""" +/spark/bin/spark-submit \ + --jars /root/download/nebula-algo.jar \ + /root/run/algo.py +""" diff --git a/tests/integration/spark_engine_cases/query_reader.py b/tests/integration/spark_engine_cases/query_reader.py new file mode 100644 index 0000000..41388b3 --- /dev/null +++ b/tests/integration/spark_engine_cases/query_reader.py @@ -0,0 +1,24 @@ +from ng_ai import NebulaGraphConfig, NebulaReader + +# read data with spark engine, query mode +config_dict = { + "graphd_hosts": "graphd:9669", + "metad_hosts": "metad0:9559", + "user": "root", + "password": "nebula", + "space": "basketballplayer", +} +config = NebulaGraphConfig(**config_dict) +reader = NebulaReader(engine="spark", config=config) +query = """ + MATCH ()-[e:follow]->() + RETURN e LIMIT 100000 +""" +reader.query(query=query, edge="follow", props="degree") +df = reader.read() + +""" +/spark/bin/spark-submit \ + --jars /root/download/nebula-algo.jar \ + /root/run/query_reader.py +""" diff --git a/tests/integration/spark_engine_cases/scan_reader.py b/tests/integration/spark_engine_cases/scan_reader.py new file mode 100644 index 0000000..a14c2c7 --- /dev/null +++ b/tests/integration/spark_engine_cases/scan_reader.py @@ -0,0 +1,20 @@ +from ng_ai import NebulaGraphConfig, NebulaReader + +config_dict = { + "graphd_hosts": "graphd:9669", + "metad_hosts": "metad0:9559", + "user": "root", + "password": "nebula", + "space": "basketballplayer", +} +config = NebulaGraphConfig(**config_dict) +reader = NebulaReader(engine="spark", config=config) +reader.scan(edge="follow", props="degree") +df = reader.read() +df.show(2) + +""" +/spark/bin/spark-submit \ + --jars /root/download/nebula-algo.jar \ + /root/run/scan_reader.py +""" diff --git a/tests/integration/spark_engine_cases/writer.py b/tests/integration/spark_engine_cases/writer.py new file mode 100644 index 0000000..f3b7f75 --- /dev/null +++ b/tests/integration/spark_engine_cases/writer.py @@ -0,0 +1,39 @@ +from ng_ai import NebulaReader, NebulaWriter +from ng_ai.config import NebulaGraphConfig + +config_dict = { + "graphd_hosts": "graphd:9669", + "metad_hosts": "metad0:9559", + "user": "root", + "password": "nebula", + "space": "basketballplayer", +} +config = NebulaGraphConfig(**config_dict) +reader = NebulaReader(engine="spark", config=config) +reader.scan(edge="follow", props="degree") +df = reader.read() + +df_result = df.algo.label_propagation() + +writer = NebulaWriter( + data=df_result, sink="nebulagraph_vertex", config=config, engine="spark" +) + +# map column louvain into property cluster_id +properties = {"lpa": "cluster_id"} + +writer.set_options( + tag="label_propagation", + vid_field="_id", + properties=properties, + batch_size=256, + write_mode="insert", +) +# write back to NebulaGraph +writer.write() + +""" +/spark/bin/spark-submit \ + --jars /root/download/nebula-algo.jar \ + /root/run/writer.py +""" diff --git a/tests/integration/test_e2e_spark_engine.py b/tests/integration/test_e2e_spark_engine.py index e69de29..44d4974 100644 --- a/tests/integration/test_e2e_spark_engine.py +++ b/tests/integration/test_e2e_spark_engine.py @@ -0,0 +1,191 @@ +import os +import subprocess +from time import sleep + +import pytest +from nebula3.Config import Config +from nebula3.gclient.net import ConnectionPool + + +@pytest.fixture(scope="session", autouse=True) +def prepare_data(): + print("Setup data...") + subprocess.run( + "docker run --rm --network setup_ngai-net vesoft/nebula-console:v3" + ' -addr graphd -port 9669 -u root -p nebula -e " ' + 'DROP SPACE basketballplayer;" ', + shell=True, + check=True, + ) + sleep(4) + result = subprocess.run( + "docker run --rm --network setup_ngai-net vesoft/nebula-console:v3" + ' -addr graphd -port 9669 -u root -p nebula -e " ' + ':play basketballplayer" ', + shell=True, + check=True, + capture_output=True, + timeout=300, + ) + print(f":play basketballplayer: {result}") + + subprocess.run( + "docker run --rm --network setup_ngai-net vesoft/nebula-console:v3" + ' -addr graphd -port 9669 -u root -p nebula -e " ' + 'USE basketballplayer; SUBMIT JOB STATS" ', + shell=True, + check=True, + capture_output=True, + ) + + result = subprocess.run( + "docker run --rm --network setup_ngai-net vesoft/nebula-console:v3" + ' -addr graphd -port 9669 -u root -p nebula -e " ' + "USE basketballplayer; " + "CREATE TAG IF NOT EXISTS label_propagation " + '(cluster_id string NOT NULL);" ', + shell=True, + check=True, + capture_output=True, + ) + sleep(4) + assert ( + b"ERROR" not in result.stdout + ), f"ERROR during create tag: {result.stdout.decode('utf-8')}" + + result = subprocess.run( + "docker run --rm --network setup_ngai-net vesoft/nebula-console:v3" + ' -addr graphd -port 9669 -u root -p nebula -e " ' + "USE basketballplayer; " + 'SHOW STATS;" ', + shell=True, + check=True, + capture_output=True, + ) + print(f"Show stats:\n{result.stdout.decode('utf-8')}") + + os.system("mkdir -p tests/integration/setup/run") + os.system("rm -rf tests/integration/setup/run/*") + + os.system( + "cp -r tests/integration/spark_engine_cases/* tests/integration/setup/run/" + ) + sleep(10) + print("Setup data done...") + + +def test_scan_reader_spark_engine(): + """ + Just call: + /spark/bin/spark-submit \ + --jars /root/download/nebula-algo.jar \ + /root/run/scan_reader.py + in container: master_spark + assert the return value is 0 + """ + + result = subprocess.run( + "docker exec master_spark /spark/bin/spark-submit " + "--jars /root/download/nebula-algo.jar " + "/root/run/scan_reader.py", + shell=True, + check=True, + capture_output=True, + ) + + assert result.returncode == 0, f"ERROR during run scan_reader.py: {result}" + print(f"Scan reader result:\n{result.stdout.decode('utf-8')}") + + +def test_scan_reader_spark_engine_and_run_pagerank(): + """ + Just call: + /spark/bin/spark-submit \ + --jars /root/download/nebula-algo.jar \ + /root/run/algo.py + in container: master_spark + assert the return value is 0 + """ + + result = subprocess.run( + "docker exec master_spark /spark/bin/spark-submit " + "--jars /root/download/nebula-algo.jar " + "/root/run/algo.py", + shell=True, + check=True, + capture_output=True, + ) + + assert result.returncode == 0, f"ERROR during run algo.py: {result}" + print(f"Scan reader result:\n{result.stdout.decode('utf-8')}") + + +def test_query_reader_spark_engine(): + """ + Just call: + /spark/bin/spark-submit \ + --jars /root/download/nebula-algo.jar \ + /root/run/query_reader.py + in container: master_spark + assert the return value is 0 + """ + + result = subprocess.run( + "docker exec master_spark /spark/bin/spark-submit " + "--jars /root/download/nebula-algo.jar " + "/root/run/query_reader.py", + shell=True, + check=True, + capture_output=True, + ) + + assert result.returncode == 0, f"ERROR during run query_reader.py: {result}" + print(f"Query reader result:\n{result.stdout.decode('utf-8')}") + + +def test_label_propagation_spark_engine_writer(): + """ + Just call: + /spark/bin/spark-submit \ + --jars /root/download/nebula-algo.jar \ + /root/run/writer.py + in container: master_spark + assert the return value is 0 + Then query NebulaGraph on tag: label_propagation + assert the result is correct + """ + + result = subprocess.run( + "docker exec master_spark /spark/bin/spark-submit " + "--jars /root/download/nebula-algo.jar " + "/root/run/writer.py", + shell=True, + check=True, + capture_output=True, + ) + + assert result.returncode == 0, f"ERROR during run writer.py: {result}" + print(f"Label propagation result:\n{result.stdout.decode('utf-8')}") + + nebula_config = Config() + connection_pool = ConnectionPool() + connection_pool.init([("127.0.0.1", 39669)], nebula_config) + + with connection_pool.session_context("root", "nebula") as session: + session.execute("USE basketballplayer") + result = session.execute( + "MATCH (v:player) RETURN v.label_propagation.cluster_id LIMIT 1" + ) + print(result) + connection_pool.close() + + assert result.is_succeeded(), f"ERROR during query NebulaGraph: {result}" + assert ( + not result.is_empty() + ), f"label_propagation not written to NebulaGraph result: {result}" + assert ( + result.column_values("v.label_propagation.cluster_id")[0] + .cast() + .startswith("player") + ), f"label_propagation value is not correct result: {result}" + print(f"Label propagation result:\n{result}")