feat: add e2e integration tests #26

feat: add e2e integration tests
wey-gu · Mar 22, 2023 · 5c63133 · 5c63133
2 parents 5feb5b7 + 2e1c648
commit 5c63133
Show file tree

Hide file tree

Showing 21 changed files with 1,221 additions and 41 deletions.
diff --git a/.gitignore b/.gitignore
@@ -137,4 +137,17 @@ requirements.txt
 
 # ide
 
-.vscode
+.vscode
+
+# integration
+
+.bash_history
+.jupyter
+.ipython
+.local
+data
+logs
+download
+run
+*.gz
+*.whl
diff --git a/HACKING.md b/HACKING.md
@@ -0,0 +1,23 @@
+## Hacking
+
+```bash
+pip3 install pdm
+# build and install ng_ai
+pdm install
+# run unit tests
+pdm run test
+# run integration tests
+pdm run int-test
+# lint
+pdm run lint
+# format
+pdm run format
+
+# integration tests environment setup
+pdm run dockerup
+pdm run dockerdown
+pdm run dockerstatus
+
+# integration tests environment teardown
+pdm run teardown
+```
diff --git a/README.md b/README.md
@@ -1,4 +1,4 @@
-<img alt="NebulaGraph Data Intelligence Suite(ng_ai)" src="https://user-images.githubusercontent.com/1651790/226242809-fe488ff2-bb4a-4e7d-b23a-70865a7b3228.png">
+<img alt="NebulaGraph AI Suite(ng_ai)" src="https://user-images.githubusercontent.com/1651790/226242809-fe488ff2-bb4a-4e7d-b23a-70865a7b3228.png">
 
 <p align="center">
     <em>NebulaGraph AI Suite with 4 line code to run Graph Algo on NebulaGraph</em>
@@ -136,7 +136,7 @@ ng_ai is an unified abstraction layer for different engines, the current impleme
     Engine└───────────────────────────────────────────────────┘
        │  ┌────────────────────────────────────────────────────┬──────────┐
        └──┤                                                    │          │
-          │   NebulaGraph Data Intelligence Suite(ngai)        │ ngai-api │◀─┐
+          │   NebulaGraph AI Suite(ngai)                       │ ngai-api │◀─┐
           │                                                    │          │  │
           │                                                    └──────────┤  │
           │     ┌────────┐    ┌──────┐    ┌────────┐   ┌─────┐            │  │
@@ -196,17 +196,7 @@ ng_ai is an unified abstraction layer for different engines, the current impleme
 
 ## Contributing
 
-```bash
-pip3 install pdm
-# build and install ng_ai
-pdm install
-# run tests
-pdm run test
-# lint
-pdm run lint
-# format
-pdm run format
-```
+See HACKING.md for details.
 
 ## License
 

diff --git a/examples/ng_ai_from_ngql_udf.ipynb b/examples/ng_ai_from_ngql_udf.ipynb
@@ -79,7 +79,7 @@
    "source": [
     "from ng_ai import ng_ai_api_app as app\n",
     "\n",
-    "app.run(port=9999, host='0.0.0.0')"
+    "app.run(port=9999, host=\"0.0.0.0\")"
    ]
   },
   {

diff --git a/examples/run_ng_ai_api.py b/examples/run_ng_ai_api.py
@@ -1,3 +1,3 @@
 from ng_ai import ng_ai_api_app as app
 
-app.run(port=9999, host='0.0.0.0')
+app.run(port=9999, host="0.0.0.0")
diff --git a/examples/spark_engine.ipynb b/examples/spark_engine.ipynb
@@ -20,20 +20,22 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "5b4e4143",
    "metadata": {},
    "source": [
-    "## Data Intelligence Suite Spark Engine Examples\n",
+    "## AI Suite Spark Engine Examples\n",
     "### read data with spark engine, scan mode"
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "f17abcf8",
    "metadata": {},
    "source": [
-    "In this example, we are leveraging the Spark Engine of NebulaGraph DI Suite, with the Storage Scan mode.\n",
+    "In this example, we are leveraging the Spark Engine of NebulaGraph AI Suite, with the Storage Scan mode.\n",
     "\n",
     "#### Step 1, get dataframe by scanning the Graph\n",
     "\n",
@@ -50,7 +52,7 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "\r",
+      "\r\n",
       "[Stage 0:>                                                          (0 + 1) / 1]"
      ]
     },
@@ -72,13 +74,14 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "\r",
+      "\r\n",
       "                                                                                \r"
      ]
     }
    ],
    "source": [
     "from ng_ai import NebulaReader\n",
+    "\n",
     "# read data with spark engine, scan mode\n",
     "reader = NebulaReader(engine=\"spark\")\n",
     "reader.scan(edge=\"follow\", props=\"degree\")\n",
@@ -246,14 +249,15 @@
    ],
    "source": [
     "from ng_ai import NebulaReader\n",
+    "\n",
     "# read data with spark engine, query mode\n",
     "reader = NebulaReader(engine=\"spark\")\n",
     "query = \"\"\"\n",
     "    MATCH ()-[e:follow]->()\n",
     "    RETURN e LIMIT 100000\n",
     "\"\"\"\n",
     "reader.query(query=query, edge=\"follow\", props=\"degree\")\n",
-    "df = reader.read() # this will take some time\n",
+    "df = reader.read()  # this will take some time\n",
     "df.show(2)"
    ]
   },
@@ -517,12 +521,12 @@
     "from ng_ai.config import NebulaGraphConfig\n",
     "\n",
     "config = NebulaGraphConfig()\n",
-    "writer = NebulaWriter(data=df_result, sink=\"nebulagraph_vertex\", config=config, engine=\"spark\")\n",
+    "writer = NebulaWriter(\n",
+    "    data=df_result, sink=\"nebulagraph_vertex\", config=config, engine=\"spark\"\n",
+    ")\n",
     "\n",
     "# map column louvain into property cluster_id\n",
-    "properties = {\n",
-    "    \"lpa\": \"cluster_id\"\n",
-    "}\n",
+    "properties = {\"lpa\": \"cluster_id\"}\n",
     "\n",
     "writer.set_options(\n",
     "    tag=\"label_propagation\",\n",

diff --git a/ng_ai/engines.py b/ng_ai/engines.py
@@ -38,13 +38,13 @@ def __init__(self, config):
         self.shuffle_partitions = DEFAULT_SHUFFLE_PARTITIONS
         self.executor_memory = DEFAULT_EXECUTOR_MEMORY
         self.driver_memory = DEFAULT_DRIVER_MEMORY
-        self.encode_vertex_id = ENCODE_VERTEX_ID
+        self.encode_vid = ENCODE_VERTEX_ID
         self.parse_config()
 
         from pyspark.sql import SparkSession
 
         self.spark = (
-            SparkSession.builder.appName("NebulaGraph Data Intelligence")
+            SparkSession.builder.appName("NebulaGraph AI")
             .config("spark.sql.shuffle.partitions", self.shuffle_partitions)
             .config("spark.executor.memory", self.executor_memory)
             .config("spark.driver.memory", self.driver_memory)
@@ -76,8 +76,8 @@ def parse_config(self):
         if self.config.driver_memory is not None:
             self.driver_memory = self.config.driver_memory
 
-        if self.config.encode_vertex_id is not None:
-            self.encode_vertex_id = self.config.encode_vertex_id
+        if self.config.encode_vid is not None:
+            self.encode_vid = self.config.encode_vid
 
     def prepare(self):
         self.java_import = self._get_java_import()