From ff11f3ac47726f986d26d7e203bd60b4cb2883df Mon Sep 17 00:00:00 2001 From: unknowntpo Date: Sat, 17 Aug 2024 19:14:31 +0800 Subject: [PATCH] refactor(jupyter/gravitino-fileset-example.ipynb): get host ip by env var --- init/jupyter/gravitino-fileset-example.ipynb | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/init/jupyter/gravitino-fileset-example.ipynb b/init/jupyter/gravitino-fileset-example.ipynb index 60c3ae24..9d4beac0 100644 --- a/init/jupyter/gravitino-fileset-example.ipynb +++ b/init/jupyter/gravitino-fileset-example.ipynb @@ -18,9 +18,12 @@ "outputs": [], "source": [ "from hdfs import InsecureClient\n", + "import os\n", + "\n", + "hive_host_ip=os.getenv('HIVE_HOST_IP')\n", "\n", "# Create a HDFS connector client\n", - "hdfs_client = InsecureClient('http://hive:50070', user='root')\n", + "hdfs_client = InsecureClient(f\"http://{hive_host_ip}:50070\", user='root')\n", "\n", "# List HDFS file and directories\n", "print(hdfs_client.list('/user/datastrato'))\n", @@ -47,9 +50,12 @@ "source": [ "from typing import Dict, List\n", "from gravitino import NameIdentifier, GravitinoAdminClient, GravitinoClient, Catalog, Fileset, FilesetChange\n", + "import os \n", + "\n", + "gravitino_host_ip=os.getenv('GRAVITINO_HOST_IP')\n", "\n", "# Create Gravitino admin client\n", - "gravitino_admin_client = GravitinoAdminClient(uri=\"http://gravitino:8090\")\n", + "gravitino_admin_client = GravitinoAdminClient(uri=f\"http://{gravitino_host_ip}:8090\")\n", "\n", "# Create metalake via Gravitino admin client\n", "metalake_name=\"default\"\n", @@ -68,7 +74,7 @@ "outputs": [], "source": [ "# Create Gravitino client\n", - "gravitino_client = GravitinoClient(uri=\"http://gravitino:8090\", metalake_name=metalake_name)" + "gravitino_client = GravitinoClient(uri=f\"http://{gravitino_host_ip}:8090\", metalake_name=metalake_name)" ] }, { @@ -127,7 +133,7 @@ "# Create schema entity via Gravition client\n", "schema_name=\"schema\"\n", "schema_path=\"/user/datastrato/\"+schema_name\n", - "schema_hdfs_path=\"hdfs://hive:9000\"+schema_path\n", + "schema_hdfs_path=f\"hdfs://{hive_host_ip}:9000{schema_path}\"\n", "\n", "schema_ident: NameIdentifier = NameIdentifier.of_schema(metalake_name, catalog_name, schema_name)\n", "catalog.as_schemas().create_schema(ident=schema_ident, \n", @@ -150,10 +156,11 @@ "metadata": {}, "outputs": [], "source": [ + "\n", "# Create a managed type of Fileset\n", "managed_fileset_name=\"managed_fileset\"\n", "managed_fileset_path=\"/user/datastrato/\"+schema_name+\"/\"+managed_fileset_name\n", - "managed_fileset_hdfs_path=\"hdfs://hive:9000\"+managed_fileset_path\n", + "managed_fileset_hdfs_path=f\"hdfs://{hive_host_ip}:9000{managed_fileset_path}\"\n", "\n", "managed_fileset_ident: NameIdentifier = NameIdentifier.of_fileset(metalake_name, catalog_name, schema_name, managed_fileset_name)\n", "catalog.as_fileset_catalog().create_fileset(ident=managed_fileset_ident,\n", @@ -180,7 +187,7 @@ "source": [ "external_fileset_name=\"external_fileset\"\n", "external_fileset_path=\"/user/datastrato/\"+schema_name+\"/\"+external_fileset_name\n", - "external_fileset_hdfs_path=\"hdfs://hive:9000\"+external_fileset_path\n", + "external_fileset_hdfs_path=f\"hdfs://{hive_host_ip}:9000{external_fileset_path}\"\n", "\n", "# Create a fileset path in HDFS in advance\n", "hdfs_client.makedirs(external_fileset_path)\n",