Skip to content

Commit

Permalink
refactor(jupyter/gravitino-fileset-example.ipynb): get host ip by env…
Browse files Browse the repository at this point in the history
… var
  • Loading branch information
unknowntpo committed Aug 20, 2024
1 parent e656caa commit ff11f3a
Showing 1 changed file with 13 additions and 6 deletions.
19 changes: 13 additions & 6 deletions init/jupyter/gravitino-fileset-example.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,12 @@
"outputs": [],
"source": [
"from hdfs import InsecureClient\n",
"import os\n",
"\n",
"hive_host_ip=os.getenv('HIVE_HOST_IP')\n",
"\n",
"# Create a HDFS connector client\n",
"hdfs_client = InsecureClient('http://hive:50070', user='root')\n",
"hdfs_client = InsecureClient(f\"http://{hive_host_ip}:50070\", user='root')\n",
"\n",
"# List HDFS file and directories\n",
"print(hdfs_client.list('/user/datastrato'))\n",
Expand All @@ -47,9 +50,12 @@
"source": [
"from typing import Dict, List\n",
"from gravitino import NameIdentifier, GravitinoAdminClient, GravitinoClient, Catalog, Fileset, FilesetChange\n",
"import os \n",
"\n",
"gravitino_host_ip=os.getenv('GRAVITINO_HOST_IP')\n",
"\n",
"# Create Gravitino admin client\n",
"gravitino_admin_client = GravitinoAdminClient(uri=\"http://gravitino:8090\")\n",
"gravitino_admin_client = GravitinoAdminClient(uri=f\"http://{gravitino_host_ip}:8090\")\n",
"\n",
"# Create metalake via Gravitino admin client\n",
"metalake_name=\"default\"\n",
Expand All @@ -68,7 +74,7 @@
"outputs": [],
"source": [
"# Create Gravitino client\n",
"gravitino_client = GravitinoClient(uri=\"http://gravitino:8090\", metalake_name=metalake_name)"
"gravitino_client = GravitinoClient(uri=f\"http://{gravitino_host_ip}:8090\", metalake_name=metalake_name)"
]
},
{
Expand Down Expand Up @@ -127,7 +133,7 @@
"# Create schema entity via Gravition client\n",
"schema_name=\"schema\"\n",
"schema_path=\"/user/datastrato/\"+schema_name\n",
"schema_hdfs_path=\"hdfs://hive:9000\"+schema_path\n",
"schema_hdfs_path=f\"hdfs://{hive_host_ip}:9000{schema_path}\"\n",
"\n",
"schema_ident: NameIdentifier = NameIdentifier.of_schema(metalake_name, catalog_name, schema_name)\n",
"catalog.as_schemas().create_schema(ident=schema_ident, \n",
Expand All @@ -150,10 +156,11 @@
"metadata": {},
"outputs": [],
"source": [
"\n",
"# Create a managed type of Fileset\n",
"managed_fileset_name=\"managed_fileset\"\n",
"managed_fileset_path=\"/user/datastrato/\"+schema_name+\"/\"+managed_fileset_name\n",
"managed_fileset_hdfs_path=\"hdfs://hive:9000\"+managed_fileset_path\n",
"managed_fileset_hdfs_path=f\"hdfs://{hive_host_ip}:9000{managed_fileset_path}\"\n",
"\n",
"managed_fileset_ident: NameIdentifier = NameIdentifier.of_fileset(metalake_name, catalog_name, schema_name, managed_fileset_name)\n",
"catalog.as_fileset_catalog().create_fileset(ident=managed_fileset_ident,\n",
Expand All @@ -180,7 +187,7 @@
"source": [
"external_fileset_name=\"external_fileset\"\n",
"external_fileset_path=\"/user/datastrato/\"+schema_name+\"/\"+external_fileset_name\n",
"external_fileset_hdfs_path=\"hdfs://hive:9000\"+external_fileset_path\n",
"external_fileset_hdfs_path=f\"hdfs://{hive_host_ip}:9000{external_fileset_path}\"\n",
"\n",
"# Create a fileset path in HDFS in advance\n",
"hdfs_client.makedirs(external_fileset_path)\n",
Expand Down

0 comments on commit ff11f3a

Please sign in to comment.