From d6b7f50f9b0585e44df5edfadd12c78fc019feb5 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Thu, 2 Nov 2023 03:27:23 +0100 Subject: [PATCH] cli: Add `ctk {cluster,shell,list-jobs}` subcommands --- cratedb_toolkit/cli.py | 9 +++++ cratedb_toolkit/cluster/cli.py | 3 ++ cratedb_toolkit/io/README.md | 62 +++++++++++++++++++++++++++--- cratedb_toolkit/job/cli.py | 19 +++++++-- cratedb_toolkit/job/croud.py | 4 +- cratedb_toolkit/shell/__init__.py | 0 cratedb_toolkit/shell/cli.py | 64 +++++++++++++++++++++++++++++++ 7 files changed, 150 insertions(+), 11 deletions(-) create mode 100644 cratedb_toolkit/shell/__init__.py create mode 100644 cratedb_toolkit/shell/cli.py diff --git a/cratedb_toolkit/cli.py b/cratedb_toolkit/cli.py index 803b099c..26c0ef65 100644 --- a/cratedb_toolkit/cli.py +++ b/cratedb_toolkit/cli.py @@ -3,7 +3,11 @@ from cratedb_toolkit.util.cli import boot_click +from .cluster.cli import cli as cloud_cli from .io.cli import cli as io_cli +from .job.cli import cli_list_jobs +from .shell.cli import cli as shell_cli + @click.group(cls=ClickAliasedGroup) # type: ignore[arg-type] @click.option("--verbose", is_flag=True, required=False, help="Turn on logging") @@ -12,4 +16,9 @@ @click.pass_context def cli(ctx: click.Context, verbose: bool, debug: bool): return boot_click(ctx, verbose, debug) + + +cli.add_command(cloud_cli, name="cluster") cli.add_command(io_cli, name="load") +cli.add_command(shell_cli, name="shell") +cli.add_command(cli_list_jobs) diff --git a/cratedb_toolkit/cluster/cli.py b/cratedb_toolkit/cluster/cli.py index 73686404..f18a9597 100644 --- a/cratedb_toolkit/cluster/cli.py +++ b/cratedb_toolkit/cluster/cli.py @@ -15,6 +15,9 @@ @click.version_option() @click.pass_context def cli(ctx: click.Context, verbose: bool, debug: bool): + """ + Run cluster operations on CrateDB and CrateDB Cloud. + """ return boot_click(ctx, verbose, debug) diff --git a/cratedb_toolkit/io/README.md b/cratedb_toolkit/io/README.md index b304f4d7..a89db233 100644 --- a/cratedb_toolkit/io/README.md +++ b/cratedb_toolkit/io/README.md @@ -1,16 +1,68 @@ # Load and extract data into/from CrateDB -## Synopsis +## About -Define the cluster id of your CrateDB Cloud Cluster, and connection credentials +A one-stop command `ctk load table` to load data into CrateDB database tables. + + +## General Notes + +By default, the table name will be derived from the name of the input resource. +If you would like to specify a different table name, use the `--table` command +line option, or the `CRATEDB_TABLE` environment variable. + +When aiming to write into a table in a different database schema, use the +`--schema` command line option, or the `CRATEDB_SCHEMA` environment variable. +When omitting this parameter, the default value `doc` will be used. + + +## Cloud Import API + +Using the [CrateDB Cloud] Import API, you can import files in CSV, JSON, and +Parquet formats. + +### Prerequisites +Authenticate with CrateDB Cloud using one of those identity providers: +Cognito, Azure AD, GitHub, Google. +```shell +croud login --idp azuread +``` + +To discover the list of available database clusters. ```shell -export CRATEDB_CLOUD_CLUSTER_ID=e1e38d92-a650-48f1-8a70-8133f2d5c400 -export CRATEDB_USERNAME=admin -export CRATEDB_PASSWORD=3$MJ5fALP8bNOYCYBMLOrzd& +croud clusters list ``` +Define the cluster id of your CrateDB Cloud Cluster you are aiming to connect +to, and its connection credentials. +```shell +export CRATEDB_CLOUD_CLUSTER_ID='e1e38d92-a650-48f1-8a70-8133f2d5c400' +``` + +Define authentication credentials, to satisfy the `ctk shell` command you will +use to inspect the database content after transferring data. +```shell +export CRATEDB_USERNAME='admin' +export CRATEDB_PASSWORD='3$MJ5fALP8bNOYCYBMLOrzd&' +``` + +### Usage Load data. ```shell ctk load table https://github.com/crate/cratedb-datasets/raw/main/cloud-tutorials/data_weather.csv.gz +ctk load table https://github.com/crate/cratedb-datasets/raw/main/cloud-tutorials/data_marketing.json.gz +ctk load table https://github.com/daq-tools/skeem/raw/main/tests/testdata/basic.parquet ``` + +Inquire data. +```shell +ctk shell --command="SELECT * FROM data_weather LIMIT 10;" +ctk shell --command="SELECT * FROM data_weather LIMIT 10;" --format=json +``` + +### Backlog +- Exercise data imports from AWS S3 and other Object Storage providers. + + +[CrateDB Cloud]: https://console.cratedb.cloud/ diff --git a/cratedb_toolkit/job/cli.py b/cratedb_toolkit/job/cli.py index 91a19383..20053f63 100644 --- a/cratedb_toolkit/job/cli.py +++ b/cratedb_toolkit/job/cli.py @@ -1,16 +1,27 @@ +import sys + import click from cratedb_toolkit.job.croud import jobs_list -from cratedb_toolkit.util import jd +from cratedb_toolkit.util.croud import get_croud_output_formats + +output_formats = get_croud_output_formats() @click.command(name="list-jobs") @click.option( "--cluster-id", envvar="CRATEDB_CLOUD_CLUSTER_ID", type=str, required=True, help="CrateDB Cloud cluster identifier" ) +@click.option( + "--format", + "format_", + type=click.Choice(output_formats, case_sensitive=False), + required=False, + help="The output format for the result of the operation", +) @click.pass_context -def cli_list_jobs(ctx: click.Context, cluster_id: str): +def cli_list_jobs(ctx: click.Context, cluster_id: str, format_: str): """ - List running jobs. + List CrateDB Cloud jobs. """ - jd(jobs_list(cluster_id)) + print(jobs_list(cluster_id, output_format=format_, decode_output=False), file=sys.stdout) # noqa: T201 diff --git a/cratedb_toolkit/job/croud.py b/cratedb_toolkit/job/croud.py index fcba03f0..611477df 100644 --- a/cratedb_toolkit/job/croud.py +++ b/cratedb_toolkit/job/croud.py @@ -1,7 +1,7 @@ from cratedb_toolkit.util.croud import CroudCall, CroudWrapper -def jobs_list(cratedb_cloud_cluster_id: str): +def jobs_list(cratedb_cloud_cluster_id: str, output_format: str = None, decode_output: bool = True): from croud.clusters.commands import import_jobs_list from croud.parser import Argument @@ -13,5 +13,5 @@ def jobs_list(cratedb_cloud_cluster_id: str): ], ) - wr = CroudWrapper(call) + wr = CroudWrapper(call, output_format=output_format, decode_output=decode_output) return wr.invoke() diff --git a/cratedb_toolkit/shell/__init__.py b/cratedb_toolkit/shell/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cratedb_toolkit/shell/cli.py b/cratedb_toolkit/shell/cli.py new file mode 100644 index 00000000..0df97bbb --- /dev/null +++ b/cratedb_toolkit/shell/cli.py @@ -0,0 +1,64 @@ +import click + +from cratedb_toolkit.cluster.util import get_cluster_info +from cratedb_toolkit.util.cli import boot_click +from cratedb_toolkit.util.crash import get_crash_output_formats, run_crash + +output_formats = get_crash_output_formats() + + +@click.command() +@click.option( + "--cluster-id", envvar="CRATEDB_CLOUD_CLUSTER_ID", type=str, required=True, help="CrateDB Cloud cluster identifier" +) +@click.option("--username", envvar="CRATEDB_USERNAME", type=str, required=False, help="Username for CrateDB cluster") +@click.option("--password", envvar="CRATEDB_PASSWORD", type=str, required=False, help="Password for CrateDB cluster") +@click.option( + "--schema", + envvar="CRATEDB_SCHEMA", + type=str, + required=False, + help="Default schema for statements if schema is not explicitly stated within queries", +) +@click.option("--command", type=str, required=False, help="SQL command") +@click.option( + "--format", + "format_", + type=click.Choice(output_formats, case_sensitive=False), + required=False, + help="The output format of the database response", +) +@click.option("--verbose", is_flag=True, required=False, help="Turn on logging") +@click.option("--debug", is_flag=True, required=False, help="Turn on logging with debug level") +@click.version_option() +@click.pass_context +def cli( + ctx: click.Context, + cluster_id: str, + username: str, + password: str, + schema: str, + command: str, + format_: str, + verbose: bool, + debug: bool, +): + """ + Start an interactive database shell, or invoke SQL commands. + + TODO: Only talks to CrateDB Cloud for now. Also implement for standalone CrateDB servers. + TODO: Learn/forward more options of `crash`. + """ + boot_click(ctx, verbose, debug) + + cluster_info = get_cluster_info(cluster_id=cluster_id) + cratedb_http_url = cluster_info.cloud["url"] + + run_crash( + hosts=cratedb_http_url, + username=username, + password=password, + schema=schema, + command=command, + output_format=format_, + )