cli: Add ctk {cluster,shell,list-jobs} subcommands

cratedb_toolkit/cli.py

@@ -3,7 +3,11 @@
 
 from cratedb_toolkit.util.cli import boot_click
 
+from .cluster.cli import cli as cloud_cli
 from .io.cli import cli as io_cli
+from .job.cli import cli_list_jobs
+from .shell.cli import cli as shell_cli
+
 
 @click.group(cls=ClickAliasedGroup)  # type: ignore[arg-type]
 @click.option("--verbose", is_flag=True, required=False, help="Turn on logging")
@@ -12,4 +16,9 @@
 @click.pass_context
 def cli(ctx: click.Context, verbose: bool, debug: bool):
     return boot_click(ctx, verbose, debug)
+
+
+cli.add_command(cloud_cli, name="cluster")
 cli.add_command(io_cli, name="load")
+cli.add_command(shell_cli, name="shell")
+cli.add_command(cli_list_jobs)

cratedb_toolkit/cluster/cli.py

@@ -15,6 +15,9 @@
 @click.version_option()
 @click.pass_context
 def cli(ctx: click.Context, verbose: bool, debug: bool):
+    """
+    Run cluster operations on CrateDB and CrateDB Cloud.
+    """
     return boot_click(ctx, verbose, debug)
 
 

cratedb_toolkit/io/README.md

@@ -1,16 +1,68 @@
 # Load and extract data into/from CrateDB
 
 
-## Synopsis
+## About
 
-Define the cluster id of your CrateDB Cloud Cluster, and connection credentials
+A one-stop command `ctk load table` to load data into CrateDB database tables.
+
+
+## General Notes
+
+By default, the table name will be derived from the name of the input resource.
+If you would like to specify a different table name, use the `--table` command
+line option, or the `CRATEDB_TABLE` environment variable.
+
+When aiming to write into a table in a different database schema, use the
+`--schema` command line option, or the `CRATEDB_SCHEMA` environment variable.
+When omitting this parameter, the default value `doc` will be used.
+
+
+## Cloud Import API
+
+Using the [CrateDB Cloud] Import API, you can import files in CSV, JSON, and
+Parquet formats.
+
+### Prerequisites
+Authenticate with CrateDB Cloud using one of those identity providers:
+Cognito, Azure AD, GitHub, Google.
+```shell
+croud login --idp azuread
+```
+
+To discover the list of available database clusters.
 ```shell
-export CRATEDB_CLOUD_CLUSTER_ID=e1e38d92-a650-48f1-8a70-8133f2d5c400
-export CRATEDB_USERNAME=admin
-export CRATEDB_PASSWORD=3$MJ5fALP8bNOYCYBMLOrzd&
+croud clusters list
 ```
 
+Define the cluster id of your CrateDB Cloud Cluster you are aiming to connect
+to, and its connection credentials.
+```shell
+export CRATEDB_CLOUD_CLUSTER_ID='e1e38d92-a650-48f1-8a70-8133f2d5c400'
+```
+
+Define authentication credentials, to satisfy the `ctk shell` command you will
+use to inspect the database content after transferring data.
+```shell
+export CRATEDB_USERNAME='admin'
+export CRATEDB_PASSWORD='3$MJ5fALP8bNOYCYBMLOrzd&'
+```
+
+### Usage
 Load data.
 ```shell
 ctk load table https://github.com/crate/cratedb-datasets/raw/main/cloud-tutorials/data_weather.csv.gz
+ctk load table https://github.com/crate/cratedb-datasets/raw/main/cloud-tutorials/data_marketing.json.gz
+ctk load table https://github.com/daq-tools/skeem/raw/main/tests/testdata/basic.parquet
 ```
+
+Inquire data.
+```shell
+ctk shell --command="SELECT * FROM data_weather LIMIT 10;"
+ctk shell --command="SELECT * FROM data_weather LIMIT 10;" --format=json
+```
+
+### Backlog
+- Exercise data imports from AWS S3 and other Object Storage providers.
+
+
+[CrateDB Cloud]: https://console.cratedb.cloud/

cratedb_toolkit/job/cli.py

@@ -1,16 +1,27 @@
+import sys
+
 import click
 
 from cratedb_toolkit.job.croud import jobs_list
-from cratedb_toolkit.util import jd
+from cratedb_toolkit.util.croud import get_croud_output_formats
+
+output_formats = get_croud_output_formats()
 
 
 @click.command(name="list-jobs")
 @click.option(
     "--cluster-id", envvar="CRATEDB_CLOUD_CLUSTER_ID", type=str, required=True, help="CrateDB Cloud cluster identifier"
 )
+@click.option(
+    "--format",
+    "format_",
+    type=click.Choice(output_formats, case_sensitive=False),
+    required=False,
+    help="The output format for the result of the operation",
+)
 @click.pass_context
-def cli_list_jobs(ctx: click.Context, cluster_id: str):
+def cli_list_jobs(ctx: click.Context, cluster_id: str, format_: str):
     """
-    List running jobs.
+    List CrateDB Cloud jobs.
     """
-    jd(jobs_list(cluster_id))
+    print(jobs_list(cluster_id, output_format=format_, decode_output=False), file=sys.stdout)  # noqa: T201

cratedb_toolkit/job/croud.py

@@ -1,7 +1,7 @@
 from cratedb_toolkit.util.croud import CroudCall, CroudWrapper
 
 
-def jobs_list(cratedb_cloud_cluster_id: str):
+def jobs_list(cratedb_cloud_cluster_id: str, output_format: str = None, decode_output: bool = True):
     from croud.clusters.commands import import_jobs_list
     from croud.parser import Argument
 
@@ -13,5 +13,5 @@ def jobs_list(cratedb_cloud_cluster_id: str):
         ],
     )
 
-    wr = CroudWrapper(call)
+    wr = CroudWrapper(call, output_format=output_format, decode_output=decode_output)
     return wr.invoke()

cratedb_toolkit/shell/cli.py

@@ -0,0 +1,64 @@
+import click
+
+from cratedb_toolkit.cluster.util import get_cluster_info
+from cratedb_toolkit.util.cli import boot_click
+from cratedb_toolkit.util.crash import get_crash_output_formats, run_crash
+
+output_formats = get_crash_output_formats()
+
+
+@click.command()
+@click.option(
+    "--cluster-id", envvar="CRATEDB_CLOUD_CLUSTER_ID", type=str, required=True, help="CrateDB Cloud cluster identifier"
+)
+@click.option("--username", envvar="CRATEDB_USERNAME", type=str, required=False, help="Username for CrateDB cluster")
+@click.option("--password", envvar="CRATEDB_PASSWORD", type=str, required=False, help="Password for CrateDB cluster")
+@click.option(
+    "--schema",
+    envvar="CRATEDB_SCHEMA",
+    type=str,
+    required=False,
+    help="Default schema for statements if schema is not explicitly stated within queries",
+)
+@click.option("--command", type=str, required=False, help="SQL command")
+@click.option(
+    "--format",
+    "format_",
+    type=click.Choice(output_formats, case_sensitive=False),
+    required=False,
+    help="The output format of the database response",
+)
+@click.option("--verbose", is_flag=True, required=False, help="Turn on logging")
+@click.option("--debug", is_flag=True, required=False, help="Turn on logging with debug level")
+@click.version_option()
+@click.pass_context
+def cli(
+    ctx: click.Context,
+    cluster_id: str,
+    username: str,
+    password: str,
+    schema: str,
+    command: str,
+    format_: str,
+    verbose: bool,
+    debug: bool,
+):
+    """
+    Start an interactive database shell, or invoke SQL commands.
+
+    TODO: Only talks to CrateDB Cloud for now. Also implement for standalone CrateDB servers.
+    TODO: Learn/forward more options of `crash`.
+    """
+    boot_click(ctx, verbose, debug)
+
+    cluster_info = get_cluster_info(cluster_id=cluster_id)
+    cratedb_http_url = cluster_info.cloud["url"]
+
+    run_crash(
+        hosts=cratedb_http_url,
+        username=username,
+        password=password,
+        schema=schema,
+        command=command,
+        output_format=format_,
+    )