From e23546d4a9f117d6cee20380e96a236ea3f6f65a Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Thu, 2 Nov 2023 12:18:14 +0100 Subject: [PATCH] Chore: Update backlog --- doc/backlog.md | 41 +++++++++++++++++++++------ doc/bugs.md | 75 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 9 deletions(-) create mode 100644 doc/bugs.md diff --git a/doc/backlog.md b/doc/backlog.md index 30cdc1b2..dafea121 100644 --- a/doc/backlog.md +++ b/doc/backlog.md @@ -1,21 +1,43 @@ # Backlog ## Iteration +1 -- Refactoring towards `cratedb-toolkit`. +- UX: Refactoring towards `cratedb-toolkit`. +- UX: `ctk load`: Clearly disambiguate between loading data into + RDBMS database tables, blob tables, or filesystem objects. + ```shell + ctk load table https://s3.amazonaws.com/my.import.data.gz + ``` + ```shell + ctk load blob /path/to/image.png + ``` + ```shell + ctk load object /local/path/to/image.png /dbfs/assets + ``` +- Improve retention subsystem API. ```shell ctk retention create-policy lalala ctk materialized create lalala ctk schedule add lalala ``` -## Iteration +1.25 +## Iteration +2 - CLI: Use proper exit codes. -- Make `--cutoff-day` optional, use `today()` as default. -- Refactor "partition"-based strategies into subfamily/category, in order to +- Retention: Make `--cutoff-day` optional, use `today()` as default. +- Retention: Refactor "partition"-based strategies into subfamily/category, in order to make room for other types of strategies not necessarily using partitioned tables. +- Cloud: Use `keyring` for storing CrateDB Cloud Cluster credentials +- `ctk load table`: Accept `offset`/`limit` and `start`/`stop` options + - Humanized: https://github.com/panodata/aika +- UX: Unlock `testdata://` data sources from `influxio` +- UX: No stack traces when `cratedb_toolkit.util.croud.CroudException: 401 - Unauthorized` +- UX: Explain `cratedb_toolkit.util.croud.CroudException: Another cluster operation is currently in progress, please try again later.` +- UX: Explain `cratedb_toolkit.util.croud.CroudException: Resource not found.` when accessing unknown cluster id. +- UX: Make `ctk list-jobs` respect `"status": "SUCCEEDED"` etc. +- UX: Improve textual report from `ctk load table` +- UX: Accept alias `--format {jsonl,ndjson}` for `--format json_row` -## Iteration +1.5 +## Iteration +3 - CI: Nightly builds, to verify regressions on CrateDB - CI: Also build OCI images for ARM, maybe only on PRs to `main`, and releases? - CI: Add code coverage tracking and reporting. @@ -33,7 +55,7 @@ cratedb-retention refresh-materialized-view doc.raw_metrics ``` -## Iteration +1.75 +## Iteration +4 Add two non-partition-based strategies. Category: `timerange`. - Add a shortcut interface for adding policies. @@ -69,10 +91,11 @@ Add two non-partition-based strategies. Category: `timerange`. st ttl doc.sensor_readings:time_month snapshot:export_cold 4w ``` -## Iteration +2 +## Iteration +5 - Periodic/recurrent queries via scheduling. - https://github.com/crate/crate/issues/11182 - https://github.com/crate/crate-insights/issues/75 + - Humanized: https://github.com/panodata/aika Either use classic cron, or systemd-timers, or use one of `APScheduler`, `schedule`, or `scheduler`. @@ -100,7 +123,7 @@ Add two non-partition-based strategies. Category: `timerange`. - https://github.com/crate/crate/issues/14298 - https://github.com/crate/crate/pull/14346 -## Iteration +3 +## Iteration +6 - Review SQL queries: What about details like `ORDER BY 5 ASC`? - Use SQLAlchemy as query builder, to prevent SQL injection (S608), see `render_delete.py` spike. @@ -112,7 +135,7 @@ Add two non-partition-based strategies. Category: `timerange`. - Document usage with Kubernetes, and Nomad/Waypoint. - Job progress -## Iteration +4 +## Iteration +7 - More packaging: Use `fpm` - More packaging: What about an Ubuntu Snap, a Helm chart, or a Nomad Pack? - Clarify how to interpret the `--cutoff-day` option. diff --git a/doc/bugs.md b/doc/bugs.md new file mode 100644 index 00000000..96b89b87 --- /dev/null +++ b/doc/bugs.md @@ -0,0 +1,75 @@ +# Bugs + +## + +``` +$ croud clusters import-jobs list --cluster-id=$CRATEDB_CLOUD_CLUSTER_ID --format=table +Traceback (most recent call last): + File "/Users/amo/dev/crate/ecosystem/cratedb-retentions/.venv/bin/croud", line 8, in + sys.exit(main()) + ^^^^^^ + File "/Users/amo/dev/crate/ecosystem/cratedb-retentions/.venv/lib/python3.11/site-packages/croud/__main__.py", line 1302, in main + fn(params) + File "/Users/amo/dev/crate/ecosystem/cratedb-retentions/.venv/lib/python3.11/site-packages/croud/clusters/commands.py", line 292, in import_jobs_list + print_response( + File "/Users/amo/dev/crate/ecosystem/cratedb-retentions/.venv/lib/python3.11/site-packages/croud/printer.py", line 71, in print_response + print_format(data, output_fmt, keys, transforms) + File "/Users/amo/dev/crate/ecosystem/cratedb-retentions/.venv/lib/python3.11/site-packages/croud/printer.py", line 46, in print_format + printer.print_rows(rows) + File "/Users/amo/dev/crate/ecosystem/cratedb-retentions/.venv/lib/python3.11/site-packages/croud/printer.py", line 122, in print_rows + print(self.format_rows(rows)) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/Users/amo/dev/crate/ecosystem/cratedb-retentions/.venv/lib/python3.11/site-packages/croud/printer.py", line 165, in format_rows + values = [ + ^ + File "/Users/amo/dev/crate/ecosystem/cratedb-retentions/.venv/lib/python3.11/site-packages/croud/printer.py", line 166, in + [ + File "/Users/amo/dev/crate/ecosystem/cratedb-retentions/.venv/lib/python3.11/site-packages/croud/printer.py", line 168, in + row[header] + ~~~^^^^^^^^ +KeyError: 'url' +``` + + +``` +$ ctk load table https://github.com/crate/cratedb-datasets/raw/main/cloud-tutorials/data_weather.csv.gz --schema foo +==> Info: Status: REGISTERED (Your import job was received and is pending processing.) +==> Info: Status: SENT (Your creation request was sent to the region.) +==> Info: Done importing 70.00K records +==> Success: Operation completed. +Traceback (most recent call last): + File "/Users/amo/dev/crate/ecosystem/cratedb-retentions/.venv/bin/ctk", line 8, in + sys.exit(cli()) + ^^^^^ + File "/Users/amo/dev/crate/ecosystem/cratedb-retentions/.venv/lib/python3.11/site-packages/click/core.py", line 1157, in __call__ + return self.main(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Users/amo/dev/crate/ecosystem/cratedb-retentions/.venv/lib/python3.11/site-packages/click/core.py", line 1078, in main + rv = self.invoke(ctx) + ^^^^^^^^^^^^^^^^ + File "/Users/amo/dev/crate/ecosystem/cratedb-retentions/.venv/lib/python3.11/site-packages/click/core.py", line 1688, in invoke + return _process_result(sub_ctx.command.invoke(sub_ctx)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Users/amo/dev/crate/ecosystem/cratedb-retentions/.venv/lib/python3.11/site-packages/click/core.py", line 1688, in invoke + return _process_result(sub_ctx.command.invoke(sub_ctx)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Users/amo/dev/crate/ecosystem/cratedb-retentions/.venv/lib/python3.11/site-packages/click/core.py", line 1434, in invoke + return ctx.invoke(self.callback, **ctx.params) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Users/amo/dev/crate/ecosystem/cratedb-retentions/.venv/lib/python3.11/site-packages/click/core.py", line 783, in invoke + return __callback(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Users/amo/dev/crate/ecosystem/cratedb-retentions/.venv/lib/python3.11/site-packages/click/decorators.py", line 33, in new_func + return f(get_current_context(), *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Users/amo/dev/crate/ecosystem/cratedb-retentions/cratedb_toolkit/io/cli.py", line 90, in load_table + job_info, success = cio.load_resource(url=url, schema=schema, table=table) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Users/amo/dev/crate/ecosystem/cratedb-retentions/cratedb_toolkit/io/croud.py", line 28, in load_resource + outcome, success, found = self.find_job(job_id=job_id) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Users/amo/dev/crate/ecosystem/cratedb-retentions/cratedb_toolkit/io/croud.py", line 45, in find_job + if job["id"] == job_id: + ~~~^^^^^^ +TypeError: string indices must be integers, not 'str' +```