From 5c298a174fd11e6d8a119f1914fbe8d342559927 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Wed, 4 Sep 2024 18:30:40 +0200 Subject: [PATCH] [Inference] Implement NL-to-ESQL task (#190433) Implements the NL-to-ESQL task and migrates the Observability AI Assistant to use the new task. Most of the files are simply generated documentation. I've also included two scripts: one to generate the documentation, and another to evaluate the task against a real LLM. TBD: run evaluation framework in Observability AI Assistant to ensure there are no performance regressions. --------- Co-authored-by: Elastic Machine Co-authored-by: pgayvallet Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../generate_fake_tool_call_id.ts | 0 .../inference/common/chat_complete/index.ts | 6 +- .../is_chat_completion_chunk_event.ts | 14 + .../chat_complete/is_chat_completion_event.ts | 17 + .../is_chat_completion_message_event.ts | 15 + .../common/chat_complete/tool_schema.ts | 25 +- .../inference/common/chat_complete/tools.ts | 8 +- .../inference/common/ensure_multi_turn.ts | 31 ++ x-pack/plugins/inference/common/errors.ts | 9 +- x-pack/plugins/inference/common/index.ts | 31 ++ .../common/{tasks.ts => inference_task.ts} | 0 .../common/output/create_output_api.ts | 24 +- .../plugins/inference/common/output/index.ts | 14 +- .../common/output/is_output_complete_event.ts | 14 + .../common/output/is_output_event.ts | 15 + .../common/output/is_output_update_event.ts | 14 + .../common/tasks/nl_to_esql}/constants.ts | 0 .../correct_common_esql_mistakes.test.ts | 0 .../correct_common_esql_mistakes.ts | 0 .../correct_query_with_actions.test.ts | 46 ++ .../nl_to_esql/correct_query_with_actions.ts | 62 +++ .../get_errors_with_commands.test.ts | 25 + .../nl_to_esql/get_errors_with_commands.ts | 26 + x-pack/plugins/inference/jest.config.js | 6 +- .../http_response_into_observable.test.ts | 2 +- .../util/http_response_into_observable.ts | 2 +- .../scripts/evaluation/.eslintrc.json | 17 + .../inference/scripts/evaluation/README.md | 31 ++ .../inference/scripts/evaluation/cli.ts | 45 ++ .../scripts/evaluation/evaluation.ts | 190 +++++++ .../scripts/evaluation/evaluation_client.ts | 181 +++++++ .../inference/scripts/evaluation/index.js | 10 + .../evaluation/scenarios/esql/index.spec.ts | 409 +++++++++++++++ .../scripts/evaluation/services/index.ts | 49 ++ .../scripts/evaluation/table_renderer.ts | 100 ++++ .../inference/scripts/evaluation/types.ts | 31 ++ .../scripts/load_esql_docs/README.md | 10 + .../load_esql_docs/extract_sections.ts | 0 .../scripts/load_esql_docs/index.js | 0 .../scripts/load_esql_docs/load_esql_docs.ts | 64 ++- .../inference/scripts/util/cli_options.ts | 30 ++ .../scripts/util/get_service_urls.ts | 148 ++++++ .../inference/scripts/util/kibana_client.ts | 226 +++++++++ .../scripts/util/read_kibana_config.ts | 44 ++ .../scripts/util/select_connector.ts | 47 ++ .../bedrock/bedrock_claude_adapter.test.ts | 19 + .../bedrock/bedrock_claude_adapter.ts | 75 ++- .../adapters/gemini/gemini_adapter.test.ts | 12 +- .../adapters/gemini/gemini_adapter.ts | 18 +- .../adapters/gemini/process_vertex_stream.ts | 2 +- .../chat_complete/adapters/openai/index.ts | 179 +------ .../{index.test.ts => openai_adapter.test.ts} | 10 +- .../adapters/openai/openai_adapter.ts | 189 +++++++ .../inference/server/chat_complete/api.ts | 17 +- .../inference/server/chat_complete/types.ts | 2 + .../utils/chunks_into_message.test.ts | 75 +-- .../utils/chunks_into_message.ts | 13 +- .../server/chat_complete/utils/index.ts | 1 - x-pack/plugins/inference/server/index.ts | 2 + .../server/inference_client/index.ts | 9 +- x-pack/plugins/inference/server/plugin.ts | 9 +- .../inference/server/routes/chat_complete.ts | 8 +- .../plugins/inference/server/routes/index.ts | 5 +- .../tasks/nl_to_esql/esql_docs/esql-abs.txt | 16 + .../tasks/nl_to_esql/esql_docs/esql-acos.txt | 15 + .../tasks/nl_to_esql/esql_docs/esql-asin.txt | 15 + .../tasks/nl_to_esql/esql_docs/esql-atan.txt | 15 + .../tasks/nl_to_esql/esql_docs/esql-atan2.txt | 15 + .../tasks/nl_to_esql/esql_docs/esql-avg.txt | 15 + .../nl_to_esql/esql_docs/esql-bucket.txt | 66 +++ .../tasks/nl_to_esql/esql_docs/esql-case.txt | 37 ++ .../tasks/nl_to_esql/esql_docs/esql-cbrt.txt | 15 + .../tasks/nl_to_esql/esql_docs/esql-ceil.txt | 16 + .../nl_to_esql/esql_docs/esql-cidr_match.txt | 17 + .../nl_to_esql/esql_docs/esql-coalesce.txt | 15 + .../nl_to_esql/esql_docs/esql-concat.txt | 16 + .../tasks/nl_to_esql/esql_docs/esql-cos.txt | 15 + .../tasks/nl_to_esql/esql_docs/esql-cosh.txt | 15 + .../tasks/nl_to_esql/esql_docs/esql-count.txt | 32 ++ .../esql_docs/esql-count_distinct.txt | 31 ++ .../nl_to_esql/esql_docs/esql-date_diff.txt | 15 + .../esql_docs/esql-date_extract.txt | 15 + .../nl_to_esql/esql_docs/esql-date_format.txt | 17 + .../nl_to_esql/esql_docs/esql-date_parse.txt | 15 + .../nl_to_esql/esql_docs/esql-date_trunc.txt | 30 ++ .../nl_to_esql/esql_docs/esql-dissect.txt | 54 ++ .../tasks/nl_to_esql/esql_docs/esql-drop.txt | 58 +++ .../tasks/nl_to_esql/esql_docs/esql-e.txt | 15 + .../nl_to_esql/esql_docs/esql-ends_with.txt | 17 + .../nl_to_esql/esql_docs/esql-enrich.txt | 48 ++ .../tasks/nl_to_esql/esql_docs/esql-eval.txt | 55 ++ .../tasks/nl_to_esql/esql_docs/esql-exp.txt | 15 + .../tasks/nl_to_esql/esql_docs/esql-floor.txt | 16 + .../tasks/nl_to_esql/esql_docs/esql-from.txt | 53 ++ .../nl_to_esql/esql_docs/esql-from_base64.txt | 15 + .../nl_to_esql/esql_docs/esql-greatest.txt | 15 + .../tasks/nl_to_esql/esql_docs/esql-grok.txt | 55 ++ .../nl_to_esql/esql_docs/esql-ip_prefix.txt | 16 + .../tasks/nl_to_esql/esql_docs/esql-keep.txt | 64 +++ .../tasks/nl_to_esql/esql_docs/esql-least.txt | 15 + .../tasks/nl_to_esql/esql_docs/esql-left.txt | 19 + .../nl_to_esql/esql_docs/esql-length.txt | 16 + .../tasks/nl_to_esql/esql_docs/esql-limit.txt | 68 +++ .../nl_to_esql/esql_docs/esql-locate.txt | 25 + .../tasks/nl_to_esql/esql_docs/esql-log.txt | 15 + .../tasks/nl_to_esql/esql_docs/esql-log10.txt | 15 + .../nl_to_esql/esql_docs/esql-lookup.txt | 101 ++++ .../tasks/nl_to_esql/esql_docs/esql-ltrim.txt | 19 + .../tasks/nl_to_esql/esql_docs/esql-max.txt | 15 + .../nl_to_esql/esql_docs/esql-median.txt | 15 + .../esql-median_absolute_deviation.txt | 15 + .../tasks/nl_to_esql/esql_docs/esql-min.txt | 15 + .../nl_to_esql/esql_docs/esql-mv_append.txt | 17 + .../nl_to_esql/esql_docs/esql-mv_avg.txt | 15 + .../nl_to_esql/esql_docs/esql-mv_concat.txt | 15 + .../nl_to_esql/esql_docs/esql-mv_count.txt | 15 + .../nl_to_esql/esql_docs/esql-mv_dedupe.txt | 15 + .../nl_to_esql/esql_docs/esql-mv_expand.txt | 41 ++ .../nl_to_esql/esql_docs/esql-mv_first.txt | 15 + .../nl_to_esql/esql_docs/esql-mv_last.txt | 15 + .../nl_to_esql/esql_docs/esql-mv_max.txt | 15 + .../nl_to_esql/esql_docs/esql-mv_median.txt | 17 + .../nl_to_esql/esql_docs/esql-mv_min.txt | 15 + .../esql-mv_pseries_weighted_sum.txt | 17 + .../nl_to_esql/esql_docs/esql-mv_slice.txt | 15 + .../nl_to_esql/esql_docs/esql-mv_sort.txt | 15 + .../nl_to_esql/esql_docs/esql-mv_sum.txt | 15 + .../nl_to_esql/esql_docs/esql-mv_zip.txt | 17 + .../tasks/nl_to_esql/esql_docs/esql-now.txt | 14 + .../nl_to_esql/esql_docs/esql-operators.txt | 216 ++++++++ .../nl_to_esql/esql_docs/esql-overview.txt | 144 ++++++ .../nl_to_esql/esql_docs/esql-percentile.txt | 26 + .../tasks/nl_to_esql/esql_docs/esql-pi.txt | 15 + .../tasks/nl_to_esql/esql_docs/esql-pow.txt | 15 + .../nl_to_esql/esql_docs/esql-rename.txt | 37 ++ .../nl_to_esql/esql_docs/esql-repeat.txt | 15 + .../nl_to_esql/esql_docs/esql-replace.txt | 19 + .../tasks/nl_to_esql/esql_docs/esql-right.txt | 19 + .../tasks/nl_to_esql/esql_docs/esql-round.txt | 17 + .../tasks/nl_to_esql/esql_docs/esql-row.txt | 32 ++ .../tasks/nl_to_esql/esql_docs/esql-rtrim.txt | 13 + .../tasks/nl_to_esql/esql_docs/esql-show.txt | 24 + .../nl_to_esql/esql_docs/esql-signum.txt | 15 + .../tasks/nl_to_esql/esql_docs/esql-sin.txt | 15 + .../tasks/nl_to_esql/esql_docs/esql-sinh.txt | 15 + .../tasks/nl_to_esql/esql_docs/esql-sort.txt | 53 ++ .../tasks/nl_to_esql/esql_docs/esql-split.txt | 15 + .../tasks/nl_to_esql/esql_docs/esql-sqrt.txt | 16 + .../esql_docs/esql-st_centroid_agg.txt | 15 + .../nl_to_esql/esql_docs/esql-st_contains.txt | 17 + .../nl_to_esql/esql_docs/esql-st_disjoint.txt | 17 + .../nl_to_esql/esql_docs/esql-st_distance.txt | 19 + .../esql_docs/esql-st_intersects.txt | 16 + .../nl_to_esql/esql_docs/esql-st_within.txt | 17 + .../tasks/nl_to_esql/esql_docs/esql-st_x.txt | 15 + .../tasks/nl_to_esql/esql_docs/esql-st_y.txt | 15 + .../nl_to_esql/esql_docs/esql-starts_with.txt | 18 + .../tasks/nl_to_esql/esql_docs/esql-stats.txt | 71 +++ .../nl_to_esql/esql_docs/esql-substring.txt | 39 ++ .../tasks/nl_to_esql/esql_docs/esql-sum.txt | 15 + .../nl_to_esql/esql_docs/esql-syntax.txt | 181 +++++++ .../tasks/nl_to_esql/esql_docs/esql-tan.txt | 15 + .../tasks/nl_to_esql/esql_docs/esql-tanh.txt | 15 + .../tasks/nl_to_esql/esql_docs/esql-tau.txt | 15 + .../nl_to_esql/esql_docs/esql-to_base64.txt | 15 + .../nl_to_esql/esql_docs/esql-to_boolean.txt | 15 + .../esql_docs/esql-to_cartesianpoint.txt | 17 + .../esql_docs/esql-to_cartesianshape.txt | 9 +- .../nl_to_esql/esql_docs/esql-to_datetime.txt | 19 + .../nl_to_esql/esql_docs/esql-to_degrees.txt | 15 + .../nl_to_esql/esql_docs/esql-to_double.txt | 14 + .../nl_to_esql/esql_docs/esql-to_geopoint.txt | 15 + .../nl_to_esql/esql_docs/esql-to_geoshape.txt | 15 + .../nl_to_esql/esql_docs/esql-to_integer.txt | 12 + .../tasks/nl_to_esql/esql_docs/esql-to_ip.txt | 21 + .../nl_to_esql/esql_docs/esql-to_long.txt | 17 + .../nl_to_esql/esql_docs/esql-to_lower.txt | 17 + .../nl_to_esql/esql_docs/esql-to_radians.txt | 15 + .../nl_to_esql/esql_docs/esql-to_string.txt | 15 + .../esql_docs/esql-to_unsigned_long.txt | 20 + .../nl_to_esql/esql_docs/esql-to_upper.txt | 17 + .../nl_to_esql/esql_docs/esql-to_version.txt | 14 + .../tasks/nl_to_esql/esql_docs/esql-top.txt | 25 + .../tasks/nl_to_esql/esql_docs/esql-trim.txt | 17 + .../nl_to_esql/esql_docs/esql-values.txt | 16 + .../esql_docs/esql-weighted_avg.txt | 21 + .../tasks/nl_to_esql/esql_docs/esql-where.txt | 77 +++ .../server/tasks/nl_to_esql/index.ts | 315 +++++++++--- .../server/tasks/nl_to_esql/load_documents.ts | 55 ++ .../tasks/nl_to_esql}/system_message.txt | 75 +-- .../tasks/nl_to_esql/validate_esql_query.ts | 72 +++ ...bservable_into_event_source_stream.test.ts | 9 +- .../observable_into_event_source_stream.ts | 10 +- x-pack/plugins/inference/tsconfig.json | 13 +- .../common/types.ts | 7 +- .../chat_function_client/index.test.ts | 2 + .../service/chat_function_client/index.ts | 5 +- .../server/service/client/index.test.ts | 1 + .../server/service/client/index.ts | 1 + .../client/operators/continue_conversation.ts | 15 +- .../server/service/types.ts | 5 +- .../observability_ai_assistant/tsconfig.json | 1 + .../common/convert_messages_for_inference.ts | 75 +++ .../kibana.jsonc | 3 +- .../scripts/evaluation/kibana_client.ts | 2 +- .../functions/query/esql_docs/esql-abs.txt | 30 -- .../functions/query/esql_docs/esql-acos.txt | 29 -- .../functions/query/esql_docs/esql-asin.txt | 21 - .../functions/query/esql_docs/esql-atan.txt | 29 -- .../functions/query/esql_docs/esql-atan2.txt | 30 -- .../functions/query/esql_docs/esql-avg.txt | 21 - .../functions/query/esql_docs/esql-bucket.txt | 22 - .../functions/query/esql_docs/esql-case.txt | 38 -- .../functions/query/esql_docs/esql-cbrt.txt | 29 -- .../functions/query/esql_docs/esql-ceil.txt | 21 - .../query/esql_docs/esql-cidr_match.txt | 32 -- .../query/esql_docs/esql-coalesce.txt | 30 -- .../functions/query/esql_docs/esql-concat.txt | 22 - .../functions/query/esql_docs/esql-cos.txt | 21 - .../functions/query/esql_docs/esql-cosh.txt | 21 - .../functions/query/esql_docs/esql-count.txt | 35 -- .../query/esql_docs/esql-count_distinct.txt | 37 -- .../query/esql_docs/esql-date_diff.txt | 33 -- .../query/esql_docs/esql-date_extract.txt | 28 -- .../query/esql_docs/esql-date_format.txt | 32 -- .../query/esql_docs/esql-date_parse.txt | 30 -- .../query/esql_docs/esql-date_trunc.txt | 43 -- .../query/esql_docs/esql-dissect.txt | 47 -- .../functions/query/esql_docs/esql-drop.txt | 46 -- .../functions/query/esql_docs/esql-e.txt | 19 - .../query/esql_docs/esql-ends_with.txt | 33 -- .../functions/query/esql_docs/esql-enrich.txt | 50 -- .../functions/query/esql_docs/esql-eval.txt | 44 -- .../functions/query/esql_docs/esql-floor.txt | 29 -- .../functions/query/esql_docs/esql-from.txt | 53 -- .../query/esql_docs/esql-from_base64.txt | 19 - .../query/esql_docs/esql-greatest.txt | 30 -- .../functions/query/esql_docs/esql-grok.txt | 42 -- .../functions/query/esql_docs/esql-keep.txt | 48 -- .../functions/query/esql_docs/esql-least.txt | 30 -- .../functions/query/esql_docs/esql-left.txt | 33 -- .../functions/query/esql_docs/esql-length.txt | 29 -- .../functions/query/esql_docs/esql-limit.txt | 42 -- .../functions/query/esql_docs/esql-locate.txt | 35 -- .../functions/query/esql_docs/esql-log.txt | 30 -- .../functions/query/esql_docs/esql-log10.txt | 21 - .../functions/query/esql_docs/esql-lookup.txt | 82 --- .../functions/query/esql_docs/esql-ltrim.txt | 29 -- .../functions/query/esql_docs/esql-max.txt | 29 -- .../functions/query/esql_docs/esql-median.txt | 21 - .../esql-median_absolute_deviation.txt | 33 -- .../functions/query/esql_docs/esql-min.txt | 29 -- .../functions/query/esql_docs/esql-mv_avg.txt | 29 -- .../query/esql_docs/esql-mv_concat.txt | 23 - .../query/esql_docs/esql-mv_count.txt | 29 -- .../query/esql_docs/esql-mv_dedupe.txt | 29 -- .../query/esql_docs/esql-mv_expand.txt | 41 -- .../query/esql_docs/esql-mv_first.txt | 31 -- .../query/esql_docs/esql-mv_last.txt | 23 - .../functions/query/esql_docs/esql-mv_max.txt | 23 - .../query/esql_docs/esql-mv_median.txt | 29 -- .../functions/query/esql_docs/esql-mv_min.txt | 21 - .../query/esql_docs/esql-mv_slice.txt | 31 -- .../query/esql_docs/esql-mv_sort.txt | 30 -- .../functions/query/esql_docs/esql-mv_sum.txt | 29 -- .../functions/query/esql_docs/esql-mv_zip.txt | 33 -- .../functions/query/esql_docs/esql-now.txt | 20 - .../query/esql_docs/esql-operators.txt | 170 ------- .../query/esql_docs/esql-overview.txt | 52 -- .../query/esql_docs/esql-percentile.txt | 34 -- .../functions/query/esql_docs/esql-pi.txt | 20 - .../functions/query/esql_docs/esql-pow.txt | 21 - .../functions/query/esql_docs/esql-rename.txt | 46 -- .../functions/query/esql_docs/esql-repeat.txt | 30 -- .../query/esql_docs/esql-replace.txt | 33 -- .../functions/query/esql_docs/esql-right.txt | 33 -- .../functions/query/esql_docs/esql-round.txt | 31 -- .../functions/query/esql_docs/esql-row.txt | 42 -- .../functions/query/esql_docs/esql-rtrim.txt | 23 - .../functions/query/esql_docs/esql-show.txt | 19 - .../functions/query/esql_docs/esql-signum.txt | 29 -- .../functions/query/esql_docs/esql-sin.txt | 21 - .../functions/query/esql_docs/esql-sinh.txt | 21 - .../functions/query/esql_docs/esql-sort.txt | 45 -- .../functions/query/esql_docs/esql-split.txt | 30 -- .../functions/query/esql_docs/esql-sqrt.txt | 29 -- .../query/esql_docs/esql-st_centroid_agg.txt | 21 - .../query/esql_docs/esql-st_contains.txt | 34 -- .../query/esql_docs/esql-st_disjoint.txt | 31 -- .../query/esql_docs/esql-st_intersects.txt | 30 -- .../query/esql_docs/esql-st_within.txt | 34 -- .../functions/query/esql_docs/esql-st_x.txt | 21 - .../functions/query/esql_docs/esql-st_y.txt | 26 - .../query/esql_docs/esql-starts_with.txt | 33 -- .../functions/query/esql_docs/esql-stats.txt | 92 ---- .../query/esql_docs/esql-substring.txt | 41 -- .../functions/query/esql_docs/esql-sum.txt | 21 - .../functions/query/esql_docs/esql-syntax.txt | 121 ----- .../functions/query/esql_docs/esql-tan.txt | 29 -- .../functions/query/esql_docs/esql-tanh.txt | 29 -- .../functions/query/esql_docs/esql-tau.txt | 20 - .../query/esql_docs/esql-to_base64.txt | 29 -- .../query/esql_docs/esql-to_boolean.txt | 29 -- .../esql_docs/esql-to_cartesianpoint.txt | 22 - .../query/esql_docs/esql-to_datetime.txt | 21 - .../query/esql_docs/esql-to_degrees.txt | 29 -- .../query/esql_docs/esql-to_double.txt | 21 - .../query/esql_docs/esql-to_geopoint.txt | 21 - .../query/esql_docs/esql-to_geoshape.txt | 21 - .../query/esql_docs/esql-to_integer.txt | 24 - .../functions/query/esql_docs/esql-to_ip.txt | 22 - .../query/esql_docs/esql-to_long.txt | 21 - .../query/esql_docs/esql-to_lower.txt | 29 -- .../query/esql_docs/esql-to_radians.txt | 29 -- .../query/esql_docs/esql-to_string.txt | 21 - .../query/esql_docs/esql-to_unsigned_long.txt | 21 - .../query/esql_docs/esql-to_upper.txt | 34 -- .../query/esql_docs/esql-to_version.txt | 19 - .../functions/query/esql_docs/esql-trim.txt | 30 -- .../functions/query/esql_docs/esql-values.txt | 24 - .../functions/query/esql_docs/esql-where.txt | 65 --- .../query/get_errors_with_commands.ts | 2 +- .../server/functions/query/index.ts | 474 ++++-------------- .../functions/query/validate_esql_query.ts | 2 +- .../server/types.ts | 3 + .../tsconfig.json | 3 +- 326 files changed, 6257 insertions(+), 4632 deletions(-) rename x-pack/plugins/inference/{server/chat_complete/utils => common/chat_complete}/generate_fake_tool_call_id.ts (100%) create mode 100644 x-pack/plugins/inference/common/chat_complete/is_chat_completion_chunk_event.ts create mode 100644 x-pack/plugins/inference/common/chat_complete/is_chat_completion_event.ts create mode 100644 x-pack/plugins/inference/common/chat_complete/is_chat_completion_message_event.ts create mode 100644 x-pack/plugins/inference/common/ensure_multi_turn.ts create mode 100644 x-pack/plugins/inference/common/index.ts rename x-pack/plugins/inference/common/{tasks.ts => inference_task.ts} (100%) create mode 100644 x-pack/plugins/inference/common/output/is_output_complete_event.ts create mode 100644 x-pack/plugins/inference/common/output/is_output_event.ts create mode 100644 x-pack/plugins/inference/common/output/is_output_update_event.ts rename x-pack/plugins/{observability_solution/observability_ai_assistant_app/server/functions/query => inference/common/tasks/nl_to_esql}/constants.ts (100%) rename x-pack/plugins/{observability_solution/observability_ai_assistant_app/server/functions/query => inference/common/tasks/nl_to_esql}/correct_common_esql_mistakes.test.ts (100%) rename x-pack/plugins/{observability_solution/observability_ai_assistant_app/server/functions/query => inference/common/tasks/nl_to_esql}/correct_common_esql_mistakes.ts (100%) create mode 100644 x-pack/plugins/inference/common/tasks/nl_to_esql/correct_query_with_actions.test.ts create mode 100644 x-pack/plugins/inference/common/tasks/nl_to_esql/correct_query_with_actions.ts create mode 100644 x-pack/plugins/inference/common/tasks/nl_to_esql/get_errors_with_commands.test.ts create mode 100644 x-pack/plugins/inference/common/tasks/nl_to_esql/get_errors_with_commands.ts create mode 100644 x-pack/plugins/inference/scripts/evaluation/.eslintrc.json create mode 100644 x-pack/plugins/inference/scripts/evaluation/README.md create mode 100644 x-pack/plugins/inference/scripts/evaluation/cli.ts create mode 100644 x-pack/plugins/inference/scripts/evaluation/evaluation.ts create mode 100644 x-pack/plugins/inference/scripts/evaluation/evaluation_client.ts create mode 100644 x-pack/plugins/inference/scripts/evaluation/index.js create mode 100644 x-pack/plugins/inference/scripts/evaluation/scenarios/esql/index.spec.ts create mode 100644 x-pack/plugins/inference/scripts/evaluation/services/index.ts create mode 100644 x-pack/plugins/inference/scripts/evaluation/table_renderer.ts create mode 100644 x-pack/plugins/inference/scripts/evaluation/types.ts create mode 100644 x-pack/plugins/inference/scripts/load_esql_docs/README.md rename x-pack/plugins/{observability_solution/observability_ai_assistant_app => inference}/scripts/load_esql_docs/extract_sections.ts (100%) rename x-pack/plugins/{observability_solution/observability_ai_assistant_app => inference}/scripts/load_esql_docs/index.js (100%) rename x-pack/plugins/{observability_solution/observability_ai_assistant_app => inference}/scripts/load_esql_docs/load_esql_docs.ts (91%) create mode 100644 x-pack/plugins/inference/scripts/util/cli_options.ts create mode 100644 x-pack/plugins/inference/scripts/util/get_service_urls.ts create mode 100644 x-pack/plugins/inference/scripts/util/kibana_client.ts create mode 100644 x-pack/plugins/inference/scripts/util/read_kibana_config.ts create mode 100644 x-pack/plugins/inference/scripts/util/select_connector.ts rename x-pack/plugins/inference/server/chat_complete/adapters/openai/{index.test.ts => openai_adapter.test.ts} (96%) create mode 100644 x-pack/plugins/inference/server/chat_complete/adapters/openai/openai_adapter.ts create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-abs.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-acos.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-asin.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-atan.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-atan2.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-avg.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-bucket.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-case.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-cbrt.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-ceil.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-cidr_match.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-coalesce.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-concat.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-cos.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-cosh.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-count.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-count_distinct.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-date_diff.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-date_extract.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-date_format.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-date_parse.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-date_trunc.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-dissect.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-drop.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-e.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-ends_with.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-enrich.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-eval.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-exp.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-floor.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-from.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-from_base64.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-greatest.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-grok.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-ip_prefix.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-keep.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-least.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-left.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-length.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-limit.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-locate.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-log.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-log10.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-lookup.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-ltrim.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-max.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-median.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-median_absolute_deviation.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-min.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_append.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_avg.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_concat.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_count.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_dedupe.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_expand.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_first.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_last.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_max.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_median.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_min.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_pseries_weighted_sum.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_slice.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_sort.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_sum.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_zip.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-now.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-operators.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-overview.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-percentile.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-pi.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-pow.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-rename.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-repeat.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-replace.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-right.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-round.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-row.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-rtrim.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-show.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-signum.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-sin.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-sinh.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-sort.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-split.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-sqrt.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-st_centroid_agg.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-st_contains.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-st_disjoint.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-st_distance.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-st_intersects.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-st_within.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-st_x.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-st_y.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-starts_with.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-stats.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-substring.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-sum.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-syntax.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-tan.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-tanh.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-tau.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_base64.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_boolean.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_cartesianpoint.txt rename x-pack/plugins/{observability_solution/observability_ai_assistant_app/server/functions/query => inference/server/tasks/nl_to_esql}/esql_docs/esql-to_cartesianshape.txt (52%) create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_datetime.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_degrees.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_double.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_geopoint.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_geoshape.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_integer.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_ip.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_long.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_lower.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_radians.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_string.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_unsigned_long.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_upper.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_version.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-top.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-trim.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-values.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-weighted_avg.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-where.txt create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/load_documents.ts rename x-pack/plugins/{observability_solution/observability_ai_assistant_app/server/functions/query => inference/server/tasks/nl_to_esql}/system_message.txt (86%) create mode 100644 x-pack/plugins/inference/server/tasks/nl_to_esql/validate_esql_query.ts create mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/common/convert_messages_for_inference.ts delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-abs.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-acos.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-asin.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-atan.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-atan2.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-avg.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-bucket.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-case.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-cbrt.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-ceil.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-cidr_match.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-coalesce.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-concat.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-cos.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-cosh.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-count.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-count_distinct.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-date_diff.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-date_extract.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-date_format.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-date_parse.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-date_trunc.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-dissect.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-drop.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-e.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-ends_with.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-enrich.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-eval.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-floor.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-from.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-from_base64.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-greatest.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-grok.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-keep.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-least.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-left.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-length.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-limit.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-locate.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-log.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-log10.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-lookup.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-ltrim.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-max.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-median.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-median_absolute_deviation.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-min.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_avg.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_concat.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_count.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_dedupe.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_expand.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_first.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_last.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_max.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_median.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_min.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_slice.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_sort.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_sum.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_zip.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-now.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-operators.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-overview.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-percentile.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-pi.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-pow.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-rename.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-repeat.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-replace.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-right.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-round.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-row.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-rtrim.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-show.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-signum.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-sin.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-sinh.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-sort.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-split.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-sqrt.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-st_centroid_agg.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-st_contains.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-st_disjoint.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-st_intersects.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-st_within.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-st_x.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-st_y.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-starts_with.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-stats.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-substring.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-sum.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-syntax.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-tan.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-tanh.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-tau.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_base64.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_boolean.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_cartesianpoint.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_datetime.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_degrees.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_double.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_geopoint.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_geoshape.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_integer.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_ip.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_long.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_lower.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_radians.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_string.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_unsigned_long.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_upper.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_version.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-trim.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-values.txt delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-where.txt diff --git a/x-pack/plugins/inference/server/chat_complete/utils/generate_fake_tool_call_id.ts b/x-pack/plugins/inference/common/chat_complete/generate_fake_tool_call_id.ts similarity index 100% rename from x-pack/plugins/inference/server/chat_complete/utils/generate_fake_tool_call_id.ts rename to x-pack/plugins/inference/common/chat_complete/generate_fake_tool_call_id.ts diff --git a/x-pack/plugins/inference/common/chat_complete/index.ts b/x-pack/plugins/inference/common/chat_complete/index.ts index abad6a4372595..b42c2217c0177 100644 --- a/x-pack/plugins/inference/common/chat_complete/index.ts +++ b/x-pack/plugins/inference/common/chat_complete/index.ts @@ -6,7 +6,7 @@ */ import type { Observable } from 'rxjs'; -import type { InferenceTaskEventBase } from '../tasks'; +import type { InferenceTaskEventBase } from '../inference_task'; import type { ToolCall, ToolCallsOf, ToolOptions } from './tools'; export enum MessageRole { @@ -34,7 +34,7 @@ export type ToolMessage | unknown> = export type Message = UserMessage | AssistantMessage | ToolMessage; -export type ChatCompletionMessageEvent = +export type ChatCompletionMessageEvent = InferenceTaskEventBase & { content: string; } & { toolCalls: ToolCallsOf['toolCalls'] }; @@ -87,7 +87,7 @@ export type ChatCompletionEvent * @param {ToolChoice} [options.toolChoice] Force the LLM to call a (specific) tool, or no tool * @param {Record} [options.tools] A map of tools that can be called by the LLM */ -export type ChatCompleteAPI = ( +export type ChatCompleteAPI = ( options: { connectorId: string; system?: string; diff --git a/x-pack/plugins/inference/common/chat_complete/is_chat_completion_chunk_event.ts b/x-pack/plugins/inference/common/chat_complete/is_chat_completion_chunk_event.ts new file mode 100644 index 0000000000000..1630d765ab81e --- /dev/null +++ b/x-pack/plugins/inference/common/chat_complete/is_chat_completion_chunk_event.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ChatCompletionChunkEvent, ChatCompletionEvent, ChatCompletionEventType } from '.'; + +export function isChatCompletionChunkEvent( + event: ChatCompletionEvent +): event is ChatCompletionChunkEvent { + return event.type === ChatCompletionEventType.ChatCompletionChunk; +} diff --git a/x-pack/plugins/inference/common/chat_complete/is_chat_completion_event.ts b/x-pack/plugins/inference/common/chat_complete/is_chat_completion_event.ts new file mode 100644 index 0000000000000..d4d9305cac94b --- /dev/null +++ b/x-pack/plugins/inference/common/chat_complete/is_chat_completion_event.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ChatCompletionEvent, ChatCompletionEventType } from '.'; +import { InferenceTaskEvent } from '../inference_task'; + +export function isChatCompletionEvent(event: InferenceTaskEvent): event is ChatCompletionEvent { + return ( + event.type === ChatCompletionEventType.ChatCompletionChunk || + event.type === ChatCompletionEventType.ChatCompletionMessage || + event.type === ChatCompletionEventType.ChatCompletionTokenCount + ); +} diff --git a/x-pack/plugins/inference/common/chat_complete/is_chat_completion_message_event.ts b/x-pack/plugins/inference/common/chat_complete/is_chat_completion_message_event.ts new file mode 100644 index 0000000000000..172e55df9e4b4 --- /dev/null +++ b/x-pack/plugins/inference/common/chat_complete/is_chat_completion_message_event.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ChatCompletionEvent, ChatCompletionEventType, ChatCompletionMessageEvent } from '.'; +import type { ToolOptions } from './tools'; + +export function isChatCompletionMessageEvent>( + event: ChatCompletionEvent +): event is ChatCompletionMessageEvent { + return event.type === ChatCompletionEventType.ChatCompletionMessage; +} diff --git a/x-pack/plugins/inference/common/chat_complete/tool_schema.ts b/x-pack/plugins/inference/common/chat_complete/tool_schema.ts index b23c03aaad775..2a2c61f8e9b70 100644 --- a/x-pack/plugins/inference/common/chat_complete/tool_schema.ts +++ b/x-pack/plugins/inference/common/chat_complete/tool_schema.ts @@ -13,7 +13,7 @@ interface ToolSchemaFragmentBase { interface ToolSchemaTypeObject extends ToolSchemaFragmentBase { type: 'object'; - properties: Record; + properties?: Record; required?: string[] | readonly string[]; } @@ -47,16 +47,19 @@ export type ToolSchemaType = | ToolSchemaTypeNumber | ToolSchemaTypeArray; -type FromToolSchemaObject = Required< - { - [key in keyof TToolSchemaObject['properties']]?: FromToolSchema< - TToolSchemaObject['properties'][key] - >; - }, - TToolSchemaObject['required'] extends string[] | readonly string[] - ? ValuesType - : never ->; +type FromToolSchemaObject = + TToolSchemaObject extends { properties: Record } + ? Required< + { + [key in keyof TToolSchemaObject['properties']]?: FromToolSchema< + TToolSchemaObject['properties'][key] + >; + }, + TToolSchemaObject['required'] extends string[] | readonly string[] + ? ValuesType + : never + > + : never; type FromToolSchemaArray = Array< FromToolSchema diff --git a/x-pack/plugins/inference/common/chat_complete/tools.ts b/x-pack/plugins/inference/common/chat_complete/tools.ts index 85fb4cd9d7020..a5db86c7c996d 100644 --- a/x-pack/plugins/inference/common/chat_complete/tools.ts +++ b/x-pack/plugins/inference/common/chat_complete/tools.ts @@ -48,11 +48,9 @@ export type ToolCallsOf = TToolOptions extends ? TToolOptions extends { toolChoice: ToolChoiceType.none } ? { toolCalls: [] } : { - toolCalls: ToolResponsesOf< - Assert, Record | undefined> - >; + toolCalls: ToolResponsesOf>; } - : { toolCalls: never[] }; + : { toolCalls: never }; export enum ToolChoiceType { none = 'none', @@ -70,7 +68,7 @@ export interface UnvalidatedToolCall { export interface ToolCall< TName extends string = string, - TArguments extends Record | undefined = undefined + TArguments extends Record | undefined = Record | undefined > { toolCallId: string; function: { diff --git a/x-pack/plugins/inference/common/ensure_multi_turn.ts b/x-pack/plugins/inference/common/ensure_multi_turn.ts new file mode 100644 index 0000000000000..8d222564f3e72 --- /dev/null +++ b/x-pack/plugins/inference/common/ensure_multi_turn.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Message, MessageRole } from './chat_complete'; + +function isUserMessage(message: Message): boolean { + return message.role !== MessageRole.Assistant; +} + +export function ensureMultiTurn(messages: Message[]): Message[] { + const next: Message[] = []; + + messages.forEach((message) => { + const prevMessage = next[next.length - 1]; + + if (prevMessage && isUserMessage(prevMessage) === isUserMessage(message)) { + next.push({ + content: '-', + role: isUserMessage(message) ? MessageRole.Assistant : MessageRole.User, + }); + } + + next.push(message); + }); + + return next; +} diff --git a/x-pack/plugins/inference/common/errors.ts b/x-pack/plugins/inference/common/errors.ts index fa063e1669936..e8bcd4cf60aaf 100644 --- a/x-pack/plugins/inference/common/errors.ts +++ b/x-pack/plugins/inference/common/errors.ts @@ -5,7 +5,7 @@ * 2.0. */ import { i18n } from '@kbn/i18n'; -import { InferenceTaskEventBase, InferenceTaskEventType } from './tasks'; +import { InferenceTaskEventBase, InferenceTaskEventType } from './inference_task'; export enum InferenceTaskErrorCode { internalError = 'internalError', @@ -42,7 +42,7 @@ export type InferenceTaskErrorEvent = InferenceTaskEventBase >; export type InferenceTaskRequestError = InferenceTaskError< @@ -53,9 +53,10 @@ export type InferenceTaskRequestError = InferenceTaskError< export function createInferenceInternalError( message: string = i18n.translate('xpack.inference.internalError', { defaultMessage: 'An internal error occurred', - }) + }), + meta?: Record ): InferenceTaskInternalError { - return new InferenceTaskError(InferenceTaskErrorCode.internalError, message, {}); + return new InferenceTaskError(InferenceTaskErrorCode.internalError, message, meta ?? {}); } export function createInferenceRequestError( diff --git a/x-pack/plugins/inference/common/index.ts b/x-pack/plugins/inference/common/index.ts new file mode 100644 index 0000000000000..58c84a47c1804 --- /dev/null +++ b/x-pack/plugins/inference/common/index.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { + correctCommonEsqlMistakes, + splitIntoCommands, +} from './tasks/nl_to_esql/correct_common_esql_mistakes'; + +export { isChatCompletionChunkEvent } from './chat_complete/is_chat_completion_chunk_event'; +export { isChatCompletionMessageEvent } from './chat_complete/is_chat_completion_message_event'; +export { isChatCompletionEvent } from './chat_complete/is_chat_completion_event'; + +export { isOutputUpdateEvent } from './output/is_output_update_event'; +export { isOutputCompleteEvent } from './output/is_output_complete_event'; +export { isOutputEvent } from './output/is_output_event'; + +export type { ToolSchema } from './chat_complete/tool_schema'; + +export { + type Message, + MessageRole, + type ToolMessage, + type AssistantMessage, + type UserMessage, +} from './chat_complete'; + +export { generateFakeToolCallId } from './chat_complete/generate_fake_tool_call_id'; diff --git a/x-pack/plugins/inference/common/tasks.ts b/x-pack/plugins/inference/common/inference_task.ts similarity index 100% rename from x-pack/plugins/inference/common/tasks.ts rename to x-pack/plugins/inference/common/inference_task.ts diff --git a/x-pack/plugins/inference/common/output/create_output_api.ts b/x-pack/plugins/inference/common/output/create_output_api.ts index 9842f9635dea8..0da2a84c53c5b 100644 --- a/x-pack/plugins/inference/common/output/create_output_api.ts +++ b/x-pack/plugins/inference/common/output/create_output_api.ts @@ -9,22 +9,29 @@ import { map } from 'rxjs'; import { ChatCompleteAPI, ChatCompletionEventType, MessageRole } from '../chat_complete'; import { withoutTokenCountEvents } from '../chat_complete/without_token_count_events'; import { OutputAPI, OutputEvent, OutputEventType } from '.'; +import { ensureMultiTurn } from '../ensure_multi_turn'; export function createOutputApi(chatCompleteApi: ChatCompleteAPI): OutputAPI { - return (id, { connectorId, input, schema, system }) => { + return (id, { connectorId, input, schema, system, previousMessages }) => { return chatCompleteApi({ connectorId, system, - messages: [ + messages: ensureMultiTurn([ + ...(previousMessages || []), { role: MessageRole.User, content: input, }, - ], + ]), ...(schema ? { - tools: { output: { description: `Output your response in the this format`, schema } }, - toolChoice: { function: 'output' }, + tools: { + output: { + description: `Use the following schema to respond to the user's request in structured data, so it can be parsed and handled.`, + schema, + }, + }, + toolChoice: { function: 'output' as const }, } : {}), }).pipe( @@ -37,10 +44,15 @@ export function createOutputApi(chatCompleteApi: ChatCompleteAPI): OutputAPI { content: event.content, }; } + return { id, type: OutputEventType.OutputComplete, - output: event.toolCalls[0].function.arguments, + output: + event.toolCalls.length && 'arguments' in event.toolCalls[0].function + ? event.toolCalls[0].function.arguments + : undefined, + content: event.content, }; }) ); diff --git a/x-pack/plugins/inference/common/output/index.ts b/x-pack/plugins/inference/common/output/index.ts index 69df7fb3ecd9d..12636498ab925 100644 --- a/x-pack/plugins/inference/common/output/index.ts +++ b/x-pack/plugins/inference/common/output/index.ts @@ -7,14 +7,15 @@ import { Observable } from 'rxjs'; import { FromToolSchema, ToolSchema } from '../chat_complete/tool_schema'; -import { InferenceTaskEventBase } from '../tasks'; +import { InferenceTaskEventBase } from '../inference_task'; +import { Message } from '../chat_complete'; export enum OutputEventType { OutputUpdate = 'output', OutputComplete = 'complete', } -type Output = Record | undefined; +type Output = Record | undefined | unknown; export type OutputUpdateEvent = InferenceTaskEventBase & { @@ -28,6 +29,7 @@ export type OutputCompleteEvent< > = InferenceTaskEventBase & { id: TId; output: TOutput; + content?: string; }; export type OutputEvent = @@ -39,7 +41,8 @@ export type OutputEvent Observable< OutputEvent : undefined> @@ -59,11 +63,13 @@ export type OutputAPI = < export function createOutputCompleteEvent( id: TId, - output: TOutput + output: TOutput, + content?: string ): OutputCompleteEvent { return { id, type: OutputEventType.OutputComplete, output, + content, }; } diff --git a/x-pack/plugins/inference/common/output/is_output_complete_event.ts b/x-pack/plugins/inference/common/output/is_output_complete_event.ts new file mode 100644 index 0000000000000..bac3443b8258c --- /dev/null +++ b/x-pack/plugins/inference/common/output/is_output_complete_event.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { OutputEvent, OutputEventType, OutputUpdateEvent } from '.'; + +export function isOutputCompleteEvent( + event: TOutputEvent +): event is Exclude { + return event.type === OutputEventType.OutputComplete; +} diff --git a/x-pack/plugins/inference/common/output/is_output_event.ts b/x-pack/plugins/inference/common/output/is_output_event.ts new file mode 100644 index 0000000000000..dad2b0967a6ac --- /dev/null +++ b/x-pack/plugins/inference/common/output/is_output_event.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { OutputEvent, OutputEventType } from '.'; +import type { InferenceTaskEvent } from '../inference_task'; + +export function isOutputEvent(event: InferenceTaskEvent): event is OutputEvent { + return ( + event.type === OutputEventType.OutputComplete || event.type === OutputEventType.OutputUpdate + ); +} diff --git a/x-pack/plugins/inference/common/output/is_output_update_event.ts b/x-pack/plugins/inference/common/output/is_output_update_event.ts new file mode 100644 index 0000000000000..459436e64014e --- /dev/null +++ b/x-pack/plugins/inference/common/output/is_output_update_event.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { OutputEvent, OutputEventType, OutputUpdateEvent } from '.'; + +export function isOutputUpdateEvent( + event: OutputEvent +): event is OutputUpdateEvent { + return event.type === OutputEventType.OutputComplete; +} diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/constants.ts b/x-pack/plugins/inference/common/tasks/nl_to_esql/constants.ts similarity index 100% rename from x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/constants.ts rename to x-pack/plugins/inference/common/tasks/nl_to_esql/constants.ts diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/correct_common_esql_mistakes.test.ts b/x-pack/plugins/inference/common/tasks/nl_to_esql/correct_common_esql_mistakes.test.ts similarity index 100% rename from x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/correct_common_esql_mistakes.test.ts rename to x-pack/plugins/inference/common/tasks/nl_to_esql/correct_common_esql_mistakes.test.ts diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/correct_common_esql_mistakes.ts b/x-pack/plugins/inference/common/tasks/nl_to_esql/correct_common_esql_mistakes.ts similarity index 100% rename from x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/correct_common_esql_mistakes.ts rename to x-pack/plugins/inference/common/tasks/nl_to_esql/correct_common_esql_mistakes.ts diff --git a/x-pack/plugins/inference/common/tasks/nl_to_esql/correct_query_with_actions.test.ts b/x-pack/plugins/inference/common/tasks/nl_to_esql/correct_query_with_actions.test.ts new file mode 100644 index 0000000000000..818f4854e038d --- /dev/null +++ b/x-pack/plugins/inference/common/tasks/nl_to_esql/correct_query_with_actions.test.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { correctQueryWithActions } from './correct_query_with_actions'; + +describe('correctQueryWithActions', () => { + it(`fixes errors correctly for a query with one syntax error for stats`, async () => { + const fixedQuery = await correctQueryWithActions('from logstash-* | stats aveg(bytes)'); + expect(fixedQuery).toBe('from logstash-* | stats avg(bytes)'); + }); + + it(`fixes errors correctly for a query with 2 syntax errors for stats`, async () => { + const fixedQuery = await correctQueryWithActions( + 'from logstash-* | stats aveg(bytes), max2(bytes)' + ); + expect(fixedQuery).toBe('from logstash-* | stats avg(bytes), max(bytes)'); + }); + + it(`fixes errors correctly for a query with one syntax error for eval`, async () => { + const fixedQuery = await correctQueryWithActions( + 'from logstash-* | stats var0 = max(bytes) | eval ab(var0) | limit 1' + ); + expect(fixedQuery).toBe('from logstash-* | stats var0 = max(bytes) | eval abs(var0) | limit 1'); + }); + + it(`fixes errors correctly for a query with two syntax error for eval`, async () => { + const fixedQuery = await correctQueryWithActions( + 'from logstash-* | stats var0 = max2(bytes) | eval ab(var0) | limit 1' + ); + expect(fixedQuery).toBe('from logstash-* | stats var0 = max(bytes) | eval abs(var0) | limit 1'); + }); + + it(`doesnt complain for @timestamp column`, async () => { + const queryWithTimestamp = `FROM logstash-* + | WHERE @timestamp >= NOW() - 15 minutes + | EVAL bucket = DATE_TRUNC(1 minute, @timestamp) + | STATS avg_cpu = AVG(system.cpu.total.norm.pct) BY service.name, bucket + | SORT avg_cpu DESC + | LIMIT 10`; + const fixedQuery = await correctQueryWithActions(queryWithTimestamp); + expect(fixedQuery).toBe(queryWithTimestamp); + }); +}); diff --git a/x-pack/plugins/inference/common/tasks/nl_to_esql/correct_query_with_actions.ts b/x-pack/plugins/inference/common/tasks/nl_to_esql/correct_query_with_actions.ts new file mode 100644 index 0000000000000..15b050c3a3897 --- /dev/null +++ b/x-pack/plugins/inference/common/tasks/nl_to_esql/correct_query_with_actions.ts @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { validateQuery, getActions } from '@kbn/esql-validation-autocomplete'; +import { getAstAndSyntaxErrors } from '@kbn/esql-ast'; + +const fixedQueryByOneAction = async (queryString: string) => { + const { errors } = await validateQuery(queryString, getAstAndSyntaxErrors, { + ignoreOnMissingCallbacks: true, + }); + + const actions = await getActions(queryString, errors, getAstAndSyntaxErrors, { + relaxOnMissingCallbacks: true, + }); + + if (actions.length) { + const [firstAction] = actions; + const range = firstAction.edits[0].range; + const correctText = firstAction.edits[0].text; + const problematicString = queryString.substring(range.startColumn - 1, range.endColumn - 1); + const fixedQuery = queryString.replace(problematicString, correctText); + + return { + query: fixedQuery, + shouldRunAgain: Boolean(actions.length), + }; + } + return { + query: queryString, + shouldRunAgain: false, + }; +}; + +/** + * @param queryString + * @returns corrected queryString + * The cases that are handled are: + * - Query stats / eval functions have typos e.g. aveg instead of avg + * - Unquoted fields e.g. keep field-1 instead of keep `field-1` + * - Unquoted fields in stats or eval e.g. stats avg(field-1) instead of stats avg(`field-1`) + * - Combination of the above + */ + +export const correctQueryWithActions = async (queryString: string) => { + let shouldCorrectQuery = true; + let fixedQuery = queryString; + // this is an escape hatch, the loop will end automatically if the ast doesnt return more actions + // in case it goes wrong, we allow it to loop 10 times + let limit = 10; + + while (shouldCorrectQuery && limit >= 0) { + const { query, shouldRunAgain } = await fixedQueryByOneAction(fixedQuery); + shouldCorrectQuery = shouldRunAgain; + fixedQuery = query; + limit--; + } + + return fixedQuery; +}; diff --git a/x-pack/plugins/inference/common/tasks/nl_to_esql/get_errors_with_commands.test.ts b/x-pack/plugins/inference/common/tasks/nl_to_esql/get_errors_with_commands.test.ts new file mode 100644 index 0000000000000..04d99aede62b8 --- /dev/null +++ b/x-pack/plugins/inference/common/tasks/nl_to_esql/get_errors_with_commands.test.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { getErrorsWithCommands } from './get_errors_with_commands'; + +describe('getErrorsWithCommands', () => { + it('returns the command associated with the error', () => { + expect( + getErrorsWithCommands(`FROM logs-* | WHERE @timestamp <= NOW() | STATS BY host.name`, [ + { + type: 'error', + text: 'Syntax error', + code: '', + location: { + min: 24, + max: 36, + }, + }, + ]) + ).toEqual(['Error in `| WHERE @timestamp <= NOW()`:\n Syntax error']); + }); +}); diff --git a/x-pack/plugins/inference/common/tasks/nl_to_esql/get_errors_with_commands.ts b/x-pack/plugins/inference/common/tasks/nl_to_esql/get_errors_with_commands.ts new file mode 100644 index 0000000000000..b25c79161f79b --- /dev/null +++ b/x-pack/plugins/inference/common/tasks/nl_to_esql/get_errors_with_commands.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { EditorError, ESQLMessage } from '@kbn/esql-ast'; +import { splitIntoCommands } from './correct_common_esql_mistakes'; + +export function getErrorsWithCommands(query: string, errors: Array) { + const asCommands = splitIntoCommands(query); + + const errorMessages = errors.map((error) => { + if ('location' in error) { + const commandsUntilEndOfError = splitIntoCommands(query.substring(0, error.location.max)); + const lastCompleteCommand = asCommands[commandsUntilEndOfError.length - 1]; + if (lastCompleteCommand) { + return `Error in \`| ${lastCompleteCommand.command}\`:\n ${error.text}`; + } + } + return 'text' in error ? error.text : error.message; + }); + + return errorMessages; +} diff --git a/x-pack/plugins/inference/jest.config.js b/x-pack/plugins/inference/jest.config.js index 3bc2142bcdfc3..8b20bd72f17b2 100644 --- a/x-pack/plugins/inference/jest.config.js +++ b/x-pack/plugins/inference/jest.config.js @@ -8,7 +8,11 @@ module.exports = { preset: '@kbn/test', rootDir: '../../..', - roots: ['/x-pack/plugins/inference/public', '/x-pack/plugins/inference/server'], + roots: [ + '/x-pack/plugins/inference/public', + '/x-pack/plugins/inference/server', + '/x-pack/plugins/inference/common', + ], setupFiles: [], collectCoverage: true, collectCoverageFrom: [ diff --git a/x-pack/plugins/inference/public/util/http_response_into_observable.test.ts b/x-pack/plugins/inference/public/util/http_response_into_observable.test.ts index f50ec402bdd77..2b99b6f1db6f5 100644 --- a/x-pack/plugins/inference/public/util/http_response_into_observable.test.ts +++ b/x-pack/plugins/inference/public/util/http_response_into_observable.test.ts @@ -9,7 +9,7 @@ import { lastValueFrom, of, toArray } from 'rxjs'; import { httpResponseIntoObservable } from './http_response_into_observable'; import type { StreamedHttpResponse } from './create_observable_from_http_response'; import { ChatCompletionEventType } from '../../common/chat_complete'; -import { InferenceTaskEventType } from '../../common/tasks'; +import { InferenceTaskEventType } from '../../common/inference_task'; import { InferenceTaskErrorCode } from '../../common/errors'; function toSse(...events: Array>) { diff --git a/x-pack/plugins/inference/public/util/http_response_into_observable.ts b/x-pack/plugins/inference/public/util/http_response_into_observable.ts index 5b0929762e25d..53fdf302076a9 100644 --- a/x-pack/plugins/inference/public/util/http_response_into_observable.ts +++ b/x-pack/plugins/inference/public/util/http_response_into_observable.ts @@ -6,7 +6,7 @@ */ import { map, OperatorFunction, pipe, switchMap, tap } from 'rxjs'; -import { InferenceTaskEvent, InferenceTaskEventType } from '../../common/tasks'; +import { InferenceTaskEvent, InferenceTaskEventType } from '../../common/inference_task'; import { createObservableFromHttpResponse, StreamedHttpResponse, diff --git a/x-pack/plugins/inference/scripts/evaluation/.eslintrc.json b/x-pack/plugins/inference/scripts/evaluation/.eslintrc.json new file mode 100644 index 0000000000000..4eef2a5557280 --- /dev/null +++ b/x-pack/plugins/inference/scripts/evaluation/.eslintrc.json @@ -0,0 +1,17 @@ +{ + "overrides": [ + { + "files": [ + "**/*.spec.ts" + ], + "rules": { + "@kbn/imports/require_import": [ + "error", + "@kbn/ambient-ftr-types" + ], + "@typescript-eslint/triple-slash-reference": "off", + "spaced-comment": "off" + } + } + ] +} diff --git a/x-pack/plugins/inference/scripts/evaluation/README.md b/x-pack/plugins/inference/scripts/evaluation/README.md new file mode 100644 index 0000000000000..39ba3426ba0d4 --- /dev/null +++ b/x-pack/plugins/inference/scripts/evaluation/README.md @@ -0,0 +1,31 @@ +# Evaluation Framework + +## Overview + +This tool is developed for the teams working on anything related to inference. It simplifies scripting and evaluating various scenarios with the Large Language Model (LLM) integration. + +## Setup requirements + +- An Elasticsearch instance +- A Kibana instance +- At least one generative AI connector set up + +## Running evaluations + +Run the tool using: + +`$ node x-pack/plugins/inference/scripts/evaluation/index.js` + +This will evaluate all existing scenarios, and write the evaluation results to the terminal. + +### Configuration + +#### Kibana and Elasticsearch + +By default, the tool will look for a Kibana instance running locally (at `http://localhost:5601`, which is the default address for running Kibana in development mode). It will also attempt to read the Kibana config file for the Elasticsearch address & credentials. If you want to override these settings, use `--kibana` and `--es`. Only basic auth is supported, e.g. `--kibana http://username:password@localhost:5601`. If you want to use a specific space, use `--spaceId` + +#### Connector + +Use `--connectorId` to specify a generative AI connector to use. If none are given, it will prompt you to select a connector based on the ones that are available. If only a single supported connector is found, it will be used without prompting. + +Use `--evaluateWith` to specify the gen AI connector to use for evaluating the output of the task. By default, the same connector will be used. \ No newline at end of file diff --git a/x-pack/plugins/inference/scripts/evaluation/cli.ts b/x-pack/plugins/inference/scripts/evaluation/cli.ts new file mode 100644 index 0000000000000..d1cf133b0c4c0 --- /dev/null +++ b/x-pack/plugins/inference/scripts/evaluation/cli.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Argv } from 'yargs'; +import { connectorIdOption, elasticsearchOption, kibanaOption } from '../util/cli_options'; + +export enum EvaluateWith { + same = 'same', + other = 'other', +} + +export function options(y: Argv) { + return y + .option('files', { + string: true as const, + array: true, + describe: 'A file or list of files containing the scenarios to evaluate. Defaults to all', + }) + .option('grep', { + string: true, + array: false, + describe: 'A string or regex to filter scenarios by', + }) + .option('kibana', kibanaOption) + .option('spaceId', { + describe: 'The space to use.', + string: true, + array: false, + }) + .option('elasticsearch', elasticsearchOption) + .option('connectorId', connectorIdOption) + .option('logLevel', { + describe: 'Log level', + default: 'info', + }) + .option('evaluateWith', { + describe: + 'The connector ID to evaluate with. Leave empty for the same connector, use "other" to get a selection menu', + default: EvaluateWith.same, + }); +} diff --git a/x-pack/plugins/inference/scripts/evaluation/evaluation.ts b/x-pack/plugins/inference/scripts/evaluation/evaluation.ts new file mode 100644 index 0000000000000..0c70bf81eddad --- /dev/null +++ b/x-pack/plugins/inference/scripts/evaluation/evaluation.ts @@ -0,0 +1,190 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Client } from '@elastic/elasticsearch'; +import { run } from '@kbn/dev-cli-runner'; +import * as fastGlob from 'fast-glob'; +import yargs from 'yargs'; +import { castArray } from 'lodash'; +// @ts-expect-error +import Mocha from 'mocha'; +import Path from 'path'; +import { EvaluateWith, options } from './cli'; +import { getServiceUrls } from '../util/get_service_urls'; +import { KibanaClient } from '../util/kibana_client'; +import { initServices } from './services'; +import { EvaluationResult } from './types'; +import { selectConnector } from '../util/select_connector'; +import { createInferenceEvaluationClient } from './evaluation_client'; +import { createResultRenderer, renderFailedScenarios } from './table_renderer'; + +function runEvaluations() { + yargs(process.argv.slice(2)) + .command('*', 'Run Inference evaluations', options, (argv) => { + run( + async ({ log }) => { + const serviceUrls = await getServiceUrls({ + log, + elasticsearch: argv.elasticsearch, + kibana: argv.kibana, + }); + + const kibanaClient = new KibanaClient(log, serviceUrls.kibanaUrl, argv.spaceId); + const esClient = new Client({ + node: serviceUrls.esUrl, + }); + + await kibanaClient.createSpaceIfNeeded(); + + const connectors = await kibanaClient.getConnectors(); + if (!connectors.length) { + throw new Error('No connectors found'); + } + + const connector = await selectConnector({ + connectors, + preferredId: argv.connectorId, + log, + }); + + log.info(`Using connector ${connector.connectorId}`); + + const evaluationConnector = + argv.evaluateWith === EvaluateWith.same + ? connector + : await selectConnector({ + connectors, + preferredId: + argv.evaluateWith === EvaluateWith.other ? undefined : argv.evaluateWith, + log, + message: 'Select a connector for evaluation', + }); + + log.info(`Using connector ${evaluationConnector.connectorId} for evaluation`); + + const scenarios = + (argv.files !== undefined && + castArray(argv.files).map((file) => Path.join(process.cwd(), file))) || + fastGlob.sync(Path.join(__dirname, './scenarios/**/*.spec.ts')); + + if (!scenarios.length) { + throw new Error('No scenarios to run'); + } + + const mocha = new Mocha({ + grep: argv.grep, + timeout: '10m', + }); + + const chatClient = kibanaClient.createInferenceClient({ + connectorId: connector.connectorId, + }); + + const evaluationConnectorId = evaluationConnector.connectorId || connector.connectorId; + + const evaluationClient = createInferenceEvaluationClient({ + connectorId: connector.connectorId, + evaluationConnectorId, + outputApi: (id, parameters) => + chatClient.output(id, { + ...parameters, + connectorId: evaluationConnectorId, + }) as any, + suite: mocha.suite, + }); + + const renderer = createResultRenderer(); + const results: EvaluationResult[] = []; + const failedResults: EvaluationResult[] = []; + + evaluationClient.onResult((result) => { + results.push(result); + if (result.scores.filter((score) => score.score < 1).length) { + failedResults.push(result); + } + + log.write(renderer.render({ result })); + }); + + initServices({ + kibanaClient, + esClient, + chatClient, + evaluationClient, + logger: log, + }); + + for (const filename of scenarios) { + mocha.addFile(filename); + } + + return new Promise((resolve, reject) => { + mocha.run((failures: any) => { + if (failures) { + log.write(renderFailedScenarios(failedResults)); + reject(new Error(`Some tests failed`)); + return; + } + resolve(); + }); + }).finally(() => { + const modelScore = results + .flatMap((result) => result.scores) + .reduce( + (prev, result) => { + prev.score += result.score; + prev.total += 1; + return prev; + }, + { score: 0, total: 0 } + ); + + log.write('-------------------------------------------'); + log.write( + `Model ${connector.connectorId} scored ${modelScore.score} out of ${modelScore.total}` + ); + log.write('-------------------------------------------'); + + const scoresByCategory = results.reduce<{ + [key: string]: { + score: number; + total: number; + }; + }>((acc, result) => { + const category = result.category; + if (!acc[category]) { + acc[category] = { score: 0, total: 0 }; + } + result.scores.forEach((score) => { + acc[category].score += score.score; + acc[category].total += 1; + }); + return acc; + }, {}); + + log.write('-------------------------------------------'); + log.write(`Model ${connector.connectorId} scores per category`); + Object.entries(scoresByCategory).forEach(([category, { score, total }]) => { + log.write(`- category: ${category} - scored ${score} out of ${total}`); + }); + log.write('-------------------------------------------'); + }); + }, + { + log: { + defaultLevel: argv.logLevel as any, + }, + flags: { + allowUnexpected: true, + }, + } + ); + }) + .parse(); +} + +runEvaluations(); diff --git a/x-pack/plugins/inference/scripts/evaluation/evaluation_client.ts b/x-pack/plugins/inference/scripts/evaluation/evaluation_client.ts new file mode 100644 index 0000000000000..acf2fece1d0ff --- /dev/null +++ b/x-pack/plugins/inference/scripts/evaluation/evaluation_client.ts @@ -0,0 +1,181 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { remove } from 'lodash'; +import { lastValueFrom } from 'rxjs'; +import type { OutputAPI } from '../../common/output'; +import { withoutOutputUpdateEvents } from '../../common/output/without_output_update_events'; +import type { EvaluationResult } from './types'; + +export interface InferenceEvaluationClient { + getEvaluationConnectorId: () => string; + evaluate: (options: { + input: string; + criteria?: string[]; + system?: string; + }) => Promise; + getResults: () => EvaluationResult[]; + onResult: (cb: (result: EvaluationResult) => void) => () => void; + output: OutputAPI; +} + +export function createInferenceEvaluationClient({ + connectorId, + evaluationConnectorId, + suite, + outputApi, +}: { + connectorId: string; + evaluationConnectorId: string; + suite?: Mocha.Suite; + outputApi: OutputAPI; +}): InferenceEvaluationClient { + let currentTitle: string = ''; + let firstSuiteName: string = ''; + + // get the suite name + if (suite) { + suite.beforeEach(function () { + const currentTest: Mocha.Test = this.currentTest!; + const titles: string[] = []; + titles.push(this.currentTest!.title); + let parent = currentTest.parent; + while (parent) { + titles.push(parent.title); + parent = parent.parent; + } + currentTitle = titles.reverse().join(' '); + firstSuiteName = titles.filter((item) => item !== '')[0]; + }); + + suite.afterEach(function () { + currentTitle = ''; + }); + } + + const onResultCallbacks: Array<{ + callback: (result: EvaluationResult) => void; + unregister: () => void; + }> = []; + + const results: EvaluationResult[] = []; + + return { + output: outputApi, + getEvaluationConnectorId: () => evaluationConnectorId, + evaluate: async ({ input, criteria = [], system }) => { + const evaluation = await lastValueFrom( + outputApi('evaluate', { + connectorId, + system: withAdditionalSystemContext( + `You are a helpful, respected assistant for evaluating task + inputs and outputs in the Elastic Platform. + + Your goal is to verify whether the output of a task + succeeded, given the criteria. + + For each criterion, calculate a *score* between 0 (criterion fully failed) + and 1 (criterion fully succeeded), Fractional results (e.g. 0.5) are allowed, + if only part of the criterion succeeded. Explain your *reasoning* for the score, by + describing what the assistant did right, and describing and + quoting what the assistant did wrong, where it could improve, + and what the root cause was in case of a failure. + `, + system + ), + + input: ` + ## Criteria + + ${criteria + .map((criterion, index) => { + return `${index}: ${criterion}`; + }) + .join('\n')} + + ## Input + + ${input}`, + schema: { + type: 'object', + properties: { + criteria: { + type: 'array', + items: { + type: 'object', + properties: { + index: { + type: 'number', + description: 'The number of the criterion', + }, + score: { + type: 'number', + description: + 'The score you calculated for the criterion, between 0 (criterion fully failed) and 1 (criterion fully succeeded).', + }, + reasoning: { + type: 'string', + description: + 'Your reasoning for the score. Explain your score by mentioning what you expected to happen and what did happen.', + }, + }, + required: ['index', 'score', 'reasoning'], + }, + }, + }, + required: ['criteria'], + } as const, + }).pipe(withoutOutputUpdateEvents()) + ); + + const scoredCriteria = evaluation.output.criteria; + + const scores = scoredCriteria.map(({ index, score, reasoning }) => { + return { + criterion: criteria[index], + score, + reasoning, + }; + }); + + const result: EvaluationResult = { + name: currentTitle, + category: firstSuiteName, + input, + passed: scoredCriteria.every(({ score }) => score >= 1), + scores, + }; + + results.push(result); + + onResultCallbacks.forEach(({ callback }) => { + callback(result); + }); + + return result; + }, + getResults: () => results, + onResult: (callback) => { + const unregister = () => { + remove(onResultCallbacks, { callback }); + }; + onResultCallbacks.push({ callback, unregister }); + return unregister; + }, + }; +} + +const withAdditionalSystemContext = (system: string, additionalContext?: string): string => { + if (!additionalContext) { + return system; + } + + return [ + system, + `Here is some additional context that should help you for your evaluation:` + additionalContext, + ].join('\n\n'); +}; diff --git a/x-pack/plugins/inference/scripts/evaluation/index.js b/x-pack/plugins/inference/scripts/evaluation/index.js new file mode 100644 index 0000000000000..963e1a2ecfbed --- /dev/null +++ b/x-pack/plugins/inference/scripts/evaluation/index.js @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +require('@kbn/babel-register').install(); + +require('./evaluation'); diff --git a/x-pack/plugins/inference/scripts/evaluation/scenarios/esql/index.spec.ts b/x-pack/plugins/inference/scripts/evaluation/scenarios/esql/index.spec.ts new file mode 100644 index 0000000000000..dffca52b10836 --- /dev/null +++ b/x-pack/plugins/inference/scripts/evaluation/scenarios/esql/index.spec.ts @@ -0,0 +1,409 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/// + +import expect from '@kbn/expect'; +import { mapValues, pick } from 'lodash'; +import { firstValueFrom, lastValueFrom, filter } from 'rxjs'; +import { naturalLanguageToEsql } from '../../../../server/tasks/nl_to_esql'; +import { chatClient, evaluationClient, logger } from '../../services'; +import { loadDocuments } from '../../../../server/tasks/nl_to_esql/load_documents'; +import { isOutputCompleteEvent } from '../../../../common'; + +interface TestCase { + title: string; + question: string; + expected?: string; + criteria?: string[]; + only?: boolean; +} + +interface Section { + title: string; + tests: TestCase[]; +} + +const callNaturalLanguageToEsql = async (question: string) => { + return await lastValueFrom( + naturalLanguageToEsql({ + client: { + output: chatClient.output, + chatComplete: chatClient.chatComplete, + }, + connectorId: chatClient.getConnectorId(), + input: question, + logger: { + debug: (source) => { + logger.debug(typeof source === 'function' ? source() : source); + }, + }, + }) + ); +}; + +const expectedQueryCriteria = (expected: string) => { + return `The answer provides a ES|QL query that is functionally equivalent to: + + \`\`\`esql + ${expected} + \`\`\` + + It's OK if column names are slightly different, or if the used functions or operators are different, + as long as the expected end result is the same.`; +}; + +const retrieveUsedCommands = async ({ + question, + answer, + esqlDescription, +}: { + question: string; + answer: string; + esqlDescription: string; +}) => { + const commandsListOutput = await firstValueFrom( + evaluationClient + .output('retrieve_commands', { + connectorId: evaluationClient.getEvaluationConnectorId(), + system: ` + You are a helpful, respected Elastic ES|QL assistant. + + Your role is to enumerate the list of ES|QL commands and functions that were used + on a question and its answer. + + Only return each command or function once, even if they were used multiple times. + + The following extract of the ES|QL documentation lists all the commands and functions available: + + ${esqlDescription} + `, + input: ` + # Question + ${question} + + # Answer + ${answer} + `, + schema: { + type: 'object', + properties: { + commands: { + description: + 'The list of commands that were used in the provided ES|QL question and answer', + type: 'array', + items: { type: 'string' }, + }, + functions: { + description: + 'The list of functions that were used in the provided ES|QL question and answer', + type: 'array', + items: { type: 'string' }, + }, + }, + required: ['commands', 'functions'], + } as const, + }) + .pipe(filter(isOutputCompleteEvent)) + ); + + const output = commandsListOutput.output; + + const keywords = [ + ...(output.commands ?? []), + ...(output.functions ?? []), + 'SYNTAX', + 'OVERVIEW', + 'OPERATORS', + ].map((keyword) => keyword.toUpperCase()); + + return keywords; +}; + +async function evaluateEsqlQuery({ + question, + expected, + criteria = [], +}: { + question: string; + expected?: string; + criteria?: string[]; +}): Promise { + logger.debug(`Evaluation: ${question}`); + + const generateEvent = await callNaturalLanguageToEsql(question); + const answer = generateEvent.content!; + + logger.debug(`Received response: ${answer}`); + + const [systemMessage, esqlDocs] = await loadDocuments(); + + const usedCommands = await retrieveUsedCommands({ + question, + answer, + esqlDescription: systemMessage, + }); + + const requestedDocumentation = mapValues(pick(esqlDocs, usedCommands), ({ data }) => data); + + const evaluation = await evaluationClient.evaluate({ + input: ` + # Question + + ${question} + + # Answer + + ${generateEvent.content} + + `, + criteria: [...(expected ? [expectedQueryCriteria(expected)] : []), ...criteria], + system: ` + The assistant was asked to generate an ES|QL query based on the question from the user. + + Here is the documentation about the commands and function that are being used + in the ES|QL queries present in the question and the answer. + + ${Object.values(requestedDocumentation).join('\n\n')} + `, + }); + + expect(evaluation.passed).to.be(true); +} + +const buildTestDefinitions = (): Section[] => { + const testDefinitions: Section[] = [ + { + title: 'ES|QL query generation', + tests: [ + { + title: 'Generates a query to show the volume of logs over time', + question: `From the "kibana_sample_data_logs" index, show me the volume of logs per day over the last 10 days + Assume the following fields: + - @timestamp`, + expected: `FROM kibana_sample_data_logs + | WHERE @timestamp > NOW() - 10 days + | STATS volume = COUNT(*) BY BUCKET(@timestamp, 1 day)`, + }, + { + title: 'Generates a query to show employees filtered by name and grouped by hire_date', + question: `From the employees index, I want to see how many employees with a "B" in their first name + where hired each month over the past 2 years. + Assume the following fields: + - hire_date + - first_name + - last_name`, + expected: `FROM employees + | WHERE first_name LIKE "*B*" AND hire_date >= NOW() - 2 years + | STATS COUNT(*) BY BUCKET(hire_date, 1 month) + | SORT hire_date`, + }, + { + title: 'Generates a query to show employees which have a palindrome as last name', + question: `From the employees index, I want to find all employees with a palindrome as last name + (which can be read the same backward and forward), and then return their last name and first name + - last_name + - first_name`, + expected: `FROM employees + | EVAL reversed_last_name = REVERSE(last_name) + | WHERE TO_LOWER(last_name) == TO_LOWER(reversed_last_name) + | KEEP last_name, first_name`, + }, + { + title: 'Generates a query to show the top 10 domains by doc count', + question: `For standard Elastic ECS compliant packetbeat data view (\`packetbeat-*\`), + show me the top 10 unique destination.domain with the most docs. + Assume the following fields: + - destination.domain`, + expected: `FROM packetbeat-* + | STATS doc_count = COUNT(*) BY destination.domain + | SORT doc_count DESC + | LIMIT 10`, + }, + { + title: 'Generates a query to show the top 5 employees sorted ascending by hire date', + question: `From employees, I want to see the 5 earliest employees (hire_date), I want to display + only the month and the year that they were hired in and their employee number (emp_no). + Format the date as e.g. "September 2019".`, + expected: `FROM employees + | EVAL hire_date_formatted = DATE_FORMAT("MMMM YYYY", hire_date) + | SORT hire_date + | KEEP emp_no, hire_date_formatted + | LIMIT 5`, + }, + { + title: 'Explains that pagination is not supported when requested', + question: `From employees, I want to sort the documents by \`salary\`, + and then return 10 results per page, and then see the second page`, + criteria: [ + `The assistant should clearly mention that pagination is currently not supported in ES|QL. + It might provide a workaround, even if this should not be considered mandatory for this criteria.`, + ], + }, + { + title: + 'Generates a query to extract the year and shows 10 employees that were hired in 2024', + question: `From employees, extract the year from hire_date and show 10 employees hired in 2024`, + expected: `FROM employees + | WHERE DATE_EXTRACT("year", hire_date) == 2024 + | LIMIT 10`, + }, + { + title: 'Generates a query to calculate the avg cpu usage over the last 15m per service', + question: `Assume my metrics data is in \`metrics-*\`. I want to see what + a query would look like that gets the average CPU per service, + limit it to the top 10 results, in 1m buckets, and only the last 15m. + Assume the following fields: + - @timestamp + - system.cpu.total.norm.pct + - service.name`, + expected: `FROM metrics-* + | WHERE @timestamp >= NOW() - 15 minutes + | STATS avg_cpu = AVG(system.cpu.total.norm.pct) BY BUCKET(@timestamp, 1m), service.name + | SORT avg_cpu DESC + | LIMIT 10`, + }, + { + title: 'Generates a query to show logs volume with some complex multi-field filter', + question: `From the "sample_logs" index, show me the volume of daily logs + from source "foo" or "bar", with message longer than 62 chars and not containing "dolly", and with INFO level. + Assume the following fields: + - source + - message + - level + - @timestamp`, + expected: `│ FROM sample_logs + | WHERE source IN ("foo", "bar") AND LENGTH(message) > 62 AND message NOT LIKE "*dolly*" AND level == "INFO" + | STATS COUNT(*) BY day = BUCKET(@timestamp, 1d) + | SORT day ASC`, + }, + { + title: 'Generates a query to show normalized CPU per host', + question: `My data is in \`metricbeat*\`. Show me a query to see the percentage + of CPU time (system.cpu.system.pct) normalized by the number of CPU cores + (system.cpu.cores), broken down by host name. + Assume the following fields: + - system.cpu.system.pct + - system.cpu.cores + - host.name`, + expected: `FROM metricbeat* + | EVAL system_pct_normalized = TO_DOUBLE(system.cpu.system.pct) / system.cpu.cores + | STATS avg_system_pct_normalized = AVG(system_pct_normalized) BY host.name + | SORT host.name ASC`, + }, + { + title: 'Generates a query to show normalized CPU per host', + question: `From the access_logs index, I want to see the 50, 95 and 99 percentile + of response_time, break down by month over the past year. + Assume the following fields: + - @timestamp + - response_time + - status_code`, + expected: ` FROM access_logs + | WHERE @timestamp > NOW() - 1 year + | STATS p50 = PERCENTILE(response_time, 50), p95 = PERCENTILE(response_time, 95), p99 = PERCENTILE(response_time, 99) + BY month = BUCKET(@timestamp, 1 month) + | SORT month`, + }, + { + title: 'Generates a query using DISSECT to parse a postgres log message', + question: `Show me an example ESQL query to extract the query duration + from postgres log messages in postgres-logs*, with this format: + \`2021-01-01 00:00:00 UTC [12345]: [1-1] user=postgres,db=mydb,app=[unknown],client=127.0.0.1 LOG: duration: 123.456 ms statement: SELECT * FROM my_table\`. + Calculate the average. + Assume the following fields: + - message`, + expected: `FROM postgres-logs* + | DISSECT message "%{}: duration: %{query_duration} ms %{}" + | EVAL duration_double = TO_DOUBLE(duration) + | STATS AVG(duration_double)`, + }, + { + title: 'Generates a query using GROK to parse a postgres log message', + question: `Consider the postgres-logs index, with the message field having the following format: + \`2023-01-23T12:15:00.000Z ip=127.0.0.1 email=some.email@foo.com userid=42 [some message]\`. + Using GROK, please count the number of log entries for the email "test@org.com" for each month over last year + Assume the following fields: + - message`, + expected: `FROM postgres-logs + | GROK message "%{TIMESTAMP_ISO8601:timestamp} ip=%{IP:ip} email=%{EMAILADDRESS:email} userid=%{NUMBER:userid} [%{GREEDYDATA:log_message}]" + | WHERE email == "test@org.com" AND timestamp > NOW() - 1 year + | STATS COUNT(*) BY month = BUCKET(@timestamp, 1m) + | SORT month`, + }, + ], + }, + { + title: 'SPL to ESQL', + tests: [ + { + title: 'Converts a simple count query', + question: `Can you convert this SPL query to ESQL? \`index=network_firewall "SYN Timeout" | stats count by dest\``, + expected: `FROM network_firewall + | WHERE _raw == "SYN Timeout" + | STATS count = count(*) by dest`, + }, + { + title: 'Converts if/len to CASE/LENGTH', + question: `Can you convert this SPL query to ESQL? + \`index=prod_web + | eval length=len(message) + | eval k255=if((length>255),1,0) + | eval k2=if((length>2048),1,0) + | eval k4=if((length>4096),1,0) + | eval k16=if((length>16384),1,0) + | stats count, sum(k255), sum(k2),sum(k4),sum(k16), sum(length)\``, + expected: `FROM prod_web + | EVAL length = length(message), k255 = CASE(length > 255, 1, 0), k2 = CASE(length > 2048, 1, 0), k4 = CASE(length > 4096, 1, 0), k16 = CASE(length > 16384, 1, 0) + | STATS COUNT(*), SUM(k255), SUM(k2), SUM(k4), SUM(k16), SUM(length)`, + criteria: [ + 'The query provided by the Assistant uses the ESQL functions LENGTH and CASE, not the SPL functions len and if', + ], + }, + { + title: 'Converts matchers to NOT LIKE', + question: `Can you convert this SPL query to ESQL? + \`index=prod_web NOT "Connection reset" NOT "[acm-app] created a ThreadLocal" + sourcetype!=prod_urlf_east_logs sourcetype!=prod_urlf_west_logs + host!="dbs-tools-*" NOT "Public] in context with path [/global]" + host!="*dev*" host!="*qa*" host!="*uat*"\``, + expected: `FROM prod_web + | WHERE _raw NOT LIKE "Connection reset" + AND _raw NOT LIKE "[acm-app] created a ThreadLocal" + AND sourcetype != "prod_urlf_east_logs" + AND sourcetype != "prod_urlf_west_logs" + AND host NOT LIKE "dbs-tools-*" + AND _raw NOT LIKE "Public] in context with path [/global]" + AND host NOT LIKE "*dev*" + AND host NOT LIKE "*qa*" + AND host NOT LIKE "*uat*"`, + }, + ], + }, + ]; + + return testDefinitions; +}; + +const generateTestSuite = () => { + const testDefinitions = buildTestDefinitions(); + for (const section of testDefinitions) { + describe(`${section.title}`, () => { + for (const test of section.tests) { + (test.only ? it.only : it)(`${test.title}`, async () => { + await evaluateEsqlQuery({ + question: test.question, + expected: test.expected, + criteria: test.criteria, + }); + }); + } + }); + } +}; + +generateTestSuite(); diff --git a/x-pack/plugins/inference/scripts/evaluation/services/index.ts b/x-pack/plugins/inference/scripts/evaluation/services/index.ts new file mode 100644 index 0000000000000..5685a24e0df8b --- /dev/null +++ b/x-pack/plugins/inference/scripts/evaluation/services/index.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Client } from '@elastic/elasticsearch'; +import type { ToolingLog } from '@kbn/tooling-log'; +import type { ScriptInferenceClient, KibanaClient } from '../../util/kibana_client'; +import type { InferenceEvaluationClient } from '../evaluation_client'; + +// make services globals so they can more easily be used in the tests + +function createErrorThrowingProxy(name: string): any { + return new Proxy( + {}, + { + get: () => { + throw new Error(`${name} has not been instantiated yet`); + }, + set: () => { + throw new Error(`${name} has not been instantiated yet`); + }, + } + ); +} + +export let chatClient: ScriptInferenceClient = createErrorThrowingProxy('evaluationClient'); +export let evaluationClient: InferenceEvaluationClient = + createErrorThrowingProxy('evaluationClient'); + +export let esClient: Client = createErrorThrowingProxy('esClient'); +export let kibanaClient: KibanaClient = createErrorThrowingProxy('kibanaClient'); +export let logger: ToolingLog = createErrorThrowingProxy('logger'); + +export const initServices = (services: { + chatClient: ScriptInferenceClient; + evaluationClient: InferenceEvaluationClient; + esClient: Client; + kibanaClient: KibanaClient; + logger: ToolingLog; +}) => { + chatClient = services.chatClient; + evaluationClient = services.evaluationClient; + esClient = services.esClient; + kibanaClient = services.kibanaClient; + logger = services.logger; +}; diff --git a/x-pack/plugins/inference/scripts/evaluation/table_renderer.ts b/x-pack/plugins/inference/scripts/evaluation/table_renderer.ts new file mode 100644 index 0000000000000..48e09b96457dc --- /dev/null +++ b/x-pack/plugins/inference/scripts/evaluation/table_renderer.ts @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as table from 'table'; +import chalk from 'chalk'; +import type { TableUserConfig } from 'table'; +import type { EvaluationResult } from './types'; + +interface ResultRenderer { + render: (params: { result: EvaluationResult }) => string; +} + +export const createResultRenderer = (): ResultRenderer => { + const config = { + ...baseConfig, + spanningCells: [ + { row: 0, col: 0, colSpan: 3 }, + { row: 1, col: 0, colSpan: 3 }, + ], + columns: [{ wrapWord: true, width: 60 }, { wrapWord: true }, { wrapWord: true, width: 60 }], + }; + const header = [chalk.bold('Criterion'), chalk.bold('Result'), chalk.bold('Reasoning')]; + + return { + render: ({ result }) => { + const rows: string[][] = [[sanitize(result.input), '', ''], ['', '', ''], header]; + result.scores.forEach((score) => { + rows.push([ + sanitize(score.criterion), + score.score < 1 + ? chalk.redBright(String(score.score)) + : chalk.greenBright(String(score.score)), + sanitize(score.reasoning), + ]); + }); + return table.table(rows, config); + }, + }; +}; + +export const renderFailedScenarios = (failedScenario: EvaluationResult[]): string => { + const config = { + ...baseConfig, + spanningCells: [], + columns: [{ wrapWord: true, width: 60 }, { wrapWord: true }, { wrapWord: true, width: 60 }], + }; + const rows: string[][] = [ + ['Failed Tests', '', ''], + ['Scenario', 'Scores', 'Reasoning'], + ]; + + failedScenario.forEach((result) => { + const totalResults = result.scores.length; + const failedResults = result.scores.filter((score) => score.score < 1).length; + + const reasoningConcat = result.scores.map((score) => sanitize(score.reasoning)).join(' '); + rows.push([ + `${result.name}`, + `Average score ${Math.round( + (result.scores.reduce((total, next) => total + next.score, 0) * 100) / totalResults + )}. Failed ${failedResults} tests out of ${totalResults}`, + `Reasoning: ${reasoningConcat}`, + ]); + }); + + return table.table(rows, config); +}; + +const baseConfig: TableUserConfig = { + singleLine: false, + border: { + topBody: `─`, + topJoin: `┬`, + topLeft: `┌`, + topRight: `┐`, + + bottomBody: `─`, + bottomJoin: `┴`, + bottomLeft: `└`, + bottomRight: `┘`, + + bodyLeft: `│`, + bodyRight: `│`, + bodyJoin: `│`, + + joinBody: `─`, + joinLeft: `├`, + joinRight: `┤`, + joinJoin: `┼`, + }, +}; + +const sanitize = (text: string) => { + // table really doesn't like leading whitespaces and empty lines... + return text.replace(/^ +/gm, '').replace(/ +$/gm, '').replace(/^\n+/g, '').replace(/\n+$/g, ''); +}; diff --git a/x-pack/plugins/inference/scripts/evaluation/types.ts b/x-pack/plugins/inference/scripts/evaluation/types.ts new file mode 100644 index 0000000000000..cd6485306344e --- /dev/null +++ b/x-pack/plugins/inference/scripts/evaluation/types.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Client } from '@elastic/elasticsearch'; +import type { ScriptInferenceClient, KibanaClient } from '../util/kibana_client'; +import type { InferenceEvaluationClient } from './evaluation_client'; + +export interface ScenarioOptions { + esClient: Client; + kibanaClient: KibanaClient; + chatClient: ScriptInferenceClient; + evaluationClient: InferenceEvaluationClient; +} + +export interface EvaluationResult { + name: string; + category: string; + input: string; + passed: boolean; + scores: Array<{ + criterion: string; + reasoning: string; + score: number; + }>; +} + +export type EvaluationFunction = (options: ScenarioOptions) => Promise; diff --git a/x-pack/plugins/inference/scripts/load_esql_docs/README.md b/x-pack/plugins/inference/scripts/load_esql_docs/README.md new file mode 100644 index 0000000000000..06b6cf22aa49d --- /dev/null +++ b/x-pack/plugins/inference/scripts/load_esql_docs/README.md @@ -0,0 +1,10 @@ +# Load ES|QL docs + +This script loads ES|QL documentation from the `built-docs` repo, and rewrites it with help of the LLM. +The generated documentation is validated and will emit warnings when invalid queries have been generated. + +## Requirements + +- checked out `built-docs` repo in the same folder as the `kibana` repository +- a running Kibana instance +- an installed Generative AI connector diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/scripts/load_esql_docs/extract_sections.ts b/x-pack/plugins/inference/scripts/load_esql_docs/extract_sections.ts similarity index 100% rename from x-pack/plugins/observability_solution/observability_ai_assistant_app/scripts/load_esql_docs/extract_sections.ts rename to x-pack/plugins/inference/scripts/load_esql_docs/extract_sections.ts diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/scripts/load_esql_docs/index.js b/x-pack/plugins/inference/scripts/load_esql_docs/index.js similarity index 100% rename from x-pack/plugins/observability_solution/observability_ai_assistant_app/scripts/load_esql_docs/index.js rename to x-pack/plugins/inference/scripts/load_esql_docs/index.js diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/scripts/load_esql_docs/load_esql_docs.ts b/x-pack/plugins/inference/scripts/load_esql_docs/load_esql_docs.ts similarity index 91% rename from x-pack/plugins/observability_solution/observability_ai_assistant_app/scripts/load_esql_docs/load_esql_docs.ts rename to x-pack/plugins/inference/scripts/load_esql_docs/load_esql_docs.ts index b8a90c2b62e46..3250d06906905 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/scripts/load_esql_docs/load_esql_docs.ts +++ b/x-pack/plugins/inference/scripts/load_esql_docs/load_esql_docs.ts @@ -5,30 +5,31 @@ * 2.0. */ import { run } from '@kbn/dev-cli-runner'; +import { ESQLMessage, EditorError, getAstAndSyntaxErrors } from '@kbn/esql-ast'; +import { validateQuery } from '@kbn/esql-validation-autocomplete'; import $, { load } from 'cheerio'; import { SingleBar } from 'cli-progress'; import FastGlob from 'fast-glob'; import Fs from 'fs/promises'; -import { once, partition, compact } from 'lodash'; +import { compact, once, partition } from 'lodash'; import pLimit from 'p-limit'; import Path from 'path'; import git, { SimpleGitProgressEvent } from 'simple-git'; import yargs, { Argv } from 'yargs'; -import { MessageRole } from '@kbn/observability-ai-assistant-plugin/common'; -import { validateQuery } from '@kbn/esql-validation-autocomplete'; -import { EditorError, ESQLMessage, getAstAndSyntaxErrors } from '@kbn/esql-ast'; -import { connectorIdOption, elasticsearchOption, kibanaOption } from '../evaluation/cli'; -import { getServiceUrls } from '../evaluation/get_service_urls'; -import { KibanaClient } from '../evaluation/kibana_client'; -import { selectConnector } from '../evaluation/select_connector'; +import { lastValueFrom } from 'rxjs'; +import { REPO_ROOT } from '@kbn/repo-info'; +import { INLINE_ESQL_QUERY_REGEX } from '../../common/tasks/nl_to_esql/constants'; +import { correctCommonEsqlMistakes } from '../../common/tasks/nl_to_esql/correct_common_esql_mistakes'; +import { connectorIdOption, elasticsearchOption, kibanaOption } from '../util/cli_options'; +import { getServiceUrls } from '../util/get_service_urls'; +import { KibanaClient } from '../util/kibana_client'; +import { selectConnector } from '../util/select_connector'; import { extractSections } from './extract_sections'; -import { correctCommonEsqlMistakes } from '../../server/functions/query/correct_common_esql_mistakes'; -import { INLINE_ESQL_QUERY_REGEX } from '../../server/functions/query/constants'; yargs(process.argv.slice(2)) .command( '*', - 'Extract ES|QL documentation for the Observability AI Assistant', + 'Extract ES|QL documentation', (y: Argv) => y .option('logLevel', { @@ -73,15 +74,13 @@ yargs(process.argv.slice(2)) log, }); - const chatClient = kibanaClient.createChatClient({ - connectorId: connector.id, - evaluationConnectorId: connector.id, - persist: false, + const chatClient = kibanaClient.createInferenceClient({ + connectorId: connector.connectorId, }); - log.info(`Using connector ${connector.id}`); + log.info(`Using connector ${connector.connectorId}`); - const builtDocsDir = Path.join(__dirname, '../../../../../../../built-docs'); + const builtDocsDir = Path.join(REPO_ROOT, '../built-docs'); log.debug(`Looking in ${builtDocsDir} for built-docs repository`); @@ -329,10 +328,7 @@ yargs(process.argv.slice(2)) return !isOverviewArticle; }); - const outDir = Path.join( - __dirname, - '../../../observability_ai_assistant_app/server/functions/query/esql_docs' - ); + const outDir = Path.join(__dirname, '../../server/tasks/nl_to_esql/esql_docs'); if (!argv.dryRun) { log.info(`Writing ${flattened.length} documents to disk to ${outDir}`); @@ -341,9 +337,12 @@ yargs(process.argv.slice(2)) if (!argv.only && !argv.dryRun) { log.debug(`Clearing ${outDir}`); - await Fs.rm(outDir, { recursive: true }).catch((error) => - error.code === 'ENOENT' ? Promise.resolve() : error - ); + await Fs.readdir(outDir, { recursive: true }) + .then((filesInDir) => { + const limiter = pLimit(10); + return Promise.all(filesInDir.map((file) => limiter(() => Fs.unlink(file)))); + }) + .catch((error) => (error.code === 'ENOENT' ? Promise.resolve() : error)); } if (!argv.dryRun) { @@ -428,10 +427,10 @@ yargs(process.argv.slice(2)) return chatLimiter(async () => { try { - const response = await chatClient.chat([ - { - role: MessageRole.System, - content: `## System instructions + const response = await lastValueFrom( + chatClient.output('generate_markdown', { + connectorId: chatClient.getConnectorId(), + system: `## System instructions Your job is to generate Markdown documentation off of content that is scraped from the Elasticsearch website. @@ -462,10 +461,7 @@ yargs(process.argv.slice(2)) ${allContent} `, - }, - { - role: MessageRole.User, - content: `Generate Markdown for the following document: + input: `Generate Markdown for the following document: ## ${doc.title} @@ -476,8 +472,8 @@ yargs(process.argv.slice(2)) ### Content of file ${doc.content}`, - }, - ]); + }) + ); return fsLimiter(() => writeFile({ title: doc.title, content: response.content! }) diff --git a/x-pack/plugins/inference/scripts/util/cli_options.ts b/x-pack/plugins/inference/scripts/util/cli_options.ts new file mode 100644 index 0000000000000..13bac131922ff --- /dev/null +++ b/x-pack/plugins/inference/scripts/util/cli_options.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { format, parse } from 'url'; +import { readKibanaConfig } from './read_kibana_config'; + +const config = readKibanaConfig(); + +export const kibanaOption = { + describe: 'Where Kibana is running', + string: true as const, + default: process.env.KIBANA_HOST || 'http://localhost:5601', +}; +export const elasticsearchOption = { + alias: 'es', + describe: 'Where Elasticsearch is running', + string: true as const, + default: format({ + ...parse(config['elasticsearch.hosts']), + auth: `${config['elasticsearch.username']}:${config['elasticsearch.password']}`, + }), +}; + +export const connectorIdOption = { + describe: 'The ID of the connector', + string: true as const, +}; diff --git a/x-pack/plugins/inference/scripts/util/get_service_urls.ts b/x-pack/plugins/inference/scripts/util/get_service_urls.ts new file mode 100644 index 0000000000000..859b47987a470 --- /dev/null +++ b/x-pack/plugins/inference/scripts/util/get_service_urls.ts @@ -0,0 +1,148 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ToolingLog } from '@kbn/tooling-log'; +import { omit } from 'lodash'; +import fetch from 'node-fetch'; +import { format, parse, Url } from 'url'; + +async function discoverAuth(parsedTarget: Url, log: ToolingLog) { + const possibleCredentials = [`admin:changeme`, `elastic:changeme`]; + for (const auth of possibleCredentials) { + const url = format({ + ...parsedTarget, + auth, + }); + let status: number; + try { + log.debug(`Fetching ${url}`); + const response = await fetch(url); + status = response.status; + } catch (err) { + log.debug(`${url} resulted in ${err.message}`); + status = 0; + } + + if (status === 200) { + return auth; + } + } + + throw new Error(`Failed to authenticate user for ${format(parsedTarget)}`); +} + +async function getKibanaUrl({ kibana, log }: { kibana: string; log: ToolingLog }) { + try { + const isCI = process.env.CI?.toLowerCase() === 'true'; + + const parsedKibanaUrl = parse(kibana); + + const kibanaUrlWithoutAuth = format(omit(parsedKibanaUrl, 'auth')); + + log.debug(`Checking Kibana URL ${kibanaUrlWithoutAuth} for a redirect`); + + const unredirectedResponse = await fetch(kibanaUrlWithoutAuth, { + headers: { + ...(parsedKibanaUrl.auth + ? { Authorization: `Basic ${Buffer.from(parsedKibanaUrl.auth).toString('base64')}` } + : {}), + }, + method: 'HEAD', + follow: 1, + redirect: 'manual', + }); + + log.debug('Unredirected response', unredirectedResponse.headers.get('location')); + + const discoveredKibanaUrl = + unredirectedResponse.headers + .get('location') + ?.replace('/spaces/enter', '') + ?.replace('spaces/space_selector', '') || kibanaUrlWithoutAuth; + + log.debug(`Discovered Kibana URL at ${discoveredKibanaUrl}`); + + const parsedTarget = parse(kibana); + + const parsedDiscoveredUrl = parse(discoveredKibanaUrl); + + const discoveredKibanaUrlWithAuth = format({ + ...parsedDiscoveredUrl, + auth: parsedTarget.auth, + }); + + const redirectedResponse = await fetch(discoveredKibanaUrlWithAuth, { + method: 'HEAD', + }); + + if (redirectedResponse.status !== 200) { + throw new Error( + `Expected HTTP 200 from ${discoveredKibanaUrlWithAuth}, got ${redirectedResponse.status}` + ); + } + + const discoveredKibanaUrlWithoutAuth = format({ + ...parsedDiscoveredUrl, + auth: undefined, + }); + + log.info( + `Discovered kibana running at: ${ + isCI ? discoveredKibanaUrlWithoutAuth : discoveredKibanaUrlWithAuth + }` + ); + + return discoveredKibanaUrlWithAuth.replace(/\/$/, ''); + } catch (error) { + throw new Error(`Could not connect to Kibana: ` + error.message); + } +} + +export async function getServiceUrls({ + log, + elasticsearch, + kibana, +}: { + elasticsearch: string; + kibana: string; + log: ToolingLog; +}) { + if (!elasticsearch) { + // assume things are running locally + kibana = kibana || 'http://127.0.0.1:5601'; + elasticsearch = 'http://127.0.0.1:9200'; + } + + const parsedTarget = parse(elasticsearch); + + let auth = parsedTarget.auth; + + if (!parsedTarget.auth) { + auth = await discoverAuth(parsedTarget, log); + } + + const formattedEsUrl = format({ + ...parsedTarget, + auth, + }); + + const suspectedKibanaUrl = kibana || elasticsearch.replace('.es', '.kb'); + + const parsedKibanaUrl = parse(suspectedKibanaUrl); + + const kibanaUrlWithAuth = format({ + ...parsedKibanaUrl, + auth: parsedKibanaUrl.auth || auth, + }); + + const validatedKibanaUrl = await getKibanaUrl({ kibana: kibanaUrlWithAuth, log }); + + return { + kibanaUrl: validatedKibanaUrl, + esUrl: formattedEsUrl, + }; +} diff --git a/x-pack/plugins/inference/scripts/util/kibana_client.ts b/x-pack/plugins/inference/scripts/util/kibana_client.ts new file mode 100644 index 0000000000000..f8e96cfcf4ae3 --- /dev/null +++ b/x-pack/plugins/inference/scripts/util/kibana_client.ts @@ -0,0 +1,226 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ToolingLog } from '@kbn/tooling-log'; +import axios, { AxiosInstance, AxiosResponse, isAxiosError } from 'axios'; +import { IncomingMessage } from 'http'; +import { omit, pick } from 'lodash'; +import { from, map, switchMap, throwError } from 'rxjs'; +import { UrlObject, format, parse } from 'url'; +import { inspect } from 'util'; +import { isReadable } from 'stream'; +import type { ChatCompleteAPI, ChatCompletionEvent } from '../../common/chat_complete'; +import { ChatCompleteRequestBody } from '../../common/chat_complete/request'; +import type { InferenceConnector } from '../../common/connectors'; +import { + InferenceTaskError, + InferenceTaskErrorEvent, + createInferenceInternalError, +} from '../../common/errors'; +import { InferenceTaskEventType } from '../../common/inference_task'; +import type { OutputAPI } from '../../common/output'; +import { createOutputApi } from '../../common/output/create_output_api'; +import { withoutOutputUpdateEvents } from '../../common/output/without_output_update_events'; +import { eventSourceStreamIntoObservable } from '../../server/util/event_source_stream_into_observable'; + +// eslint-disable-next-line spaced-comment +/// + +export interface ScriptInferenceClient { + getConnectorId: () => string; + chatComplete: ChatCompleteAPI; + output: OutputAPI; +} + +export class KibanaClient { + axios: AxiosInstance; + constructor( + private readonly log: ToolingLog, + private readonly url: string, + private readonly spaceId?: string + ) { + this.axios = axios.create({ + headers: { + 'kbn-xsrf': 'foo', + }, + }); + } + + private getUrl(props: { query?: UrlObject['query']; pathname: string; ignoreSpaceId?: boolean }) { + const parsed = parse(this.url); + + const baseUrl = parsed.pathname?.replaceAll('/', '') ?? ''; + + const url = format({ + ...parsed, + pathname: `/${[ + ...(baseUrl ? [baseUrl] : []), + ...(props.ignoreSpaceId || !this.spaceId ? [] : ['s', this.spaceId]), + props.pathname.startsWith('/') ? props.pathname.substring(1) : props.pathname, + ].join('/')}`, + query: props.query, + }); + + return url; + } + + callKibana( + method: string, + props: { query?: UrlObject['query']; pathname: string; ignoreSpaceId?: boolean }, + data?: any + ) { + const url = this.getUrl(props); + return this.axios({ + method, + url, + data: data || {}, + headers: { + 'kbn-xsrf': 'true', + 'x-elastic-internal-origin': 'foo', + }, + }).catch((error) => { + if (isAxiosError(error)) { + const interestingPartsOfError = { + ...omit(error, 'request', 'response', 'config'), + ...pick( + error, + 'response.data', + 'response.headers', + 'response.status', + 'response.statusText' + ), + }; + this.log.error(inspect(interestingPartsOfError, { depth: 10 })); + } + throw error; + }); + } + + async createSpaceIfNeeded() { + if (!this.spaceId) { + return; + } + + this.log.debug(`Checking if space ${this.spaceId} exists`); + + const spaceExistsResponse = await this.callKibana<{ + id?: string; + }>('GET', { + pathname: `/api/spaces/space/${this.spaceId}`, + ignoreSpaceId: true, + }).catch((error) => { + if (isAxiosError(error) && error.response?.status === 404) { + return { + status: 404, + data: { + id: undefined, + }, + }; + } + throw error; + }); + + if (spaceExistsResponse.data.id) { + this.log.debug(`Space id ${this.spaceId} found`); + return; + } + + this.log.info(`Creating space ${this.spaceId}`); + + const spaceCreatedResponse = await this.callKibana<{ id: string }>( + 'POST', + { + pathname: '/api/spaces/space', + ignoreSpaceId: true, + }, + { + id: this.spaceId, + name: this.spaceId, + } + ); + + if (spaceCreatedResponse.status === 200) { + this.log.info(`Created space ${this.spaceId}`); + } else { + throw new Error( + `Error creating space: ${spaceCreatedResponse.status} - ${spaceCreatedResponse.data}` + ); + } + } + + createInferenceClient({ connectorId }: { connectorId: string }): ScriptInferenceClient { + function stream(responsePromise: Promise) { + return from(responsePromise).pipe( + switchMap((response) => { + if (isReadable(response.data)) { + return eventSourceStreamIntoObservable(response.data as IncomingMessage); + } + return throwError(() => createInferenceInternalError('Unexpected error', response.data)); + }), + map((line) => { + return JSON.parse(line) as ChatCompletionEvent | InferenceTaskErrorEvent; + }), + map((line) => { + if (line.type === InferenceTaskEventType.error) { + throw new InferenceTaskError(line.error.code, line.error.message, line.error.meta); + } + return line; + }) + ); + } + + const chatCompleteApi: ChatCompleteAPI = ({ + connectorId: chatCompleteConnectorId, + messages, + system, + toolChoice, + tools, + }) => { + const body: ChatCompleteRequestBody = { + connectorId: chatCompleteConnectorId, + system, + messages, + toolChoice, + tools, + }; + + return stream( + this.axios.post( + this.getUrl({ + pathname: `/internal/inference/chat_complete`, + }), + body, + { responseType: 'stream', timeout: NaN } + ) + ); + }; + + const outputApi: OutputAPI = createOutputApi(chatCompleteApi); + + return { + getConnectorId: () => connectorId, + chatComplete: (options) => { + return chatCompleteApi({ + ...options, + }); + }, + output: (id, options) => { + return outputApi(id, { ...options }).pipe(withoutOutputUpdateEvents()); + }, + }; + } + + async getConnectors() { + const connectors: AxiosResponse<{ connectors: InferenceConnector[] }> = await axios.get( + this.getUrl({ + pathname: '/internal/inference/connectors', + }) + ); + + return connectors.data.connectors; + } +} diff --git a/x-pack/plugins/inference/scripts/util/read_kibana_config.ts b/x-pack/plugins/inference/scripts/util/read_kibana_config.ts new file mode 100644 index 0000000000000..5b64bb2f56189 --- /dev/null +++ b/x-pack/plugins/inference/scripts/util/read_kibana_config.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import path from 'path'; +import fs from 'fs'; +import yaml from 'js-yaml'; +import { identity, pickBy } from 'lodash'; + +export type KibanaConfig = ReturnType; + +export const readKibanaConfig = () => { + const kibanaConfigDir = path.join(__filename, '../../../../../../config'); + const kibanaDevConfig = path.join(kibanaConfigDir, 'kibana.dev.yml'); + const kibanaConfig = path.join(kibanaConfigDir, 'kibana.yml'); + + const loadedKibanaConfig = (yaml.safeLoad( + fs.readFileSync(fs.existsSync(kibanaDevConfig) ? kibanaDevConfig : kibanaConfig, 'utf8') + ) || {}) as {}; + + const cliEsCredentials = pickBy( + { + 'elasticsearch.username': process.env.ELASTICSEARCH_USERNAME, + 'elasticsearch.password': process.env.ELASTICSEARCH_PASSWORD, + 'elasticsearch.hosts': process.env.ELASTICSEARCH_HOST, + }, + identity + ) as { + 'elasticsearch.username'?: string; + 'elasticsearch.password'?: string; + 'elasticsearch.hosts'?: string; + }; + + return { + 'elasticsearch.hosts': 'http://localhost:9200', + 'elasticsearch.username': 'elastic', + 'elasticsearch.password': 'changeme', + ...loadedKibanaConfig, + ...cliEsCredentials, + }; +}; diff --git a/x-pack/plugins/inference/scripts/util/select_connector.ts b/x-pack/plugins/inference/scripts/util/select_connector.ts new file mode 100644 index 0000000000000..b5dc8ae2d4549 --- /dev/null +++ b/x-pack/plugins/inference/scripts/util/select_connector.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import inquirer from 'inquirer'; +import { ToolingLog } from '@kbn/tooling-log'; +import { KibanaClient } from './kibana_client'; + +export async function selectConnector({ + connectors, + preferredId, + log, + message = 'Select a connector', +}: { + connectors: Awaited>; + preferredId?: string; + log: ToolingLog; + message?: string; +}) { + let connector = connectors.find((item) => item.connectorId === preferredId); + + if (!connector && preferredId) { + log.warning(`Could not find connector ${preferredId}`); + } + + if (!connector && connectors.length === 1) { + connector = connectors[0]; + log.debug('Using the only connector found'); + } else if (!connector) { + const connectorChoice = await inquirer.prompt({ + type: 'list', + name: 'connector', + message, + choices: connectors.map((item) => ({ + name: `${item.name} (${item.connectorId})`, + value: item.connectorId, + })), + }); + + connector = connectors.find((item) => item.connectorId === connectorChoice.connector)!; + } + + return connector; +} diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/bedrock_claude_adapter.test.ts b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/bedrock_claude_adapter.test.ts index 1d25f09dce3bc..d34b8693cb85f 100644 --- a/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/bedrock_claude_adapter.test.ts +++ b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/bedrock_claude_adapter.test.ts @@ -6,6 +6,7 @@ */ import { PassThrough } from 'stream'; +import { loggerMock } from '@kbn/logging-mocks'; import type { InferenceExecutor } from '../../utils/inference_executor'; import { MessageRole } from '../../../../common/chat_complete'; import { ToolChoiceType } from '../../../../common/chat_complete/tools'; @@ -13,6 +14,7 @@ import { bedrockClaudeAdapter } from './bedrock_claude_adapter'; import { addNoToolUsageDirective } from './prompts'; describe('bedrockClaudeAdapter', () => { + const logger = loggerMock.create(); const executorMock = { invoke: jest.fn(), } as InferenceExecutor & { invoke: jest.MockedFn }; @@ -41,6 +43,7 @@ describe('bedrockClaudeAdapter', () => { describe('#chatComplete()', () => { it('calls `executor.invoke` with the right fixed parameters', () => { bedrockClaudeAdapter.chatComplete({ + logger, executor: executorMock, messages: [ { @@ -62,6 +65,16 @@ describe('bedrockClaudeAdapter', () => { ], temperature: 0, stopSequences: ['\n\nHuman:'], + tools: [ + { + description: 'Do not call this tool, it is strictly forbidden', + input_schema: { + properties: {}, + type: 'object', + }, + name: 'do_not_call_this_tool', + }, + ], }, }); }); @@ -69,6 +82,7 @@ describe('bedrockClaudeAdapter', () => { it('correctly format tools', () => { bedrockClaudeAdapter.chatComplete({ executor: executorMock, + logger, messages: [ { role: MessageRole.User, @@ -127,6 +141,7 @@ describe('bedrockClaudeAdapter', () => { it('correctly format messages', () => { bedrockClaudeAdapter.chatComplete({ executor: executorMock, + logger, messages: [ { role: MessageRole.User, @@ -225,6 +240,7 @@ describe('bedrockClaudeAdapter', () => { it('correctly format system message', () => { bedrockClaudeAdapter.chatComplete({ executor: executorMock, + logger, system: 'Some system message', messages: [ { @@ -243,6 +259,7 @@ describe('bedrockClaudeAdapter', () => { it('correctly format tool choice', () => { bedrockClaudeAdapter.chatComplete({ executor: executorMock, + logger, messages: [ { role: MessageRole.User, @@ -263,6 +280,7 @@ describe('bedrockClaudeAdapter', () => { it('correctly format tool choice for named function', () => { bedrockClaudeAdapter.chatComplete({ executor: executorMock, + logger, messages: [ { role: MessageRole.User, @@ -284,6 +302,7 @@ describe('bedrockClaudeAdapter', () => { it('correctly adapt the request for ToolChoiceType.None', () => { bedrockClaudeAdapter.chatComplete({ executor: executorMock, + logger, system: 'some system instruction', messages: [ { diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/bedrock_claude_adapter.ts b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/bedrock_claude_adapter.ts index 5a03dc04347b1..a0b48e6fc8631 100644 --- a/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/bedrock_claude_adapter.ts +++ b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/bedrock_claude_adapter.ts @@ -7,7 +7,6 @@ import { filter, from, map, switchMap, tap } from 'rxjs'; import { Readable } from 'stream'; -import type { InvokeAIActionParams } from '@kbn/stack-connectors-plugin/common/bedrock/types'; import { parseSerdeChunkMessage } from './serde_utils'; import { Message, MessageRole } from '../../../../common/chat_complete'; import { createInferenceInternalError } from '../../../../common/errors'; @@ -20,15 +19,16 @@ import { } from './serde_eventstream_into_observable'; import { processCompletionChunks } from './process_completion_chunks'; import { addNoToolUsageDirective } from './prompts'; +import { ToolSchemaType } from '../../../../common/chat_complete/tool_schema'; export const bedrockClaudeAdapter: InferenceConnectorAdapter = { chatComplete: ({ executor, system, messages, toolChoice, tools }) => { const noToolUsage = toolChoice === ToolChoiceType.none; - const connectorInvokeRequest: InvokeAIActionParams = { + const subActionParams = { system: noToolUsage ? addNoToolUsageDirective(system) : system, messages: messagesToBedrock(messages), - tools: noToolUsage ? [] : toolsToBedrock(tools), + tools: noToolUsage ? [] : toolsToBedrock(tools, messages), toolChoice: toolChoiceToBedrock(toolChoice), temperature: 0, stopSequences: ['\n\nHuman:'], @@ -37,7 +37,7 @@ export const bedrockClaudeAdapter: InferenceConnectorAdapter = { return from( executor.invoke({ subAction: 'invokeStream', - subActionParams: connectorInvokeRequest, + subActionParams, }) ).pipe( switchMap((response) => { @@ -82,19 +82,64 @@ const toolChoiceToBedrock = ( return undefined; }; -const toolsToBedrock = (tools: ToolOptions['tools']) => { - return tools - ? Object.entries(tools).map(([toolName, toolDef]) => { - return { - name: toolName, - description: toolDef.description, - input_schema: toolDef.schema ?? { +const toolsToBedrock = (tools: ToolOptions['tools'], messages: Message[]) => { + function walkSchema(schemaPart: T): T { + if (schemaPart.type === 'object' && schemaPart.properties) { + return { + ...schemaPart, + properties: Object.fromEntries( + Object.entries(schemaPart.properties).map(([key, childSchemaPart]) => { + return [key, walkSchema(childSchemaPart)]; + }) + ), + }; + } + + if (schemaPart.type === 'array') { + return { + ...schemaPart, + // Claude is prone to ignoring the "array" part of an array type + description: schemaPart.description + '. Must be provided as a JSON array', + items: walkSchema(schemaPart.items), + }; + } + + return schemaPart; + } + + if (tools) { + return Object.entries(tools).map(([toolName, toolDef]) => { + return { + name: toolName, + description: toolDef.description, + input_schema: walkSchema( + toolDef.schema ?? { type: 'object' as const, properties: {}, - }, - }; - }) - : undefined; + } + ), + }; + }); + } + + const hasToolUse = messages.filter( + (message) => + message.role === MessageRole.Tool || + (message.role === MessageRole.Assistant && message.toolCalls?.length) + ); + + if (hasToolUse) { + return [ + { + name: 'do_not_call_this_tool', + description: 'Do not call this tool, it is strictly forbidden', + input_schema: { + type: 'object', + properties: {}, + }, + }, + ]; + } }; const messagesToBedrock = (messages: Message[]): BedRockMessage[] => { diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/gemini/gemini_adapter.test.ts b/x-pack/plugins/inference/server/chat_complete/adapters/gemini/gemini_adapter.test.ts index 3fe8c917c0015..a9f4305a3c532 100644 --- a/x-pack/plugins/inference/server/chat_complete/adapters/gemini/gemini_adapter.test.ts +++ b/x-pack/plugins/inference/server/chat_complete/adapters/gemini/gemini_adapter.test.ts @@ -8,6 +8,7 @@ import { processVertexStreamMock } from './gemini_adapter.test.mocks'; import { PassThrough } from 'stream'; import { noop, tap, lastValueFrom, toArray, Subject } from 'rxjs'; +import { loggerMock } from '@kbn/logging-mocks'; import type { InferenceExecutor } from '../../utils/inference_executor'; import { observableIntoEventSourceStream } from '../../../util/observable_into_event_source_stream'; import { MessageRole } from '../../../../common/chat_complete'; @@ -15,6 +16,7 @@ import { ToolChoiceType } from '../../../../common/chat_complete/tools'; import { geminiAdapter } from './gemini_adapter'; describe('geminiAdapter', () => { + const logger = loggerMock.create(); const executorMock = { invoke: jest.fn(), } as InferenceExecutor & { invoke: jest.MockedFn }; @@ -47,6 +49,7 @@ describe('geminiAdapter', () => { it('calls `executor.invoke` with the right fixed parameters', () => { geminiAdapter.chatComplete({ + logger, executor: executorMock, messages: [ { @@ -75,6 +78,7 @@ describe('geminiAdapter', () => { it('correctly format tools', () => { geminiAdapter.chatComplete({ + logger, executor: executorMock, messages: [ { @@ -138,6 +142,7 @@ describe('geminiAdapter', () => { it('correctly format messages', () => { geminiAdapter.chatComplete({ + logger, executor: executorMock, messages: [ { @@ -236,6 +241,7 @@ describe('geminiAdapter', () => { it('groups messages from the same user', () => { geminiAdapter.chatComplete({ + logger, executor: executorMock, messages: [ { @@ -304,6 +310,7 @@ describe('geminiAdapter', () => { it('correctly format system message', () => { geminiAdapter.chatComplete({ + logger, executor: executorMock, system: 'Some system message', messages: [ @@ -322,6 +329,7 @@ describe('geminiAdapter', () => { it('correctly format tool choice', () => { geminiAdapter.chatComplete({ + logger, executor: executorMock, messages: [ { @@ -340,6 +348,7 @@ describe('geminiAdapter', () => { it('correctly format tool choice for named function', () => { geminiAdapter.chatComplete({ + logger, executor: executorMock, messages: [ { @@ -366,11 +375,12 @@ describe('geminiAdapter', () => { return { actionId: '', status: 'ok', - data: observableIntoEventSourceStream(source$), + data: observableIntoEventSourceStream(source$, logger), }; }); const response$ = geminiAdapter.chatComplete({ + logger, executor: executorMock, messages: [ { diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/gemini/gemini_adapter.ts b/x-pack/plugins/inference/server/chat_complete/adapters/gemini/gemini_adapter.ts index 8f6d02da0e3f8..2e86adcc82a85 100644 --- a/x-pack/plugins/inference/server/chat_complete/adapters/gemini/gemini_adapter.ts +++ b/x-pack/plugins/inference/server/chat_complete/adapters/gemini/gemini_adapter.ts @@ -106,12 +106,16 @@ function toolSchemaToGemini({ schema }: { schema: ToolSchema }): Gemini.Function type: Gemini.FunctionDeclarationSchemaType.OBJECT, description: def.description, required: def.required as string[], - properties: Object.entries(def.properties).reduce< - Record - >((properties, [key, prop]) => { - properties[key] = convertSchemaType({ def: prop }) as Gemini.FunctionDeclarationSchema; - return properties; - }, {}), + properties: def.properties + ? Object.entries(def.properties).reduce< + Record + >((properties, [key, prop]) => { + properties[key] = convertSchemaType({ + def: prop, + }) as Gemini.FunctionDeclarationSchema; + return properties; + }, {}) + : undefined, }; case 'string': return { @@ -137,7 +141,7 @@ function toolSchemaToGemini({ schema }: { schema: ToolSchema }): Gemini.Function return { type: Gemini.FunctionDeclarationSchemaType.OBJECT, required: schema.required as string[], - properties: Object.entries(schema.properties).reduce< + properties: Object.entries(schema.properties ?? {}).reduce< Record >((properties, [key, def]) => { properties[key] = convertSchemaType({ def }); diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/gemini/process_vertex_stream.ts b/x-pack/plugins/inference/server/chat_complete/adapters/gemini/process_vertex_stream.ts index ec3fa8e82eed6..e2a6c74a0447f 100644 --- a/x-pack/plugins/inference/server/chat_complete/adapters/gemini/process_vertex_stream.ts +++ b/x-pack/plugins/inference/server/chat_complete/adapters/gemini/process_vertex_stream.ts @@ -11,7 +11,7 @@ import { ChatCompletionTokenCountEvent, ChatCompletionEventType, } from '../../../../common/chat_complete'; -import { generateFakeToolCallId } from '../../utils'; +import { generateFakeToolCallId } from '../../../../common'; import type { GenerateContentResponseChunk } from './types'; export function processVertexStream() { diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/openai/index.ts b/x-pack/plugins/inference/server/chat_complete/adapters/openai/index.ts index d72ceb2020e8a..9aa1d94e01a52 100644 --- a/x-pack/plugins/inference/server/chat_complete/adapters/openai/index.ts +++ b/x-pack/plugins/inference/server/chat_complete/adapters/openai/index.ts @@ -5,181 +5,4 @@ * 2.0. */ -import OpenAI from 'openai'; -import type { - ChatCompletionAssistantMessageParam, - ChatCompletionMessageParam, - ChatCompletionSystemMessageParam, - ChatCompletionToolMessageParam, - ChatCompletionUserMessageParam, -} from 'openai/resources'; -import { filter, from, map, switchMap, tap } from 'rxjs'; -import { Readable } from 'stream'; -import { - ChatCompletionChunkEvent, - ChatCompletionEventType, - Message, - MessageRole, -} from '../../../../common/chat_complete'; -import type { ToolOptions } from '../../../../common/chat_complete/tools'; -import { createTokenLimitReachedError } from '../../../../common/chat_complete/errors'; -import { createInferenceInternalError } from '../../../../common/errors'; -import { eventSourceStreamIntoObservable } from '../../../util/event_source_stream_into_observable'; -import type { InferenceConnectorAdapter } from '../../types'; - -export const openAIAdapter: InferenceConnectorAdapter = { - chatComplete: ({ executor, system, messages, toolChoice, tools }) => { - const stream = true; - - const request: Omit & { model?: string } = { - stream, - messages: messagesToOpenAI({ system, messages }), - tool_choice: toolChoiceToOpenAI(toolChoice), - tools: toolsToOpenAI(tools), - temperature: 0, - }; - - return from( - executor.invoke({ - subAction: 'stream', - subActionParams: { - body: JSON.stringify(request), - stream, - }, - }) - ).pipe( - switchMap((response) => { - const readable = response.data as Readable; - return eventSourceStreamIntoObservable(readable); - }), - filter((line) => !!line && line !== '[DONE]'), - map( - (line) => JSON.parse(line) as OpenAI.ChatCompletionChunk | { error: { message: string } } - ), - tap((line) => { - if ('error' in line) { - throw createInferenceInternalError(line.error.message); - } - if ( - 'choices' in line && - line.choices.length && - line.choices[0].finish_reason === 'length' - ) { - throw createTokenLimitReachedError(); - } - }), - filter( - (line): line is OpenAI.ChatCompletionChunk => - 'object' in line && line.object === 'chat.completion.chunk' - ), - map((chunk): ChatCompletionChunkEvent => { - const delta = chunk.choices[0].delta; - - return { - type: ChatCompletionEventType.ChatCompletionChunk, - content: delta.content ?? '', - tool_calls: - delta.tool_calls?.map((toolCall) => { - return { - function: { - name: toolCall.function?.name ?? '', - arguments: toolCall.function?.arguments ?? '', - }, - toolCallId: toolCall.id ?? '', - index: toolCall.index, - }; - }) ?? [], - }; - }) - ); - }, -}; - -function toolsToOpenAI(tools: ToolOptions['tools']): OpenAI.ChatCompletionCreateParams['tools'] { - return tools - ? Object.entries(tools).map(([toolName, { description, schema }]) => { - return { - type: 'function', - function: { - name: toolName, - description, - parameters: (schema ?? { - type: 'object' as const, - properties: {}, - }) as unknown as Record, - }, - }; - }) - : undefined; -} - -function toolChoiceToOpenAI( - toolChoice: ToolOptions['toolChoice'] -): OpenAI.ChatCompletionCreateParams['tool_choice'] { - return typeof toolChoice === 'string' - ? toolChoice - : toolChoice - ? { - function: { - name: toolChoice.function, - }, - type: 'function' as const, - } - : undefined; -} - -function messagesToOpenAI({ - system, - messages, -}: { - system?: string; - messages: Message[]; -}): OpenAI.ChatCompletionMessageParam[] { - const systemMessage: ChatCompletionSystemMessageParam | undefined = system - ? { role: 'system', content: system } - : undefined; - - return [ - ...(systemMessage ? [systemMessage] : []), - ...messages.map((message): ChatCompletionMessageParam => { - const role = message.role; - - switch (role) { - case MessageRole.Assistant: - const assistantMessage: ChatCompletionAssistantMessageParam = { - role: 'assistant', - content: message.content, - tool_calls: message.toolCalls?.map((toolCall) => { - return { - function: { - name: toolCall.function.name, - arguments: - 'arguments' in toolCall.function - ? JSON.stringify(toolCall.function.arguments) - : '{}', - }, - id: toolCall.toolCallId, - type: 'function', - }; - }), - }; - return assistantMessage; - - case MessageRole.User: - const userMessage: ChatCompletionUserMessageParam = { - role: 'user', - content: message.content, - }; - return userMessage; - - case MessageRole.Tool: - const toolMessage: ChatCompletionToolMessageParam = { - role: 'tool', - content: JSON.stringify(message.response), - tool_call_id: message.toolCallId, - }; - return toolMessage; - } - }), - ]; -} +export { openAIAdapter } from './openai_adapter'; diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/openai/index.test.ts b/x-pack/plugins/inference/server/chat_complete/adapters/openai/openai_adapter.test.ts similarity index 96% rename from x-pack/plugins/inference/server/chat_complete/adapters/openai/index.test.ts rename to x-pack/plugins/inference/server/chat_complete/adapters/openai/openai_adapter.test.ts index f3b7c423ea42f..813e88760de8c 100644 --- a/x-pack/plugins/inference/server/chat_complete/adapters/openai/index.test.ts +++ b/x-pack/plugins/inference/server/chat_complete/adapters/openai/openai_adapter.test.ts @@ -10,6 +10,8 @@ import { v4 } from 'uuid'; import { PassThrough } from 'stream'; import { pick } from 'lodash'; import { lastValueFrom, Subject, toArray } from 'rxjs'; +import type { Logger } from '@kbn/logging'; +import { loggerMock } from '@kbn/logging-mocks'; import { ChatCompletionEventType, MessageRole } from '../../../../common/chat_complete'; import { observableIntoEventSourceStream } from '../../../util/observable_into_event_source_stream'; import { InferenceExecutor } from '../../utils/inference_executor'; @@ -43,12 +45,18 @@ describe('openAIAdapter', () => { invoke: jest.fn(), } as InferenceExecutor & { invoke: jest.MockedFn }; + const logger = { + debug: jest.fn(), + error: jest.fn(), + } as unknown as Logger; + beforeEach(() => { executorMock.invoke.mockReset(); }); const defaultArgs = { executor: executorMock, + logger: loggerMock.create(), }; describe('when creating the request', () => { @@ -255,7 +263,7 @@ describe('openAIAdapter', () => { return { actionId: '', status: 'ok', - data: observableIntoEventSourceStream(source$), + data: observableIntoEventSourceStream(source$, logger), }; }); }); diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/openai/openai_adapter.ts b/x-pack/plugins/inference/server/chat_complete/adapters/openai/openai_adapter.ts new file mode 100644 index 0000000000000..3a89f100f2879 --- /dev/null +++ b/x-pack/plugins/inference/server/chat_complete/adapters/openai/openai_adapter.ts @@ -0,0 +1,189 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import OpenAI from 'openai'; +import type { + ChatCompletionAssistantMessageParam, + ChatCompletionMessageParam, + ChatCompletionSystemMessageParam, + ChatCompletionToolMessageParam, + ChatCompletionUserMessageParam, +} from 'openai/resources'; +import { filter, from, map, switchMap, tap, throwError } from 'rxjs'; +import { Readable, isReadable } from 'stream'; +import { + ChatCompletionChunkEvent, + ChatCompletionEventType, + Message, + MessageRole, +} from '../../../../common/chat_complete'; +import type { ToolOptions } from '../../../../common/chat_complete/tools'; +import { createTokenLimitReachedError } from '../../../../common/chat_complete/errors'; +import { createInferenceInternalError } from '../../../../common/errors'; +import { eventSourceStreamIntoObservable } from '../../../util/event_source_stream_into_observable'; +import type { InferenceConnectorAdapter } from '../../types'; + +export const openAIAdapter: InferenceConnectorAdapter = { + chatComplete: ({ executor, system, messages, toolChoice, tools }) => { + const stream = true; + + const request: Omit & { model?: string } = { + stream, + messages: messagesToOpenAI({ system, messages }), + tool_choice: toolChoiceToOpenAI(toolChoice), + tools: toolsToOpenAI(tools), + temperature: 0, + }; + + return from( + executor.invoke({ + subAction: 'stream', + subActionParams: { + body: JSON.stringify(request), + stream, + }, + }) + ).pipe( + switchMap((response) => { + if (isReadable(response.data as any)) { + return eventSourceStreamIntoObservable(response.data as Readable); + } + return throwError(() => + createInferenceInternalError('Unexpected error', response.data as Record) + ); + }), + filter((line) => !!line && line !== '[DONE]'), + map( + (line) => JSON.parse(line) as OpenAI.ChatCompletionChunk | { error: { message: string } } + ), + tap((line) => { + if ('error' in line) { + throw createInferenceInternalError(line.error.message); + } + if ( + 'choices' in line && + line.choices.length && + line.choices[0].finish_reason === 'length' + ) { + throw createTokenLimitReachedError(); + } + }), + filter( + (line): line is OpenAI.ChatCompletionChunk => + 'object' in line && line.object === 'chat.completion.chunk' + ), + map((chunk): ChatCompletionChunkEvent => { + const delta = chunk.choices[0].delta; + + return { + type: ChatCompletionEventType.ChatCompletionChunk, + content: delta.content ?? '', + tool_calls: + delta.tool_calls?.map((toolCall) => { + return { + function: { + name: toolCall.function?.name ?? '', + arguments: toolCall.function?.arguments ?? '', + }, + toolCallId: toolCall.id ?? '', + index: toolCall.index, + }; + }) ?? [], + }; + }) + ); + }, +}; + +function toolsToOpenAI(tools: ToolOptions['tools']): OpenAI.ChatCompletionCreateParams['tools'] { + return tools + ? Object.entries(tools).map(([toolName, { description, schema }]) => { + return { + type: 'function', + function: { + name: toolName, + description, + parameters: (schema ?? { + type: 'object' as const, + properties: {}, + }) as unknown as Record, + }, + }; + }) + : undefined; +} + +function toolChoiceToOpenAI( + toolChoice: ToolOptions['toolChoice'] +): OpenAI.ChatCompletionCreateParams['tool_choice'] { + return typeof toolChoice === 'string' + ? toolChoice + : toolChoice + ? { + function: { + name: toolChoice.function, + }, + type: 'function' as const, + } + : undefined; +} + +function messagesToOpenAI({ + system, + messages, +}: { + system?: string; + messages: Message[]; +}): OpenAI.ChatCompletionMessageParam[] { + const systemMessage: ChatCompletionSystemMessageParam | undefined = system + ? { role: 'system', content: system } + : undefined; + + return [ + ...(systemMessage ? [systemMessage] : []), + ...messages.map((message): ChatCompletionMessageParam => { + const role = message.role; + + switch (role) { + case MessageRole.Assistant: + const assistantMessage: ChatCompletionAssistantMessageParam = { + role: 'assistant', + content: message.content, + tool_calls: message.toolCalls?.map((toolCall) => { + return { + function: { + name: toolCall.function.name, + arguments: + 'arguments' in toolCall.function + ? JSON.stringify(toolCall.function.arguments) + : '{}', + }, + id: toolCall.toolCallId, + type: 'function', + }; + }), + }; + return assistantMessage; + + case MessageRole.User: + const userMessage: ChatCompletionUserMessageParam = { + role: 'user', + content: message.content, + }; + return userMessage; + + case MessageRole.Tool: + const toolMessage: ChatCompletionToolMessageParam = { + role: 'tool', + content: JSON.stringify(message.response), + tool_call_id: message.toolCallId, + }; + return toolMessage; + } + }), + ]; +} diff --git a/x-pack/plugins/inference/server/chat_complete/api.ts b/x-pack/plugins/inference/server/chat_complete/api.ts index 17bf0e5214300..fe879392cd4de 100644 --- a/x-pack/plugins/inference/server/chat_complete/api.ts +++ b/x-pack/plugins/inference/server/chat_complete/api.ts @@ -5,8 +5,10 @@ * 2.0. */ -import type { KibanaRequest } from '@kbn/core-http-server'; +import { last } from 'lodash'; import { defer, switchMap, throwError } from 'rxjs'; +import type { Logger } from '@kbn/logging'; +import type { KibanaRequest } from '@kbn/core-http-server'; import type { ChatCompleteAPI, ChatCompletionResponse } from '../../common/chat_complete'; import { createInferenceRequestError } from '../../common/errors'; import type { InferenceStartDependencies } from '../types'; @@ -17,9 +19,11 @@ import { createInferenceExecutor, chunksIntoMessage } from './utils'; export function createChatCompleteApi({ request, actions, + logger, }: { request: KibanaRequest; actions: InferenceStartDependencies['actions']; + logger: Logger; }) { const chatCompleteAPI: ChatCompleteAPI = ({ connectorId, @@ -44,17 +48,24 @@ export function createChatCompleteApi({ ); } + logger.debug(() => `Sending request: ${JSON.stringify(last(messages))}`); + logger.trace(() => JSON.stringify({ messages, toolChoice, tools, system })); + return inferenceAdapter.chatComplete({ system, executor, messages, toolChoice, tools, + logger, }); }), chunksIntoMessage({ - toolChoice, - tools, + toolOptions: { + toolChoice, + tools, + }, + logger, }) ); }; diff --git a/x-pack/plugins/inference/server/chat_complete/types.ts b/x-pack/plugins/inference/server/chat_complete/types.ts index fff902f7e885e..5ef28fdbdc808 100644 --- a/x-pack/plugins/inference/server/chat_complete/types.ts +++ b/x-pack/plugins/inference/server/chat_complete/types.ts @@ -6,6 +6,7 @@ */ import type { Observable } from 'rxjs'; +import type { Logger } from '@kbn/logging'; import type { ChatCompletionChunkEvent, ChatCompletionTokenCountEvent, @@ -26,6 +27,7 @@ export interface InferenceConnectorAdapter { messages: Message[]; system?: string; executor: InferenceExecutor; + logger: Logger; } & ToolOptions ) => Observable; } diff --git a/x-pack/plugins/inference/server/chat_complete/utils/chunks_into_message.test.ts b/x-pack/plugins/inference/server/chat_complete/utils/chunks_into_message.test.ts index a3fdcd33bee94..0c5552a0113b8 100644 --- a/x-pack/plugins/inference/server/chat_complete/utils/chunks_into_message.test.ts +++ b/x-pack/plugins/inference/server/chat_complete/utils/chunks_into_message.test.ts @@ -12,15 +12,21 @@ import { } from '../../../common/chat_complete'; import { ToolChoiceType } from '../../../common/chat_complete/tools'; import { chunksIntoMessage } from './chunks_into_message'; +import type { Logger } from '@kbn/logging'; describe('chunksIntoMessage', () => { function fromEvents(...events: Array) { return of(...events); } + const logger = { + debug: jest.fn(), + error: jest.fn(), + } as unknown as Logger; + it('concatenates content chunks into a single message', async () => { const message = await lastValueFrom( - chunksIntoMessage({})( + chunksIntoMessage({ logger, toolOptions: {} })( fromEvents( { content: 'Hey', @@ -51,21 +57,24 @@ describe('chunksIntoMessage', () => { it('parses tool calls', async () => { const message = await lastValueFrom( chunksIntoMessage({ - toolChoice: ToolChoiceType.auto, - tools: { - myFunction: { - description: 'myFunction', - schema: { - type: 'object', - properties: { - foo: { - type: 'string', - const: 'bar', + toolOptions: { + toolChoice: ToolChoiceType.auto, + tools: { + myFunction: { + description: 'myFunction', + schema: { + type: 'object', + properties: { + foo: { + type: 'string', + const: 'bar', + }, }, }, }, }, }, + logger, })( fromEvents( { @@ -135,21 +144,24 @@ describe('chunksIntoMessage', () => { async function getMessage() { return await lastValueFrom( chunksIntoMessage({ - toolChoice: ToolChoiceType.auto, - tools: { - myFunction: { - description: 'myFunction', - schema: { - type: 'object', - properties: { - foo: { - type: 'string', - const: 'bar', + toolOptions: { + toolChoice: ToolChoiceType.auto, + tools: { + myFunction: { + description: 'myFunction', + schema: { + type: 'object', + properties: { + foo: { + type: 'string', + const: 'bar', + }, }, }, }, }, }, + logger, })( fromEvents({ content: '', @@ -177,20 +189,23 @@ describe('chunksIntoMessage', () => { it('concatenates multiple tool calls into a single message', async () => { const message = await lastValueFrom( chunksIntoMessage({ - toolChoice: ToolChoiceType.auto, - tools: { - myFunction: { - description: 'myFunction', - schema: { - type: 'object', - properties: { - foo: { - type: 'string', + toolOptions: { + toolChoice: ToolChoiceType.auto, + tools: { + myFunction: { + description: 'myFunction', + schema: { + type: 'object', + properties: { + foo: { + type: 'string', + }, }, }, }, }, }, + logger, })( fromEvents( { diff --git a/x-pack/plugins/inference/server/chat_complete/utils/chunks_into_message.ts b/x-pack/plugins/inference/server/chat_complete/utils/chunks_into_message.ts index 786a4c4ff7fb3..902289182a37a 100644 --- a/x-pack/plugins/inference/server/chat_complete/utils/chunks_into_message.ts +++ b/x-pack/plugins/inference/server/chat_complete/utils/chunks_into_message.ts @@ -6,6 +6,7 @@ */ import { last, map, merge, OperatorFunction, scan, share } from 'rxjs'; +import type { Logger } from '@kbn/logging'; import type { UnvalidatedToolCall, ToolOptions } from '../../../common/chat_complete/tools'; import { ChatCompletionChunkEvent, @@ -16,9 +17,13 @@ import { import { withoutTokenCountEvents } from '../../../common/chat_complete/without_token_count_events'; import { validateToolCalls } from '../../util/validate_tool_calls'; -export function chunksIntoMessage( - toolOptions: TToolOptions -): OperatorFunction< +export function chunksIntoMessage({ + logger, + toolOptions, +}: { + toolOptions: TToolOptions; + logger: Pick; +}): OperatorFunction< ChatCompletionChunkEvent | ChatCompletionTokenCountEvent, | ChatCompletionChunkEvent | ChatCompletionTokenCountEvent @@ -63,6 +68,8 @@ export function chunksIntoMessage( ), last(), map((concatenatedChunk): ChatCompletionMessageEvent => { + logger.debug(() => `Received completed message: ${JSON.stringify(concatenatedChunk)}`); + const validatedToolCalls = validateToolCalls({ ...toolOptions, toolCalls: concatenatedChunk.tool_calls, diff --git a/x-pack/plugins/inference/server/chat_complete/utils/index.ts b/x-pack/plugins/inference/server/chat_complete/utils/index.ts index d9344164eff46..dea2ac65f4755 100644 --- a/x-pack/plugins/inference/server/chat_complete/utils/index.ts +++ b/x-pack/plugins/inference/server/chat_complete/utils/index.ts @@ -12,4 +12,3 @@ export { type InferenceExecutor, } from './inference_executor'; export { chunksIntoMessage } from './chunks_into_message'; -export { generateFakeToolCallId } from './generate_fake_tool_call_id'; diff --git a/x-pack/plugins/inference/server/index.ts b/x-pack/plugins/inference/server/index.ts index 721aa05d06023..e45ae303d2833 100644 --- a/x-pack/plugins/inference/server/index.ts +++ b/x-pack/plugins/inference/server/index.ts @@ -18,6 +18,8 @@ export { withoutTokenCountEvents } from '../common/chat_complete/without_token_c export { withoutChunkEvents } from '../common/chat_complete/without_chunk_events'; export { withoutOutputUpdateEvents } from '../common/output/without_output_update_events'; +export { naturalLanguageToEsql } from './tasks/nl_to_esql'; + export type { InferenceServerSetup, InferenceServerStart }; export const plugin: PluginInitializer< diff --git a/x-pack/plugins/inference/server/inference_client/index.ts b/x-pack/plugins/inference/server/inference_client/index.ts index d9d52a8e41ec1..25208bebc54bb 100644 --- a/x-pack/plugins/inference/server/inference_client/index.ts +++ b/x-pack/plugins/inference/server/inference_client/index.ts @@ -5,6 +5,7 @@ * 2.0. */ +import type { Logger } from '@kbn/logging'; import type { KibanaRequest } from '@kbn/core-http-server'; import type { InferenceClient, InferenceStartDependencies } from '../types'; import { createChatCompleteApi } from '../chat_complete'; @@ -14,8 +15,12 @@ import { getConnectorById } from '../util/get_connector_by_id'; export function createInferenceClient({ request, actions, -}: { request: KibanaRequest } & Pick): InferenceClient { - const chatComplete = createChatCompleteApi({ request, actions }); + logger, +}: { request: KibanaRequest; logger: Logger } & Pick< + InferenceStartDependencies, + 'actions' +>): InferenceClient { + const chatComplete = createChatCompleteApi({ request, actions, logger }); return { chatComplete, output: createOutputApi(chatComplete), diff --git a/x-pack/plugins/inference/server/plugin.ts b/x-pack/plugins/inference/server/plugin.ts index 1b17eb4a66d35..2b1a7be0a165c 100644 --- a/x-pack/plugins/inference/server/plugin.ts +++ b/x-pack/plugins/inference/server/plugin.ts @@ -26,7 +26,7 @@ export class InferencePlugin InferenceStartDependencies > { - logger: Logger; + private logger: Logger; constructor(context: PluginInitializerContext) { this.logger = context.logger.get(); @@ -40,6 +40,7 @@ export class InferencePlugin registerRoutes({ router, coreSetup, + logger: this.logger, }); return {}; @@ -48,7 +49,11 @@ export class InferencePlugin start(core: CoreStart, pluginsStart: InferenceStartDependencies): InferenceServerStart { return { getClient: ({ request }) => { - return createInferenceClient({ request, actions: pluginsStart.actions }); + return createInferenceClient({ + request, + actions: pluginsStart.actions, + logger: this.logger.get('client'), + }); }, }; } diff --git a/x-pack/plugins/inference/server/routes/chat_complete.ts b/x-pack/plugins/inference/server/routes/chat_complete.ts index bfa95fdbb9213..5a9c0aae50958 100644 --- a/x-pack/plugins/inference/server/routes/chat_complete.ts +++ b/x-pack/plugins/inference/server/routes/chat_complete.ts @@ -6,7 +6,7 @@ */ import { schema, Type } from '@kbn/config-schema'; -import type { CoreSetup, IRouter, RequestHandlerContext } from '@kbn/core/server'; +import type { CoreSetup, IRouter, Logger, RequestHandlerContext } from '@kbn/core/server'; import { MessageRole } from '../../common/chat_complete'; import type { ChatCompleteRequestBody } from '../../common/chat_complete/request'; import { ToolCall, ToolChoiceType } from '../../common/chat_complete/tools'; @@ -76,9 +76,11 @@ const chatCompleteBodySchema: Type = schema.object({ export function registerChatCompleteRoute({ coreSetup, router, + logger, }: { coreSetup: CoreSetup; router: IRouter; + logger: Logger; }) { router.post( { @@ -92,7 +94,7 @@ export function registerChatCompleteRoute({ .getStartServices() .then(([coreStart, pluginsStart]) => pluginsStart.actions); - const client = createInferenceClient({ request, actions }); + const client = createInferenceClient({ request, actions, logger }); const { connectorId, messages, system, toolChoice, tools } = request.body; @@ -105,7 +107,7 @@ export function registerChatCompleteRoute({ }); return response.ok({ - body: observableIntoEventSourceStream(chatCompleteResponse), + body: observableIntoEventSourceStream(chatCompleteResponse, logger), }); } ); diff --git a/x-pack/plugins/inference/server/routes/index.ts b/x-pack/plugins/inference/server/routes/index.ts index 0f6891ace1223..8191f358250c1 100644 --- a/x-pack/plugins/inference/server/routes/index.ts +++ b/x-pack/plugins/inference/server/routes/index.ts @@ -5,6 +5,7 @@ * 2.0. */ +import type { Logger } from '@kbn/logging'; import type { CoreSetup, IRouter } from '@kbn/core/server'; import type { InferenceServerStart, InferenceStartDependencies } from '../types'; import { registerChatCompleteRoute } from './chat_complete'; @@ -12,11 +13,13 @@ import { registerConnectorsRoute } from './connectors'; export const registerRoutes = ({ router, + logger, coreSetup, }: { router: IRouter; + logger: Logger; coreSetup: CoreSetup; }) => { - registerChatCompleteRoute({ router, coreSetup }); + registerChatCompleteRoute({ router, coreSetup, logger: logger.get('chatComplete') }); registerConnectorsRoute({ router, coreSetup }); }; diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-abs.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-abs.txt new file mode 100644 index 0000000000000..6a970dc5700fe --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-abs.txt @@ -0,0 +1,16 @@ +## ABS + +The `ABS` function returns the absolute value of a numeric expression. If the input is null, the function returns null. + +### Examples + +```esql +ROW number = -1.0 +| EVAL abs_number = ABS(number) +``` + +```esql +FROM employees +| KEEP first_name, last_name, height +| EVAL abs_height = ABS(0.0 - height) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-acos.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-acos.txt new file mode 100644 index 0000000000000..3460483c15870 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-acos.txt @@ -0,0 +1,15 @@ +## ACOS + +The `ACOS` function returns the arccosine of a number as an angle, expressed in radians. The input number must be between -1 and 1. If the input is null, the function returns null. + +### Examples + +```esql +ROW a = .9 +| EVAL acos = ACOS(a) +``` + +```esql +ROW b = -0.5 +| EVAL acos_b = ACOS(b) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-asin.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-asin.txt new file mode 100644 index 0000000000000..ad4fb8fe8d310 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-asin.txt @@ -0,0 +1,15 @@ +## ASIN + +The `ASIN` function returns the arcsine of the input numeric expression as an angle, expressed in radians. + +### Examples + +```esql +ROW a = .9 +| EVAL asin = ASIN(a) +``` + +```esql +ROW a = -.5 +| EVAL asin = ASIN(a) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-atan.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-atan.txt new file mode 100644 index 0000000000000..fbeee5e84f2f3 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-atan.txt @@ -0,0 +1,15 @@ +## ATAN + +The `ATAN` function returns the arctangent of the input numeric expression as an angle, expressed in radians. + +### Examples + +```esql +ROW a=12.9 +| EVAL atan = ATAN(a) +``` + +```esql +ROW x=5.0, y=3.0 +| EVAL atan_yx = ATAN(y / x) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-atan2.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-atan2.txt new file mode 100644 index 0000000000000..f4da581885ef7 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-atan2.txt @@ -0,0 +1,15 @@ +## ATAN2 + +The `ATAN2` function calculates the angle between the positive x-axis and the ray from the origin to the point (x, y) in the Cartesian plane, expressed in radians. + +### Examples + +```esql +ROW y=12.9, x=.6 +| EVAL atan2 = ATAN2(y, x) +``` + +```esql +ROW y=5.0, x=3.0 +| EVAL atan2 = ATAN2(y, x) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-avg.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-avg.txt new file mode 100644 index 0000000000000..943a12c4aaa90 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-avg.txt @@ -0,0 +1,15 @@ +## AVG + +The `AVG` function calculates the average of a numeric field. + +### Examples + +```esql +FROM employees +| STATS AVG(height) +``` + +```esql +FROM employees +| STATS avg_salary_change = ROUND(AVG(MV_AVG(salary_change)), 10) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-bucket.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-bucket.txt new file mode 100644 index 0000000000000..945a4328d7728 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-bucket.txt @@ -0,0 +1,66 @@ +## BUCKET + +The `BUCKET` function creates groups of values—buckets—out of a datetime or numeric input. The size of the buckets can either be provided directly or chosen based on a recommended count and values range. + +### Examples + +```esql +FROM employees +| WHERE hire_date >= "1985-01-01T00:00:00Z" AND hire_date < "1986-01-01T00:00:00Z" +| STATS hire_date = MV_SORT(VALUES(hire_date)) BY month = BUCKET(hire_date, 20, "1985-01-01T00:00:00Z", "1986-01-01T00:00:00Z") +| SORT hire_date +``` + +```esql +FROM employees +| WHERE hire_date >= "1985-01-01T00:00:00Z" AND hire_date < "1986-01-01T00:00:00Z" +| STATS hires_per_month = COUNT(*) BY month = BUCKET(hire_date, 20, "1985-01-01T00:00:00Z", "1986-01-01T00:00:00Z") +| SORT month +``` + +```esql +FROM employees +| WHERE hire_date >= "1985-01-01T00:00:00Z" AND hire_date < "1986-01-01T00:00:00Z" +| STATS hires_per_week = COUNT(*) BY week = BUCKET(hire_date, 100, "1985-01-01T00:00:00Z", "1986-01-01T00:00:00Z") +| SORT week +``` + +```esql +FROM employees +| WHERE hire_date >= "1985-01-01T00:00:00Z" AND hire_date < "1986-01-01T00:00:00Z" +| STATS hires_per_week = COUNT(*) BY week = BUCKET(hire_date, 1 week) +| SORT week +``` + +```esql +FROM employees +| STATS COUNT(*) BY bs = BUCKET(salary, 20, 25324, 74999) +| SORT bs +``` + +```esql +FROM employees +| WHERE hire_date >= "1985-01-01T00:00:00Z" AND hire_date < "1986-01-01T00:00:00Z" +| STATS c = COUNT(1) BY b = BUCKET(salary, 5000.) +| SORT b +``` + +```esql +FROM sample_data +| WHERE @timestamp >= NOW() - 1 day and @timestamp < NOW() +| STATS COUNT(*) BY bucket = BUCKET(@timestamp, 25, NOW() - 1 day, NOW()) +``` + +```esql +FROM employees +| WHERE hire_date >= "1985-01-01T00:00:00Z" AND hire_date < "1986-01-01T00:00:00Z" +| STATS AVG(salary) BY bucket = BUCKET(hire_date, 20, "1985-01-01T00:00:00Z", "1986-01-01T00:00:00Z") +| SORT bucket +``` + +```esql +FROM employees +| STATS s1 = BUCKET(salary / 1000 + 999, 50.) + 2 BY b1 = BUCKET(salary / 100 + 99, 50.), b2 = BUCKET(salary / 1000 + 999, 50.) +| SORT b1, b2 +| KEEP b1, s1, b2 +``` diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-case.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-case.txt new file mode 100644 index 0000000000000..4c9cc07e669db --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-case.txt @@ -0,0 +1,37 @@ +## CASE + +The `CASE` function accepts pairs of conditions and values. The function returns the value that belongs to the first condition that evaluates to true. If the number of arguments is odd, the last argument is the default value which is returned when no condition matches. If the number of arguments is even, and no condition matches, the function returns null. + +### Examples + +Determine whether employees are monolingual, bilingual, or polyglot: + +```esql +FROM employees +| EVAL type = CASE( + languages <= 1, "monolingual", + languages <= 2, "bilingual", + "polyglot") +| KEEP emp_no, languages, type +``` + +Calculate the total connection success rate based on log messages: + +```esql +FROM sample_data +| EVAL successful = CASE( + STARTS_WITH(message, "Connected to"), 1, + message == "Connection error", 0 + ) +| STATS success_rate = AVG(successful) +``` + +Calculate an hourly error rate as a percentage of the total number of log messages: + +```esql +FROM sample_data +| EVAL error = CASE(message LIKE "*error*", 1, 0) +| EVAL hour = DATE_TRUNC(1 hour, @timestamp) +| STATS error_rate = AVG(error) BY hour +| SORT hour +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-cbrt.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-cbrt.txt new file mode 100644 index 0000000000000..44ecddefc290d --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-cbrt.txt @@ -0,0 +1,15 @@ +## CBRT + +The `CBRT` function returns the cube root of a number. The input can be any numeric value, and the return value is always a double. Cube roots of infinities are null. + +### Examples + +```esql +ROW d = 1000.0 +| EVAL c = CBRT(d) +``` + +```esql +ROW value = 27.0 +| EVAL cube_root = CBRT(value) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-ceil.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-ceil.txt new file mode 100644 index 0000000000000..3713fa2cf4cba --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-ceil.txt @@ -0,0 +1,16 @@ +## CEIL + +The `CEIL` function rounds a number up to the nearest integer. This operation is a no-op for long (including unsigned) and integer types. For double types, it picks the closest double value to the integer, similar to `Math.ceil`. + +### Examples + +```esql +ROW a=1.8 +| EVAL a = CEIL(a) +``` + +```esql +FROM employees +| KEEP first_name, last_name, height +| EVAL height_ceil = CEIL(height) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-cidr_match.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-cidr_match.txt new file mode 100644 index 0000000000000..2e5e306d01c01 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-cidr_match.txt @@ -0,0 +1,17 @@ +## CIDR_MATCH + +The `CIDR_MATCH` function returns true if the provided IP is contained in one of the provided CIDR blocks. + +### Examples + +```esql +FROM hosts +| WHERE CIDR_MATCH(ip1, "127.0.0.2/32", "127.0.0.3/32") +| KEEP card, host, ip0, ip1 +``` + +```esql +FROM network_logs +| WHERE CIDR_MATCH(source_ip, "192.168.1.0/24", "10.0.0.0/8") +| KEEP timestamp, source_ip, destination_ip, action +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-coalesce.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-coalesce.txt new file mode 100644 index 0000000000000..057efa96da3bd --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-coalesce.txt @@ -0,0 +1,15 @@ +## COALESCE + +The `COALESCE` function returns the first of its arguments that is not null. If all arguments are null, it returns null. + +### Examples + +```esql +ROW a=null, b="b" +| EVAL COALESCE(a, b) +``` + +```esql +ROW x=null, y=null, z="z" +| EVAL first_non_null = COALESCE(x, y, z) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-concat.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-concat.txt new file mode 100644 index 0000000000000..435a8458ff05c --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-concat.txt @@ -0,0 +1,16 @@ +## CONCAT + +The `CONCAT` function concatenates two or more strings. + +### Examples + +```esql +FROM employees +| KEEP first_name, last_name +| EVAL fullname = CONCAT(first_name, " ", last_name) +``` + +```esql +ROW part1 = "Hello", part2 = "World" +| EVAL greeting = CONCAT(part1, " ", part2) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-cos.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-cos.txt new file mode 100644 index 0000000000000..e554a886c5cab --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-cos.txt @@ -0,0 +1,15 @@ +## COS + +The `COS` function returns the cosine of an angle, expressed in radians. If the input angle is null, the function returns null. + +### Examples + +```esql +ROW a=1.8 +| EVAL cos = COS(a) +``` + +```esql +ROW angle=0.5 +| EVAL cosine_value = COS(angle) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-cosh.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-cosh.txt new file mode 100644 index 0000000000000..c1eda78d10f2b --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-cosh.txt @@ -0,0 +1,15 @@ +## COSH + +Returns the hyperbolic cosine of an angle. + +### Examples + +```esql +ROW a=1.8 +| EVAL cosh = COSH(a) +``` + +```esql +ROW angle=0.5 +| EVAL hyperbolic_cosine = COSH(angle) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-count.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-count.txt new file mode 100644 index 0000000000000..407caa4c0f0c6 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-count.txt @@ -0,0 +1,32 @@ +## COUNT + +The `COUNT` function returns the total number (count) of input values. If the `field` parameter is omitted, it is equivalent to `COUNT(*)`, which counts the number of rows. + +### Examples + +```esql +FROM employees +| STATS COUNT(height) +``` + +```esql +FROM employees +| STATS count = COUNT(*) BY languages +| SORT languages DESC +``` + +```esql +ROW words="foo;bar;baz;qux;quux;foo" +| STATS word_count = COUNT(SPLIT(words, ";")) +``` + +```esql +ROW n=1 +| WHERE n < 0 +| STATS COUNT(n) +``` + +```esql +ROW n=1 +| STATS COUNT(n > 0 OR NULL), COUNT(n < 0 OR NULL) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-count_distinct.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-count_distinct.txt new file mode 100644 index 0000000000000..ec7c373e340be --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-count_distinct.txt @@ -0,0 +1,31 @@ +## COUNT_DISTINCT + +The `COUNT_DISTINCT` function returns the approximate number of distinct values in a column or literal. It uses the HyperLogLog++ algorithm to count based on the hashes of the values, providing configurable precision to trade memory for accuracy. This function is particularly useful for high-cardinality sets and large values, as it maintains fixed memory usage regardless of the number of unique values. + +### Examples + +```esql +FROM hosts +| STATS COUNT_DISTINCT(ip0), COUNT_DISTINCT(ip1) +``` + +```esql +FROM hosts +| STATS COUNT_DISTINCT(ip0, 80000), COUNT_DISTINCT(ip1, 5) +``` + +```esql +ROW words="foo;bar;baz;qux;quux;foo" +| STATS distinct_word_count = COUNT_DISTINCT(SPLIT(words, ";")) +``` + +### Additional Information + +- **Precision Threshold**: The `COUNT_DISTINCT` function takes an optional second parameter to configure the precision threshold. The maximum supported value is 40000, and the default value is 3000. This threshold allows you to trade memory for accuracy, defining a unique count below which counts are expected to be close to accurate. Above this value, counts might become a bit more fuzzy. +- **Algorithm**: The function is based on the HyperLogLog++ algorithm, which provides excellent accuracy on low-cardinality sets and fixed memory usage. The memory usage depends on the configured precision, requiring about `c * 8` bytes for a precision threshold of `c`. + +### Notes + +- Computing exact counts requires loading values into a set and returning its size, which doesn't scale well for high-cardinality sets or large values due to memory usage and communication overhead. +- The HyperLogLog++ algorithm's accuracy depends on the leading zeros of hashed values, and the exact distributions of hashes in a dataset can affect the accuracy of the cardinality. +- Even with a low threshold, the error remains very low (1-6%) even when counting millions of items. \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-date_diff.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-date_diff.txt new file mode 100644 index 0000000000000..20a261e53a100 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-date_diff.txt @@ -0,0 +1,15 @@ +## DATE_DIFF + +The `DATE_DIFF` function subtracts the `startTimestamp` from the `endTimestamp` and returns the difference in multiples of the specified unit. If `startTimestamp` is later than the `endTimestamp`, negative values are returned. Note that while there is an overlap between the function’s supported units and ES|QL’s supported time span literals, these sets are distinct and not interchangeable. Similarly, the supported abbreviations are conveniently shared with implementations of this function in other established products and not necessarily common with the date-time nomenclature used by Elasticsearch. + +### Examples + +```esql +ROW date1 = TO_DATETIME("2023-12-02T11:00:00.000Z"), date2 = TO_DATETIME("2023-12-02T11:00:00.001Z") +| EVAL dd_ms = DATE_DIFF("microseconds", date1, date2) +``` + +```esql +ROW date1 = TO_DATETIME("2023-01-01T00:00:00.000Z"), date2 = TO_DATETIME("2023-12-31T23:59:59.999Z") +| EVAL dd_days = DATE_DIFF("days", date1, date2) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-date_extract.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-date_extract.txt new file mode 100644 index 0000000000000..e064e1e09a91b --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-date_extract.txt @@ -0,0 +1,15 @@ +## DATE_EXTRACT + +The `DATE_EXTRACT` function extracts specific parts of a date, such as the year, month, day, or hour. It can be used to retrieve various components of a date based on the specified `datePart`. + +### Examples + +```esql +ROW date = DATE_PARSE("yyyy-MM-dd", "2022-05-06") +| EVAL year = DATE_EXTRACT("year", date) +``` + +```esql +FROM sample_data +| WHERE DATE_EXTRACT("hour_of_day", @timestamp) < 9 AND DATE_EXTRACT("hour_of_day", @timestamp) >= 17 +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-date_format.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-date_format.txt new file mode 100644 index 0000000000000..26149e8ce0d28 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-date_format.txt @@ -0,0 +1,17 @@ +## DATE_FORMAT + +The `DATE_FORMAT` function returns a string representation of a date in the provided format. If no format is specified, the default format `yyyy-MM-dd'T'HH:mm:ss.SSSZ` is used. If the date expression is null, the function returns null. + +### Examples + +```esql +FROM employees +| KEEP first_name, last_name, hire_date +| EVAL hired = DATE_FORMAT("YYYY-MM-dd", hire_date) +``` + +```esql +FROM employees +| KEEP first_name, last_name, hire_date +| EVAL hired = DATE_FORMAT("yyyy/MM/dd", hire_date) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-date_parse.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-date_parse.txt new file mode 100644 index 0000000000000..4d2843deed440 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-date_parse.txt @@ -0,0 +1,15 @@ +## DATE_PARSE + +The `DATE_PARSE` function returns a date by parsing the second argument using the format specified in the first argument. + +### Examples + +```esql +ROW date_string = "2022-05-06" +| EVAL date = DATE_PARSE("yyyy-MM-dd", date_string) +``` + +```esql +ROW date_string = "2023-12-25" +| EVAL date = DATE_PARSE("yyyy-MM-dd", date_string) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-date_trunc.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-date_trunc.txt new file mode 100644 index 0000000000000..28c15f62c5c53 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-date_trunc.txt @@ -0,0 +1,30 @@ +## DATE_TRUNC + +The `DATE_TRUNC` function rounds down a date to the closest interval. + +### Examples + +```esql +FROM employees +| KEEP first_name, last_name, hire_date +| EVAL year_hired = DATE_TRUNC(1 year, hire_date) +``` + +Combine `DATE_TRUNC` with `STATS ... BY` to create date histograms. For example, the number of hires per year: + +```esql +FROM employees +| EVAL year = DATE_TRUNC(1 year, hire_date) +| STATS hires = COUNT(emp_no) BY year +| SORT year +``` + +Or an hourly error rate: + +```esql +FROM sample_data +| EVAL error = CASE(message LIKE "*error*", 1, 0) +| EVAL hour = DATE_TRUNC(1 hour, @timestamp) +| STATS error_rate = AVG(error) BY hour +| SORT hour +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-dissect.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-dissect.txt new file mode 100644 index 0000000000000..5ce173f0e801d --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-dissect.txt @@ -0,0 +1,54 @@ +## DISSECT + +DISSECT enables you to extract structured data out of a string. It matches the string against a delimiter-based pattern and extracts the specified keys as columns. This command is particularly useful for parsing log files, structured text, or any other string data where fields are separated by specific delimiters. + +### Use Cases +- **Log Parsing**: Extracting timestamps, log levels, and messages from log entries. +- **Data Transformation**: Converting unstructured text data into structured columns for further analysis. +- **Data Cleaning**: Removing or reformatting specific parts of a string to make the data more usable. + +### Limitations +- If a field name conflicts with an existing column, the existing column is dropped. +- If a field name is used more than once, only the rightmost duplicate creates a column. +- DISSECT does not support reference keys. + +### Syntax + +`DISSECT input "pattern" [APPEND_SEPARATOR=""]` + +### Parameters +- **input**: The column that contains the string you want to structure. If the column has multiple values, DISSECT will process each value. +- **pattern**: A dissect pattern. +- ****: A string used as the separator between appended values, when using the append modifier. + +### Examples + +#### Example 1: Basic Usage +The following example parses a string that contains a timestamp, some text, and an IP address: + +```esql +ROW a = "2023-01-23T12:15:00.000Z - some text - 127.0.0.1" +| DISSECT a "%{date} - %{msg} - %{ip}" +| KEEP date, msg, ip +``` + +#### Example 2: Type Conversion +By default, DISSECT outputs keyword string columns. To convert to another type, use Type conversion functions: + +```esql +ROW a = "2023-01-23T12:15:00.000Z - some text - 127.0.0.1" +| DISSECT a "%{date} - %{msg} - %{ip}" +| KEEP date, msg, ip +| EVAL date = TO_DATETIME(date) +``` + +#### Example 3: Using Append Separator +In this example, we use the `APPEND_SEPARATOR` to concatenate values with a custom separator: + +```esql +ROW a = "2023-01-23T12:15:00.000Z - some text - 127.0.0.1" +| DISSECT a "%{date} - %{msg} - %{ip}" APPEND_SEPARATOR=" | " +| KEEP date, msg, ip +``` + +These examples showcase different ways to use the DISSECT command to parse and transform string data in Elasticsearch. \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-drop.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-drop.txt new file mode 100644 index 0000000000000..9bc678ef29c2f --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-drop.txt @@ -0,0 +1,58 @@ +## DROP + +The `DROP` processing command in ES|QL is used to remove one or more columns from the result set. This command is particularly useful when you want to exclude certain fields from your query results, either to simplify the output or to reduce the amount of data being processed and transferred. The `DROP` command supports the use of wildcards, allowing you to remove multiple columns that match a specific pattern. + +### Use Cases +- **Simplifying Output:** Remove unnecessary columns to make the result set easier to read and analyze. +- **Data Reduction:** Exclude large or irrelevant fields to reduce the amount of data processed and transferred. +- **Pattern Matching:** Use wildcards to efficiently drop multiple columns that share a common naming pattern. + +### Limitations +- The `DROP` command does not support nested fields. +- It cannot be used to drop columns of unsupported types as specified in the ES|QL limitations. + +### Examples + +#### Example 1: Dropping a Single Column +This example demonstrates how to drop a single column named `height` from the `employees` index. + +```esql +FROM employees +| DROP height +``` + +#### Example 2: Dropping Multiple Columns Using Wildcards +This example shows how to use wildcards to drop all columns that start with `height`. + +```esql +FROM employees +| DROP height* +``` + +#### Example 3: Dropping Multiple Specific Columns +This example demonstrates how to drop multiple specific columns by listing them in a comma-separated format. + +```esql +FROM employees +| DROP height, weight, age +``` + +#### Example 4: Dropping Columns with Complex Patterns +This example shows how to drop columns that match a more complex pattern using wildcards. + +```esql +FROM employees +| DROP emp_* +``` + +#### Example 5: Combining DROP with Other Commands +This example demonstrates how to use the `DROP` command in conjunction with other commands like `KEEP` and `SORT`. + +```esql +FROM employees +| KEEP first_name, last_name, height, weight +| DROP weight +| SORT height DESC +``` + +By using the `DROP` command, you can effectively manage the columns in your result set, making your ES|QL queries more efficient and easier to work with. \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-e.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-e.txt new file mode 100644 index 0000000000000..7f81d56ab63c2 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-e.txt @@ -0,0 +1,15 @@ +## E + +The `E` function returns Euler’s number, which is a mathematical constant approximately equal to 2.71828. It is the base of the natural logarithm. + +### Examples + +```esql +ROW E() +``` + +```esql +FROM employees +| EVAL euler_number = E() +| KEEP euler_number +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-ends_with.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-ends_with.txt new file mode 100644 index 0000000000000..7607666f70213 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-ends_with.txt @@ -0,0 +1,17 @@ +## ENDS_WITH + +The `ENDS_WITH` function returns a boolean that indicates whether a keyword string ends with another string. + +### Examples + +```esql +FROM employees +| KEEP last_name +| EVAL ln_E = ENDS_WITH(last_name, "d") +``` + +```esql +FROM employees +| KEEP first_name +| EVAL fn_E = ENDS_WITH(first_name, "a") +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-enrich.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-enrich.txt new file mode 100644 index 0000000000000..0db6c10e0d44f --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-enrich.txt @@ -0,0 +1,48 @@ +## ENRICH + +ENRICH enables you to add data from existing indices as new columns using an enrich policy. This command is useful for enriching your dataset with additional information from other indices, which can be particularly beneficial for data analysis and reporting. Before using the ENRICH command, you need to create and execute an enrich policy. + +### Use Cases +- **Data Enrichment**: Add supplementary data to your existing dataset for more comprehensive analysis. +- **Cross-Cluster Enrichment**: Enrich data across multiple clusters using the `mode` parameter. +- **Custom Column Names**: Rename columns to avoid conflicts or for better readability. + +### Limitations +- The ENRICH command only supports enrich policies of type `match`. +- ENRICH only supports enriching on a column of type `keyword`. + +### Examples + +#### Example 1: Basic Enrichment +The following example uses the `languages_policy` enrich policy to add a new column for each enrich field defined in the policy. The match is performed using the `match_field` defined in the enrich policy and requires that the input table has a column with the same name (`language_code` in this example). + +```esql +ROW language_code = "1" +| ENRICH languages_policy +``` + +#### Example 2: Using a Different Match Field +To use a column with a different name than the `match_field` defined in the policy as the match field, use the `ON` parameter. + +```esql +ROW a = "1" +| ENRICH languages_policy ON a +``` + +#### Example 3: Selecting Specific Enrich Fields +By default, each of the enrich fields defined in the policy is added as a column. To explicitly select the enrich fields that are added, use the `WITH` parameter. + +```esql +ROW a = "1" +| ENRICH languages_policy ON a WITH language_name +``` + +#### Example 4: Renaming Enrich Fields +You can rename the columns that are added using the `WITH new_name=` syntax. + +```esql +ROW a = "1" +| ENRICH languages_policy ON a WITH name = language_name +``` + +In case of name collisions, the newly created columns will override existing columns. \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-eval.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-eval.txt new file mode 100644 index 0000000000000..a7ad446cbbde9 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-eval.txt @@ -0,0 +1,55 @@ +## EVAL + +The `EVAL` processing command enables you to append new columns with calculated values. This command is useful for creating new data points derived from existing columns, such as performing arithmetic operations, applying functions, or using expressions. + +### Use Cases +- **Data Transformation**: Create new columns based on existing data, such as converting units or calculating derived metrics. +- **Data Enrichment**: Add additional context to your data by computing new values. +- **Data Cleaning**: Standardize or normalize data by applying transformations. + +### Limitations +- If a column with the same name already exists, the existing column is dropped. +- If a column name is used more than once, only the rightmost duplicate creates a column. + +### Examples + +#### Example 1: Converting Height to Different Units +This example demonstrates how to convert the height from meters to feet and centimeters. + +```esql +FROM employees +| SORT emp_no +| KEEP first_name, last_name, height +| EVAL height_feet = height * 3.281, height_cm = height * 100 +``` + +#### Example 2: Overwriting an Existing Column +In this example, the `height` column is overwritten with its value in feet. + +```esql +FROM employees +| SORT emp_no +| KEEP first_name, last_name, height +| EVAL height = height * 3.281 +``` + +#### Example 3: Using an Expression as Column Name +Here, a new column is created with a name equal to the expression used to calculate its value. + +```esql +FROM employees +| SORT emp_no +| KEEP first_name, last_name, height +| EVAL height * 3.281 +``` + +#### Example 4: Using Special Characters in Column Names +This example shows how to handle special characters in column names by quoting them with backticks. + +```esql +FROM employees +| EVAL height * 3.281 +| STATS avg_height_feet = AVG(`height * 3.281`) +``` + +These examples illustrate the versatility of the `EVAL` command in transforming and enriching your data within Elasticsearch. \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-exp.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-exp.txt new file mode 100644 index 0000000000000..89a2c612b08b7 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-exp.txt @@ -0,0 +1,15 @@ +## EXP + +The `EXP` function returns the value of Euler's number (e) raised to the power of the given numeric expression. If the input is null, the function returns null. + +### Examples + +```esql +ROW d = 5.0 +| EVAL s = EXP(d) +``` + +```esql +ROW value = 2.0 +| EVAL result = EXP(value) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-floor.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-floor.txt new file mode 100644 index 0000000000000..d3f50c55d0091 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-floor.txt @@ -0,0 +1,16 @@ +## FLOOR + +The `FLOOR` function rounds a number down to the nearest integer. This operation is a no-op for long (including unsigned) and integer types. For double types, it picks the closest double value to the integer, similar to `Math.floor`. + +### Examples + +```esql +ROW a=1.8 +| EVAL a = FLOOR(a) +``` + +```esql +FROM employees +| KEEP first_name, last_name, height +| EVAL height_floor = FLOOR(height) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-from.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-from.txt new file mode 100644 index 0000000000000..7847e7c847655 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-from.txt @@ -0,0 +1,53 @@ +## FROM + +The `FROM` source command returns a table with data from a data stream, index, or alias. Each row in the resulting table represents a document, and each column corresponds to a field that can be accessed by the name of that field. This command is fundamental for querying data in Elasticsearch using ES|QL. + +### Use Cases + +- **Basic Data Retrieval**: Fetch data from a specific index or data stream. +- **Time Series Data**: Use date math to access indices relevant to specific time periods. +- **Multiple Indices**: Query multiple data streams, indices, or aliases using comma-separated lists or wildcards. +- **Remote Clusters**: Query data streams and indices on remote clusters. +- **Metadata Retrieval**: Retrieve specific metadata fields using the `METADATA` directive. + +### Limitations + +- By default, an ES|QL query without an explicit `LIMIT` uses an implicit limit of 1000 rows. This applies to the `FROM` command as well. +- Queries do not return more than 10,000 rows, regardless of the `LIMIT` command’s value. + +### Examples + +#### Basic Data Retrieval +```esql +FROM employees +``` + +#### Time Series Data +Use date math to refer to indices, aliases, and data streams. This can be useful for time series data, for example, to access today’s index: +```esql +FROM +``` + +#### Multiple Indices +Use comma-separated lists or wildcards to query multiple data streams, indices, or aliases: +```esql +FROM employees-00001,other-employees-* +``` + +#### Remote Clusters +Use the format `:` to query data streams and indices on remote clusters: +```esql +FROM cluster_one:employees-00001,cluster_two:other-employees-* +``` + +#### Metadata Retrieval +Use the optional `METADATA` directive to enable metadata fields: +```esql +FROM employees METADATA _id +``` + +#### Escaping Special Characters +Use enclosing double quotes (") or three enclosing double quotes (""") to escape index names that contain special characters: +```esql +FROM "this=that","""this[that""" +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-from_base64.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-from_base64.txt new file mode 100644 index 0000000000000..19090768a9db4 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-from_base64.txt @@ -0,0 +1,15 @@ +## FROM_BASE64 + +Decodes a base64 string. + +### Examples + +```esql +ROW a = "ZWxhc3RpYw==" +| EVAL d = FROM_BASE64(a) +``` + +```esql +ROW encoded = "U29tZSBzYW1wbGUgdGV4dA==" +| EVAL decoded = FROM_BASE64(encoded) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-greatest.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-greatest.txt new file mode 100644 index 0000000000000..17217e8e84682 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-greatest.txt @@ -0,0 +1,15 @@ +## GREATEST + +The `GREATEST` function returns the maximum value from multiple columns. This is similar to `MV_MAX` except it is intended to run on multiple columns at once. When run on keyword or text fields, this function returns the last string in alphabetical order. When run on boolean columns, it will return `true` if any values are `true`. + +### Examples + +```esql +ROW a = 10, b = 20 +| EVAL g = GREATEST(a, b) +``` + +```esql +ROW x = "apple", y = "banana", z = "cherry" +| EVAL max_fruit = GREATEST(x, y, z) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-grok.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-grok.txt new file mode 100644 index 0000000000000..cc357b986a58b --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-grok.txt @@ -0,0 +1,55 @@ +## GROK + +GROK enables you to extract structured data out of a string. It matches the string against patterns based on regular expressions and extracts the specified patterns as columns. This command is useful for parsing logs, extracting fields from text, and structuring unstructured data. + +### Use Cases +- **Log Parsing**: Extracting timestamps, IP addresses, and other fields from log entries. +- **Data Structuring**: Converting unstructured text data into structured columns. +- **Field Extraction**: Extracting specific fields from a string for further analysis. + +### Limitations +- If a field name conflicts with an existing column, the existing column is discarded. +- If a field name is used more than once, a multi-valued column will be created with one value per each occurrence of the field name. +- The `GROK` command does not support configuring custom patterns or multiple patterns. +- The `GROK` command is not subject to Grok watchdog settings. + +### Examples + +#### Example 1: Basic GROK Usage +This example parses a string that contains a timestamp, an IP address, an email address, and a number. + +```esql +ROW a = "2023-01-23T12:15:00.000Z 127.0.0.1 some.email@foo.com 42" +| GROK a "%{TIMESTAMP_ISO8601:date} %{IP:ip} %{EMAILADDRESS:email} %{NUMBER:num}" +| KEEP date, ip, email, num +``` + +#### Example 2: Type Conversion with GROK +By default, GROK outputs keyword string columns. To convert to other types, append `:type` to the semantics in the pattern. + +```esql +ROW a = "2023-01-23T12:15:00.000Z 127.0.0.1 some.email@foo.com 42" +| GROK a "%{TIMESTAMP_ISO8601:date} %{IP:ip} %{EMAILADDRESS:email} %{NUMBER:num:int}" +| KEEP date, ip, email, num +``` + +#### Example 3: Using Type Conversion Functions +For other type conversions, use Type conversion functions. + +```esql +ROW a = "2023-01-23T12:15:00.000Z 127.0.0.1 some.email@foo.com 42" +| GROK a "%{TIMESTAMP_ISO8601:date} %{IP:ip} %{EMAILADDRESS:email} %{NUMBER:num:int}" +| KEEP date, ip, email, num +| EVAL date = TO_DATETIME(date) +``` + +#### Example 4: Handling Multi-Valued Columns +If a field name is used more than once, GROK creates a multi-valued column. + +```esql +FROM addresses +| KEEP city.name, zip_code +| GROK zip_code "%{WORD:zip_parts} %{WORD:zip_parts}" +``` + +These examples showcase different usages of the GROK command, from basic extraction to handling type conversions and multi-valued columns. \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-ip_prefix.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-ip_prefix.txt new file mode 100644 index 0000000000000..65d4ccbf5d4b3 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-ip_prefix.txt @@ -0,0 +1,16 @@ +## IP_PREFIX + +The `IP_PREFIX` function truncates an IP address to a given prefix length. It supports both IPv4 and IPv6 addresses. + +### Examples + +```esql +ROW ip4 = TO_IP("1.2.3.4"), ip6 = TO_IP("fe80::cae2:65ff:fece:feb9") +| EVAL ip4_prefix = IP_PREFIX(ip4, 24, 0), ip6_prefix = IP_PREFIX(ip6, 0, 112) +``` + +```esql +FROM network_logs +| EVAL truncated_ip = IP_PREFIX(ip_address, 16, 0) +| KEEP ip_address, truncated_ip +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-keep.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-keep.txt new file mode 100644 index 0000000000000..fbf2466d26c6e --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-keep.txt @@ -0,0 +1,64 @@ +## KEEP + +The `KEEP` processing command in ES|QL enables you to specify which columns are returned and the order in which they are returned. This command is particularly useful when you want to focus on specific fields in your dataset, either by explicitly naming them or by using wildcard patterns. The `KEEP` command supports a variety of use cases, such as filtering out unnecessary columns, reordering columns for better readability, and ensuring that only relevant data is processed in subsequent commands. + +### Use Cases +- **Selective Column Retrieval**: Retrieve only the columns you need for analysis, reducing the amount of data processed. +- **Column Reordering**: Specify the order in which columns should appear in the result set. +- **Wildcard Support**: Use wildcards to include multiple columns that match a pattern, simplifying queries when dealing with numerous fields. + +### Limitations +- **Precedence Rules**: When a field name matches multiple expressions, precedence rules are applied. Complete field names take the highest precedence, followed by partial wildcard expressions, and finally, the wildcard `*`. +- **Column Conflicts**: If a field matches two expressions with the same precedence, the rightmost expression wins. + +### Examples + +#### Example 1: Specifying Columns Explicitly +This example demonstrates how to explicitly specify the columns to be returned. + +```esql +FROM employees +| KEEP emp_no, first_name, last_name, height +``` + +#### Example 2: Using Wildcards to Match Column Names +This example shows how to use wildcards to return all columns that match a specific pattern. + +```esql +FROM employees +| KEEP h* +``` + +#### Example 3: Combining Wildcards and Explicit Column Names +This example illustrates how to combine wildcards and explicit column names, and how precedence rules are applied. + +```esql +FROM employees +| KEEP h*, * +``` + +#### Example 4: Precedence Rules with Complete Field Names +This example demonstrates how complete field names take precedence over wildcard expressions. + +```esql +FROM employees +| KEEP first_name, last_name, first_name* +``` + +#### Example 5: Wildcard Expressions with Same Priority +This example shows how the last wildcard expression wins when multiple wildcard expressions have the same priority. + +```esql +FROM employees +| KEEP first_name*, last_name, first_na* +``` + +#### Example 6: Simple Wildcard Expression with Lowest Precedence +This example illustrates how the simple wildcard expression `*` has the lowest precedence. + +```esql +FROM employees +| KEEP *, first_name +``` + +These examples showcase the versatility and utility of the `KEEP` command in various scenarios, making it a powerful tool for data manipulation in ES|QL. \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-least.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-least.txt new file mode 100644 index 0000000000000..f756820f7840d --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-least.txt @@ -0,0 +1,15 @@ +## LEAST + +Returns the minimum value from multiple columns. This is similar to `MV_MIN` except it is intended to run on multiple columns at once. + +### Examples + +```esql +ROW a = 10, b = 20 +| EVAL l = LEAST(a, b) +``` + +```esql +ROW x = 5, y = 15, z = 10 +| EVAL min_value = LEAST(x, y, z) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-left.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-left.txt new file mode 100644 index 0000000000000..5164a100ea22b --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-left.txt @@ -0,0 +1,19 @@ +## LEFT + +The `LEFT` function returns the substring that extracts a specified number of characters from a string, starting from the left. + +### Examples + +```esql +FROM employees +| KEEP last_name +| EVAL left = LEFT(last_name, 3) +| SORT last_name ASC +| LIMIT 5 +``` + +```esql +ROW full_name = "John Doe" +| EVAL first_name = LEFT(full_name, 4) +| KEEP first_name +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-length.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-length.txt new file mode 100644 index 0000000000000..ea692e7fe9eae --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-length.txt @@ -0,0 +1,16 @@ +## LENGTH + +The `LENGTH` function returns the character length of a string. If the input string is null, the function returns null. + +### Examples + +```esql +FROM employees +| KEEP first_name, last_name +| EVAL fn_length = LENGTH(first_name) +``` + +```esql +ROW message = "Hello, World!" +| EVAL message_length = LENGTH(message) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-limit.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-limit.txt new file mode 100644 index 0000000000000..da1a0f85a8782 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-limit.txt @@ -0,0 +1,68 @@ +## LIMIT + +The `LIMIT` processing command in ES|QL is used to restrict the number of rows returned by a query. This is particularly useful when you want to control the volume of data retrieved, either for performance reasons or to focus on a specific subset of the data. + +### Use Cases +- **Performance Optimization**: By limiting the number of rows returned, you can improve query performance and reduce the load on the Elasticsearch cluster. +- **Data Sampling**: Useful for retrieving a sample of data for analysis or debugging. +- **Pagination**: Helps in implementing pagination by limiting the number of rows per page. + +### Limitations +- **Maximum Rows**: Queries do not return more than 10,000 rows, regardless of the `LIMIT` command’s value. This limit only applies to the number of rows that are retrieved by the query. Queries and aggregations run on the full data set. +- **Overcoming Limitations**: To overcome this limitation, you can: + - Reduce the result set size by modifying the query to only return relevant data using the `WHERE` command. + - Shift any post-query processing to the query itself using the `STATS ... BY` command to aggregate data in the query. +- **Dynamic Cluster Settings**: The default and maximum limits can be changed using these dynamic cluster settings: + - `esql.query.result_truncation_default_size` + - `esql.query.result_truncation_max_size` + +### Examples + +#### Example 1: Basic Usage +This example demonstrates how to limit the number of rows returned to 5. + +```esql +FROM employees +| SORT emp_no ASC +| LIMIT 5 +``` + +#### Example 2: Limiting Rows After Filtering +This example shows how to limit the number of rows after applying a filter. + +```esql +FROM employees +| WHERE department == "Engineering" +| LIMIT 10 +``` + +#### Example 3: Limiting Rows with Aggregation +This example demonstrates limiting the number of rows after performing an aggregation. + +```esql +FROM employees +| STATS avg_salary = AVG(salary) BY department +| LIMIT 3 +``` + +#### Example 4: Limiting Rows with Sorting +This example shows how to limit the number of rows after sorting the data. + +```esql +FROM employees +| SORT hire_date DESC +| LIMIT 7 +``` + +#### Example 5: Limiting Rows with Multiple Commands +This example demonstrates the use of `LIMIT` in conjunction with multiple other commands. + +```esql +FROM employees +| WHERE hire_date > "2020-01-01" +| SORT salary DESC +| KEEP first_name, last_name, salary +| LIMIT 5 +``` + +By using the `LIMIT` command, you can effectively manage the volume of data returned by your ES|QL queries, ensuring better performance and more focused results. \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-locate.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-locate.txt new file mode 100644 index 0000000000000..e62ea05fcc3ab --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-locate.txt @@ -0,0 +1,25 @@ +## LOCATE + +The `LOCATE` function returns an integer that indicates the position of a keyword substring within another string. + +### Syntax + +`LOCATE(string, substring, start)` + +### Parameters + +- `string`: An input string. +- `substring`: A substring to locate in the input string. +- `start`: The start index. + +### Examples + +```esql +ROW a = "hello" +| EVAL a_ll = LOCATE(a, "ll") +``` + +```esql +ROW phrase = "Elasticsearch is powerful" +| EVAL position = LOCATE(phrase, "powerful") +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-log.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-log.txt new file mode 100644 index 0000000000000..b41fef3adc86d --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-log.txt @@ -0,0 +1,15 @@ +## LOG + +The `LOG` function returns the logarithm of a value to a specified base. The input can be any numeric value, and the return value is always a double. Logs of zero, negative numbers, and base of one return null as well as a warning. + +### Examples + +```esql +ROW base = 2.0, value = 8.0 +| EVAL s = LOG(base, value) +``` + +```esql +ROW value = 100 +| EVAL s = LOG(value) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-log10.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-log10.txt new file mode 100644 index 0000000000000..f8b9748577624 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-log10.txt @@ -0,0 +1,15 @@ +## LOG10 + +The `LOG10` function returns the logarithm of a value to base 10. The input can be any numeric value, and the return value is always a double. Logs of 0 and negative numbers return null as well as a warning. + +### Examples + +```esql +ROW d = 1000.0 +| EVAL s = LOG10(d) +``` + +```esql +ROW value = 100 +| EVAL log_value = LOG10(value) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-lookup.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-lookup.txt new file mode 100644 index 0000000000000..fc9312674db81 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-lookup.txt @@ -0,0 +1,101 @@ +## LOOKUP + +The `LOOKUP` command in ES|QL is highly experimental and only available in SNAPSHOT versions. It matches values from the input against a table provided in the request, adding the other fields from the table to the output. This command is useful for enriching your dataset with additional information from a predefined table. However, it is important to note that if the table’s column names conflict with existing columns, the existing columns will be dropped. + +### Examples + +Here are some example ES|QL queries using the `LOOKUP` command: + +1. **Basic Lookup Example:** + ```esql +FROM library +| SORT page_count DESC +| KEEP name, author +| LOOKUP era ON author +| LIMIT 5 +``` + +2. **Lookup with Multiple Match Fields:** + ```esql +FROM library +| SORT page_count DESC +| KEEP name, author, genre +| LOOKUP era ON author, genre +| LIMIT 5 +``` + +3. **Lookup with Different Table:** + ```esql +FROM library +| SORT page_count DESC +| KEEP name, author +| LOOKUP awards ON author +| LIMIT 5 +``` + +### Content of file + +```plaintext +LOOKUP + +LOOKUP is highly experimental and only available in SNAPSHOT versions. +LOOKUP matches values from the input against a table provided in the request, +adding the other fields from the table to the output. + +Syntax: +LOOKUP table ON match_field1[, match_field2, ...] + +Parameters: +- table: The name of the table provided in the request to match. If the table’s column names conflict with existing columns, the existing columns will be dropped. +- match_field: The fields in the input to match against the table. + +Examples: +const response = await client.esql.query({ + format: "txt", + query: + "\n FROM library\n | SORT page_count DESC\n | KEEP name, author\n | LOOKUP era ON author\n | LIMIT 5\n ", + tables: { + era: { + author: { + keyword: [ + "Frank Herbert", + "Peter F. Hamilton", + "Vernor Vinge", + "Alastair Reynolds", + "James S.A. Corey", + ], + }, + era: { + keyword: ["The New Wave", "Diamond", "Diamond", "Diamond", "Hadron"], + }, + }, + }, +}); +console.log(response); + +POST /_query?format=txt +{ + "query": """ + FROM library + | SORT page_count DESC + | KEEP name, author + | LOOKUP era ON author + | LIMIT 5 + """, + "tables": { + "era": { + "author": {"keyword": ["Frank Herbert", "Peter F. Hamilton", "Vernor Vinge", "Alastair Reynolds", "James S.A. Corey"]}, + "era": {"keyword": [ "The New Wave", "Diamond", "Diamond", "Diamond", "Hadron"]} + } + } +} + +Which returns: +name | author | era +--------------------+-----------------+--------------- +Pandora's Star |Peter F. Hamilton|Diamond +A Fire Upon the Deep|Vernor Vinge |Diamond +Dune |Frank Herbert |The New Wave +Revelation Space |Alastair Reynolds|Diamond +Leviathan Wakes |James S.A. Corey |Hadron +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-ltrim.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-ltrim.txt new file mode 100644 index 0000000000000..7a34fe57f9801 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-ltrim.txt @@ -0,0 +1,19 @@ +## LTRIM + +Removes leading whitespaces from a string. + +### Examples + +```esql +ROW message = " some text ", color = " red " +| EVAL message = LTRIM(message) +| EVAL color = LTRIM(color) +| EVAL message = CONCAT("'", message, "'") +| EVAL color = CONCAT("'", color, "'") +``` + +```esql +ROW text = " example text " +| EVAL trimmed_text = LTRIM(text) +| EVAL formatted_text = CONCAT("Trimmed: '", trimmed_text, "'") +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-max.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-max.txt new file mode 100644 index 0000000000000..381c66afa9bb1 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-max.txt @@ -0,0 +1,15 @@ +## MAX + +The `MAX` function returns the maximum value of a specified field. + +### Examples + +```esql +FROM employees +| STATS MAX(languages) +``` + +```esql +FROM employees +| STATS max_avg_salary_change = MAX(MV_AVG(salary_change)) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-median.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-median.txt new file mode 100644 index 0000000000000..5da7a9be4fdb3 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-median.txt @@ -0,0 +1,15 @@ +## MEDIAN + +The `MEDIAN` function returns the value that is greater than half of all values and less than half of all values, also known as the 50% PERCENTILE. Like `PERCENTILE`, `MEDIAN` is usually approximate. It is also non-deterministic, meaning you can get slightly different results using the same data. + +### Examples + +```esql +FROM employees +| STATS MEDIAN(salary), PERCENTILE(salary, 50) +``` + +```esql +FROM employees +| STATS median_max_salary_change = MEDIAN(MV_MAX(salary_change)) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-median_absolute_deviation.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-median_absolute_deviation.txt new file mode 100644 index 0000000000000..07da947d5494c --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-median_absolute_deviation.txt @@ -0,0 +1,15 @@ +## MEDIAN_ABSOLUTE_DEVIATION + +The `MEDIAN_ABSOLUTE_DEVIATION` function returns the median absolute deviation, a measure of variability. It is a robust statistic, meaning that it is useful for describing data that may have outliers, or may not be normally distributed. For such data, it can be more descriptive than standard deviation. It is calculated as the median of each data point’s deviation from the median of the entire sample. That is, for a random variable X, the median absolute deviation is median(|median(X) - X|). Like `PERCENTILE`, `MEDIAN_ABSOLUTE_DEVIATION` is usually approximate. + +### Examples + +```esql +FROM employees +| STATS MEDIAN(salary), MEDIAN_ABSOLUTE_DEVIATION(salary) +``` + +```esql +FROM employees +| STATS m_a_d_max_salary_change = MEDIAN_ABSOLUTE_DEVIATION(MV_MAX(salary_change)) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-min.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-min.txt new file mode 100644 index 0000000000000..043ad01280ad8 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-min.txt @@ -0,0 +1,15 @@ +## MIN + +The `MIN` function returns the minimum value of a specified field. + +### Examples + +```esql +FROM employees +| STATS MIN(languages) +``` + +```esql +FROM employees +| STATS min_avg_salary_change = MIN(MV_AVG(salary_change)) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_append.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_append.txt new file mode 100644 index 0000000000000..9926157ce96c4 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_append.txt @@ -0,0 +1,17 @@ +## MV_APPEND + +The `MV_APPEND` function concatenates values of two multi-value fields. + +### Examples + +```esql +ROW a = ["foo", "bar"], b = ["baz", "qux"] +| EVAL c = MV_APPEND(a, b) +| KEEP a, b, c +``` + +```esql +ROW x = [1, 2, 3], y = [4, 5, 6] +| EVAL z = MV_APPEND(x, y) +| KEEP x, y, z +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_avg.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_avg.txt new file mode 100644 index 0000000000000..431c4ec6b2891 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_avg.txt @@ -0,0 +1,15 @@ +## MV_AVG + +The `MV_AVG` function converts a multivalued field into a single-valued field containing the average of all the values. + +### Examples + +```esql +ROW a=[3, 5, 1, 6] +| EVAL avg_a = MV_AVG(a) +``` + +```esql +ROW scores=[10, 20, 30, 40] +| EVAL average_score = MV_AVG(scores) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_concat.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_concat.txt new file mode 100644 index 0000000000000..32c029703257d --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_concat.txt @@ -0,0 +1,15 @@ +## MV_CONCAT + +Converts a multivalued string expression into a single valued column containing the concatenation of all values separated by a delimiter. + +### Examples + +```esql +ROW a=["foo", "zoo", "bar"] +| EVAL j = MV_CONCAT(a, ", ") +``` + +```esql +ROW a=[10, 9, 8] +| EVAL j = MV_CONCAT(TO_STRING(a), ", ") +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_count.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_count.txt new file mode 100644 index 0000000000000..a8f8d0c5149ad --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_count.txt @@ -0,0 +1,15 @@ +## MV_COUNT + +The `MV_COUNT` function converts a multivalued expression into a single-valued column containing a count of the number of values. + +### Examples + +```esql +ROW a=["foo", "zoo", "bar"] +| EVAL count_a = MV_COUNT(a) +``` + +```esql +ROW b=["apple", "banana", "cherry", "date"] +| EVAL count_b = MV_COUNT(b) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_dedupe.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_dedupe.txt new file mode 100644 index 0000000000000..297179f995dff --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_dedupe.txt @@ -0,0 +1,15 @@ +## MV_DEDUPE + +Removes duplicate values from a multivalued field. `MV_DEDUPE` may, but won’t always, sort the values in the column. + +### Examples + +```esql +ROW a=["foo", "foo", "bar", "foo"] +| EVAL dedupe_a = MV_DEDUPE(a) +``` + +```esql +ROW b=["apple", "apple", "banana", "apple", "banana"] +| EVAL dedupe_b = MV_DEDUPE(b) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_expand.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_expand.txt new file mode 100644 index 0000000000000..76528b5e22654 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_expand.txt @@ -0,0 +1,41 @@ +## MV_EXPAND + +The `MV_EXPAND` processing command expands multivalued columns into one row per value, duplicating other columns. This command is useful when you need to normalize data that contains multivalued fields, making it easier to perform operations on each individual value. + +### Use Cases +- **Normalization**: Transform multivalued fields into single-valued rows for easier analysis and processing. +- **Data Transformation**: Prepare data for further operations like sorting, filtering, or aggregating by expanding multivalued fields. +- **Data Cleaning**: Simplify complex data structures by breaking down multivalued fields into individual rows. + +### Limitations +- This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features. + +### Examples + +#### Example 1: Basic Expansion +Expanding a multivalued column `a` into individual rows. + +```esql +ROW a=[1,2,3], b="b", j=["a","b"] +| MV_EXPAND a +``` + +#### Example 2: Expanding Multiple Columns +Expanding two multivalued columns `a` and `j` into individual rows. + +```esql +ROW a=[1,2,3], b="b", j=["a","b"] +| MV_EXPAND a +| MV_EXPAND j +``` + +#### Example 3: Combining with Other Commands +Expanding a multivalued column and then filtering the results. + +```esql +ROW a=[1,2,3,4,5], b="b" +| MV_EXPAND a +| WHERE a > 2 +``` + +These examples demonstrate different ways to use the `MV_EXPAND` command to transform and analyze data with multivalued fields. \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_first.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_first.txt new file mode 100644 index 0000000000000..1969ad30226ac --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_first.txt @@ -0,0 +1,15 @@ +## MV_FIRST + +The `MV_FIRST` function converts a multivalued expression into a single-valued column containing the first value. This is most useful when reading from a function that emits multivalued columns in a known order like `SPLIT`. The order that multivalued fields are read from underlying storage is not guaranteed. It is frequently ascending, but don’t rely on that. If you need the minimum value, use `MV_MIN` instead of `MV_FIRST`. `MV_MIN` has optimizations for sorted values so there isn’t a performance benefit to `MV_FIRST`. + +### Examples + +```esql +ROW a="foo;bar;baz" +| EVAL first_a = MV_FIRST(SPLIT(a, ";")) +``` + +```esql +ROW b="apple;banana;cherry" +| EVAL first_b = MV_FIRST(SPLIT(b, ";")) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_last.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_last.txt new file mode 100644 index 0000000000000..f6331ab55a7eb --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_last.txt @@ -0,0 +1,15 @@ +## MV_LAST + +The `MV_LAST` function converts a multivalue expression into a single valued column containing the last value. This is most useful when reading from a function that emits multivalued columns in a known order like `SPLIT`. The order that multivalued fields are read from underlying storage is not guaranteed. It is frequently ascending, but don’t rely on that. If you need the maximum value, use `MV_MAX` instead of `MV_LAST`. `MV_MAX` has optimizations for sorted values so there isn’t a performance benefit to `MV_LAST`. + +### Examples + +```esql +ROW a="foo;bar;baz" +| EVAL last_a = MV_LAST(SPLIT(a, ";")) +``` + +```esql +ROW a="apple;banana;cherry" +| EVAL last_fruit = MV_LAST(SPLIT(a, ";")) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_max.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_max.txt new file mode 100644 index 0000000000000..4c6d50ec151ee --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_max.txt @@ -0,0 +1,15 @@ +## MV_MAX + +The `MV_MAX` function converts a multivalued expression into a single valued column containing the maximum value. + +### Examples + +```esql +ROW a=[3, 5, 1] +| EVAL max_a = MV_MAX(a) +``` + +```esql +ROW a=["foo", "zoo", "bar"] +| EVAL max_a = MV_MAX(a) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_median.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_median.txt new file mode 100644 index 0000000000000..6702441a82bca --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_median.txt @@ -0,0 +1,17 @@ +## MV_MEDIAN + +The `MV_MEDIAN` function converts a multivalued field into a single valued field containing the median value. + +### Examples + +```esql +ROW a=[3, 5, 1] +| EVAL median_a = MV_MEDIAN(a) +``` + +If the row has an even number of values for a column, the result will be the average of the middle two entries. If the column is not floating point, the average rounds down: + +```esql +ROW a=[3, 7, 1, 6] +| EVAL median_a = MV_MEDIAN(a) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_min.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_min.txt new file mode 100644 index 0000000000000..386f5d424cef8 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_min.txt @@ -0,0 +1,15 @@ +## MV_MIN + +The `MV_MIN` function converts a multivalued expression into a single valued column containing the minimum value. + +### Examples + +```esql +ROW a=[2, 1] +| EVAL min_a = MV_MIN(a) +``` + +```esql +ROW a=["foo", "bar"] +| EVAL min_a = MV_MIN(a) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_pseries_weighted_sum.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_pseries_weighted_sum.txt new file mode 100644 index 0000000000000..1b1fc706b8d3d --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_pseries_weighted_sum.txt @@ -0,0 +1,17 @@ +## MV_PSERIES_WEIGHTED_SUM + +Converts a multivalued expression into a single-valued column by multiplying every element on the input list by its corresponding term in P-Series and computing the sum. + +### Examples + +```esql +ROW a = [70.0, 45.0, 21.0, 21.0, 21.0] +| EVAL sum = MV_PSERIES_WEIGHTED_SUM(a, 1.5) +| KEEP sum +``` + +```esql +ROW b = [10.0, 20.0, 30.0, 40.0, 50.0] +| EVAL weighted_sum = MV_PSERIES_WEIGHTED_SUM(b, 2.0) +| KEEP weighted_sum +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_slice.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_slice.txt new file mode 100644 index 0000000000000..4b93d9703095b --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_slice.txt @@ -0,0 +1,15 @@ +## MV_SLICE + +The `MV_SLICE` function returns a subset of the multivalued field using the start and end index values. + +### Examples + +```esql +ROW a = [1, 2, 2, 3] +| EVAL a1 = MV_SLICE(a, 1), a2 = MV_SLICE(a, 2, 3) +``` + +```esql +ROW a = [1, 2, 2, 3] +| EVAL a1 = MV_SLICE(a, -2), a2 = MV_SLICE(a, -3, -1) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_sort.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_sort.txt new file mode 100644 index 0000000000000..14d41a8fd8d56 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_sort.txt @@ -0,0 +1,15 @@ +## MV_SORT + +The `MV_SORT` function sorts a multivalued field in lexicographical order. The valid options for the sort order are `ASC` (ascending) and `DESC` (descending), with the default being `ASC`. + +### Examples + +```esql +ROW a = [4, 2, -3, 2] +| EVAL sa = mv_sort(a), sd = mv_sort(a, "DESC") +``` + +```esql +ROW names = ["Alice", "Bob", "Charlie"] +| EVAL sorted_names = mv_sort(names) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_sum.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_sum.txt new file mode 100644 index 0000000000000..8ee548edccc9c --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_sum.txt @@ -0,0 +1,15 @@ +## MV_SUM + +The `MV_SUM` function converts a multivalued field into a single valued field containing the sum of all of the values. + +### Examples + +```esql +ROW a=[3, 5, 6] +| EVAL sum_a = MV_SUM(a) +``` + +```esql +ROW numbers=[1, 2, 3, 4, 5] +| EVAL total_sum = MV_SUM(numbers) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_zip.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_zip.txt new file mode 100644 index 0000000000000..953519b4bd3fe --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_zip.txt @@ -0,0 +1,17 @@ +## MV_ZIP + +The `MV_ZIP` function combines the values from two multivalued fields with a delimiter that joins them together. + +### Examples + +```esql +ROW a = ["x", "y", "z"], b = ["1", "2"] +| EVAL c = mv_zip(a, b, "-") +| KEEP a, b, c +``` + +```esql +ROW names = ["Alice", "Bob", "Charlie"], ids = ["001", "002", "003"] +| EVAL combined = mv_zip(names, ids, ":") +| KEEP names, ids, combined +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-now.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-now.txt new file mode 100644 index 0000000000000..165bcfe7af1dd --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-now.txt @@ -0,0 +1,14 @@ +## NOW + +The `NOW` function returns the current date and time. + +### Examples + +```esql +ROW current_date = NOW() +``` + +```esql +FROM sample_data +| WHERE @timestamp > NOW() - 1 hour +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-operators.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-operators.txt new file mode 100644 index 0000000000000..cc6c7f5bdf348 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-operators.txt @@ -0,0 +1,216 @@ +# ES|QL Operators + +## Binary Operators + +### Equality (`==`) +Check if two fields are equal. If either field is multivalued, the result is null. This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an index and doc_values. + +#### Example: +```esql +FROM employees +| WHERE first_name == "John" +| KEEP first_name, last_name +``` + +### Inequality (`!=`) +Check if two fields are unequal. If either field is multivalued, the result is null. This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an index and doc_values. + +#### Example: +```esql +FROM employees +| WHERE first_name != "John" +| KEEP first_name, last_name +``` + +### Less than (`<`) +Check if one field is less than another. If either field is multivalued, the result is null. This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an index and doc_values. + +#### Example: +```esql +FROM employees +| WHERE age < 30 +| KEEP first_name, last_name, age +``` + +### Less than or equal to (`<=`) +Check if one field is less than or equal to another. If either field is multivalued, the result is null. This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an index and doc_values. + +#### Example: +```esql +FROM employees +| WHERE age <= 30 +| KEEP first_name, last_name, age +``` + +### Greater than (`>`) +Check if one field is greater than another. If either field is multivalued, the result is null. This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an index and doc_values. + +#### Example: +```esql +FROM employees +| WHERE age > 30 +| KEEP first_name, last_name, age +``` + +### Greater than or equal to (`>=`) +Check if one field is greater than or equal to another. If either field is multivalued, the result is null. This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an index and doc_values. + +#### Example: +```esql +FROM employees +| WHERE age >= 30 +| KEEP first_name, last_name, age +``` + +### Add (`+`) +Add two numbers together. If either field is multivalued, the result is null. + +#### Example: +```esql +FROM employees +| EVAL total_salary = base_salary + bonus +| KEEP first_name, last_name, total_salary +``` + +### Subtract (`-`) +Subtract one number from another. If either field is multivalued, the result is null. + +#### Example: +```esql +FROM employees +| EVAL net_salary = gross_salary - tax +| KEEP first_name, last_name, net_salary +``` + +### Multiply (`*`) +Multiply two numbers together. If either field is multivalued, the result is null. + +#### Example: +```esql +FROM employees +| EVAL annual_salary = monthly_salary * 12 +| KEEP first_name, last_name, annual_salary +``` + +### Divide (`/`) +Divide one number by another. If either field is multivalued, the result is null. Division of two integer types will yield an integer result, rounding towards 0. If you need floating point division, cast one of the arguments to a `DOUBLE`. + +#### Example: +```esql +FROM employees +| EVAL average_salary = total_salary / months_worked +| KEEP first_name, last_name, average_salary +``` + +### Modulus (`%`) +Divide one number by another and return the remainder. If either field is multivalued, the result is null. + +#### Example: +```esql +FROM employees +| EVAL remainder = total_days % 7 +| KEEP first_name, last_name, remainder +``` + +## Unary Operators + +### Negation (`-`) +The only unary operator is negation. + +#### Example: +```esql +FROM employees +| EVAL negative_salary = -salary +| KEEP first_name, last_name, negative_salary +``` + +## Logical Operators + +### AND +Logical AND operator. + +#### Example: +```esql +FROM employees +| WHERE age > 30 AND department == "Engineering" +| KEEP first_name, last_name, age, department +``` + +### OR +Logical OR operator. + +#### Example: +```esql +FROM employees +| WHERE age > 30 OR department == "Engineering" +| KEEP first_name, last_name, age, department +``` + +### NOT +Logical NOT operator. + +#### Example: +```esql +FROM employees +| WHERE NOT (age > 30) +| KEEP first_name, last_name, age +``` + +## Other Operators + +### IS NULL and IS NOT NULL +For NULL comparison, use the `IS NULL` and `IS NOT NULL` predicates. + +#### Example: +```esql +FROM employees +| WHERE birth_date IS NULL +| KEEP first_name, last_name +| SORT first_name +| LIMIT 3 +``` + +```esql +FROM employees +| WHERE is_rehired IS NOT NULL +| STATS COUNT(emp_no) +``` + +### Cast (`::`) +The `::` operator provides a convenient alternative syntax to the `TO_` conversion functions. + +#### Example: +```esql +ROW ver = CONCAT(("0"::INT + 1)::STRING, ".2.3")::VERSION +``` + +### IN +The `IN` operator allows testing whether a field or expression equals an element in a list of literals, fields, or expressions. + +#### Example: +```esql +ROW a = 1, b = 4, c = 3 +| WHERE c-a IN (3, b / 2, a) +``` + +### LIKE +Use `LIKE` to filter data based on string patterns using wildcards. The following wildcard characters are supported: +- `*` matches zero or more characters. +- `?` matches one character. + +#### Example: +```esql +FROM employees +| WHERE first_name LIKE "?b*" +| KEEP first_name, last_name +``` + +### RLIKE +Use `RLIKE` to filter data based on string patterns using regular expressions. + +#### Example: +```esql +FROM employees +| WHERE first_name RLIKE ".leja.*" +| KEEP first_name, last_name +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-overview.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-overview.txt new file mode 100644 index 0000000000000..32e82c7986480 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-overview.txt @@ -0,0 +1,144 @@ +## Overview + +### ES|QL + +The Elasticsearch Query Language (ES|QL) provides a powerful way to filter, transform, and analyze data stored in Elasticsearch, and in the future in other runtimes. It is designed to be easy to learn and use by end users, SRE teams, application developers, and administrators. + +Users can author ES|QL queries to find specific events, perform statistical analysis, and generate visualizations. It supports a wide range of commands and functions that enable users to perform various data operations, such as filtering, aggregation, time-series analysis, and more. + +ES|QL makes use of "pipes" (`|`) to manipulate and transform data in a step-by-step fashion. This approach allows users to compose a series of operations, where the output of one operation becomes the input for the next, enabling complex data transformations and analysis. + +### The ES|QL Compute Engine + +ES|QL is more than a language: it represents a significant investment in new compute capabilities within Elasticsearch. To achieve both the functional and performance requirements for ES|QL, it was necessary to build an entirely new compute architecture. ES|QL search, aggregation, and transformation functions are directly executed within Elasticsearch itself. Query expressions are not transpiled to Query DSL for execution. This approach allows ES|QL to be extremely performant and versatile. + +The new ES|QL execution engine was designed with performance in mind — it operates on blocks at a time instead of per row, targets vectorization and cache locality, and embraces specialization and multi-threading. It is a separate component from the existing Elasticsearch aggregation framework with different performance characteristics. + +### Known Limitations + +#### Result Set Size Limit + +By default, an ES|QL query returns up to 1000 rows. You can increase the number of rows up to 10,000 using the `LIMIT` command. Queries do not return more than 10,000 rows, regardless of the `LIMIT` command’s value. This limit only applies to the number of rows that are retrieved by the query. Queries and aggregations run on the full data set. + +To overcome this limitation: +- Reduce the result set size by modifying the query to only return relevant data. Use `WHERE` to select a smaller subset of the data. +- Shift any post-query processing to the query itself. You can use the ES|QL `STATS ... BY` command to aggregate data in the query. + +The default and maximum limits can be changed using these dynamic cluster settings: +- `esql.query.result_truncation_default_size` +- `esql.query.result_truncation_max_size` + +#### Field Types + +ES|QL currently supports the following field types: +- `alias` +- `boolean` +- `date` +- `double` (`float`, `half_float`, `scaled_float` are represented as `double`) +- `ip` +- `keyword` family including `keyword`, `constant_keyword`, and `wildcard` +- `int` (`short` and `byte` are represented as `int`) +- `long` +- `null` +- `text` +- `unsigned_long` (preview) +- `version` + +Spatial types: +- `geo_point` +- `geo_shape` +- `point` +- `shape` + +Unsupported types: +- TSDB metrics: `counter`, `position`, `aggregate_metric_double` +- Date/time: `date_nanos`, `date_range` +- Other types: `binary`, `completion`, `dense_vector`, `double_range`, `flattened`, `float_range`, `histogram`, `integer_range`, `ip_range`, `long_range`, `nested`, `rank_feature`, `rank_features`, `search_as_you_type` + +Querying a column with an unsupported type returns an error. If a column with an unsupported type is not explicitly used in a query, it is returned with `null` values, with the exception of nested fields. Nested fields are not returned at all. + +#### _source Availability + +ES|QL does not support configurations where the `_source` field is disabled. ES|QL’s support for synthetic `_source` is currently experimental. + +#### Full-Text Search + +Because of the way ES|QL treats `text` values, full-text search is not yet supported. Queries on `text` fields are like queries on `keyword` fields: they are case-sensitive and need to match the full string. + +#### Time Series Data Streams + +ES|QL does not support querying time series data streams (TSDS). + +#### Date Math Limitations + +Date math expressions work well when the leftmost expression is a datetime. However, using parentheses or putting the datetime to the right is not always supported yet. Date math does not allow subtracting two datetimes. + +#### Timezone Support + +ES|QL only supports the UTC timezone. + +### Cross-Cluster Querying + +Using ES|QL across clusters allows you to execute a single query across multiple clusters. This feature is in technical preview and may be changed or removed in a future release. + +#### Prerequisites + +- Remote clusters must be configured. +- The local coordinating node must have the `remote_cluster_client` node role. +- Security privileges must be configured appropriately. + +#### Querying Across Clusters + +In the `FROM` command, specify data streams and indices on remote clusters using the format `:`. For example: + +```esql +FROM cluster_one:my-index-000001 +| LIMIT 10 +``` + +### Using ES|QL in Kibana + +ES|QL can be used in Kibana to query and aggregate data, create visualizations, and set up alerts. + +#### Important Information + +- ES|QL is enabled by default in Kibana. +- The query bar in Discover allows you to write and execute ES|QL queries. +- The results table shows up to 10,000 rows, and Discover shows no more than 50 columns. +- You can create visualizations and alerts based on ES|QL queries. + +### Using the REST API + +The ES|QL query API allows you to execute ES|QL queries via REST API. + +#### Example + +```javascript +const response = await client.esql.query({ + query: ` + FROM library + | EVAL year = DATE_TRUNC(1 YEARS, release_date) + | STATS MAX(page_count) BY year + | SORT year + | LIMIT 5 + `, +}); +console.log(response); +``` + +#### Request + +`POST /_query` + +#### Request Body + +- `query` (Required): The ES|QL query to run. +- `format` (Optional): Format for the response. +- `params` (Optional): Values for parameters in the query. +- `profile` (Optional): If `true`, includes a `profile` object with information about query execution. + +#### Response + +- `columns`: Column `name` and `type` for each column returned in `values`. +- `rows`: Values for the search results. +- `profile`: Profile describing the execution of the query (if `profile` was sent in the request). \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-percentile.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-percentile.txt new file mode 100644 index 0000000000000..4873cace16392 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-percentile.txt @@ -0,0 +1,26 @@ +## PERCENTILE + +The `PERCENTILE` function returns the value at which a certain percentage of observed values occur. For example, the 95th percentile is the value which is greater than 95% of the observed values and the 50th percentile is the MEDIAN. + +### Examples + +```esql +FROM employees +| STATS p0 = PERCENTILE(salary, 0), p50 = PERCENTILE(salary, 50), p99 = PERCENTILE(salary, 99) +``` + +```esql +FROM employees +| STATS p80_max_salary_change = PERCENTILE(MV_MAX(salary_change), 80) +``` + +PERCENTILE is usually approximate. There are many different algorithms to calculate percentiles. The naive implementation simply stores all the values in a sorted array. To find the 50th percentile, you simply find the value that is at `my_array[count(my_array) * 0.5]`. Clearly, the naive implementation does not scale — the sorted array grows linearly with the number of values in your dataset. To calculate percentiles across potentially billions of values in an Elasticsearch cluster, approximate percentiles are calculated. The algorithm used by the percentile metric is called TDigest (introduced by Ted Dunning in Computing Accurate Quantiles using T-Digests). + +When using this metric, there are a few guidelines to keep in mind: +- Accuracy is proportional to q(1-q). This means that extreme percentiles (e.g. 99%) are more accurate than less extreme percentiles, such as the median. +- For small sets of values, percentiles are highly accurate (and potentially 100% accurate if the data is small enough). +- As the quantity of values in a bucket grows, the algorithm begins to approximate the percentiles. It is effectively trading accuracy for memory savings. The exact level of inaccuracy is difficult to generalize, since it depends on your data distribution and volume of data being aggregated. + +The following chart shows the relative error on a uniform distribution depending on the number of collected values and the requested percentile. It shows how precision is better for extreme percentiles. The reason why error diminishes for a large number of values is that the law of large numbers makes the distribution of values more and more uniform and the t-digest tree can do a better job at summarizing it. It would not be the case on more skewed distributions. + +PERCENTILE is also non-deterministic. This means you can get slightly different results using the same data. \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-pi.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-pi.txt new file mode 100644 index 0000000000000..2afabb44200ea --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-pi.txt @@ -0,0 +1,15 @@ +## PI + +The `PI` function returns Pi, the ratio of a circle’s circumference to its diameter. + +### Examples + +```esql +ROW PI() +``` + +```esql +FROM employees +| EVAL pi_value = PI() +| KEEP pi_value +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-pow.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-pow.txt new file mode 100644 index 0000000000000..43e021e4883e4 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-pow.txt @@ -0,0 +1,15 @@ +## POW + +The `POW` function returns the value of a base raised to the power of an exponent. It is still possible to overflow a double result here; in that case, null will be returned. + +### Examples + +```esql +ROW base = 2.0, exponent = 2 +| EVAL result = POW(base, exponent) +``` + +```esql +ROW base = 4, exponent = 0.5 +| EVAL s = POW(base, exponent) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-rename.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-rename.txt new file mode 100644 index 0000000000000..0e9dd3258b3c8 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-rename.txt @@ -0,0 +1,37 @@ +## RENAME + +The `RENAME` processing command in ES|QL is used to rename one or more columns in a dataset. This command is particularly useful when you need to standardize column names, make them more readable, or avoid conflicts with existing column names. If a column with the new name already exists, it will be replaced by the new column. If multiple columns are renamed to the same name, all but the rightmost column with the same new name are dropped. + +### Examples + +Here are some example ES|QL queries using the `RENAME` command: + +1. **Renaming a single column:** + + ```esql +FROM employees +| KEEP first_name, last_name, still_hired +| RENAME still_hired AS employed +``` + +2. **Renaming multiple columns in a single command:** + + ```esql +FROM employees +| KEEP first_name, last_name +| RENAME first_name AS fn, last_name AS ln +``` + +### Syntax + +`RENAME old_name1 AS new_name1[, ..., old_nameN AS new_nameN]` + +### Parameters + +- **old_nameX**: The name of a column you want to rename. +- **new_nameX**: The new name of the column. If it conflicts with an existing column name, the existing column is dropped. If multiple columns are renamed to the same name, all but the rightmost column with the same new name are dropped. + +### Limitations + +- If a column with the new name already exists, it will be replaced by the new column. +- If multiple columns are renamed to the same name, all but the rightmost column with the same new name are dropped. diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-repeat.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-repeat.txt new file mode 100644 index 0000000000000..aa702c052e1b7 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-repeat.txt @@ -0,0 +1,15 @@ +## REPEAT + +The `REPEAT` function returns a string constructed by concatenating the input string with itself the specified number of times. + +### Examples + +```esql +ROW a = "Hello!" +| EVAL triple_a = REPEAT(a, 3) +``` + +```esql +ROW greeting = "Hi" +| EVAL repeated_greeting = REPEAT(greeting, 5) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-replace.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-replace.txt new file mode 100644 index 0000000000000..931fcab1d25b9 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-replace.txt @@ -0,0 +1,19 @@ +## REPLACE + +The `REPLACE` function substitutes in the string `str` any match of the regular expression `regex` with the replacement string `newStr`. + +### Examples + +```esql +ROW str = "Hello World" +| EVAL str = REPLACE(str, "World", "Universe") +| KEEP str +``` + +Another example could be replacing digits in a string with a specific character: + +```esql +ROW str = "User123" +| EVAL str = REPLACE(str, "\\d", "*") +| KEEP str +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-right.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-right.txt new file mode 100644 index 0000000000000..99e1fbf2d3c1b --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-right.txt @@ -0,0 +1,19 @@ +## RIGHT + +The `RIGHT` function returns a substring that extracts a specified number of characters from a string, starting from the right. + +### Examples + +```esql +FROM employees +| KEEP last_name +| EVAL right = RIGHT(last_name, 3) +| SORT last_name ASC +| LIMIT 5 +``` + +```esql +ROW full_name = "John Doe" +| EVAL last_part = RIGHT(full_name, 4) +| KEEP last_part +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-round.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-round.txt new file mode 100644 index 0000000000000..a3efefb84d2d0 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-round.txt @@ -0,0 +1,17 @@ +## ROUND + +The `ROUND` function rounds a number to the specified number of decimal places. By default, it rounds to 0 decimal places, which returns the nearest integer. If the precision is a negative number, it rounds to the number of digits left of the decimal point. If the input value is null, the function returns null. + +### Examples + +```esql +FROM employees +| KEEP first_name, last_name, height +| EVAL height_ft = ROUND(height * 3.281, 1) +``` + +```esql +FROM sales +| KEEP product_name, revenue +| EVAL rounded_revenue = ROUND(revenue, -2) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-row.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-row.txt new file mode 100644 index 0000000000000..079668328f76d --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-row.txt @@ -0,0 +1,32 @@ +## ROW + +The `ROW` source command produces a row with one or more columns with values that you specify. This can be useful for testing. The command allows you to create a row with specified column names and values, which can be literals, expressions, or functions. In case of duplicate column names, only the rightmost duplicate creates a column. + +### Examples + +Here are some example ES|QL queries using the `ROW` command: + +1. Creating a row with simple literal values: + ```esql +ROW a = 1, b = "two", c = null +``` + +2. Creating a row with multi-value columns using square brackets: + ```esql +ROW a = [2, 1] +``` + +3. Creating a row with a function: + ```esql +ROW a = ROUND(1.23, 0) +``` + +4. Combining literals, multi-value columns, and functions: + ```esql +ROW x = 5, y = [3, 4], z = TO_STRING(123) +``` + +5. Using nested functions within a row: + ```esql +ROW a = ABS(-10), b = CONCAT("Hello", " ", "World"), c = TO_BOOLEAN("true") +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-rtrim.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-rtrim.txt new file mode 100644 index 0000000000000..8060580a76ae0 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-rtrim.txt @@ -0,0 +1,13 @@ +## RTRIM + +Removes trailing whitespaces from a string. + +### Examples + +```esql +ROW message = " some text ", color = " red " +| EVAL message = RTRIM(message) +| EVAL color = RTRIM(color) +| EVAL message = CONCAT("'", message, "'") +| EVAL color = CONCAT("'", color, "'") +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-show.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-show.txt new file mode 100644 index 0000000000000..ed27f65613931 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-show.txt @@ -0,0 +1,24 @@ +## SHOW + +The `SHOW` source command returns information about the deployment and its capabilities. This command is useful for retrieving metadata about the Elasticsearch deployment, such as the version, build date, and hash. It is particularly helpful for administrators and developers who need to verify the deployment details or troubleshoot issues. The `SHOW` command has a limitation in that it can only be used with the `INFO` item. + +### Examples + +Here are some example ES|QL queries using the `SHOW` command: + +1. Retrieve the deployment’s version, build date, and hash: + ```esql +SHOW INFO +``` + +2. Use the `SHOW` command in a multi-line query for better readability: + ```esql +SHOW INFO +``` + +3. Another example of using the `SHOW` command to get deployment information: + ```esql +SHOW INFO +``` + +These examples demonstrate the primary usage of the `SHOW` command to retrieve deployment information. \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-signum.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-signum.txt new file mode 100644 index 0000000000000..4a1bab62699af --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-signum.txt @@ -0,0 +1,15 @@ +## SIGNUM + +The `SIGNUM` function returns the sign of the given number. It returns -1 for negative numbers, 0 for 0, and 1 for positive numbers. + +### Examples + +```esql +ROW d = 100.0 +| EVAL s = SIGNUM(d) +``` + +```esql +ROW d = -50.0 +| EVAL s = SIGNUM(d) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-sin.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-sin.txt new file mode 100644 index 0000000000000..2083ea8f29dad --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-sin.txt @@ -0,0 +1,15 @@ +## SIN + +The `SIN` function returns the sine trigonometric function of an angle, expressed in radians. If the input angle is null, the function returns null. + +### Examples + +```esql +ROW a=1.8 +| EVAL sin = SIN(a) +``` + +```esql +ROW angle=0.5 +| EVAL sine_value = SIN(angle) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-sinh.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-sinh.txt new file mode 100644 index 0000000000000..189fdb8a8b82f --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-sinh.txt @@ -0,0 +1,15 @@ +## SINH + +The `SINH` function returns the hyperbolic sine of an angle. + +### Examples + +```esql +ROW a=1.8 +| EVAL sinh = SINH(a) +``` + +```esql +ROW angle=0.5 +| EVAL hyperbolic_sine = SINH(angle) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-sort.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-sort.txt new file mode 100644 index 0000000000000..0d68d505a5d78 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-sort.txt @@ -0,0 +1,53 @@ +## SORT + +The `SORT` processing command in ES|QL is used to sort a table based on one or more columns. This command is essential for organizing data in a specific order, which can be particularly useful for reporting, data analysis, and visualization. The default sort order is ascending, but you can specify descending order using `DESC`. Additionally, you can handle null values explicitly by using `NULLS FIRST` or `NULLS LAST`. + +### Use Cases +- **Organizing Data**: Sort data to make it easier to read and analyze. +- **Reporting**: Generate reports where data needs to be presented in a specific order. +- **Data Analysis**: Facilitate data analysis by sorting data based on key metrics. +- **Visualization**: Prepare data for visualizations that require sorted input. + +### Limitations +- **Multivalued Columns**: When sorting on multivalued columns, the lowest value is used for ascending order and the highest value for descending order. +- **Null Values**: By default, null values are treated as larger than any other value. This can be changed using `NULLS FIRST` or `NULLS LAST`. + +### Examples + +#### Basic Sorting +Sort the `employees` table by the `height` column in ascending order: + +```esql +FROM employees +| KEEP first_name, last_name, height +| SORT height +``` + +#### Explicit Ascending Order +Sort the `employees` table by the `height` column in descending order: + +```esql +FROM employees +| KEEP first_name, last_name, height +| SORT height DESC +``` + +#### Multiple Sort Expressions +Sort the `employees` table by the `height` column in descending order and use `first_name` as a tie breaker in ascending order: + +```esql +FROM employees +| KEEP first_name, last_name, height +| SORT height DESC, first_name ASC +``` + +#### Sorting Null Values First +Sort the `employees` table by the `first_name` column in ascending order, placing null values first: + +```esql +FROM employees +| KEEP first_name, last_name, height +| SORT first_name ASC NULLS FIRST +``` + +These examples demonstrate the versatility of the `SORT` command in organizing data for various analytical and reporting needs. \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-split.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-split.txt new file mode 100644 index 0000000000000..d18750ac146f6 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-split.txt @@ -0,0 +1,15 @@ +## SPLIT + +The `SPLIT` function splits a single-valued string into multiple strings based on a specified delimiter. + +### Examples + +```esql +ROW words="foo;bar;baz;qux;quux;corge" +| EVAL word = SPLIT(words, ";") +``` + +```esql +ROW sentence="hello world;this is ES|QL" +| EVAL words = SPLIT(sentence, " ") +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-sqrt.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-sqrt.txt new file mode 100644 index 0000000000000..4988c31564633 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-sqrt.txt @@ -0,0 +1,16 @@ +## SQRT + +The `SQRT` function returns the square root of a number. The input can be any numeric value, and the return value is always a double. Square roots of negative numbers and infinities are null. + +### Examples + +```esql +ROW d = 100.0 +| EVAL s = SQRT(d) +``` + +```esql +FROM employees +| KEEP first_name, last_name, height +| EVAL sqrt_height = SQRT(height) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-st_centroid_agg.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-st_centroid_agg.txt new file mode 100644 index 0000000000000..a58a2d6550e8a --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-st_centroid_agg.txt @@ -0,0 +1,15 @@ +## ST_CENTROID_AGG + +The `ST_CENTROID_AGG` function calculates the spatial centroid over a field with spatial point geometry type. + +### Examples + +```esql +FROM airports +| STATS centroid = ST_CENTROID_AGG(location) +``` + +```esql +FROM city_boundaries +| STATS city_centroid = ST_CENTROID_AGG(boundary) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-st_contains.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-st_contains.txt new file mode 100644 index 0000000000000..8d1dc8da115fc --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-st_contains.txt @@ -0,0 +1,17 @@ +## ST_CONTAINS + +Returns whether the first geometry contains the second geometry. This is the inverse of the `ST_WITHIN` function. + +### Examples + +```esql +FROM airport_city_boundaries +| WHERE ST_CONTAINS(city_boundary, TO_GEOSHAPE("POLYGON((109.35 18.3, 109.45 18.3, 109.45 18.4, 109.35 18.4, 109.35 18.3))")) +| KEEP abbrev, airport, region, city, city_location +``` + +```esql +FROM regions +| WHERE ST_CONTAINS(region_boundary, TO_GEOSHAPE("POLYGON((30 10, 40 40, 20 40, 10 20, 30 10))")) +| KEEP region_name, region_code, region_boundary +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-st_disjoint.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-st_disjoint.txt new file mode 100644 index 0000000000000..7f22061024330 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-st_disjoint.txt @@ -0,0 +1,17 @@ +## ST_DISJOINT + +The `ST_DISJOINT` function returns whether two geometries or geometry columns are disjoint. This is the inverse of the `ST_INTERSECTS` function. In mathematical terms: `ST_Disjoint(A, B) ⇔ A ⋂ B = ∅`. + +### Examples + +```esql +FROM airport_city_boundaries +| WHERE ST_DISJOINT(city_boundary, TO_GEOSHAPE("POLYGON((-10 -60, 120 -60, 120 60, -10 60, -10 -60))")) +| KEEP abbrev, airport, region, city, city_location +``` + +```esql +FROM airport_city_boundaries +| WHERE ST_DISJOINT(city_boundary, TO_GEOSHAPE("POLYGON((30 10, 40 40, 20 40, 10 20, 30 10))")) +| KEEP abbrev, airport, region, city, city_location +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-st_distance.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-st_distance.txt new file mode 100644 index 0000000000000..a1d3e05842c4a --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-st_distance.txt @@ -0,0 +1,19 @@ +## ST_DISTANCE + +The `ST_DISTANCE` function computes the distance between two points. For cartesian geometries, this is the pythagorean distance in the same units as the original coordinates. For geographic geometries, this is the circular distance along the great circle in meters. + +### Examples + +```esql +FROM airports +| WHERE abbrev == "CPH" +| EVAL distance = ST_DISTANCE(location, city_location) +| KEEP abbrev, name, location, city_location, distance +``` + +```esql +FROM airports +| WHERE abbrev == "JFK" +| EVAL distance = ST_DISTANCE(location, city_location) +| KEEP abbrev, name, location, city_location, distance +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-st_intersects.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-st_intersects.txt new file mode 100644 index 0000000000000..46df07d8d9f67 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-st_intersects.txt @@ -0,0 +1,16 @@ +## ST_INTERSECTS + +The `ST_INTERSECTS` function returns true if two geometries intersect. They intersect if they have any point in common, including their interior points (points along lines or within polygons). This is the inverse of the `ST_DISJOINT` function. In mathematical terms: `ST_Intersects(A, B) ⇔ A ⋂ B ≠ ∅`. + +### Examples + +```esql +FROM airports +| WHERE ST_INTERSECTS(location, TO_GEOSHAPE("POLYGON((42 14, 43 14, 43 15, 42 15, 42 14))")) +``` + +```esql +FROM city_boundaries +| WHERE ST_INTERSECTS(boundary, TO_GEOSHAPE("POLYGON((10 10, 20 10, 20 20, 10 20, 10 10))")) +| KEEP city_name, boundary +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-st_within.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-st_within.txt new file mode 100644 index 0000000000000..24883a731e24b --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-st_within.txt @@ -0,0 +1,17 @@ +## ST_WITHIN + +The `ST_WITHIN` function returns whether the first geometry is within the second geometry. This is the inverse of the `ST_CONTAINS` function. + +### Examples + +```esql +FROM airport_city_boundaries +| WHERE ST_WITHIN(city_boundary, TO_GEOSHAPE("POLYGON((109.1 18.15, 109.6 18.15, 109.6 18.65, 109.1 18.65, 109.1 18.15))")) +| KEEP abbrev, airport, region, city, city_location +``` + +```esql +FROM parks +| WHERE ST_WITHIN(park_boundary, TO_GEOSHAPE("POLYGON((40.7128 -74.0060, 40.7128 -73.9352, 40.7306 -73.9352, 40.7306 -74.0060, 40.7128 -74.0060))")) +| KEEP park_name, park_boundary +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-st_x.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-st_x.txt new file mode 100644 index 0000000000000..11b569d7db065 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-st_x.txt @@ -0,0 +1,15 @@ +## ST_X + +The `ST_X` function extracts the x coordinate from the supplied point. If the point is of type `geo_point`, this is equivalent to extracting the longitude value. + +### Examples + +```esql +ROW point = TO_GEOPOINT("POINT(42.97109629958868 14.7552534006536)") +| EVAL x = ST_X(point), y = ST_Y(point) +``` + +```esql +ROW point = TO_CARTESIANPOINT("POINT(100.0 200.0)") +| EVAL x = ST_X(point), y = ST_Y(point) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-st_y.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-st_y.txt new file mode 100644 index 0000000000000..be0b96539bf15 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-st_y.txt @@ -0,0 +1,15 @@ +## ST_Y + +The `ST_Y` function extracts the y coordinate from the supplied point. If the point is of type `geo_point`, this is equivalent to extracting the latitude value. + +### Examples + +```esql +ROW point = TO_GEOPOINT("POINT(42.97109629958868 14.7552534006536)") +| EVAL x = ST_X(point), y = ST_Y(point) +``` + +```esql +ROW point = TO_GEOPOINT("POINT(34.052235 -118.243683)") +| EVAL latitude = ST_Y(point) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-starts_with.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-starts_with.txt new file mode 100644 index 0000000000000..a293c13d1d706 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-starts_with.txt @@ -0,0 +1,18 @@ +## STARTS_WITH + +The `STARTS_WITH` function returns a boolean that indicates whether a keyword string starts with another string. + +### Examples + +```esql +FROM employees +| KEEP last_name +| EVAL ln_S = STARTS_WITH(last_name, "B") +``` + +```esql +FROM employees +| KEEP first_name, last_name +| EVAL fn_S = STARTS_WITH(first_name, "A") +| WHERE fn_S +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-stats.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-stats.txt new file mode 100644 index 0000000000000..a85669b4b3fa1 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-stats.txt @@ -0,0 +1,71 @@ +## STATS + +The `STATS ... BY` processing command in ES|QL groups rows according to a common value and calculates one or more aggregated values over the grouped rows. This command is highly useful for performing statistical analysis and aggregations on datasets. It supports a variety of aggregation functions such as `AVG`, `COUNT`, `COUNT_DISTINCT`, `MAX`, `MEDIAN`, `MIN`, `SUM`, and more. + +### Use Cases +- **Statistical Analysis**: Calculate average, sum, count, and other statistical measures over grouped data. +- **Data Aggregation**: Aggregate data based on specific fields to derive meaningful insights. +- **Time-Series Analysis**: Group data by time intervals to analyze trends over time. + +### Limitations +- **Performance**: `STATS` without any groups is much faster than adding a group. Grouping on a single expression is more optimized than grouping on multiple expressions. +- **Multivalue Fields**: If the grouping key is multivalued, the input row is included in all groups. +- **Technical Preview**: Some functions like `PERCENTILE`, `ST_CENTROID_AGG`, `VALUES`, and `WEIGHTED_AVG` are in technical preview and may change in future releases. + +### Examples + +#### Example 1: Grouping by a Single Column +Calculate the count of employees grouped by languages: + +```esql +FROM employees +| STATS count = COUNT(emp_no) BY languages +| SORT languages +``` + +#### Example 2: Aggregation Without Grouping +Calculate the average number of languages spoken by employees: + +```esql +FROM employees +| STATS avg_lang = AVG(languages) +``` + +#### Example 3: Multiple Aggregations +Calculate both the average and maximum number of languages spoken by employees: + +```esql +FROM employees +| STATS avg_lang = AVG(languages), max_lang = MAX(languages) +``` + +#### Example 4: Grouping by Multiple Values +Calculate the average salary grouped by the year of hire and language: + +```esql +FROM employees +| EVAL hired = DATE_FORMAT("YYYY", hire_date) +| STATS avg_salary = AVG(salary) BY hired, languages.long +| EVAL avg_salary = ROUND(avg_salary) +| SORT hired, languages.long +``` + +#### Example 5: Grouping by an Expression +Group employees by the first letter of their last name and count them: + +```esql +FROM employees +| STATS my_count = COUNT() BY LEFT(last_name, 1) +| SORT `LEFT(last_name, 1)` +``` + +#### Example 6: Using Multivalue Columns +Calculate the minimum value of a multivalue column: + +```esql +ROW i=1, a=["a", "b"], b=[2, 3] +| STATS MIN(i) BY a, b +| SORT a ASC, b ASC +``` + +These examples showcase the versatility and power of the `STATS ... BY` command in performing various types of data aggregations and statistical analyses. \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-substring.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-substring.txt new file mode 100644 index 0000000000000..a03574e7d2cef --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-substring.txt @@ -0,0 +1,39 @@ +## SUBSTRING + +The `SUBSTRING` function returns a substring of a string, specified by a start position and an optional length. + +### Syntax + +`SUBSTRING(string, start, [length])` + +### Parameters + +- `string`: String expression. If null, the function returns null. +- `start`: Start position. +- `length`: Length of the substring from the start position. Optional; if omitted, all positions after start are returned. + +### Examples + +This example returns the first three characters of every last name: + +```esql +FROM employees +| KEEP last_name +| EVAL ln_sub = SUBSTRING(last_name, 1, 3) +``` + +A negative start position is interpreted as being relative to the end of the string. This example returns the last three characters of every last name: + +```esql +FROM employees +| KEEP last_name +| EVAL ln_sub = SUBSTRING(last_name, -3, 3) +``` + +If length is omitted, `SUBSTRING` returns the remainder of the string. This example returns all characters except for the first: + +```esql +FROM employees +| KEEP last_name +| EVAL ln_sub = SUBSTRING(last_name, 2) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-sum.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-sum.txt new file mode 100644 index 0000000000000..c893aeb160d00 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-sum.txt @@ -0,0 +1,15 @@ +## SUM + +The `SUM` function calculates the sum of a numeric expression. + +### Examples + +```esql +FROM employees +| STATS SUM(languages) +``` + +```esql +FROM employees +| STATS total_salary_changes = SUM(MV_MAX(salary_change)) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-syntax.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-syntax.txt new file mode 100644 index 0000000000000..e1e339239a713 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-syntax.txt @@ -0,0 +1,181 @@ +## Syntax + +### Instructions + +Generate a description of ES|QL syntax. Be as complete as possible. +For timespan literals, generate at least five examples of full ES|QL queries, using a mix of commands and functions, using different intervals and units. +**Make sure you use timespan literals, such as `1 day` or `24h` or `7 weeks` in these examples**. +Combine ISO timestamps with time span literals and NOW(). +Make sure the example queries are using different combinations of syntax, commands, and functions for each. +When using DATE_TRUNC, make sure you DO NOT wrap the timespan in single or double quotes. +Do not use the Cast operator. + +### Content of file + +### ES|QL Syntax Reference + +#### Basic Syntax + +An ES|QL query is composed of a source command followed by an optional series of processing commands, separated by a pipe character: `|`. For example: + +``` +source-command +| processing-command1 +| processing-command2 +``` + +The result of a query is the table produced by the final processing command. For an overview of all supported commands, functions, and operators, refer to Commands and Functions and operators. + +For readability, this documentation puts each processing command on a new line. However, you can write an ES|QL query as a single line. The following query is identical to the previous one: + +``` +source-command +| processing-command1 +| processing-command2 +``` + +#### Identifiers + +Identifiers need to be quoted with backticks (```) if: +- They don’t start with a letter, `_` or `@` +- Any of the other characters is not a letter, number, or `_` + +For example: + +```esql +FROM index +| KEEP `1.field` +``` + +When referencing a function alias that itself uses a quoted identifier, the backticks of the quoted identifier need to be escaped with another backtick. For example: + +```esql +FROM index +| STATS COUNT(`1.field`) +| EVAL my_count = `COUNT(``1.field``)` +``` + +#### Literals + +ES|QL currently supports numeric and string literals. + +##### String Literals + +A string literal is a sequence of unicode characters delimited by double quotes (`"`). + +```esql +// Filter by a string value +FROM index +| WHERE first_name == "Georgi" +``` + +If the literal string itself contains quotes, these need to be escaped (`\\"`). ES|QL also supports the triple-quotes (`"""`) delimiter, for convenience: + +```esql +ROW name = """Indiana "Indy" Jones""" +``` + +The special characters CR, LF, and TAB can be provided with the usual escaping: `\r`, `\n`, `\t`, respectively. + +##### Numerical Literals + +The numeric literals are accepted in decimal and in the scientific notation with the exponent marker (`e` or `E`), starting either with a digit, decimal point `.` or the negative sign `-`: + +- `1969` -- integer notation +- `3.14` -- decimal notation +- `.1234` -- decimal notation starting with decimal point +- `4E5` -- scientific notation (with exponent marker) +- `1.2e-3` -- scientific notation with decimal point +- `-.1e2` -- scientific notation starting with the negative sign + +The integer numeric literals are implicitly converted to the `integer`, `long` or the `double` type, whichever can first accommodate the literal’s value. The floating point literals are implicitly converted to the `double` type. To obtain constant values of different types, use one of the numeric conversion functions. + +#### Comments + +ES|QL uses C++ style comments: +- Double slash `//` for single line comments +- `/*` and `*/` for block comments + +```esql +// Query the employees index +FROM employees +| WHERE height > 2 +``` + +```esql +FROM /* Query the employees index */ employees +| WHERE height > 2 +``` + +```esql +FROM employees +/* Query the + * employees + * index */ +| WHERE height > 2 +``` + +#### Timespan Literals + +Datetime intervals and timespans can be expressed using timespan literals. Timespan literals are a combination of a number and a qualifier. These qualifiers are supported: +- `millisecond`/`milliseconds`/`ms` +- `second`/`seconds`/`sec`/`s` +- `minute`/`minutes`/`min` +- `hour`/`hours`/`h` +- `day`/`days`/`d` +- `week`/`weeks`/`w` +- `month`/`months`/`mo` +- `quarter`/`quarters`/`q` +- `year`/`years`/`yr`/`y` + +Timespan literals are not whitespace sensitive. These expressions are all valid: +- `1day` +- `1 day` +- `1 day` + +#### Example Queries Using Timespan Literals + +1. Retrieve logs from the last 24 hours and calculate the average response time: + + ```esql +FROM logs-* +| WHERE @timestamp > NOW() - 24h +| STATS avg_response_time = AVG(response_time) +``` + +2. Get the count of events per day for the last 7 days: + + ```esql +FROM events +| WHERE @timestamp > NOW() - 7 days +| STATS daily_count = COUNT(*) BY day = DATE_TRUNC(1 day, @timestamp) +| SORT day +``` + +3. Find the maximum temperature recorded in the last month: + + ```esql +FROM weather_data +| WHERE @timestamp > NOW() - 1 month +| STATS max_temp = MAX(temperature) +``` + +4. Calculate the total sales for each week in the last quarter: + + ```esql +FROM sales +| WHERE @timestamp > NOW() - 1 quarter +| STATS weekly_sales = SUM(sales_amount) BY week = DATE_TRUNC(1 week, @timestamp) +| SORT week +``` + +5. Retrieve error logs from the last 15 minutes and group by error type: + + ```esql +FROM error_logs +| WHERE @timestamp > NOW() - 15 minutes +| STATS error_count = COUNT(*) BY error_type +| SORT error_count DESC +``` + +These examples demonstrate the use of timespan literals in combination with various ES|QL commands and functions to perform different types of data queries and transformations. diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-tan.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-tan.txt new file mode 100644 index 0000000000000..8541f193d89a4 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-tan.txt @@ -0,0 +1,15 @@ +## TAN + +The `TAN` function returns the Tangent trigonometric function of an angle. + +### Examples + +```esql +ROW a=1.8 +| EVAL tan = TAN(a) +``` + +```esql +ROW angle=0.5 +| EVAL tangent = TAN(angle) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-tanh.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-tanh.txt new file mode 100644 index 0000000000000..45fc52fa501a7 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-tanh.txt @@ -0,0 +1,15 @@ +## TANH + +The `TANH` function returns the hyperbolic tangent of an angle, expressed in radians. If the input angle is null, the function returns null. + +### Examples + +```esql +ROW a=1.8 +| EVAL tanh = TANH(a) +``` + +```esql +ROW angle=0.5 +| EVAL hyperbolic_tangent = TANH(angle) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-tau.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-tau.txt new file mode 100644 index 0000000000000..149d1333b2a1a --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-tau.txt @@ -0,0 +1,15 @@ +## TAU + +The `TAU` function returns the ratio of a circle’s circumference to its radius. + +### Examples + +```esql +ROW TAU() +``` + +```esql +FROM sample_data +| EVAL tau_value = TAU() +| KEEP tau_value +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_base64.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_base64.txt new file mode 100644 index 0000000000000..e0c64eba07d58 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_base64.txt @@ -0,0 +1,15 @@ +## TO_BASE64 + +Encodes a string to a base64 string. + +### Examples + +```esql +ROW a = "elastic" +| EVAL e = TO_BASE64(a) +``` + +```esql +ROW text = "Hello, World!" +| EVAL encoded_text = TO_BASE64(text) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_boolean.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_boolean.txt new file mode 100644 index 0000000000000..9a243ba128fb9 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_boolean.txt @@ -0,0 +1,15 @@ +## TO_BOOLEAN + +The `TO_BOOLEAN` function converts an input value to a boolean value. A string value of "true" will be case-insensitively converted to the Boolean `true`. For anything else, including the empty string, the function will return `false`. The numerical value of `0` will be converted to `false`, and anything else will be converted to `true`. + +### Examples + +```esql +ROW str = ["true", "TRuE", "false", "", "yes", "1"] +| EVAL bool = TO_BOOLEAN(str) +``` + +```esql +ROW num = [0, 1, 2, -1] +| EVAL bool = TO_BOOLEAN(num) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_cartesianpoint.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_cartesianpoint.txt new file mode 100644 index 0000000000000..db56fd5fc67f4 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_cartesianpoint.txt @@ -0,0 +1,17 @@ +## TO_CARTESIANPOINT + +Converts an input value to a `cartesian_point` value. A string will only be successfully converted if it respects the WKT Point format. + +### Examples + +```esql +ROW wkt = ["POINT(4297.11 -1475.53)", "POINT(7580.93 2272.77)"] +| MV_EXPAND wkt +| EVAL pt = TO_CARTESIANPOINT(wkt) +``` + +```esql +ROW wkt = ["POINT(1000.0 2000.0)", "POINT(3000.0 4000.0)"] +| MV_EXPAND wkt +| EVAL pt = TO_CARTESIANPOINT(wkt) +``` \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_cartesianshape.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_cartesianshape.txt similarity index 52% rename from x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_cartesianshape.txt rename to x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_cartesianshape.txt index 8e16ba92a8e7b..f32af0a6d4805 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_cartesianshape.txt +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_cartesianshape.txt @@ -1,18 +1,17 @@ ## TO_CARTESIANSHAPE -The `TO_CARTESIANSHAPE` function converts an input value to a `cartesian_shape` value. A string will only be successfully converted if it respects the WKT format. +Converts an input value to a `cartesian_shape` value. A string will only be successfully converted if it respects the WKT format. ### Examples -Here are a couple of examples of full ES|QL queries using the `TO_CARTESIANSHAPE` function: - ```esql -ROW wkt = "POINT(4297.11 -1475.53)" +ROW wkt = ["POINT(4297.11 -1475.53)", "POLYGON ((3339584.72 1118889.97, 4452779.63 4865942.27, 2226389.81 4865942.27, 1113194.90 2273030.92, 3339584.72 1118889.97))"] +| MV_EXPAND wkt | EVAL geom = TO_CARTESIANSHAPE(wkt) ``` ```esql -ROW wkt = ["POINT(4297.11 -1475.53)", "POLYGON ((3339584.72 1118889.97, 4452779.63 4865942.27, 2226389.81 4865942.27, 1113194.90 2273030.92, 3339584.72 1118889.97))"] +ROW wkt = ["POINT(1000.0 2000.0)", "POLYGON ((1000.0 2000.0, 2000.0 3000.0, 3000.0 4000.0, 1000.0 2000.0))"] | MV_EXPAND wkt | EVAL geom = TO_CARTESIANSHAPE(wkt) ``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_datetime.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_datetime.txt new file mode 100644 index 0000000000000..7e0cd1fc82c06 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_datetime.txt @@ -0,0 +1,19 @@ +## TO_DATETIME + +Converts an input value to a date value. A string will only be successfully converted if it’s respecting the format `yyyy-MM-dd'T'HH:mm:ss.SSS'Z'`. To convert dates in other formats, use `DATE_PARSE`. + +### Examples + +```esql +ROW string = ["1953-09-02T00:00:00.000Z", "1964-06-02T00:00:00.000Z", "1964-06-02 00:00:00"] +| EVAL datetime = TO_DATETIME(string) +``` + +Note that in this example, the last value in the source multi-valued field has not been converted. The reason being that if the date format is not respected, the conversion will result in a null value. When this happens a Warning header is added to the response. The header will provide information on the source of the failure: "Line 1:112: evaluation of [TO_DATETIME(string)] failed, treating result as null. Only first 20 failures recorded." A following header will contain the failure reason and the offending value: "java.lang.IllegalArgumentException: failed to parse date field [1964-06-02 00:00:00] with format [yyyy-MM-dd'T'HH:mm:ss.SSS'Z']". + +```esql +ROW int = [0, 1] +| EVAL dt = TO_DATETIME(int) +``` + +If the input parameter is of a numeric type, its value will be interpreted as milliseconds since the Unix epoch. \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_degrees.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_degrees.txt new file mode 100644 index 0000000000000..287e526f143e5 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_degrees.txt @@ -0,0 +1,15 @@ +## TO_DEGREES + +Converts a number in radians to degrees. + +### Examples + +```esql +ROW rad = [1.57, 3.14, 4.71] +| EVAL deg = TO_DEGREES(rad) +``` + +```esql +ROW angle_in_radians = 1.0 +| EVAL angle_in_degrees = TO_DEGREES(angle_in_radians) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_double.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_double.txt new file mode 100644 index 0000000000000..c835ee9531281 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_double.txt @@ -0,0 +1,14 @@ +## TO_DOUBLE + +Converts an input value to a double value. If the input parameter is of a date type, its value will be interpreted as milliseconds since the Unix epoch, converted to double. Boolean true will be converted to double 1.0, false to 0.0. + +### Examples + +```esql +ROW str1 = "5.20128E11", str2 = "foo" +| EVAL dbl = TO_DOUBLE("520128000000"), dbl1 = TO_DOUBLE(str1), dbl2 = TO_DOUBLE(str2) +``` + +Note that in this example, the last conversion of the string isn’t possible. When this happens, the result is a null value. In this case, a Warning header is added to the response. The header will provide information on the source of the failure: +"Line 1:115: evaluation of [TO_DOUBLE(str2)] failed, treating result as null. Only first 20 failures recorded." A following header will contain the failure reason and the offending value: +"java.lang.NumberFormatException: For input string: "foo"" \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_geopoint.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_geopoint.txt new file mode 100644 index 0000000000000..27cc2b7569742 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_geopoint.txt @@ -0,0 +1,15 @@ +## TO_GEOPOINT + +Converts an input value to a `geo_point` value. A string will only be successfully converted if it respects the WKT Point format. + +### Examples + +```esql +ROW wkt = "POINT(42.97109630194 14.7552534413725)" +| EVAL pt = TO_GEOPOINT(wkt) +``` + +```esql +ROW wkt = "POINT(34.052235 -118.243683)" +| EVAL pt = TO_GEOPOINT(wkt) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_geoshape.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_geoshape.txt new file mode 100644 index 0000000000000..9160e081f6477 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_geoshape.txt @@ -0,0 +1,15 @@ +## TO_GEOSHAPE + +Converts an input value to a `geo_shape` value. A string will only be successfully converted if it respects the WKT format. + +### Examples + +```esql +ROW wkt = "POLYGON ((30 10, 40 40, 20 40, 10 20, 30 10))" +| EVAL geom = TO_GEOSHAPE(wkt) +``` + +```esql +ROW wkt = "LINESTRING (30 10, 10 30, 40 40)" +| EVAL geom = TO_GEOSHAPE(wkt) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_integer.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_integer.txt new file mode 100644 index 0000000000000..3944c64ee004b --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_integer.txt @@ -0,0 +1,12 @@ +## TO_INTEGER + +Converts an input value to an integer value. If the input parameter is of a date type, its value will be interpreted as milliseconds since the Unix epoch, converted to integer. Boolean true will be converted to integer 1, false to 0. + +### Examples + +```esql +ROW long = [5013792, 2147483647, 501379200000] +| EVAL int = TO_INTEGER(long) +``` + +Note that in this example, the last value of the multi-valued field cannot be converted as an integer. When this happens, the result is a null value. In this case, a Warning header is added to the response. The header will provide information on the source of the failure: "Line 1:61: evaluation of [TO_INTEGER(long)] failed, treating result as null. Only first 20 failures recorded." A following header will contain the failure reason and the offending value: "org.elasticsearch.xpack.esql.core.InvalidArgumentException: [501379200000] out of [integer] range" \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_ip.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_ip.txt new file mode 100644 index 0000000000000..af3d8b754d88e --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_ip.txt @@ -0,0 +1,21 @@ +## TO_IP + +The `TO_IP` function converts an input string to an IP value. + +### Examples + +```esql +ROW str1 = "1.1.1.1", str2 = "foo" +| EVAL ip1 = TO_IP(str1), ip2 = TO_IP(str2) +| WHERE CIDR_MATCH(ip1, "1.0.0.0/8") +``` + +Note that in this example, the last conversion of the string isn’t possible. When this happens, the result is a null value. In this case, a Warning header is added to the response. The header will provide information on the source of the failure: "Line 1:68: evaluation of [TO_IP(str2)] failed, treating result as null. Only first 20 failures recorded." A following header will contain the failure reason and the offending value: "java.lang.IllegalArgumentException: 'foo' is not an IP string literal." + +```esql +ROW ip_str = "192.168.1.1" +| EVAL ip = TO_IP(ip_str) +| KEEP ip +``` + +In this example, the string "192.168.1.1" is successfully converted to an IP value and kept in the result set. \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_long.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_long.txt new file mode 100644 index 0000000000000..b5fd5788be666 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_long.txt @@ -0,0 +1,17 @@ +## TO_LONG + +Converts an input value to a long value. If the input parameter is of a date type, its value will be interpreted as milliseconds since the Unix epoch, converted to long. Boolean true will be converted to long 1, false to 0. + +### Examples + +```esql +ROW str1 = "2147483648", str2 = "2147483648.2", str3 = "foo" +| EVAL long1 = TO_LONG(str1), long2 = TO_LONG(str2), long3 = TO_LONG(str3) +``` + +Note that in this example, the last conversion of the string isn’t possible. When this happens, the result is a null value. In this case, a Warning header is added to the response. The header will provide information on the source of the failure: "Line 1:113: evaluation of [TO_LONG(str3)] failed, treating result as null. Only first 20 failures recorded." A following header will contain the failure reason and the offending value: "java.lang.NumberFormatException: For input string: "foo"" + +```esql +ROW str1 = "1234567890", str2 = "9876543210" +| EVAL long1 = TO_LONG(str1), long2 = TO_LONG(str2) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_lower.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_lower.txt new file mode 100644 index 0000000000000..26be9d76088df --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_lower.txt @@ -0,0 +1,17 @@ +## TO_LOWER + +The `TO_LOWER` function returns a new string representing the input string converted to lower case. + +### Examples + +```esql +ROW message = "Some Text" +| EVAL message_lower = TO_LOWER(message) +``` + +```esql +FROM employees +| KEEP first_name, last_name +| EVAL first_name_lower = TO_LOWER(first_name) +| EVAL last_name_lower = TO_LOWER(last_name) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_radians.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_radians.txt new file mode 100644 index 0000000000000..d0c90d3739998 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_radians.txt @@ -0,0 +1,15 @@ +## TO_RADIANS + +Converts a number in degrees to radians. + +### Examples + +```esql +ROW deg = [90.0, 180.0, 270.0] +| EVAL rad = TO_RADIANS(deg) +``` + +```esql +ROW angle_deg = 45.0 +| EVAL angle_rad = TO_RADIANS(angle_deg) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_string.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_string.txt new file mode 100644 index 0000000000000..b2ee6a7a7f106 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_string.txt @@ -0,0 +1,15 @@ +## TO_STRING + +Converts an input value into a string. + +### Examples + +```esql +ROW a=10 +| EVAL j = TO_STRING(a) +``` + +```esql +ROW a=[10, 9, 8] +| EVAL j = TO_STRING(a) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_unsigned_long.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_unsigned_long.txt new file mode 100644 index 0000000000000..7702b6f8da228 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_unsigned_long.txt @@ -0,0 +1,20 @@ +## TO_UNSIGNED_LONG + +Converts an input value to an unsigned long value. If the input parameter is of a date type, its value will be interpreted as milliseconds since the Unix epoch, converted to unsigned long. Boolean true will be converted to unsigned long 1, false to 0. + +### Examples + +```esql +ROW str1 = "2147483648", str2 = "2147483648.2", str3 = "foo" +| EVAL long1 = TO_UNSIGNED_LONG(str1), long2 = TO_ULONG(str2), long3 = TO_UL(str3) +``` + +Note that in this example, the last conversion of the string isn’t possible. When this happens, the result is a null value. In this case, a Warning header is added to the response. The header will provide information on the source of the failure: +"Line 1:133: evaluation of [TO_UL(str3)] failed, treating result as null. Only first 20 failures recorded." +A following header will contain the failure reason and the offending value: +"java.lang.NumberFormatException: Character f is neither a decimal digit number, decimal point, nor "e" notation exponential mark." + +```esql +ROW date1 = TO_DATETIME("2023-12-02T11:00:00.000Z"), date2 = TO_DATETIME("2023-12-02T11:00:00.001Z") +| EVAL long_date1 = TO_UNSIGNED_LONG(date1), long_date2 = TO_UNSIGNED_LONG(date2) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_upper.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_upper.txt new file mode 100644 index 0000000000000..d563e9efa59b4 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_upper.txt @@ -0,0 +1,17 @@ +## TO_UPPER + +The `TO_UPPER` function returns a new string representing the input string converted to upper case. + +### Examples + +```esql +ROW message = "Some Text" +| EVAL message_upper = TO_UPPER(message) +``` + +```esql +FROM employees +| KEEP first_name, last_name +| EVAL first_name_upper = TO_UPPER(first_name) +| EVAL last_name_upper = TO_UPPER(last_name) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_version.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_version.txt new file mode 100644 index 0000000000000..2fb20b0686723 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-to_version.txt @@ -0,0 +1,14 @@ +## TO_VERSION + +Converts an input string to a version value. + +### Examples + +```esql +ROW v = TO_VERSION("1.2.3") +``` + +```esql +ROW version_string = "2.3.4" +| EVAL version = TO_VERSION(version_string) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-top.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-top.txt new file mode 100644 index 0000000000000..6b18788f4b6ac --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-top.txt @@ -0,0 +1,25 @@ +## TOP + +The `TOP` function collects the top values for a specified field. It includes repeated values and allows you to specify the maximum number of values to collect and the order in which to sort them (either ascending or descending). + +### Syntax + +`TOP(field, limit, order)` + +### Parameters + +- **field**: The field to collect the top values for. +- **limit**: The maximum number of values to collect. +- **order**: The order to calculate the top values. Either `asc` or `desc`. + +### Examples + +```esql +FROM employees +| STATS top_salaries = TOP(salary, 3, "desc"), top_salary = MAX(salary) +``` + +```esql +FROM sales +| STATS top_products = TOP(product_id, 5, "asc"), max_sales = MAX(sales_amount) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-trim.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-trim.txt new file mode 100644 index 0000000000000..bec5cce253909 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-trim.txt @@ -0,0 +1,17 @@ +## TRIM + +Removes leading and trailing whitespaces from a string. + +### Examples + +```esql +ROW message = " some text ", color = " red " +| EVAL message = TRIM(message) +| EVAL color = TRIM(color) +``` + +```esql +ROW text = " example text ", label = " label " +| EVAL text = TRIM(text) +| EVAL label = TRIM(label) +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-values.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-values.txt new file mode 100644 index 0000000000000..a4acd2c397104 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-values.txt @@ -0,0 +1,16 @@ +## VALUES + +`VALUES` returns all values in a group as a multivalued field. The order of the returned values isn’t guaranteed. If you need the values returned in order, use `MV_SORT`. + +**Note:** Do not use `VALUES` in production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features. + +### Examples + +```esql +FROM employees +| EVAL first_letter = SUBSTRING(first_name, 0, 1) +| STATS first_name = MV_SORT(VALUES(first_name)) BY first_letter +| SORT first_letter +``` + +This can use a significant amount of memory and ES|QL doesn’t yet grow aggregations beyond memory. So this aggregation will work until it is used to collect more values than can fit into memory. Once it collects too many values it will fail the query with a Circuit Breaker Error. \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-weighted_avg.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-weighted_avg.txt new file mode 100644 index 0000000000000..9030159ff728c --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-weighted_avg.txt @@ -0,0 +1,21 @@ +## WEIGHTED_AVG + +The `WEIGHTED_AVG` function calculates the weighted average of a numeric expression. + +### Examples + +```esql +FROM employees +| STATS w_avg = WEIGHTED_AVG(salary, height) BY languages +| EVAL w_avg = ROUND(w_avg) +| KEEP w_avg, languages +| SORT languages +``` + +```esql +FROM sales +| STATS weighted_sales = WEIGHTED_AVG(revenue, units_sold) BY region +| EVAL weighted_sales = ROUND(weighted_sales, 2) +| KEEP weighted_sales, region +| SORT region +``` \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-where.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-where.txt new file mode 100644 index 0000000000000..7a772aed34bb3 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-where.txt @@ -0,0 +1,77 @@ +## WHERE + +The `WHERE` processing command produces a table that contains all the rows from the input table for which the provided condition evaluates to true. This command is essential for filtering data based on specific criteria, making it highly useful for narrowing down datasets to relevant records. It supports various functions and operators, including boolean expressions, date math, string patterns, and NULL comparisons. + +### Use Cases +- Filtering records based on boolean conditions. +- Retrieving data within a specific time range using date math. +- Filtering data based on string patterns using wildcards or regular expressions. +- Performing NULL comparisons. +- Testing whether a field or expression equals an element in a list of literals, fields, or expressions. + +### Limitations +- The `WHERE` command is subject to the limitations of ES|QL, such as the maximum number of rows returned (10,000) and the types of fields supported. + +### Examples + +#### Example 1: Filtering Based on Boolean Condition +```esql +FROM employees +| KEEP first_name, last_name, still_hired +| WHERE still_hired == true +``` +If `still_hired` is a boolean field, this can be simplified to: +```esql +FROM employees +| KEEP first_name, last_name, still_hired +| WHERE still_hired +``` + +#### Example 2: Retrieving Data from a Specific Time Range +```esql +FROM sample_data +| WHERE @timestamp > NOW() - 1 hour +``` + +#### Example 3: Using Functions in WHERE Clause +```esql +FROM employees +| KEEP first_name, last_name, height +| WHERE LENGTH(first_name) < 4 +``` + +#### Example 4: NULL Comparison +```esql +FROM employees +| WHERE birth_date IS NULL +| KEEP first_name, last_name +| SORT first_name +| LIMIT 3 +``` +```esql +FROM employees +| WHERE is_rehired IS NOT NULL +| STATS COUNT(emp_no) +``` + +#### Example 5: Filtering Based on String Patterns Using Wildcards +```esql +FROM employees +| WHERE first_name LIKE "?b*" +| KEEP first_name, last_name +``` + +#### Example 6: Filtering Based on String Patterns Using Regular Expressions +```esql +FROM employees +| WHERE first_name RLIKE ".leja.*" +| KEEP first_name, last_name +``` + +#### Example 7: Using IN Operator +```esql +ROW a = 1, b = 4, c = 3 +| WHERE c-a IN (3, b / 2, a) +``` + +For a complete list of all functions and operators, refer to the Functions overview and Operators documentation. \ No newline at end of file diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/index.ts b/x-pack/plugins/inference/server/tasks/nl_to_esql/index.ts index 00ce53fe5d288..2d6fe4ca7dac3 100644 --- a/x-pack/plugins/inference/server/tasks/nl_to_esql/index.ts +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/index.ts @@ -5,93 +5,260 @@ * 2.0. */ -import { switchMap, map } from 'rxjs'; -import { MessageRole } from '../../../common/chat_complete'; -import { ToolOptions } from '../../../common/chat_complete/tools'; -import { withoutChunkEvents } from '../../../common/chat_complete/without_chunk_events'; +import type { Logger } from '@kbn/logging'; +import { isEmpty, mapValues, pick } from 'lodash'; +import { Observable, from, map, merge, of, switchMap } from 'rxjs'; +import { v4 } from 'uuid'; +import { ToolSchema, isChatCompletionMessageEvent } from '../../../common'; +import { + ChatCompletionChunkEvent, + ChatCompletionMessageEvent, + Message, + MessageRole, +} from '../../../common/chat_complete'; +import { ToolChoiceType, type ToolOptions } from '../../../common/chat_complete/tools'; import { withoutTokenCountEvents } from '../../../common/chat_complete/without_token_count_events'; -import { createOutputCompleteEvent } from '../../../common/output'; +import { OutputCompleteEvent, OutputEventType } from '../../../common/output'; import { withoutOutputUpdateEvents } from '../../../common/output/without_output_update_events'; -import { InferenceClient } from '../../types'; +import { INLINE_ESQL_QUERY_REGEX } from '../../../common/tasks/nl_to_esql/constants'; +import { correctCommonEsqlMistakes } from '../../../common/tasks/nl_to_esql/correct_common_esql_mistakes'; +import type { InferenceClient } from '../../types'; +import { loadDocuments } from './load_documents'; -const ESQL_SYSTEM_MESSAGE = ''; - -async function getEsqlDocuments(documents: string[]) { - return [ - { - document: 'my-esql-function', - text: 'My ES|QL function', - }, - ]; -} +type NlToEsqlTaskEvent = + | OutputCompleteEvent< + 'request_documentation', + { keywords: string[]; requestedDocumentation: Record } + > + | ChatCompletionChunkEvent + | ChatCompletionMessageEvent; export function naturalLanguageToEsql({ client, - input, connectorId, tools, toolChoice, + logger, + ...rest }: { - client: InferenceClient; - input: string; + client: Pick; connectorId: string; -} & TToolOptions) { - return client - .output('request_documentation', { - connectorId, - system: ESQL_SYSTEM_MESSAGE, - input: `Based on the following input, request documentation - from the ES|QL handbook to help you get the right information - needed to generate a query: - ${input} - `, - schema: { - type: 'object', - properties: { - documents: { - type: 'array', - items: { - type: 'string', + logger: Pick; +} & TToolOptions & + ({ input: string } | { messages: Message[] })): Observable> { + const hasTools = !isEmpty(tools) && toolChoice !== ToolChoiceType.none; + + const requestDocumentationSchema = { + type: 'object', + properties: { + commands: { + type: 'array', + items: { + type: 'string', + }, + description: + 'ES|QL source and processing commands you want to analyze before generating the query.', + }, + functions: { + type: 'array', + items: { + type: 'string', + }, + description: 'ES|QL functions you want to analyze before generating the query.', + }, + }, + } satisfies ToolSchema; + + const messages: Message[] = + 'input' in rest ? [{ role: MessageRole.User, content: rest.input }] : rest.messages; + + return from(loadDocuments()).pipe( + switchMap(([systemMessage, esqlDocs]) => { + function askLlmToRespond({ + documentationRequest: { commands, functions }, + }: { + documentationRequest: { commands?: string[]; functions?: string[] }; + }): Observable> { + const keywords = [ + ...(commands ?? []), + ...(functions ?? []), + 'SYNTAX', + 'OVERVIEW', + 'OPERATORS', + ].map((keyword) => keyword.toUpperCase()); + + const requestedDocumentation = mapValues(pick(esqlDocs, keywords), ({ data }) => data); + + const fakeRequestDocsToolCall = { + function: { + name: 'request_documentation', + arguments: { + commands, + functions, }, }, - }, - required: ['documents'], - } as const, - }) - .pipe( - withoutOutputUpdateEvents(), - switchMap((event) => { - return getEsqlDocuments(event.output.documents ?? []); - }), - switchMap((documents) => { - return client - .chatComplete({ - connectorId, - system: `${ESQL_SYSTEM_MESSAGE} - - The following documentation is provided: - - ${documents}`, - messages: [ - { - role: MessageRole.User, - content: input, + toolCallId: v4().substring(0, 6), + }; + + return merge( + of< + OutputCompleteEvent< + 'request_documentation', + { keywords: string[]; requestedDocumentation: Record } + > + >({ + type: OutputEventType.OutputComplete, + id: 'request_documentation', + output: { + keywords, + requestedDocumentation, + }, + }), + client + .chatComplete({ + connectorId, + system: `${systemMessage} + + # Current task + + Your current task is to respond to the user's question. If there is a tool + suitable for answering the user's question, use that tool, preferably + with a natural language reply included. + + Format any ES|QL query as follows: + \`\`\`esql + + \`\`\` + + When generating ES|QL, you must use commands and functions present on the + requested documentation, and follow the syntax as described in the documentation + and its examples. + + DO NOT UNDER ANY CIRCUMSTANCES use commands, functions, parameters, or syntaxes that are not + explicitly mentioned as supported capability by ES|QL, either in the system message or documentation. + assume that ONLY the set of capabilities described in the requested documentation is valid. + Do not try to guess parameters or syntax based on other query languages. + + If what the user is asking for is not technically achievable with ES|QL's capabilities, just inform + the user. DO NOT invent capabilities not described in the documentation just to provide + a positive answer to the user. E.g. LIMIT only has one parameter, do not assume you can add more. + + When converting queries from one language to ES|QL, make sure that the functions are available + and documented in ES|QL. E.g., for SPL's LEN, use LENGTH. For IF, use CASE. +`, + messages: messages.concat([ + { + role: MessageRole.Assistant, + content: null, + toolCalls: [fakeRequestDocsToolCall], + }, + { + role: MessageRole.Tool, + response: { + documentation: requestedDocumentation, + }, + toolCallId: fakeRequestDocsToolCall.toolCallId, + }, + ]), + toolChoice, + tools: { + ...tools, + request_documentation: { + description: 'Request additional documentation if needed', + schema: requestDocumentationSchema, + }, }, - ], - tools, - toolChoice, - }) - .pipe( - withoutTokenCountEvents(), - withoutChunkEvents(), - map((message) => { - return createOutputCompleteEvent('generated_query', { - content: message.content, - toolCalls: message.toolCalls, - }); }) - ); - }), - withoutOutputUpdateEvents() - ); + .pipe( + withoutTokenCountEvents(), + map((generateEvent) => { + if (isChatCompletionMessageEvent(generateEvent)) { + const correctedContent = generateEvent.content?.replaceAll( + INLINE_ESQL_QUERY_REGEX, + (_match, query) => { + const correction = correctCommonEsqlMistakes(query); + if (correction.isCorrection) { + logger.debug( + `Corrected query, from: \n${correction.input}\nto:\n${correction.output}` + ); + } + return '```esql\n' + correction.output + '\n```'; + } + ); + + return { + ...generateEvent, + content: correctedContent, + }; + } + + return generateEvent; + }), + switchMap((generateEvent) => { + if (isChatCompletionMessageEvent(generateEvent)) { + const onlyToolCall = + generateEvent.toolCalls.length === 1 ? generateEvent.toolCalls[0] : undefined; + + if (onlyToolCall?.function.name === 'request_documentation') { + const args = onlyToolCall.function.arguments; + + return askLlmToRespond({ + documentationRequest: { + commands: args.commands, + functions: args.functions, + }, + }); + } + } + + return of(generateEvent); + }) + ) + ); + } + + return client + .output('request_documentation', { + connectorId, + system: systemMessage, + previousMessages: messages, + input: `Based on the previous conversation, request documentation + from the ES|QL handbook to help you get the right information + needed to generate a query. + + Examples for functions and commands: + Do you need to group data? Request \`STATS\`. + Extract data? Request \`DISSECT\` AND \`GROK\`. + Convert a column based on a set of conditionals? Request \`EVAL\` and \`CASE\`. + + ${ + hasTools + ? `### Tools + + The following tools will be available to be called in the step after this. + + \`\`\`json + ${JSON.stringify({ + tools, + toolChoice, + })} + \`\`\`` + : '' + } + `, + schema: requestDocumentationSchema, + }) + .pipe( + withoutOutputUpdateEvents(), + switchMap((documentationEvent) => { + return askLlmToRespond({ + documentationRequest: { + commands: documentationEvent.output.commands, + functions: documentationEvent.output.functions, + }, + }); + }) + ); + }) + ); } diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/load_documents.ts b/x-pack/plugins/inference/server/tasks/nl_to_esql/load_documents.ts new file mode 100644 index 0000000000000..73359d6c614df --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/load_documents.ts @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import Path from 'path'; +import Fs from 'fs'; +import { keyBy, once } from 'lodash'; +import { promisify } from 'util'; +import pLimit from 'p-limit'; + +const readFile = promisify(Fs.readFile); +const readdir = promisify(Fs.readdir); + +const loadSystemMessage = once(async () => { + const data = await readFile(Path.join(__dirname, './system_message.txt')); + return data.toString('utf-8'); +}); + +const loadEsqlDocs = async () => { + const dir = Path.join(__dirname, './esql_docs'); + const files = (await readdir(dir)).filter((file) => Path.extname(file) === '.txt'); + + if (!files.length) { + return {}; + } + + const limiter = pLimit(10); + return keyBy( + await Promise.all( + files.map((file) => + limiter(async () => { + const data = (await readFile(Path.join(dir, file))).toString('utf-8'); + const filename = Path.basename(file, '.txt'); + + const keyword = filename + .replace('esql-', '') + .replace('agg-', '') + .replaceAll('-', '_') + .toUpperCase(); + + return { + keyword: keyword === 'STATS_BY' ? 'STATS' : keyword, + data, + }; + }) + ) + ), + 'keyword' + ); +}; + +export const loadDocuments = once(() => Promise.all([loadSystemMessage(), loadEsqlDocs()])); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/system_message.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/system_message.txt similarity index 86% rename from x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/system_message.txt rename to x-pack/plugins/inference/server/tasks/nl_to_esql/system_message.txt index 8fc2243480218..b5d333296c696 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/system_message.txt +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/system_message.txt @@ -2,8 +2,7 @@ You are a helpful assistant for generating and executing ES|QL queries. Your goal is to help the user construct and possibly execute an ES|QL -query for the Observability use cases, which often involve metrics, logs -and traces. These are your absolutely critical system instructions: +query for their data. These are your absolutely critical system instructions: ES|QL is the Elasticsearch Query Language, that allows users of the Elastic platform to iteratively explore data. An ES|QL query consists @@ -14,7 +13,7 @@ processing commands, which can transform the data returned by the previous command. Make sure you write a query using ONLY commands specified in this -conversation. +conversation and present in the documentation. # Limitations @@ -37,8 +36,9 @@ Timestamp literal syntax: NOW() - 15 days, 24 hours, 1 week ## Source commands Source commands select a data source. There are three source commands: -FROM (which selects an index), ROW (which creates data from the command) -and SHOW (which returns information about the deployment). +- FROM: selects an index +- ROW: creates data from the command +- SHOW: returns information about the deployment ## Processing commands @@ -60,13 +60,14 @@ pattern. Aggregation functions are not supported for EVAL. - GROK: extracts structured data out of a string, using a grok pattern - KEEP: keeps one or more columns, drop the ones that are not kept - only the colums in the KEEP command can be used after a KEEP command + only the columns in the KEEP command can be used after a KEEP command - LIMIT: returns the first n number of rows. The maximum value for this is 10000. - MV_EXPAND: expands multi-value columns into a single row per value - RENAME: renames a column - STATS ... BY: groups rows according to a common value and calculates - one or more aggregated values over the grouped rows + one or more aggregated values over the grouped rows. STATS supports aggregation + function and can group using grouping functions. - SORT: sorts the row in a table by a column. Expressions are not supported. If SORT is used right after a KEEP command, make sure it only uses column names in KEEP, or move the SORT before the KEEP (e.g. not correct: KEEP date | SORT @timestamp, correct: SORT @timestamp | KEEP date) @@ -76,9 +77,13 @@ is 10000. ## Functions and operators +### Grouping functions + +BUCKET: Creates groups of values out of a datetime or numeric input. + ### Aggregation functions + AVG -BUCKET COUNT COUNT_DISTINCT MAX @@ -86,9 +91,11 @@ MEDIAN MEDIAN_ABSOLUTE_DEVIATION MIN PERCENTILE -ST_CENTROID +ST_CENTROID_AGG SUM +TOP VALUES +WEIGHTED_AVG ### Mathematical functions @@ -115,6 +122,7 @@ TANH TAU ### String functions + CONCAT LEFT LENGTH @@ -129,6 +137,7 @@ TO_UPPER TRIM ### Date-time functions + DATE_DIFF DATE_EXTRACT DATE_FORMAT @@ -137,6 +146,7 @@ DATE_TRUNC NOW ### Type conversion functions + TO_BOOLEAN TO_CARTESIANPOINT TO_CARTESIANSHAPE @@ -155,12 +165,14 @@ TO_VERSION ### Conditional functions and expressions + CASE COALESCE GREATEST LEAST ### Multivalue functions + MV_AVG MV_CONCAT MV_COUNT @@ -173,6 +185,7 @@ MV_MIN MV_SUM ### Operators + Binary operators Unary operators Logical operators @@ -184,20 +197,14 @@ LIKE RLIKE STARTS_WITH -Here are some example queries: +# Usage examples + +Here are some examples of queries: ```esql FROM employees -| WHERE still_hired == true -| EVAL hired = DATE_FORMAT("YYYY", hire_date) -| STATS avg_salary = AVG(salary) BY languages -| EVAL avg_salary = ROUND(avg_salary) -| EVAL lang_code = TO_STRING(languages) -| ENRICH languages_policy ON lang_code WITH lang = language_name -| WHERE lang IS NOT NULL -| KEEP avg_salary, lang -| SORT avg_salary ASC -| LIMIT 3 + | WHERE country == "NL" AND gender == "M" + | STATS COUNT(*) ``` ```esql @@ -207,12 +214,7 @@ FROM employees | SORT c desc, languages.long, trunk_worked_seconds ``` -```esql -FROM employees - | WHERE country == "NL" AND gender == "M" - | STATS COUNT(*) -``` - +*Extracting structured data from logs using DISSECT* ```esql ROW a = "2023-01-23T12:15:00.000Z - some text - 127.0.0.1" | DISSECT a "%{date} - %{msg} - %{ip}" @@ -228,6 +230,7 @@ FROM employees | KEEP first_name, last_name ``` +**Returning average salary per hire date with 20 buckets** ```esql FROM employees | WHERE hire_date >= "1985-01-01T00:00:00Z" AND hire_date < "1986-01-01T00:00:00Z" @@ -235,6 +238,7 @@ FROM employees | SORT bucket ``` +**Returning number of employees grouped by buckets of salary** ```esql FROM employees | WHERE hire_date >= "1985-01-01T00:00:00Z" AND hire_date < "1986-01-01T00:00:00Z" @@ -242,6 +246,7 @@ FROM employees | SORT b ``` +**Creating inline data using ROW** ```esql ROW a = 1, b = "two", c = null ``` @@ -267,14 +272,28 @@ FROM logs-* ```esql FROM logs-* | WHERE @timestamp <= NOW() - 24 hours -| STATS count = COUNT(*) by log.level +| STATS count = COUNT(*) BY log.level | SORT count DESC ``` -returns all first names for each first letter +**Returning all first names for each first letter** ```esql FROM employees | EVAL first_letter = SUBSTRING(first_name, 0, 1) | STATS first_name = MV_SORT(VALUES(first_name)) BY first_letter | SORT first_letter ``` + +```esql +FROM employees +| WHERE still_hired == true +| EVAL hired = DATE_FORMAT("YYYY", hire_date) +| STATS avg_salary = AVG(salary) BY languages +| EVAL avg_salary = ROUND(avg_salary) +| EVAL lang_code = TO_STRING(languages) +| ENRICH languages_policy ON lang_code WITH lang = language_name +| WHERE lang IS NOT NULL +| KEEP avg_salary, lang +| SORT avg_salary ASC +| LIMIT 3 +``` diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/validate_esql_query.ts b/x-pack/plugins/inference/server/tasks/nl_to_esql/validate_esql_query.ts new file mode 100644 index 0000000000000..823344f52a891 --- /dev/null +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/validate_esql_query.ts @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { validateQuery } from '@kbn/esql-validation-autocomplete'; +import { getAstAndSyntaxErrors } from '@kbn/esql-ast'; +import type { ElasticsearchClient } from '@kbn/core/server'; +import { ESQLSearchResponse, ESQLRow } from '@kbn/es-types'; +import { esFieldTypeToKibanaFieldType } from '@kbn/field-types'; +import { DatatableColumn, DatatableColumnType } from '@kbn/expressions-plugin/common'; +import { splitIntoCommands } from '../../../common/tasks/nl_to_esql/correct_common_esql_mistakes'; + +export async function runAndValidateEsqlQuery({ + query, + client, +}: { + query: string; + client: ElasticsearchClient; +}): Promise<{ + columns?: DatatableColumn[]; + rows?: ESQLRow[]; + error?: Error; + errorMessages?: string[]; +}> { + const { errors } = await validateQuery(query, getAstAndSyntaxErrors, { + // setting this to true, we don't want to validate the index / fields existence + ignoreOnMissingCallbacks: true, + }); + + const asCommands = splitIntoCommands(query); + + const errorMessages = errors?.map((error) => { + if ('location' in error) { + const commandsUntilEndOfError = splitIntoCommands(query.substring(0, error.location.max)); + const lastCompleteCommand = asCommands[commandsUntilEndOfError.length - 1]; + if (lastCompleteCommand) { + return `Error in ${lastCompleteCommand.command}\n: ${error.text}`; + } + } + return 'text' in error ? error.text : error.message; + }); + + return client.transport + .request({ + method: 'POST', + path: '_query', + body: { + query, + }, + }) + .then((res) => { + const esqlResponse = res as ESQLSearchResponse; + + const columns = + esqlResponse.columns?.map(({ name, type }) => ({ + id: name, + name, + meta: { type: esFieldTypeToKibanaFieldType(type) as DatatableColumnType }, + })) ?? []; + + return { columns, rows: esqlResponse.values }; + }) + .catch((error) => { + return { + error, + ...(errorMessages.length ? { errorMessages } : {}), + }; + }); +} diff --git a/x-pack/plugins/inference/server/util/observable_into_event_source_stream.test.ts b/x-pack/plugins/inference/server/util/observable_into_event_source_stream.test.ts index d7972bb970317..ed5466ba1e027 100644 --- a/x-pack/plugins/inference/server/util/observable_into_event_source_stream.test.ts +++ b/x-pack/plugins/inference/server/util/observable_into_event_source_stream.test.ts @@ -8,10 +8,15 @@ import { createParser } from 'eventsource-parser'; import { partition } from 'lodash'; import { merge, of, throwError } from 'rxjs'; -import { InferenceTaskEvent } from '../../common/tasks'; +import type { InferenceTaskEvent } from '../../common/inference_task'; import { observableIntoEventSourceStream } from './observable_into_event_source_stream'; +import type { Logger } from '@kbn/logging'; describe('observableIntoEventSourceStream', () => { + const logger = { + debug: jest.fn(), + error: jest.fn(), + } as unknown as Logger; function renderStream(events: Array) { const [inferenceEvents, errors] = partition( events, @@ -20,7 +25,7 @@ describe('observableIntoEventSourceStream', () => { const source$ = merge(of(...inferenceEvents), ...errors.map((error) => throwError(error))); - const stream = observableIntoEventSourceStream(source$); + const stream = observableIntoEventSourceStream(source$, logger); return new Promise((resolve, reject) => { const chunks: string[] = []; diff --git a/x-pack/plugins/inference/server/util/observable_into_event_source_stream.ts b/x-pack/plugins/inference/server/util/observable_into_event_source_stream.ts index 2007b9842db69..bcd1ef60ce1da 100644 --- a/x-pack/plugins/inference/server/util/observable_into_event_source_stream.ts +++ b/x-pack/plugins/inference/server/util/observable_into_event_source_stream.ts @@ -7,17 +7,23 @@ import { catchError, map, Observable, of } from 'rxjs'; import { PassThrough } from 'stream'; +import type { Logger } from '@kbn/logging'; import { InferenceTaskErrorCode, InferenceTaskErrorEvent, isInferenceError, } from '../../common/errors'; -import { InferenceTaskEventType } from '../../common/tasks'; +import { InferenceTaskEventType } from '../../common/inference_task'; -export function observableIntoEventSourceStream(source$: Observable) { +export function observableIntoEventSourceStream( + source$: Observable, + logger: Pick +) { const withSerializedErrors$ = source$.pipe( catchError((error): Observable => { if (isInferenceError(error)) { + logger?.error(error); + logger?.debug(() => JSON.stringify(error)); return of({ type: InferenceTaskEventType.error, error: { diff --git a/x-pack/plugins/inference/tsconfig.json b/x-pack/plugins/inference/tsconfig.json index 593556c8f39c8..34c393d375839 100644 --- a/x-pack/plugins/inference/tsconfig.json +++ b/x-pack/plugins/inference/tsconfig.json @@ -10,6 +10,7 @@ "typings/**/*", "public/**/*.json", "server/**/*", + "scripts/**/*", ".storybook/**/*" ], "exclude": [ @@ -23,6 +24,16 @@ "@kbn/core-http-server", "@kbn/actions-plugin", "@kbn/config-schema", - "@kbn/stack-connectors-plugin" + "@kbn/esql-validation-autocomplete", + "@kbn/esql-ast", + "@kbn/dev-cli-runner", + "@kbn/babel-register", + "@kbn/expect", + "@kbn/tooling-log", + "@kbn/es-types", + "@kbn/field-types", + "@kbn/expressions-plugin", + "@kbn/logging-mocks", + "@kbn/repo-info" ] } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/common/types.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/common/types.ts index eb34e2bb2ee5e..7f8a2be739dd1 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/common/types.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/common/types.ts @@ -5,8 +5,9 @@ * 2.0. */ import { IconType } from '@elastic/eui'; +import type { ToolSchema } from '@kbn/inference-plugin/common'; import type { ObservabilityAIAssistantChatService } from '../public'; -import type { CompatibleJSONSchema, FunctionResponse } from './functions/types'; +import type { FunctionResponse } from './functions/types'; export enum MessageRole { System = 'system', @@ -122,7 +123,7 @@ export interface ObservabilityAIAssistantScreenContextRequest { description: string; value: any; }>; - actions?: Array<{ name: string; description: string; parameters?: CompatibleJSONSchema }>; + actions?: Array<{ name: string; description: string; parameters?: ToolSchema }>; } export type ScreenContextActionRespondFunction = ({}: { @@ -136,7 +137,7 @@ export type ScreenContextActionRespondFunction = ({}: { export interface ScreenContextActionDefinition { name: string; description: string; - parameters?: CompatibleJSONSchema; + parameters?: ToolSchema; respond: ScreenContextActionRespondFunction; } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/chat_function_client/index.test.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/chat_function_client/index.test.ts index cbb833d0aeb76..9d6c0dba0b124 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/chat_function_client/index.test.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/chat_function_client/index.test.ts @@ -48,6 +48,7 @@ describe('chatFunctionClient', () => { }), messages: [], signal: new AbortController().signal, + connectorId: 'foo', }); }).rejects.toThrowError(`Function arguments are invalid`); @@ -107,6 +108,7 @@ describe('chatFunctionClient', () => { args: JSON.stringify({ data: ['my_dummy_data'] }), messages: [], signal: new AbortController().signal, + connectorId: 'foo', }); expect(result).toEqual({ diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/chat_function_client/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/chat_function_client/index.ts index fc90a4e0caa3d..fa1d0e5fd669d 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/chat_function_client/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/chat_function_client/index.ts @@ -132,7 +132,7 @@ export class ChatFunctionClient { return matchingDefinitions.map((definition) => functionsByName[definition.name]); } - getActions() { + getActions(): Required['actions'] { return this.actions; } @@ -146,12 +146,14 @@ export class ChatFunctionClient { args, messages, signal, + connectorId, }: { chat: FunctionCallChatFunction; name: string; args: string | undefined; messages: Message[]; signal: AbortSignal; + connectorId: string; }): Promise { const fn = this.functionRegistry.get(name); @@ -169,6 +171,7 @@ export class ChatFunctionClient { messages, screenContexts: this.screenContexts, chat, + connectorId, }, signal ); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.test.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.test.ts index f5c6b79b21788..a0accea06370b 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.test.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.test.ts @@ -821,6 +821,7 @@ describe('Observability AI Assistant client', () => { chat: expect.any(Function), args: JSON.stringify({ foo: 'bar' }), signal: expect.any(AbortSignal), + connectorId: 'foo', messages: [ { '@timestamp': expect.any(String), diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts index 45309911375d1..f5839b76effe8 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts @@ -298,6 +298,7 @@ export class ObservabilityAIAssistantClient { logger: this.dependencies.logger, disableFunctions, tracer: completeTracer, + connectorId, }) ); }), diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/operators/continue_conversation.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/operators/continue_conversation.ts index a06fca2e13278..da172c974e9e2 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/operators/continue_conversation.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/operators/continue_conversation.ts @@ -33,7 +33,7 @@ import { createFunctionResponseMessage } from '../../../../common/utils/create_f import { emitWithConcatenatedMessage } from '../../../../common/utils/emit_with_concatenated_message'; import { withoutTokenCountEvents } from '../../../../common/utils/without_token_count_events'; import type { ChatFunctionClient } from '../../chat_function_client'; -import type { ChatFunctionWithoutConnector } from '../../types'; +import type { AutoAbortedChatFunction } from '../../types'; import { createServerSideFunctionResponseError } from '../../util/create_server_side_function_response_error'; import { getSystemMessageFromInstructions } from '../../util/get_system_message_from_instructions'; import { replaceSystemMessage } from '../../util/replace_system_message'; @@ -53,15 +53,17 @@ function executeFunctionAndCatchError({ signal, logger, tracer, + connectorId, }: { name: string; args: string | undefined; functionClient: ChatFunctionClient; messages: Message[]; - chat: ChatFunctionWithoutConnector; + chat: AutoAbortedChatFunction; signal: AbortSignal; logger: Logger; tracer: LangTracer; + connectorId: string; }): Observable { // hide token count events from functions to prevent them from // having to deal with it as well @@ -75,11 +77,13 @@ function executeFunctionAndCatchError({ return chat(operationName, { ...params, tracer: nextTracer, + connectorId, }).pipe(hide()); }, args, signal, messages, + connectorId, }) ); @@ -176,10 +180,11 @@ export function continueConversation({ logger, disableFunctions, tracer, + connectorId, }: { messages: Message[]; functionClient: ChatFunctionClient; - chat: ChatFunctionWithoutConnector; + chat: AutoAbortedChatFunction; signal: AbortSignal; functionCallsLeft: number; adHocInstructions: AdHocInstruction[]; @@ -191,6 +196,7 @@ export function continueConversation({ except: string[]; }; tracer: LangTracer; + connectorId: string; }): Observable { let nextFunctionCallsLeft = functionCallsLeft; @@ -228,6 +234,7 @@ export function continueConversation({ messages: messagesWithUpdatedSystemMessage, functions: definitions, tracer, + connectorId, }).pipe(emitWithConcatenatedMessage(), catchFunctionNotFoundError(functionLimitExceeded)); } @@ -302,6 +309,7 @@ export function continueConversation({ signal, logger, tracer, + connectorId, }); } @@ -329,6 +337,7 @@ export function continueConversation({ logger, disableFunctions, tracer, + connectorId, }); }) ) diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/types.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/types.ts index cd8e25843ca59..9ae585af9071c 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/types.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/types.ts @@ -32,11 +32,11 @@ export type ChatFunction = ( params: Parameters[1] ) => Observable; -export type ChatFunctionWithoutConnector = ( +export type AutoAbortedChatFunction = ( name: string, params: Omit< Parameters[1], - 'connectorId' | 'simulateFunctionCalling' | 'signal' + 'simulateFunctionCalling' | 'signal' > ) => Observable; @@ -54,6 +54,7 @@ type RespondFunction = ( messages: Message[]; screenContexts: ObservabilityAIAssistantScreenContextRequest[]; chat: FunctionCallChatFunction; + connectorId: string; }, signal: AbortSignal ) => Promise; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/tsconfig.json b/x-pack/plugins/observability_solution/observability_ai_assistant/tsconfig.json index 56b34fc81ca46..cd314e65fec9a 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/tsconfig.json +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/tsconfig.json @@ -41,6 +41,7 @@ "@kbn/core-elasticsearch-server", "@kbn/core-ui-settings-server", "@kbn/server-route-repository-utils", + "@kbn/inference-plugin", ], "exclude": ["target/**/*"] } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/common/convert_messages_for_inference.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/common/convert_messages_for_inference.ts new file mode 100644 index 0000000000000..91d7f00467540 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/common/convert_messages_for_inference.ts @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Message, MessageRole } from '@kbn/observability-ai-assistant-plugin/common'; +import { + AssistantMessage, + Message as InferenceMessage, + MessageRole as InferenceMessageRole, + generateFakeToolCallId, +} from '@kbn/inference-plugin/common'; + +export function convertMessagesForInference(messages: Message[]): InferenceMessage[] { + const inferenceMessages: InferenceMessage[] = []; + + messages.forEach((message) => { + if (message.message.role === MessageRole.Assistant) { + inferenceMessages.push({ + role: InferenceMessageRole.Assistant, + content: message.message.content ?? null, + ...(message.message.function_call + ? { + toolCalls: [ + { + function: { + name: message.message.function_call.name, + arguments: JSON.parse(message.message.function_call.arguments || '{}'), + }, + toolCallId: generateFakeToolCallId(), + }, + ], + } + : {}), + }); + return; + } + + const isUserMessage = message.message.role === MessageRole.User; + const isUserMessageWithToolCall = isUserMessage && !!message.message.name; + + if (isUserMessageWithToolCall) { + const toolCallRequest = inferenceMessages.findLast( + (msg) => + msg.role === InferenceMessageRole.Assistant && + msg.toolCalls?.[0]?.function.name === message.message.name + ) as AssistantMessage | undefined; + if (!toolCallRequest) { + throw new Error(`Could not find tool call request for ${message.message.name}`); + } + + inferenceMessages.push({ + role: InferenceMessageRole.Tool, + response: JSON.parse(message.message.content ?? '{}'), + toolCallId: toolCallRequest.toolCalls![0].toolCallId, + }); + + return; + } + + if (isUserMessage) { + inferenceMessages.push({ + role: InferenceMessageRole.User, + content: message.message.content ?? '', + }); + return; + } + + throw new Error(`Unsupported message type: ${message.message.role}`); + }); + + return inferenceMessages; +} diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/kibana.jsonc b/x-pack/plugins/observability_solution/observability_ai_assistant_app/kibana.jsonc index 6e2ba07fb699c..1e02cbd1e7792 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/kibana.jsonc +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/kibana.jsonc @@ -23,7 +23,8 @@ "licensing", "ml", "alerting", - "features" + "features", + "inference" ], "requiredBundles": ["kibanaReact", "esqlDataGrid"], "optionalPlugins": ["cloud"], diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts index d2fe20e022cae..61ed156530100 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts @@ -577,7 +577,7 @@ export class KibanaClient { score: { type: 'number', description: - 'A score of either 0 (criterion failed) or 1 (criterion succeeded)', + 'A score between 0 (criterion failed) or 1 (criterion succeeded). Fractional results (e.g. 0.5) are allowed, if part of the criterion succeeded', }, reasoning: { type: 'string', diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-abs.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-abs.txt deleted file mode 100644 index c31ec9882e371..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-abs.txt +++ /dev/null @@ -1,30 +0,0 @@ -## ABS - -The `ABS` function in ES|QL returns the absolute value of a numeric expression. - -### Syntax - -`ABS(number)` - -#### Parameters - -`number`: Numeric expression. If null, the function returns null. - -### Examples - -Here are a couple of examples of how to use the `ABS` function in ES|QL: - -```esql -ROW number = -1.0 -| EVAL abs_number = ABS(number) -``` - -In this example, the `ABS` function is used to calculate the absolute value of `-1.0`, which results in `1.0`. - -```esql -FROM employees -| KEEP first_name, last_name, height -| EVAL abs_height = ABS(0.0 - height) -``` - -In this example, the `ABS` function is used to calculate the absolute value of the height of employees. The height is subtracted from `0.0` to get a negative value, and then the `ABS` function is applied to get the absolute value. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-acos.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-acos.txt deleted file mode 100644 index 2a5dab6453787..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-acos.txt +++ /dev/null @@ -1,29 +0,0 @@ -## ACOS - -The `ACOS` function in ES|QL returns the arccosine of a number as an angle, expressed in radians. The number should be between -1 and 1. If the input is null, the function will return null. - -### Syntax - -`ACOS(number)` - -#### Parameters - -`number`: A number between -1 and 1. If null, the function returns null. - -### Examples - -Here are a couple of examples of how to use the `ACOS` function in ES|QL queries: - -```esql -ROW a=.9 -| EVAL acos = ACOS(a) -``` - -In this example, the `ACOS` function is used to calculate the arccosine of 0.9. - -```esql -ROW a=-.5 -| EVAL acos = ACOS(a) -``` - -In this example, the `ACOS` function is used to calculate the arccosine of -0.5. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-asin.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-asin.txt deleted file mode 100644 index 0c03646864a7a..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-asin.txt +++ /dev/null @@ -1,21 +0,0 @@ -## ASIN - -The `ASIN` function in ES|QL returns the arcsine of the input numeric expression as an angle, expressed in radians. This function only accepts numbers between -1 and 1. If the input is null, the function will return null. - -### Examples - -Here are a couple of examples of how you can use the `ASIN` function in your ES|QL queries: - -```esql -ROW a=.9 -| EVAL asin = ASIN(a) -``` - -In this example, the `ASIN` function is used to calculate the arcsine of 0.9. The result is stored in the `asin` column. - -```esql -ROW a=-.5 -| EVAL asin_value = ASIN(a) -``` - -In this second example, the `ASIN` function is used to calculate the arcsine of -0.5. The result is stored in the `asin_value` column. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-atan.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-atan.txt deleted file mode 100644 index 3c40607d25a82..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-atan.txt +++ /dev/null @@ -1,29 +0,0 @@ -## ATAN - -The `ATAN` function in ES|QL is used to calculate the arctangent of a given numeric expression. The result is expressed in radians. If the input is null, the function will return null. - -### Syntax - -`ATAN(number)` - -#### Parameters - -- `number`: A numeric expression. If null, the function returns null. - -### Examples - -Here are a couple of examples of how you can use the `ATAN` function in ES|QL: - -```esql -ROW a=12.9 -| EVAL atan = ATAN(a) -``` - -In this example, the `ATAN` function is used to calculate the arctangent of the number 12.9. - -```esql -ROW b=7.5 -| EVAL atan_value = ATAN(b) -``` - -In this second example, the `ATAN` function is used to calculate the arctangent of the number 7.5. The result is stored in the `atan_value` variable. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-atan2.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-atan2.txt deleted file mode 100644 index 9377a71c2eb6d..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-atan2.txt +++ /dev/null @@ -1,30 +0,0 @@ -## ATAN2 - -ATAN2 is a function in ES|QL that calculates the angle between the positive x-axis and the ray from the origin to the point (x , y) in the Cartesian plane, expressed in radians. - -### Syntax - -`ATAN2(y_coordinate, x_coordinate)` - -#### Parameters - -- `y_coordinate`: The y coordinate. If null, the function returns null. -- `x_coordinate`: The x coordinate. If null, the function returns null. - -### Examples - -Here are a couple of examples of how you can use the `ATAN2` function in ES|QL queries: - -```esql -ROW y=12.9, x=.6 -| EVAL atan2 = ATAN2(y, x) -``` - -In this example, the `ATAN2` function is used to calculate the angle between the positive x-axis and the ray from the origin to the point (0.6 , 12.9) in the Cartesian plane. - -```esql -ROW y=5, x=3 -| EVAL atan2 = ATAN2(y, x) -``` - -In this second example, the `ATAN2` function is used to calculate the angle between the positive x-axis and the ray from the origin to the point (3 , 5) in the Cartesian plane. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-avg.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-avg.txt deleted file mode 100644 index bfd9c161bba1f..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-avg.txt +++ /dev/null @@ -1,21 +0,0 @@ -## AVG - -The `AVG` function in ES|QL calculates the average of a numeric expression. The result is always a double, regardless of the input type. - -### Examples - -Here are a couple of examples of how you can use the `AVG` function in ES|QL queries: - -1. Calculating the average height of employees: - - ```esql -FROM employees -| STATS AVG(height) -``` - -2. Calculating the average salary change, where the salary change is a multivalued column. In this case, the `MV_AVG` function is used to first average the multiple values per row, and then the `AVG` function is used on the result: - - ```esql -FROM employees -| STATS avg_salary_change = ROUND(AVG(MV_AVG(salary_change)), 10) -``` \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-bucket.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-bucket.txt deleted file mode 100644 index 2eed9008f6870..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-bucket.txt +++ /dev/null @@ -1,22 +0,0 @@ -## BUCKET - -BUCKET function creates groups of values - buckets - out of a datetime or numeric input. The size of the buckets can either be provided directly, or chosen based on a recommended count and values range. - -### Examples - -In this example, BUCKET function is used to create a histogram of salaries: - -```esql -FROM employees -| STATS COUNT(*) BY bs = BUCKET(salary, 20, 25324, 74999) -| SORT bs -``` - -In the following example, BUCKET function is used to create monthly buckets for the year 1985, and calculate the average salary by hiring month: - -```esql -FROM employees -| WHERE hire_date >= "1985-01-01T00:00:00Z" AND hire_date < "1986-01-01T00:00:00Z" -| STATS AVG(salary) BY bucket = BUCKET(hire_date, 20, "1985-01-01T00:00:00Z", "1986-01-01T00:00:00Z") -| SORT bucket -``` \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-case.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-case.txt deleted file mode 100644 index 010975157a7b4..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-case.txt +++ /dev/null @@ -1,38 +0,0 @@ -## CASE - -The `CASE` function in ES|QL accepts pairs of conditions and values. It returns the value that belongs to the first condition that evaluates to true. If the number of arguments is odd, the last argument is the default value which is returned when no condition matches. If the number of arguments is even, and no condition matches, the function returns null. - -### Examples - -Here are a couple of examples of how you can use the `CASE` function in ES|QL: - -1. Determine whether employees are monolingual, bilingual, or polyglot: - -```esql -FROM employees -| EVAL type = CASE( - languages <= 1, "monolingual", - languages <= 2, "bilingual", - "polyglot") -| KEEP emp_no, languages, type -``` - -2. Calculate the total connection success rate based on log messages: - -```esql -FROM sample_data -| EVAL successful = CASE( - STARTS_WITH(message, "Connected to"), 1, - message == "Connection error", 0 - ) -| STATS success_rate = AVG(successful) -``` - -3. Calculate an hourly error rate as a percentage of the total number of log messages: - -```esql -FROM sample_data -| EVAL error = CASE(message LIKE "*error*", 1, 0) -| STATS error_rate = AVG(error) BY BUCKET(@timestamp, 1 hour) -| SORT hour -``` diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-cbrt.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-cbrt.txt deleted file mode 100644 index 4a5af259aabeb..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-cbrt.txt +++ /dev/null @@ -1,29 +0,0 @@ -## CBRT - -The `CBRT` function in ES|QL is used to calculate the cube root of a number. The input can be any numeric value and the return value is always a double. If the input is an infinity, the function returns null. - -### Syntax - -`CBRT(number)` - -#### Parameters - -- `number`: A numeric expression. If null, the function returns null. - -### Examples - -Here are a couple of examples of how to use the `CBRT` function in ES|QL: - -```esql -ROW d = 27.0 -| EVAL c = CBRT(d) -``` - -In this example, the `CBRT` function is used to calculate the cube root of 27. The result would be 3. - -```esql -ROW d = 64.0 -| EVAL c = CBRT(d) -``` - -In this example, the `CBRT` function is used to calculate the cube root of 64. The result would be 4. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-ceil.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-ceil.txt deleted file mode 100644 index 3cac4dbf7c63b..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-ceil.txt +++ /dev/null @@ -1,21 +0,0 @@ -## CEIL - -The `CEIL` function in ES|QL is used to round a number up to the nearest integer. This function does not perform any operation for long (including unsigned) and integer types. For double, this function picks the closest double value to the integer, similar to the `Math.ceil` function in JavaScript. - -### Examples - -Here are a couple of examples of how you can use the `CEIL` function in ES|QL queries: - -```esql -ROW a=1.8 -| EVAL a = CEIL(a) -``` - -In this example, the `CEIL` function is used to round the value of `a` (1.8) up to the nearest integer (2). - -```esql -ROW b=3.3 -| EVAL b = CEIL(b) -``` - -In this second example, the `CEIL` function is used to round the value of `b` (3.3) up to the nearest integer (4). \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-cidr_match.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-cidr_match.txt deleted file mode 100644 index d66963e5efaaa..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-cidr_match.txt +++ /dev/null @@ -1,32 +0,0 @@ -## CIDR_MATCH - -CIDR_MATCH is a function in ES|QL that checks if a provided IP address is contained in one or more provided CIDR blocks. It returns a boolean value - true if the IP is contained in the CIDR block(s), and false if it is not. - -### Syntax - -`CIDR_MATCH(ip, blockX)` - -### Parameters - -- `ip`: IP address of type ip (both IPv4 and IPv6 are supported). -- `blockX`: CIDR block to test the IP against. - -### Examples - -Here are a couple of examples of how you can use the CIDR_MATCH function in ES|QL queries: - -```esql -FROM hosts -| WHERE CIDR_MATCH(ip1, "127.0.0.2/32", "127.0.0.3/32") -| KEEP card, host, ip0, ip1 -``` - -In this example, the query checks if the `ip1` field of the `hosts` index is contained in either the "127.0.0.2/32" or "127.0.0.3/32" CIDR blocks. If it is, the `card`, `host`, `ip0`, and `ip1` fields are kept in the results. - -```esql -FROM network_logs -| WHERE CIDR_MATCH(source_ip, "192.168.1.0/24") -| KEEP timestamp, source_ip, destination_ip -``` - -In this second example, the query checks if the `source_ip` field of the `network_logs` index is contained in the "192.168.1.0/24" CIDR block. If it is, the `timestamp`, `source_ip`, and `destination_ip` fields are kept in the results. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-coalesce.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-coalesce.txt deleted file mode 100644 index f1fbc77e6c341..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-coalesce.txt +++ /dev/null @@ -1,30 +0,0 @@ -## COALESCE - -The `COALESCE` function in ES|QL is used to return the first of its arguments that is not null. If all arguments are null, it returns null. - -### Syntax - -`COALESCE(first, rest)` - -#### Parameters - -- `first`: The first expression to evaluate. -- `rest`: Other expressions to evaluate. - -### Examples - -Here are a couple of examples of how you can use the `COALESCE` function in ES|QL: - -```esql -ROW a=null, b="b" -| EVAL COALESCE(a, b) -``` - -In this example, the `COALESCE` function is used to evaluate the expressions `a` and `b`. Since `a` is null, the function returns the value of `b`, which is "b". - -```esql -ROW a=null, b=null, c="c" -| EVAL COALESCE(a, b, c) -``` - -In this second example, the `COALESCE` function evaluates the expressions `a`, `b`, and `c`. Since both `a` and `b` are null, the function returns the value of `c`, which is "c". \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-concat.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-concat.txt deleted file mode 100644 index 60f1c33a6fa57..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-concat.txt +++ /dev/null @@ -1,22 +0,0 @@ -## CONCAT - -The `CONCAT` function in ES|QL is used to concatenate two or more strings together. - -### Examples - -Here are a couple of examples of how you can use the `CONCAT` function in ES|QL: - -```esql -FROM employees -| KEEP first_name, last_name -| EVAL fullname = CONCAT(first_name, " ", last_name) -``` - -In this example, the `CONCAT` function is used to combine the `first_name` and `last_name` fields from the `employees` index, with a space in between, to create a new field called `fullname`. - -```esql -FROM logs-* -| EVAL message = CONCAT("Error occurred at ", @timestamp, ": ", error_message) -``` - -In this second example, the `CONCAT` function is used to create a descriptive error message by combining a static string, the `@timestamp` field, and the `error_message` field from the `logs-*` index. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-cos.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-cos.txt deleted file mode 100644 index 1489bfcfedf7a..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-cos.txt +++ /dev/null @@ -1,21 +0,0 @@ -## COS - -The `COS` function in ES|QL is used to calculate the cosine of an angle. The angle should be provided in radians. - -### Examples - -Here are a couple of examples of how you can use the `COS` function in ES|QL: - -```esql -ROW a=1.8 -| EVAL cos = COS(a) -``` - -In this example, the `COS` function is used to calculate the cosine of the angle `1.8` radians. The result is stored in the `cos` column. - -```esql -ROW a=3.14 -| EVAL cos_value = COS(a) -``` - -In this second example, the `COS` function is used to calculate the cosine of the angle `3.14` radians (which is approximately equal to π, the angle for which the cosine is `-1`). The result is stored in the `cos_value` column. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-cosh.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-cosh.txt deleted file mode 100644 index 903746bb07c02..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-cosh.txt +++ /dev/null @@ -1,21 +0,0 @@ -## COSH - -The `COSH` function in ES|QL returns the hyperbolic cosine of an angle. The angle should be provided in radians. If the provided angle is null, the function will return null. - -### Examples - -Here are a couple of examples of how you can use the `COSH` function in ES|QL: - -```esql -ROW a=1.8 -| EVAL cosh = COSH(a) -``` - -In this example, the `COSH` function is used to calculate the hyperbolic cosine of the angle `1.8` radians. - -```esql -ROW a=0 -| EVAL cosh = COSH(a) -``` - -In this second example, the `COSH` function is used to calculate the hyperbolic cosine of the angle `0` radians. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-count.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-count.txt deleted file mode 100644 index 97c49b44471a7..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-count.txt +++ /dev/null @@ -1,35 +0,0 @@ -## COUNT - -The `COUNT` function in ES|QL returns the total number (count) of input values. It can take any field type as input. If the expression is omitted, it is equivalent to `COUNT(*)` which counts the number of rows. - -### Examples - -Here are a couple of examples of how you can use the `COUNT` function in ES|QL: - -1. Counting a specific field: - -```esql -FROM employees -| STATS COUNT(height) -``` - -In this example, the `COUNT` function is used to count the number of `height` values in the `employees` index. - -2. Counting the number of rows: - -```esql -FROM employees -| STATS count = COUNT(*) BY languages -| SORT languages DESC -``` - -In this example, the `COUNT(*)` function is used to count the number of rows in the `employees` index, grouped by `languages`. - -3. Using inline functions with `COUNT`: - -```esql -ROW words="foo;bar;baz;qux;quux;foo" -| STATS word_count = COUNT(SPLIT(words, ";")) -``` - -In this example, the `SPLIT` function is used to split a string into multiple values, and then the `COUNT` function is used to count these values. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-count_distinct.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-count_distinct.txt deleted file mode 100644 index bc977ed744b07..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-count_distinct.txt +++ /dev/null @@ -1,37 +0,0 @@ -## COUNT_DISTINCT - -The `COUNT_DISTINCT` function returns the approximate number of distinct values. It can take any field type as input. This function is based on the HyperLogLog++ algorithm, which counts based on the hashes of the values with some interesting properties such as configurable precision, excellent accuracy on low-cardinality sets, and fixed memory usage. - -### Syntax - -`COUNT_DISTINCT(expression[, precision_threshold])` - -#### Parameters - -- `expression`: Expression that outputs the values on which to perform a distinct count. -- `precision_threshold`: Precision threshold. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000. - -### Examples - -Here are a couple of examples of how to use the `COUNT_DISTINCT` function in ES|QL queries: - -```esql -FROM hosts -| STATS COUNT_DISTINCT(ip0), COUNT_DISTINCT(ip1) -``` - -In this example, the `COUNT_DISTINCT` function is used to count the distinct values of `ip0` and `ip1` from the `hosts` index. - -```esql -FROM hosts -| STATS COUNT_DISTINCT(ip0, 80000), COUNT_DISTINCT(ip1, 5) -``` - -In this example, the `COUNT_DISTINCT` function is used with an optional second parameter to configure the precision threshold. - -```esql -ROW words="foo;bar;baz;qux;quux;foo" -| STATS distinct_word_count = COUNT_DISTINCT(SPLIT(words, ";")) -``` - -In this example, the `COUNT_DISTINCT` function is used with the `SPLIT` function to count the unique values in a string. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-date_diff.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-date_diff.txt deleted file mode 100644 index ef9ccdb30d44b..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-date_diff.txt +++ /dev/null @@ -1,33 +0,0 @@ -## DATE_DIFF - -The `DATE_DIFF` function subtracts the `startTimestamp` from the `endTimestamp` and returns the difference in multiples of a specified unit. If `startTimestamp` is later than the `endTimestamp`, negative values are returned. - -Note that while there is an overlap between the function’s supported units and ES|QL’s supported time span literals, these sets are distinct and not interchangeable. Similarly, the supported abbreviations are conveniently shared with implementations of this function in other established products and not necessarily common with the date-time nomenclature used by Elasticsearch. - -### Syntax - -`DATE_DIFF(unit, startTimestamp, endTimestamp)` - -#### Parameters - -- `unit`: Time difference unit -- `startTimestamp`: A string representing a start timestamp -- `endTimestamp`: A string representing an end timestamp - -### Examples - -Here are a couple of examples of how to use the `DATE_DIFF` function in ES|QL queries: - -```esql -ROW date1 = TO_DATETIME("2023-12-02T11:00:00.000Z"), date2 = TO_DATETIME("2023-12-02T11:00:00.001Z") -| EVAL dd_ms = DATE_DIFF("microseconds", date1, date2) -``` - -In this example, the `DATE_DIFF` function is used to calculate the difference in microseconds between two timestamps. - -```esql -ROW date1 = TO_DATETIME("2023-12-02T11:00:00.000Z"), date2 = TO_DATETIME("2023-12-03T11:00:00.000Z") -| EVAL dd_days = DATE_DIFF("days", date1, date2) -``` - -In this second example, the `DATE_DIFF` function is used to calculate the difference in days between two timestamps. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-date_extract.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-date_extract.txt deleted file mode 100644 index 36c0272e3c479..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-date_extract.txt +++ /dev/null @@ -1,28 +0,0 @@ -## DATE_EXTRACT - -The `DATE_EXTRACT` function is used to extract specific parts of a date, such as the year, month, day, or hour. - -### Syntax - -`DATE_EXTRACT(datePart, date)` - -#### Parameters - -- `datePart`: Part of the date to extract. Can be: `aligned_day_of_week_in_month`, `aligned_day_of_week_in_year`, `aligned_week_of_month`, `aligned_week_of_year`, `ampm_of_day`, `clock_hour_of_ampm`, `clock_hour_of_day`, `day_of_month`, `day_of_week`, `day_of_year`, `epoch_day`, `era`, `hour_of_ampm`, `hour_of_day`, `instant_seconds`, `micro_of_day`, `micro_of_second`, `milli_of_day`, `milli_of_second`, `minute_of_day`, `minute_of_hour`, `month_of_year`, `nano_of_day`, `nano_of_second`, `offset_seconds`, `proleptic_month`, `second_of_day`, `second_of_minute`, `year`, or `year_of_era`. Refer to `java.time.temporal.ChronoField` for a description of these values. If null, the function returns null. -- `date`: Date expression. If null, the function returns null. - -### Examples - -The following ES|QL query uses the `DATE_EXTRACT` function to extract the year from a date: - -```esql -ROW date = DATE_PARSE("yyyy-MM-dd", "2022-05-06") -| EVAL year = DATE_EXTRACT("year", date) -``` - -This ES|QL query uses the `DATE_EXTRACT` function to find all events that occurred outside of business hours (before 9 AM or after 5 PM), on any given date: - -```esql -FROM sample_data -| WHERE DATE_EXTRACT("hour_of_day", @timestamp) < 9 AND DATE_EXTRACT("hour_of_day", @timestamp) >= 17 -``` \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-date_format.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-date_format.txt deleted file mode 100644 index c2221c5add437..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-date_format.txt +++ /dev/null @@ -1,32 +0,0 @@ -## DATE_FORMAT - -The `DATE_FORMAT` function in ES|QL is used to return a string representation of a date, in the provided format. If no format is specified, the `yyyy-MM-dd'T'HH:mm:ss.SSSZ` format is used. - -### Syntax - -`DATE_FORMAT(dateFormat, date)` - -#### Parameters - -- `dateFormat`: Date format (optional). If no format is specified, the `yyyy-MM-dd'T'HH:mm:ss.SSSZ` format is used. If null, the function returns null. -- `date`: Date expression. If null, the function returns null. - -### Examples - -Here are a couple of examples of how you can use the `DATE_FORMAT` function in your ES|QL queries: - -```esql -FROM employees -| KEEP first_name, last_name, hire_date -| EVAL hired = DATE_FORMAT("YYYY-MM-dd", hire_date) -``` - -In this example, the `DATE_FORMAT` function is used to format the `hire_date` field in the "YYYY-MM-dd" format. - -```esql -FROM logs-* -| WHERE @timestamp <= NOW() -| EVAL log_date = DATE_FORMAT("YYYY-MM-dd HH:mm:ss", @timestamp) -``` - -In this second example, the `DATE_FORMAT` function is used to format the `@timestamp` field in the "YYYY-MM-dd HH:mm:ss" format. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-date_parse.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-date_parse.txt deleted file mode 100644 index 0eab64ed4fcf2..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-date_parse.txt +++ /dev/null @@ -1,30 +0,0 @@ -## DATE_PARSE - -DATE_PARSE is a function in ES|QL that allows you to parse a date string using a specified format. This function is useful when you need to convert a string into a date format for further processing or analysis. - -### Syntax - -`DATE_PARSE(datePattern, dateString)` - -#### Parameters - -- `datePattern`: The date format. Refer to the DateTimeFormatter documentation for the syntax. If null, the function returns null. -- `dateString`: Date expression as a string. If null or an empty string, the function returns null. - -### Examples - -Here are a couple of examples of how you can use the DATE_PARSE function in ES|QL queries: - -```esql -ROW date_string = "2022-05-06" -| EVAL date = DATE_PARSE("yyyy-MM-dd", date_string) -``` - -In this example, the DATE_PARSE function is used to convert the string "2022-05-06" into a date format using the "yyyy-MM-dd" pattern. - -```esql -ROW date_string = "06-05-2022" -| EVAL date = DATE_PARSE("dd-MM-yyyy", date_string) -``` - -In this second example, the DATE_PARSE function is used to convert the string "06-05-2022" into a date format using the "dd-MM-yyyy" pattern. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-date_trunc.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-date_trunc.txt deleted file mode 100644 index 5e8c1318fc2c3..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-date_trunc.txt +++ /dev/null @@ -1,43 +0,0 @@ -## DATE_TRUNC - -The `DATE_TRUNC` function in ES|QL rounds down a date to the closest interval. This can be useful for creating date histograms or calculating rates over specific time intervals. - -### Syntax - -`DATE_TRUNC(interval, date)` - -#### Parameters - -- `interval`: Interval; expressed using the timespan literal syntax. -- `date`: Date expression - -### Examples - -Here are a couple of examples of how you can use the `DATE_TRUNC` function in ES|QL queries: - -1. To round down the hire date of employees to the closest year and keep the first name, last name, and hire date: - -```esql -FROM employees -| KEEP first_name, last_name, hire_date -| EVAL year_hired = DATE_TRUNC(1 year, hire_date) -``` - -2. To create a date histogram showing the number of hires per year: - -```esql -FROM employees -| EVAL year = DATE_TRUNC(1 year, hire_date) -| STATS hires = COUNT(emp_no) BY year -| SORT year -``` - -3. To calculate an hourly error rate: - -```esql -FROM sample_data -| EVAL error = CASE(message LIKE "*error*", 1, 0) -| EVAL hour = DATE_TRUNC(1 hour, @timestamp) -| STATS error_rate = AVG(error) BY hour -| SORT hour -``` \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-dissect.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-dissect.txt deleted file mode 100644 index ec764da51518b..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-dissect.txt +++ /dev/null @@ -1,47 +0,0 @@ -## DISSECT - -The `DISSECT` command in ES|QL allows you to extract structured data from a string. It matches the string against a delimiter-based pattern and extracts the specified keys as columns. This can be particularly useful when you need to parse a string that contains multiple pieces of information, such as a timestamp, some text, and an IP address. - -### Syntax - -The syntax for the `DISSECT` command is as follows: - - -`DISSECT input "pattern" [APPEND_SEPARATOR=""]` - -Here, `input` is the column that contains the string you want to structure. If the column has multiple values, `DISSECT` will process each value. `pattern` is a dissect pattern that you want to match against the string. `` is an optional string used as the separator between appended values when using the append modifier. - -### Examples - -Here are some examples of how you can use the `DISSECT` command in ES|QL: - -**Example 1:** - -```esql -ROW a = "2023-01-23T12:15:00.000Z - some text - 127.0.0.1" -| DISSECT a "%{date} - %{msg} - %{ip}" -| KEEP date, msg, ip -``` - -In this example, the `DISSECT` command is used to parse a string that contains a timestamp, some text, and an IP address. The command matches the string against the pattern `"%{date} - %{msg} - %{ip}"` and extracts the date, message, and IP address as separate columns. - -**Example 2:** - -```esql -ROW a = "2023-01-23T12:15:00.000Z - some text - 127.0.0.1" -| DISSECT a "%{date} - %{msg} - %{ip}" -| KEEP date, msg, ip -| EVAL date = TO_DATETIME(date) -``` - -This example is similar to the first one, but it also includes a `TO_DATETIME` function to convert the `date` column to a datetime type. - -**Example 3:** - -```esql -ROW a = "John Doe - john.doe@example.com - 123 Main St" -| DISSECT a "%{name} - %{email} - %{address}" -| KEEP name, email, address -``` - -In this example, the `DISSECT` command is used to parse a string that contains a name, email address, and physical address. The command matches the string against the pattern `"%{name} - %{email} - %{address}"` and extracts the name, email, and address as separate columns. diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-drop.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-drop.txt deleted file mode 100644 index 5484d69dd191e..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-drop.txt +++ /dev/null @@ -1,46 +0,0 @@ -## DROP - -The `DROP` command in ES|QL is used to remove one or more columns from the data. This can be useful in scenarios where certain columns are not needed for further data processing or analysis. - -The command supports the use of wildcards, allowing for the removal of all columns that match a specific pattern. This can be particularly useful when dealing with large datasets with numerous columns. - -### Syntax - -The syntax for the `DROP` command is as follows: - -``` -DROP columns -``` - -Here, `columns` is a comma-separated list of columns to be removed. Wildcards are supported. - -### Examples - -Here are some examples of how the `DROP` command can be used in ES|QL queries: - -1. Removing a single column: - -```esql -FROM employees -| DROP height -``` - -In this example, the `height` column is removed from the `employees` data. - -2. Removing multiple columns: - -```esql -FROM employees -| DROP height, weight, age -``` - -Here, the `height`, `weight`, and `age` columns are all removed from the `employees` data. - -3. Using wildcards to remove all columns that match a pattern: - -```esql -FROM employees -| DROP height* -``` - -In this example, all columns that start with `height` are removed from the `employees` data. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-e.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-e.txt deleted file mode 100644 index 10bb5d6cd667b..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-e.txt +++ /dev/null @@ -1,19 +0,0 @@ -## E - -The `E` function in ES|QL returns Euler's number. - -### Examples - -Here are a couple of examples of how you can use the `E` function in ES|QL queries: - -```esql -ROW E() -``` - -This query simply returns the Euler's number. - -```esql -ROW a = E() -``` - -This query assigns the Euler's number to a variable `a`. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-ends_with.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-ends_with.txt deleted file mode 100644 index 0ea4ca64a5245..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-ends_with.txt +++ /dev/null @@ -1,33 +0,0 @@ -## ENDS_WITH - -The `ENDS_WITH` function in ES|QL is used to check if a keyword string ends with another string. It returns a boolean value indicating the result of this check. - -### Syntax - -The syntax for using the `ENDS_WITH` function is as follows: - -`ENDS_WITH(str, suffix)` - -#### Parameters - -- `str`: This is a string expression. If null, the function returns null. -- `suffix`: This is a string expression. If null, the function returns null. - -### Examples - -Here are a couple of examples showing how to use the `ENDS_WITH` function in ES|QL queries: - -```esql -FROM employees -| KEEP last_name -| EVAL ln_E = ENDS_WITH(last_name, "d") -``` - -In this example, the `ENDS_WITH` function is used to check if the `last_name` of employees ends with the letter "d". The result is stored in the `ln_E` field. - -```esql -FROM logs-* -| WHERE ENDS_WITH(file_path, ".log") -``` - -In this second example, the `ENDS_WITH` function is used in a `WHERE` clause to filter out logs that don't have a file path ending with ".log". \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-enrich.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-enrich.txt deleted file mode 100644 index c18a8cddbcb86..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-enrich.txt +++ /dev/null @@ -1,50 +0,0 @@ -## ENRICH - -The `ENRICH` command in ES|QL allows you to add data from existing indices as new columns using an enrich policy. This can be particularly useful when you need to supplement your query data with additional information stored in other indices. - -Before you can use `ENRICH`, you need to create and execute an enrich policy. Refer to the [Data enrichment](https://www.elastic.co/guide/en/elasticsearch/reference/current/ingest-enriching-data.html) documentation for information about setting up a policy. - -Please note that in case of name collisions, the newly created columns will override existing columns. - -### Syntax - -`ENRICH policy [ON match_field] [WITH [new_name1 = ]field1, [new_name2 = ]field2, ...]` - -#### Parameters - -- `policy`: The name of the enrich policy. You need to create and execute the enrich policy first. -- `match_field`: The match field. ENRICH uses its value to look for records in the enrich index. If not specified, the match will be performed on the column with the same name as the match_field defined in the enrich policy. -- `fieldX`: The enrich fields from the enrich index that are added to the result as new columns. If a column with the same name as the enrich field already exists, the existing column will be replaced by the new column. If not specified, each of the enrich fields defined in the policy is added. -- `new_nameX`: Enables you to change the name of the column that’s added for each of the enrich fields. Defaults to the enrich field name. - -### Examples - -The following examples showcase different usages of the `ENRICH` command: - -1. Using the `languages_policy` enrich policy to add a new column for each enrich field defined in the policy. The match is performed using the `match_field` defined in the enrich policy and requires that the input table has a column with the same name (`language_code` in this example). - -```esql -ROW language_code = "1" -| ENRICH languages_policy -``` - -2. Using a column with a different name than the `match_field` defined in the policy as the match field: - -```esql -ROW a = "1" -| ENRICH languages_policy ON a -``` - -3. Explicitly selecting the enrich fields that are added using `WITH , , ...`: - -```esql -ROW a = "1" -| ENRICH languages_policy ON a WITH language_name -``` - -4. Renaming the columns that are added using `WITH new_name=`: - -```esql -ROW a = "1" -| ENRICH languages_policy ON a WITH name = language_name -``` \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-eval.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-eval.txt deleted file mode 100644 index f98b4987a9e06..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-eval.txt +++ /dev/null @@ -1,44 +0,0 @@ -## EVAL - -The `EVAL` command in ES|QL allows you to append new columns with calculated values to your data. It supports various functions for calculating these values. This command is particularly useful when you need to perform calculations on your data and store the results in new columns for further analysis or visualization. - -However, it's important to note that if the specified column already exists, the existing column will be dropped, and the new column will be appended to the table. - -### Examples - -Here are some examples of how you can use the `EVAL` command in ES|QL: - -1. Calculate the height of employees in feet and centimeters and store the results in new columns: - - ```esql -FROM employees -| SORT emp_no -| KEEP first_name, last_name, height -| EVAL height_feet = height * 3.281, height_cm = height * 100 -``` - -2. Overwrite an existing column with new calculated values: - - ```esql -FROM employees -| SORT emp_no -| KEEP first_name, last_name, height -| EVAL height = height * 3.281 -``` - -3. Add a new column with a name that is equal to the expression: - - ```esql -FROM employees -| SORT emp_no -| KEEP first_name, last_name, height -| EVAL height * 3.281 -``` - - Since this name contains special characters, it needs to be quoted with backticks (`) when using it in subsequent commands: - - ```esql -FROM employees -| EVAL height * 3.281 -| STATS avg_height_feet = AVG(`height * 3.281`) -``` \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-floor.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-floor.txt deleted file mode 100644 index 7daabcc3954f3..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-floor.txt +++ /dev/null @@ -1,29 +0,0 @@ -## FLOOR - -The `FLOOR` function in ES|QL is used to round a number down to the nearest integer. This operation is a no-op for long (including unsigned) and integer types. For double types, this function picks the closest double value to the integer, similar to the `Math.floor` function in JavaScript. - -### Syntax - -`FLOOR(number)` - -#### Parameters - -- `number`: Numeric expression. If null, the function returns null. - -### Examples - -Here are a couple of examples of how to use the `FLOOR` function in ES|QL: - -```esql -ROW a=1.8 -| EVAL a = FLOOR(a) -``` - -In this example, the `FLOOR` function is used to round down the value of `a` (1.8) to the nearest integer (1). - -```esql -ROW b=3.14159 -| EVAL b = FLOOR(b) -``` - -In this second example, the `FLOOR` function is used to round down the value of `b` (3.14159) to the nearest integer (3). \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-from.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-from.txt deleted file mode 100644 index 98548000a334e..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-from.txt +++ /dev/null @@ -1,53 +0,0 @@ -## FROM - -The `FROM` command in ES|QL is a source command that returns a table with data from a data stream, index, or alias. Each row in the resulting table represents a document, and each column corresponds to a field, which can be accessed by the name of that field. - -By default, an ES|QL query without an explicit `LIMIT` uses an implicit limit of 1000. This applies to `FROM` too. For example, a `FROM` command without `LIMIT`: - -```esql -FROM employees -``` - -is executed as: - -```esql -FROM employees -| LIMIT 1000 -``` - -You can use date math to refer to indices, aliases and data streams, which can be useful for time series data. For example, to access today’s index: - -```esql -FROM -``` - -You can use comma-separated lists or wildcards to query multiple data streams, indices, or aliases: - -```esql -FROM employees-00001,other-employees-* -``` - -You can also use the format `:` to query data streams and indices on remote clusters: - -```esql -FROM cluster_one:employees-00001,cluster_two:other-employees-* -``` - -The optional `METADATA` directive can be used to enable metadata fields: - -```esql -FROM employees METADATA _id -``` - -### Syntax - -`FROM index_pattern [METADATA fields]` - -#### Parameters - -- `index_pattern`: A list of indices, data streams or aliases. Supports wildcards and date math. -- `fields`: A comma-separated list of metadata fields to retrieve. - -### Limitations - -Please note that the `FROM` command does not support querying time series data streams (TSDS). For more details on the limitations of ES|QL, refer to the [ES|QL limitations](https://www.elastic.co/guide/en/elasticsearch/reference/current/sql-limitations.html) documentation. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-from_base64.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-from_base64.txt deleted file mode 100644 index 7e7c6b8ee3ab7..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-from_base64.txt +++ /dev/null @@ -1,19 +0,0 @@ -## FROM_BASE64 - -FROM_BASE64 function decodes a base64 string. - -### Examples - -Here are a couple of examples of full ES|QL queries using the FROM_BASE64 function: - -Example 1: -```esql -ROW a = "ZWxhc3RpYw==" -| EVAL d = FROM_BASE64(a) -``` - -Example 2: -```esql -ROW b = "SGVsbG8gd29ybGQ=" -| EVAL e = FROM_BASE64(b) -``` \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-greatest.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-greatest.txt deleted file mode 100644 index 5ab87d292bea4..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-greatest.txt +++ /dev/null @@ -1,30 +0,0 @@ -## GREATEST - -The `GREATEST` function in ES|QL returns the maximum value from multiple columns. This function is similar to `MV_MAX` but is intended to run on multiple columns at once. When run on keyword or text fields, this function returns the last string in alphabetical order. When run on boolean columns, this function will return true if any values are true. - -### Syntax - -`GREATEST(first, rest)` - -#### Parameters - -- `first`: First of the columns to evaluate. -- `rest`: The rest of the columns to evaluate. - -### Examples - -Here are a couple of examples of how to use the `GREATEST` function in ES|QL: - -```esql -ROW a = 10, b = 20 -| EVAL g = GREATEST(a, b) -``` - -In this example, the `GREATEST` function is used to find the maximum value between the columns `a` and `b`. - -```esql -ROW a = 10, b = 20, c = 30, d = 40 -| EVAL g = GREATEST(a, b, c, d) -``` - -In this example, the `GREATEST` function is used to find the maximum value among the columns `a`, `b`, `c`, and `d`. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-grok.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-grok.txt deleted file mode 100644 index 9c5e1354e681c..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-grok.txt +++ /dev/null @@ -1,42 +0,0 @@ -## GROK - -The `GROK` command in ES|QL enables you to extract structured data out of a string. It matches the string against patterns, based on regular expressions, and extracts the specified patterns as columns. This can be particularly useful when you need to parse a string that contains multiple pieces of information, such as a timestamp, an IP address, an email address, and a number. - -### Limitations - -By default, `GROK` outputs keyword string columns. Integer (`int`) and float types can be converted by appending `:type` to the semantics in the pattern. For other type conversions, you need to use Type conversion functions. - -### Examples - -Here are some examples of how you can use the `GROK` command in ES|QL: - -**Example 1: Parsing a string with multiple pieces of information** - -```esql -ROW a = "2023-01-23T12:15:00.000Z 127.0.0.1 some.email@foo.com 42" -| GROK a "%{TIMESTAMP_ISO8601:date} %{IP:ip} %{EMAILADDRESS:email} %{NUMBER:num}" -| KEEP date, ip, email, num -``` - -In this example, the `GROK` command is used to parse a string that contains a timestamp, an IP address, an email address, and a number. The `KEEP` command is then used to keep only the extracted date, IP, email, and number columns. - -**Example 2: Converting types with GROK** - -```esql -ROW a = "2023-01-23T12:15:00.000Z 127.0.0.1 some.email@foo.com 42" -| GROK a "%{TIMESTAMP_ISO8601:date} %{IP:ip} %{EMAILADDRESS:email} %{NUMBER:num:int}" -| KEEP date, ip, email, num -``` - -In this example, the `GROK` command is used similarly to the first example, but with an additional `:int` appended to the `NUMBER` semantic in the pattern. This converts the extracted number to an integer type. - -**Example 3: Using type conversion functions with GROK** - -```esql -ROW a = "2023-01-23T12:15:00.000Z 127.0.0.1 some.email@foo.com 42" -| GROK a "%{TIMESTAMP_ISO8601:date} %{IP:ip} %{EMAILADDRESS:email} %{NUMBER:num:int}" -| KEEP date, ip, email, num -| EVAL date = TO_DATETIME(date) -``` - -In this example, the `GROK` command is used to parse a string and convert the extracted number to an integer type. Then, the `EVAL` command is used with the `TO_DATETIME` function to convert the extracted date string to a datetime type. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-keep.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-keep.txt deleted file mode 100644 index d51104612b7cb..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-keep.txt +++ /dev/null @@ -1,48 +0,0 @@ -## KEEP - -The `KEEP` command in ES|QL allows you to specify which columns are returned and the order in which they are returned. This can be particularly useful when you want to focus on specific data in your Elasticsearch indices and ignore the rest. - -The command supports wildcards, allowing you to match and return all columns with a name that fits a certain pattern. Precedence rules are applied when a field name matches multiple expressions. If a field matches two expressions with the same precedence, the right-most expression wins. - -### Limitations - -There are no known limitations for the `KEEP` command in ES|QL. - -### Examples - -Here are some examples of how you can use the `KEEP` command in ES|QL: - -1. Return specified columns in the order they are listed: - -```esql -FROM employees -| KEEP emp_no, first_name, last_name, height -``` - -2. Use wildcards to return all columns with a name that matches a pattern: - -```esql -FROM employees -| KEEP h* -``` - -3. Use the asterisk wildcard by itself to return all columns that do not match the other arguments: - -```esql -FROM employees -| KEEP h*, * -``` - -4. Show how precedence rules work when a field name matches multiple expressions: - -```esql -FROM employees -| KEEP first_name*, last_name, first_na* -``` - -5. Use a simple wildcard expression `*` which has the lowest precedence: - -```esql -FROM employees -| KEEP *, first_name -``` \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-least.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-least.txt deleted file mode 100644 index 4c34db4d38e01..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-least.txt +++ /dev/null @@ -1,30 +0,0 @@ -## LEAST - -The `LEAST` function in ES|QL is used to return the minimum value from multiple columns. This function is similar to `MV_MIN` but is intended to run on multiple columns at once. - -### Syntax - -`LEAST(first, rest)` - -#### Parameters - -- `first`: The first column to evaluate. -- `rest`: The rest of the columns to evaluate. - -### Examples - -Here are a couple of examples of how to use the `LEAST` function in ES|QL: - -```esql -ROW a = 10, b = 20 -| EVAL l = LEAST(a, b) -``` - -In this example, the `LEAST` function is used to find the minimum value between the columns `a` and `b`. - -```esql -ROW a = 10, b = 20, c = 30, d = 40 -| EVAL l = LEAST(a, b, c, d) -``` - -In this second example, the `LEAST` function is used to find the minimum value among four columns: `a`, `b`, `c`, and `d`. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-left.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-left.txt deleted file mode 100644 index e62a50ae7a273..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-left.txt +++ /dev/null @@ -1,33 +0,0 @@ -## LEFT - -The `LEFT` function in ES|QL is used to extract a substring from a string, starting from the left. The number of characters to return is specified by the `length` parameter. - -### Syntax - -`LEFT(string, length)` - -#### Parameters - -- `string`: The string from which to return a substring. -- `length`: The number of characters to return. - -### Examples - -Here are a couple of examples of how to use the `LEFT` function in ES|QL: - -```esql -FROM employees -| KEEP last_name -| EVAL left = LEFT(last_name, 3) -| SORT last_name ASC -| LIMIT 5 -``` - -In this example, the `LEFT` function is used to extract the first three characters from the `last_name` field of the `employees` index. The query then sorts the results in ascending order by `last_name` and limits the output to the first 5 records. - -```esql -FROM logs-* -| EVAL left_chars = LEFT(message, 10) -``` - -In this second example, the `LEFT` function is used to extract the first 10 characters from the `message` field of the `logs-*` index. The result is stored in the `left_chars` field. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-length.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-length.txt deleted file mode 100644 index 8ed8bf3f31a2c..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-length.txt +++ /dev/null @@ -1,29 +0,0 @@ -## LENGTH - -The `LENGTH` function in ES|QL is used to return the character length of a string. - -### Syntax - -`LENGTH(string)` - -#### Parameters - -- `string`: This is a string expression. If null, the function returns null. - -### Examples - -Here are a couple of examples of how you can use the `LENGTH` function in ES|QL: - -```esql -FROM employees -| EVAL name_length = LENGTH(first_name) -``` - -In this example, the `LENGTH` function is used to calculate the length of the `first_name` field for each record in the `employees` index. - -```esql -FROM logs-* -| EVAL message_length = LENGTH(message) -``` - -In this second example, the `LENGTH` function is used to calculate the length of the `message` field for each record in the `logs-*` index. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-limit.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-limit.txt deleted file mode 100644 index 4d0dce6fe6edd..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-limit.txt +++ /dev/null @@ -1,42 +0,0 @@ -## LIMIT - -The `LIMIT` command in ES|QL is a processing command that allows you to limit the number of rows that are returned in a query. This can be particularly useful in scenarios where you only need a specific number of rows from a larger dataset. - -However, it's important to note that queries do not return more than 10,000 rows, regardless of the `LIMIT` command’s value. This limit only applies to the number of rows that are retrieved by the query. Queries and aggregations run on the full data set. - -To overcome this limitation, you can: - -- Reduce the result set size by modifying the query to only return relevant data. Use `WHERE` to select a smaller subset of the data. -- Shift any post-query processing to the query itself. You can use the ES|QL `STATS ... BY` command to aggregate data in the query. - -The default and maximum limits can be changed using these dynamic cluster settings: - -- `esql.query.result_truncation_default_size` -- `esql.query.result_truncation_max_size` - -### Examples - -Here are some examples of how you can use the `LIMIT` command in ES|QL: - -1. Limit the number of rows returned to 5, sorted by employee number in ascending order: - - ```esql -FROM employees -| SORT emp_no ASC -| LIMIT 5 -``` - -2. Retrieve only the top 10 employees with the highest salary: - - ```esql -FROM employees -| SORT salary DESC -| LIMIT 10 -``` - -3. Get the first 100 rows from a logs data stream: - - ```esql -FROM logs-* -| LIMIT 100 -``` \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-locate.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-locate.txt deleted file mode 100644 index 605a287841440..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-locate.txt +++ /dev/null @@ -1,35 +0,0 @@ -## LOCATE - -LOCATE function in ES|QL returns an integer that indicates the position of a keyword substring within another string. - -### Syntax - -`LOCATE(string, substring, start)` - -#### Parameters - -- `string`: An input string -- `substring`: A substring to locate in the input string -- `start`: The start index - -### Examples - -Here are a couple of examples of how you can use the LOCATE function in ES|QL: - -Example 1: - -```esql -ROW a = "hello" -| EVAL a_ll = LOCATE(a, "ll") -``` - -In this example, the LOCATE function is used to find the position of the substring "ll" in the string "hello". The result would be `3` as "ll" starts at the third position in the string "hello". - -Example 2: - -```esql -ROW a = "Elasticsearch Query Language" -| EVAL a_ll = LOCATE(a, "Query", 5) -``` - -In this example, the LOCATE function is used to find the position of the substring "Query" in the string "Elasticsearch Query Language", starting from the fifth position. The result would be `14` as "Query" starts at the fourteenth position in the string "Elasticsearch Query Language". \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-log.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-log.txt deleted file mode 100644 index 69a19c5578f14..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-log.txt +++ /dev/null @@ -1,30 +0,0 @@ -## LOG - -The `LOG` function in ES|QL returns the logarithm of a value to a base. The input can be any numeric value, and the return value is always a double. Logs of zero, negative numbers, and base of one return null as well as a warning. - -### Syntax - -`LOG(base, number)` - -#### Parameters - -- `base`: Base of logarithm. If null, the function returns null. If not provided, this function returns the natural logarithm (base e) of a value. -- `number`: Numeric expression. If null, the function returns null. - -### Examples - -Here are a couple of examples of full ES|QL queries using the `LOG` function: - -```esql -ROW base = 2.0, value = 8.0 -| EVAL s = LOG(base, value) -``` - -In this example, the `LOG` function is used to calculate the logarithm of `8.0` to the base `2.0`. - -```esql -ROW value = 100 -| EVAL s = LOG(value) -``` - -In this example, the `LOG` function is used to calculate the natural logarithm (base e) of `100` as no base is provided. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-log10.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-log10.txt deleted file mode 100644 index 78a0b43a087a8..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-log10.txt +++ /dev/null @@ -1,21 +0,0 @@ -## LOG10 - -The `LOG10` function in ES|QL is used to calculate the logarithm of a value to the base 10. The input can be any numeric value and the return value is always a double. If the input is 0 or a negative number, the function returns null and a warning. - -### Examples - -Here are a couple of examples of how you can use the `LOG10` function in ES|QL queries: - -```esql -ROW d = 1000.0 -| EVAL s = LOG10(d) -``` - -In this example, the `LOG10` function is used to calculate the base 10 logarithm of the value 1000. The result is stored in the variable `s`. - -```esql -ROW d = 100.0 -| EVAL s = LOG10(d) -``` - -In this example, the `LOG10` function is used to calculate the base 10 logarithm of the value 100. The result is stored in the variable `s`. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-lookup.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-lookup.txt deleted file mode 100644 index 46dc960ac1332..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-lookup.txt +++ /dev/null @@ -1,82 +0,0 @@ -## LOOKUP - -The `LOOKUP` command in ES|QL is a highly experimental feature that is currently only available in SNAPSHOT versions. This command is used to match values from the input against a table provided in the request, adding the other fields from the table to the output. - -### Use Cases and Limitations - -The `LOOKUP` command is particularly useful when you need to match and compare data from different sources or tables. It allows you to enrich your query results with additional data from a separate table based on matching fields. - -However, it's important to note that this command is still in the experimental stage and may not be fully stable or support all use cases. It's recommended to use this command in testing environments and not in production. - -### Examples - -Here are some examples of how to use the `LOOKUP` command in ES|QL: - -**Example 1:** - -``` -POST /_query?format=txt -{ - "query": """ - FROM library - | SORT page_count DESC - | KEEP name, author - | LOOKUP era ON author - | LIMIT 5 - """ - "tables": { - "era": { - "author:keyword": ["Frank Herbert", "Peter F. Hamilton", "Vernor Vinge", "Alastair Reynolds", "James S.A. Corey"], - "era:keyword" : [ "The New Wave", "Diamond", "Diamond", "Diamond", "Hadron"] - } - } -} -``` - -In this example, the `LOOKUP` command is used to match the `author` field from the `library` index with the `author` field in the `era` table. The matched data is then added to the output. - -**Example 2:** - -``` -POST /_query?format=txt -{ - "query": """ - FROM employees - | SORT salary DESC - | KEEP name, department - | LOOKUP departments ON department - | LIMIT 10 - """ - "tables": { - "departments": { - "department:keyword": ["Sales", "Marketing", "HR", "Engineering"], - "location:keyword" : [ "New York", "San Francisco", "London", "Berlin"] - } - } -} -``` - -In this example, the `LOOKUP` command is used to match the `department` field from the `employees` index with the `department` field in the `departments` table. The matched data is then added to the output. - -**Example 3:** - -``` -POST /_query?format=txt -{ - "query": """ - FROM orders - | SORT order_date DESC - | KEEP order_id, product_id - | LOOKUP products ON product_id - | LIMIT 20 - """ - "tables": { - "products": { - "product_id:keyword": ["P001", "P002", "P003", "P004"], - "product_name:keyword" : [ "Product 1", "Product 2", "Product 3", "Product 4"] - } - } -} -``` - -In this example, the `LOOKUP` command is used to match the `product_id` field from the `orders` index with the `product_id` field in the `products` table. The matched data is then added to the output. diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-ltrim.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-ltrim.txt deleted file mode 100644 index 9a6b50d702a98..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-ltrim.txt +++ /dev/null @@ -1,29 +0,0 @@ -## LTRIM - -The `LTRIM` function is used to remove leading whitespaces from a string. - -### Syntax - -`LTRIM(string)` - -#### Parameters - -- `string`: String expression. If null, the function returns null. - -### Examples - -Here are a couple of examples of how you can use the `LTRIM` function in ES|QL queries: - -```esql -ROW message = " some text " -| EVAL trimmed_message = LTRIM(message) -``` - -In this example, the `LTRIM` function is used to remove the leading whitespaces from the `message` string. - -```esql -ROW color = " red " -| EVAL trimmed_color = LTRIM(color) -``` - -In this example, the `LTRIM` function is used to remove the leading whitespace from the `color` string. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-max.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-max.txt deleted file mode 100644 index 32dad967274d8..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-max.txt +++ /dev/null @@ -1,29 +0,0 @@ -## MAX - -The `MAX` function in ES|QL is used to return the maximum value of a numeric expression. - -### Syntax - -`MAX(expression)` - -#### Parameters - -`expression`: The expression from which to return the maximum value. - -### Examples - -Here are a couple of examples of how the `MAX` function can be used in ES|QL queries: - -1. To find the maximum value in the `languages` field from the `employees` index, you can use the following query: - -```esql -FROM employees -| STATS MAX(languages) -``` - -2. The `MAX` function can also be used with inline functions. For instance, to calculate the maximum over an average of a multivalued column, you can first use the `MV_AVG` function to average the multiple values per row, and then use the result with the `MAX` function: - -```esql -FROM employees -| STATS max_avg_salary_change = MAX(MV_AVG(salary_change)) -``` \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-median.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-median.txt deleted file mode 100644 index b4a8810047d41..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-median.txt +++ /dev/null @@ -1,21 +0,0 @@ -## MEDIAN - -The `MEDIAN` function in ES|QL returns the value that is greater than half of all values and less than half of all values, also known as the 50% PERCENTILE. Like `PERCENTILE`, `MEDIAN` is usually approximate and is also non-deterministic. This means you can get slightly different results using the same data. - -### Examples - -Here are a couple of examples of how to use the `MEDIAN` function in ES|QL: - -```esql -FROM employees -| STATS MEDIAN(salary), PERCENTILE(salary, 50) -``` - -In this example, the `MEDIAN` function is used to calculate the median salary from the `employees` data stream or index. - -```esql -FROM employees -| STATS median_max_salary_change = MEDIAN(MV_MAX(salary_change)) -``` - -In this example, the `MEDIAN` function is used in conjunction with the `MV_MAX` function to calculate the median of the maximum values of a multivalued column `salary_change`. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-median_absolute_deviation.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-median_absolute_deviation.txt deleted file mode 100644 index 1ef1732218e11..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-median_absolute_deviation.txt +++ /dev/null @@ -1,33 +0,0 @@ -## MEDIAN_ABSOLUTE_DEVIATION - -The `MEDIAN_ABSOLUTE_DEVIATION` function is a robust statistic that is useful for describing data that may have outliers or may not be normally distributed. It provides a measure of variability by calculating the median of each data point’s deviation from the median of the entire sample. - -This function is usually approximate and non-deterministic, meaning that you can get slightly different results using the same data. - -### Syntax: - -`MEDIAN_ABSOLUTE_DEVIATION(expression)` - -#### Parameters: - -- `expression`: Expression from which to return the median absolute deviation. - -### Examples: - -Here is an example of a complete ES|QL query using the `MEDIAN_ABSOLUTE_DEVIATION` function: - -```esql -FROM employees -| STATS MEDIAN(salary), MEDIAN_ABSOLUTE_DEVIATION(salary) -``` - -In this query, the `MEDIAN_ABSOLUTE_DEVIATION` function is used to calculate the median absolute deviation of the `salary` field for the `employees` index. - -The `MEDIAN_ABSOLUTE_DEVIATION` function can also be used with inline functions. Here is an example where it is used with the `MV_MAX` function to calculate the median absolute deviation of the maximum values of a multivalued column: - -```esql -FROM employees -| STATS m_a_d_max_salary_change = MEDIAN_ABSOLUTE_DEVIATION(MV_MAX(salary_change)) -``` - -In this query, the `MV_MAX` function is first used to get the maximum value per row of the `salary_change` field. The result is then used with the `MEDIAN_ABSOLUTE_DEVIATION` function to calculate the median absolute deviation. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-min.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-min.txt deleted file mode 100644 index 737a21bb6cd99..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-min.txt +++ /dev/null @@ -1,29 +0,0 @@ -## MIN - -The `MIN` function in ES|QL is used to return the minimum value of a numeric expression. - -### Syntax: - -`MIN(expression)` - -#### Parameters: - -`expression`: The expression from which to return the minimum value. - -### Examples: - -Here are a couple of examples of how you can use the `MIN` function in ES|QL: - -1. To find the minimum value of a field, you can use the `MIN` function directly. For example, the following query returns the minimum value of the `languages` field from the `employees` index: - -```esql -FROM employees -| STATS MIN(languages) -``` - -2. You can also use the `MIN` function with other functions like `MV_AVG` to perform more complex calculations. For example, the following query calculates the average of a multivalued column `salary_change` for each row, and then finds the minimum of these averages: - -```esql -FROM employees -| STATS min_avg_salary_change = MIN(MV_AVG(salary_change)) -``` \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_avg.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_avg.txt deleted file mode 100644 index 1dcdf012ef0db..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_avg.txt +++ /dev/null @@ -1,29 +0,0 @@ -## MV_AVG - -The `MV_AVG` function in ES|QL converts a multivalued field into a single valued field containing the average of all of the values. - -### Syntax - -`MV_AVG(number)` - -#### Parameters - -`number`: Multivalue expression. - -### Examples - -Here are a couple of examples of how you can use the `MV_AVG` function in your ES|QL queries: - -```esql -ROW a=[3, 5, 1, 6] -| EVAL avg_a = MV_AVG(a) -``` - -In this example, the `MV_AVG` function is used to calculate the average of the values in the multivalued field `a`. - -```esql -ROW b=[10, 20, 30, 40] -| EVAL avg_b = MV_AVG(b) -``` - -In this second example, the `MV_AVG` function is used to calculate the average of the values in the multivalued field `b`. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_concat.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_concat.txt deleted file mode 100644 index f2699a0ac7a87..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_concat.txt +++ /dev/null @@ -1,23 +0,0 @@ -## MV_CONCAT - -The `MV_CONCAT` function in ES|QL converts a multivalued string expression into a single valued column. It does this by concatenating all values separated by a specified delimiter. - -### Examples - -Here are a couple of examples of how you can use the `MV_CONCAT` function in your ES|QL queries: - -```esql -ROW a=["foo", "zoo", "bar"] -| EVAL j = MV_CONCAT(a, ", ") -``` - -In this example, the `MV_CONCAT` function is used to concatenate the values in the array `a` with a comma separator. The result is a single string `"foo, zoo, bar"`. - -If you need to concatenate non-string columns, you can use the `TO_STRING` function first: - -```esql -ROW a=[10, 9, 8] -| EVAL j = MV_CONCAT(TO_STRING(a), ", ") -``` - -In this case, the numeric values in the array `a` are first converted to strings using the `TO_STRING` function. Then, the `MV_CONCAT` function concatenates these string values with a comma separator. The result is a single string `"10, 9, 8"`. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_count.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_count.txt deleted file mode 100644 index 1ba65c3d81fbf..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_count.txt +++ /dev/null @@ -1,29 +0,0 @@ -## MV_COUNT - -The `MV_COUNT` function in ES|QL is used to convert a multivalued expression into a single valued column containing a count of the number of values. - -### Syntax - -The syntax for using the `MV_COUNT` function is as follows: - -`MV_COUNT(field)` - -Here, `field` is a multivalue expression. - -### Examples - -Here are a couple of examples demonstrating the use of the `MV_COUNT` function: - -```esql -ROW a=["foo", "zoo", "bar"] -| EVAL count_a = MV_COUNT(a) -``` - -In this example, the `MV_COUNT` function is used to count the number of values in the array `["foo", "zoo", "bar"]`, and the result is stored in the `count_a` column. - -```esql -ROW b=[1, 2, 3, 4, 5] -| EVAL count_b = MV_COUNT(b) -``` - -In this second example, the `MV_COUNT` function is used to count the number of values in the array `[1, 2, 3, 4, 5]`, and the result is stored in the `count_b` column. diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_dedupe.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_dedupe.txt deleted file mode 100644 index 32726e09767dd..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_dedupe.txt +++ /dev/null @@ -1,29 +0,0 @@ -## MV_DEDUPE - -The `MV_DEDUPE` function is used to remove duplicate values from a multivalued field. It's important to note that while `MV_DEDUPE` may sort the values in the column, it's not guaranteed to always do so. - -### Syntax - -`MV_DEDUPE(field)` - -#### Parameters - -- `field`: Multivalue expression. - -### Examples - -Here are a couple of examples of how you can use the `MV_DEDUPE` function in your ES|QL queries: - -```esql -ROW a=["foo", "foo", "bar", "foo"] -| EVAL dedupe_a = MV_DEDUPE(a) -``` - -In this example, the `MV_DEDUPE` function is used to remove duplicate values from the multivalued field `a`. The resulting `dedupe_a` field will contain the values `["foo", "bar"]`. - -```esql -ROW b=["apple", "banana", "apple", "orange", "banana"] -| EVAL dedupe_b = MV_DEDUPE(b) -``` - -In this second example, the `MV_DEDUPE` function is used to remove duplicate values from the multivalued field `b`. The resulting `dedupe_b` field will contain the values `["apple", "banana", "orange"]`. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_expand.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_expand.txt deleted file mode 100644 index 9a3195a115713..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_expand.txt +++ /dev/null @@ -1,41 +0,0 @@ -## MV_EXPAND - -The `MV_EXPAND` command in ES|QL is a processing command that expands multivalued columns into one row per value, duplicating other columns. This command is particularly useful when dealing with data that contains multivalued fields and you want to create a separate row for each value in the multivalued field. - -This functionality is currently in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features. - -### Syntax - -`MV_EXPAND column` - -#### Parameters - -`column` -The multivalued column to expand. - -### Examples - -Here are some examples of how you can use the `MV_EXPAND` command in ES|QL: - -1. Expanding a multivalued column 'a': - -```esql -ROW a=[1,2,3], b="b" -| MV_EXPAND a -``` - -2. Expanding a multivalued column 'languages': - -```esql -FROM employees -| MV_EXPAND languages -``` - -3. Expanding a multivalued column 'tags': - -```esql -FROM blog_posts -| MV_EXPAND tags -``` - -In each of these examples, the `MV_EXPAND` command creates a new row for each value in the specified multivalued column. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_first.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_first.txt deleted file mode 100644 index fc901d41319e8..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_first.txt +++ /dev/null @@ -1,31 +0,0 @@ -## MV_FIRST - -The `MV_FIRST` function in ES|QL is used to convert a multivalued expression into a single valued column containing the first value. This function is most useful when reading from a function that emits multivalued columns in a known order like `SPLIT`. - -It's important to note that the order that multivalued fields are read from underlying storage is not guaranteed. It is frequently ascending, but this should not be relied upon. If you need the minimum value, use `MV_MIN` instead of `MV_FIRST`. `MV_MIN` has optimizations for sorted values so there isn’t a performance benefit to `MV_FIRST`. - -### Syntax: - -`MV_FIRST(field)` - -#### Parameters: - -- `field`: Multivalue expression. - -### Examples: - -Here are a couple of examples of how you can use the `MV_FIRST` function in your ES|QL queries: - -```esql -ROW a="foo;bar;baz" -| EVAL first_a = MV_FIRST(SPLIT(a, ";")) -``` - -In this example, the `SPLIT` function is used to split the string "foo;bar;baz" into a multivalued field. The `MV_FIRST` function is then used to select the first value from this multivalued field. - -```esql -ROW numbers=[10, 20, 30, 40, 50] -| EVAL first_num = MV_FIRST(numbers) -``` - -In this second example, the `MV_FIRST` function is used to select the first value from the multivalued field "numbers". \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_last.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_last.txt deleted file mode 100644 index 2abfbb5a65ee1..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_last.txt +++ /dev/null @@ -1,23 +0,0 @@ -## MV_LAST - -The `MV_LAST` function in ES|QL is used to convert a multivalue expression into a single valued column containing the last value. This function is most useful when reading from a function that emits multivalued columns in a known order like `SPLIT`. - -It's important to note that the order that multivalued fields are read from underlying storage is not guaranteed. It is frequently ascending, but this should not be relied upon. If you need the maximum value, it is recommended to use `MV_MAX` instead of `MV_LAST`. `MV_MAX` has optimizations for sorted values so there isn’t a performance benefit to `MV_LAST`. - -### Examples - -Here are a couple of examples of how you can use the `MV_LAST` function in your ES|QL queries: - -```esql -ROW a="foo;bar;baz" -| EVAL last_a = MV_LAST(SPLIT(a, ";")) -``` - -In this example, the `SPLIT` function is used to split the string "foo;bar;baz" into a multivalue expression. The `MV_LAST` function is then used to select the last value from this multivalue expression. - -```esql -ROW numbers="1;2;3;4;5" -| EVAL last_number = MV_LAST(SPLIT(numbers, ";")) -``` - -In this second example, the `SPLIT` function is used to split the string "1;2;3;4;5" into a multivalue expression. The `MV_LAST` function is then used to select the last value from this multivalue expression. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_max.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_max.txt deleted file mode 100644 index 709a1e1b747c5..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_max.txt +++ /dev/null @@ -1,23 +0,0 @@ -## MV_MAX - -The `MV_MAX` function in ES|QL is used to convert a multivalued expression into a single valued column containing the maximum value. This function can be used with any column type, including keyword columns. In the case of keyword columns, it picks the last string, comparing their utf-8 representation byte by byte. - -### Examples - -Here are a couple of examples of how you can use the `MV_MAX` function in ES|QL: - -1. To find the maximum value in a multivalued numeric field: - -```esql -ROW a=[3, 5, 1] -| EVAL max_a = MV_MAX(a) -``` - -2. To find the last string in a multivalued keyword field: - -```esql -ROW a=["foo", "zoo", "bar"] -| EVAL max_a = MV_MAX(a) -``` - -In both examples, the `MV_MAX` function is used to find the maximum value in the multivalued field `a`. The result is stored in the new field `max_a`. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_median.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_median.txt deleted file mode 100644 index 5b66fa243c009..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_median.txt +++ /dev/null @@ -1,29 +0,0 @@ -## MV_MEDIAN - -The `MV_MEDIAN` function in ES|QL converts a multivalued field into a single valued field containing the median value. If the row has an even number of values for a column, the result will be the average of the middle two entries. If the column is not floating point, the average rounds down. - -### Syntax - -`MV_MEDIAN(number)` - -#### Parameters - -`number`: Multivalue expression. - -### Examples - -Here are a couple of examples of how you can use the `MV_MEDIAN` function in ES|QL queries: - -```esql -ROW a=[3, 5, 1] -| EVAL median_a = MV_MEDIAN(a) -``` - -In this example, the `MV_MEDIAN` function calculates the median of the values in the `a` array, which are `[3, 5, 1]`. The median value is `3`. - -```esql -ROW a=[3, 7, 1, 6] -| EVAL median_a = MV_MEDIAN(a) -``` - -In this example, the `MV_MEDIAN` function calculates the median of the values in the `a` array, which are `[3, 7, 1, 6]`. Since there is an even number of values, the function calculates the average of the middle two entries, which results in `4`. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_min.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_min.txt deleted file mode 100644 index aff39c9ddbba7..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_min.txt +++ /dev/null @@ -1,21 +0,0 @@ -## MV_MIN - -The `MV_MIN` function in ES|QL is used to convert a multivalued expression into a single valued column containing the minimum value. This function can be used with any column type, including keyword columns. In the case of keyword columns, it picks the first string, comparing their utf-8 representation byte by byte. - -### Examples - -Here are a couple of examples of how to use the `MV_MIN` function in ES|QL: - -```esql -ROW a=[2, 1] -| EVAL min_a = MV_MIN(a) -``` - -In this example, the `MV_MIN` function is used to find the minimum value in the array `[2, 1]`. The result is stored in the `min_a` column. - -```esql -ROW a=["foo", "bar"] -| EVAL min_a = MV_MIN(a) -``` - -In this example, the `MV_MIN` function is used to find the minimum value in the array `["foo", "bar"]`. Since these are string values, the function compares their utf-8 representation byte by byte. The result is stored in the `min_a` column. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_slice.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_slice.txt deleted file mode 100644 index 12e98a93550c1..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_slice.txt +++ /dev/null @@ -1,31 +0,0 @@ -## MV_SLICE - -MV_SLICE is a function in ES|QL that returns a subset of a multivalued field using the start and end index values. - -### Syntax - -`MV_SLICE(field, start, end)` - -#### Parameters - -- `field`: Multivalue expression. If null, the function returns null. -- `start`: Start position. If null, the function returns null. The start argument can be negative. An index of -1 is used to specify the last value in the list. -- `end`: End position(included). Optional; if omitted, the position at start is returned. The end argument can be negative. An index of -1 is used to specify the last value in the list. - -### Examples - -Here are a couple of examples of how to use the MV_SLICE function in ES|QL: - -```esql -row a = [1, 2, 2, 3] -| eval a1 = mv_slice(a, 1), a2 = mv_slice(a, 2, 3) -``` - -In this example, the MV_SLICE function is used to get subsets of the multivalued field `a`. The subsets are stored in the new fields `a1` and `a2`. - -```esql -row a = [1, 2, 2, 3] -| eval a1 = mv_slice(a, -2), a2 = mv_slice(a, -3, -1) -``` - -In this example, the MV_SLICE function is used with negative start and end positions to get subsets of the multivalued field `a` from the end of the list. The subsets are stored in the new fields `a1` and `a2`. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_sort.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_sort.txt deleted file mode 100644 index 41480eb81afa1..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_sort.txt +++ /dev/null @@ -1,30 +0,0 @@ -## MV_SORT - -The `MV_SORT` function is used to sort a multivalued field in lexicographical order. - -### Syntax - -`MV_SORT(field, order)` - -#### Parameters - -- `field`: A multivalue expression. If null, the function returns null. -- `order`: Sort order. The valid options are `ASC` and `DESC`, the default is `ASC`. - -### Examples - -Here are a couple of examples of how you can use the `MV_SORT` function in ES|QL queries: - -```esql -ROW a = [4, 2, -3, 2] -| EVAL sa = MV_SORT(a) -``` - -In this example, the `MV_SORT` function is used to sort the values in the `a` field in ascending order. - -```esql -ROW a = [4, 2, -3, 2] -| EVAL sd = MV_SORT(a, "DESC") -``` - -In this example, the `MV_SORT` function is used to sort the values in the `a` field in descending order. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_sum.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_sum.txt deleted file mode 100644 index d5476aedffa4a..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_sum.txt +++ /dev/null @@ -1,29 +0,0 @@ -## MV_SUM - -The `MV_SUM` function in ES|QL is used to convert a multivalued field into a single valued field containing the sum of all the values. - -### Syntax - -`MV_SUM(number)` - -#### Parameters - -- `number`: A multivalued expression. - -### Examples - -Here are a couple of examples of how you can use the `MV_SUM` function in ES|QL: - -```esql -ROW a=[3, 5, 6] -| EVAL sum_a = MV_SUM(a) -``` - -In this example, the `MV_SUM` function is used to calculate the sum of the values in the multivalued field `a`. - -```esql -ROW b=[10, 20, 30, 40] -| EVAL sum_b = MV_SUM(b) -``` - -In this second example, the `MV_SUM` function is used to calculate the sum of the values in the multivalued field `b`. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_zip.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_zip.txt deleted file mode 100644 index f5028c4207da9..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-mv_zip.txt +++ /dev/null @@ -1,33 +0,0 @@ -## MV_ZIP - -The `MV_ZIP` function in ES|QL combines the values from two multivalued fields with a delimiter that joins them together. - -### Syntax - -`MV_ZIP(string1, string2, delim)` - -#### Parameters - -- `string1`: Multivalue expression. -- `string2`: Multivalue expression. -- `delim`: Delimiter. Optional; if omitted, `,` is used as a default delimiter. - -### Examples - -Here are a couple of examples of how you can use the `MV_ZIP` function in your ES|QL queries: - -```esql -ROW a = ["x", "y", "z"], b = ["1", "2"] -| EVAL c = MV_ZIP(a, b, "-") -| KEEP a, b, c -``` - -In this example, the `MV_ZIP` function is used to combine the values from the `a` and `b` fields with a `-` delimiter. The result is stored in the `c` field. - -```esql -ROW a = ["apple", "banana", "cherry"], b = ["red", "yellow", "red"] -| EVAL fruit_color = MV_ZIP(a, b, " is ") -| KEEP a, b, fruit_color -``` - -In this second example, the `MV_ZIP` function is used to combine the values from the `a` and `b` fields with ` is ` as the delimiter. The result is stored in the `fruit_color` field. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-now.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-now.txt deleted file mode 100644 index 632e698e7008a..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-now.txt +++ /dev/null @@ -1,20 +0,0 @@ -## NOW - -The `NOW` function in ES|QL returns the current date and time. - -### Examples - -Here are a couple of examples of how you can use the `NOW` function in ES|QL queries: - -1. To get the current date and time, you can use the `NOW` function in a `ROW` command: - -```esql -ROW current_date = NOW() -``` - -2. To retrieve logs from the last hour, you can use the `NOW` function in a `WHERE` clause: - -```esql -FROM sample_data -| WHERE @timestamp > NOW() - 1 hour -``` \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-operators.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-operators.txt deleted file mode 100644 index e6d88e78993b4..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-operators.txt +++ /dev/null @@ -1,170 +0,0 @@ -# ES|QL Operators - -ES|QL supports a variety of operators that can be used in queries. These operators can be categorized into binary operators, unary operators, logical operators, and others. - -## Binary Operators - -Binary operators in ES|QL include equality, inequality, less than, less than or equal to, greater than, greater than or equal to, add, subtract, multiply, divide, and modulus. - -### Equality - -The equality operator (`==`) checks if two values are equal. - -```esql -FROM employees -| WHERE first_name == "John" -``` - -### Inequality - -The inequality operator (`!=`) checks if two values are not equal. - -```esql -FROM employees -| WHERE salary != 50000 -``` - -### Less Than - -The less than operator (`<`) checks if one value is less than another. - -```esql -FROM employees -| WHERE age < 30 -``` - -### Less Than or Equal To - -The less than or equal to operator (`<=`) checks if one value is less than or equal to another. - -```esql -FROM employees -| WHERE years_of_experience <= 5 -``` - -### Greater Than - -The greater than operator (`>`) checks if one value is greater than another. - -```esql -FROM employees -| WHERE salary > 50000 -``` - -### Greater Than or Equal To - -The greater than or equal to operator (`>=`) checks if one value is greater than or equal to another. - -```esql -FROM employees -| WHERE age >= 30 -``` - -### Add - -The add operator (`+`) adds two values together. - -```esql -FROM employees -| EVAL total_compensation = salary + bonus -``` - -### Subtract - -The subtract operator (`-`) subtracts one value from another. - -```esql -FROM employees -| EVAL years_until_retirement = 65 - age -``` - -### Multiply - -The multiply operator (`*`) multiplies two values. - -```esql -FROM employees -| EVAL yearly_bonus = monthly_bonus * 12 -``` - -### Divide - -The divide operator (`/`) divides one value by another. - -```esql -FROM employees -| EVAL hourly_wage = salary / 2080 -``` - -### Modulus - -The modulus operator (`%`) returns the remainder of a division operation. - -```esql -FROM employees -| EVAL odd_or_even = employee_id % 2 -``` - -## Unary Operators - -ES|QL supports one unary operator, negation (`-`), which negates a value. - -```esql -FROM employees -| EVAL negative_salary = -salary -``` - -## Logical Operators - -ES|QL supports the logical operators `AND`, `OR`, and `NOT`. - -```esql -FROM employees -| WHERE salary > 50000 AND years_of_experience <= 5 -``` - -## Other Operators - -### IS NULL and IS NOT NULL - -The `IS NULL` and `IS NOT NULL` predicates check if a value is null or not null, respectively. - -```esql -FROM employees -| WHERE birth_date IS NULL -``` - -### Cast (::) - -The `::` operator provides a convenient alternative syntax to the `TO_` conversion functions. - -```esql -ROW ver = CONCAT(("0"::INT + 1)::STRING, ".2.3")::VERSION -``` - -### IN - -The `IN` operator checks if a field or expression equals an element in a list of literals, fields, or expressions. - -```esql -ROW a = 1, b = 4, c = 3 -| WHERE c-a IN (3, b / 2, a) -``` - -### LIKE - -The `LIKE` operator filters data based on string patterns using wildcards. - -```esql -FROM employees -| WHERE first_name LIKE "?b*" -``` - -### RLIKE - -The `RLIKE` operator filters data based on string patterns using regular expressions. - -```esql -FROM employees -| WHERE first_name RLIKE ".leja.*" -``` \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-overview.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-overview.txt deleted file mode 100644 index 36036e92ea093..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-overview.txt +++ /dev/null @@ -1,52 +0,0 @@ -# Elasticsearch Query Language (ES|QL) - -The Elasticsearch Query Language (ES|QL) is a powerful language designed to filter, transform, and analyze data stored in Elasticsearch. It is designed to be user-friendly and can be used by end users, SRE teams, application developers, and administrators. - -Users can author ES|QL queries to find specific events, perform statistical analysis, and generate visualizations. It supports a wide range of commands and functions that enable users to perform various data operations, such as filtering, aggregation, time-series analysis, and more. - -ES|QL uses "pipes" (|) to manipulate and transform data in a step-by-step fashion. This approach allows users to compose a series of operations, where the output of one operation becomes the input for the next, enabling complex data transformations and analysis. - -## ES|QL Compute Engine - -ES|QL is more than just a language. It represents a significant investment in new compute capabilities within Elasticsearch. To achieve both the functional and performance requirements for ES|QL, a new compute architecture was built. ES|QL search, aggregation, and transformation functions are directly executed within Elasticsearch itself. Query expressions are not transpiled to Query DSL for execution. This approach allows ES|QL to be extremely performant and versatile. - -The new ES|QL execution engine was designed with performance in mind. It operates on blocks at a time instead of per row, targets vectorization and cache locality, and embraces specialization and multi-threading. It is a separate component from the existing Elasticsearch aggregation framework with different performance characteristics. - -## Limitations - -There are some known limitations to ES|QL: - -- ES|QL only supports the UTC timezone. -- Full-text search is not yet supported. -- ES|QL does not support querying time series data streams (TSDS). -- Date math expressions work well when the leftmost expression is a datetime, but using parentheses or putting the datetime to the right is not always supported yet. -- ES|QL does not support configurations where the _source field is disabled. - -## Using ES|QL - -ES|QL can be used through the REST API, in Kibana, in Elastic Security, and across clusters. - -### REST API - -You can use the REST API to execute ES|QL queries. Here's an example of how to use the REST API: - -``` -POST /_query -{ - "query": """ - FROM library - | EVAL year = DATE_TRUNC(1 YEARS, release_date) - | STATS MAX(page_count) BY year - | SORT year - | LIMIT 5 - """ -} -``` - -### Kibana - -In Kibana, ES|QL can be used to query and aggregate your data, create visualizations, and set up alerts. However, there are some limitations when using ES|QL in Kibana. For example, the user interface to filter data is not enabled when Discover is in ES|QL mode. To filter data, you need to write a query that uses the `WHERE` command instead. - -### Cross Cluster - -ES|QL also supports executing a single query across multiple clusters. This can be useful for querying data from different geographical locations or separate Elasticsearch clusters. diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-percentile.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-percentile.txt deleted file mode 100644 index 0057bd045ffac..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-percentile.txt +++ /dev/null @@ -1,34 +0,0 @@ -## PERCENTILE - -The `PERCENTILE` function in ES|QL returns the value at which a certain percentage of observed values occur. For example, the 95th percentile is the value which is greater than 95% of the observed values and the 50th percentile is the median. - -### Syntax - -`PERCENTILE(expression, percentile)` - -#### Parameters - -- `expression`: Expression from which to return a percentile. -- `percentile`: A constant numeric expression. - -### Examples - -Here are a couple of examples of how to use the `PERCENTILE` function in ES|QL: - -```esql -FROM employees -| STATS p0 = PERCENTILE(salary, 0), p50 = PERCENTILE(salary, 50), p99 = PERCENTILE(salary, 99) -``` - -In this example, the `PERCENTILE` function is used to calculate the 0th, 50th, and 99th percentiles of the `salary` field in the `employees` index. - -```esql -FROM employees -| STATS p80_max_salary_change = PERCENTILE(MV_MAX(salary_change), 80) -``` - -In this example, the `PERCENTILE` function is used in conjunction with the `MV_MAX` function to calculate the 80th percentile of the maximum values of the `salary_change` field in the `employees` index. - -### Note - -The `PERCENTILE` function is usually approximate. There are many different algorithms to calculate percentiles and the naive implementation does not scale. To calculate percentiles across potentially billions of values in an Elasticsearch cluster, approximate percentiles are calculated using the TDigest algorithm. This means you can get slightly different results using the same data. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-pi.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-pi.txt deleted file mode 100644 index b62f6e50831c0..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-pi.txt +++ /dev/null @@ -1,20 +0,0 @@ -## PI - -The `PI` function in ES|QL returns the mathematical constant Pi, which is the ratio of a circle's circumference to its diameter. - -### Examples - -Here are a couple of examples of how you can use the `PI` function in ES|QL queries: - -```esql -ROW PI() -``` - -In this example, the `PI` function is used to simply return the value of Pi. - -```esql -FROM employees -| EVAL circle_area = PI() * POW(radius, 2) -``` - -In this second example, the `PI` function is used in a calculation to determine the area of a circle, given the radius stored in the `radius` field of the `employees` index. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-pow.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-pow.txt deleted file mode 100644 index 7107b2f715aa3..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-pow.txt +++ /dev/null @@ -1,21 +0,0 @@ -## POW - -The `POW` function in ES|QL returns the value of a base raised to the power of an exponent. It takes two numeric expressions as parameters: the base and the exponent. If either of these parameters is null, the function will return null. It's important to note that it is still possible to overflow a double result here; in that case, null will be returned. - -### Examples - -Here are a couple of examples of full ES|QL queries using the `POW` function: - -1. This query calculates the result of 2.0 raised to the power of 2: - -```esql -ROW base = 2.0, exponent = 2 -| EVAL result = POW(base, exponent) -``` - -2. This query calculates the square root of 4 by raising 4 to the power of 0.5: - -```esql -ROW base = 4, exponent = 0.5 -| EVAL s = POW(base, exponent) -``` \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-rename.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-rename.txt deleted file mode 100644 index 5275d2f2c846c..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-rename.txt +++ /dev/null @@ -1,46 +0,0 @@ -## RENAME - -The `RENAME` command in ES|QL is used to rename one or more columns in a table. If a column with the new name already exists, it will be replaced by the new column. This command can be useful in scenarios where you want to make column names more descriptive or to conform to a certain naming convention. - -However, it's important to note that if a column with the new name already exists, it will be replaced by the new column. Therefore, caution should be exercised to avoid unintentionally overwriting existing columns. - -### Syntax - -``` -RENAME old_name1 AS new_name1[, ..., old_nameN AS new_nameN] -``` - -#### Parameters - -- `old_nameX`: The name of a column you want to rename. -- `new_nameX`: The new name of the column. - -### Examples - -Here are some examples of how the `RENAME` command can be used in ES|QL queries: - -1. Renaming a single column: - - ```esql -FROM employees -| KEEP first_name, last_name, still_hired -| RENAME still_hired AS employed -``` - -2. Renaming multiple columns with a single `RENAME` command: - - ```esql -FROM employees -| KEEP first_name, last_name -| RENAME first_name AS fn, last_name AS ln -``` - -3. Renaming a column and using the new name in a subsequent command: - - ```esql -FROM employees -| RENAME salary AS annual_income -| WHERE annual_income > 50000 -``` - -In the third example, after renaming the `salary` column to `annual_income`, we can use the new column name in subsequent commands in the query. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-repeat.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-repeat.txt deleted file mode 100644 index 8face87f51dff..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-repeat.txt +++ /dev/null @@ -1,30 +0,0 @@ -## REPEAT - -The `REPEAT` function in ES|QL is used to construct a string by concatenating a given string with itself a specified number of times. - -### Syntax - -`REPEAT(string, number)` - -#### Parameters - -- `string`: The string expression that you want to repeat. -- `number`: The number of times you want to repeat the string. - -### Examples - -Here are a couple of examples of how you can use the `REPEAT` function in ES|QL: - -```esql -ROW a = "Hello!" -| EVAL triple_a = REPEAT(a, 3) -``` - -In this example, the string "Hello!" is repeated 3 times, resulting in "Hello!Hello!Hello!". - -```esql -ROW b = "ES|QL " -| EVAL five_b = REPEAT(b, 5) -``` - -In this example, the string "ES|QL " is repeated 5 times, resulting in "ES|QL ES|QL ES|QL ES|QL ES|QL ". diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-replace.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-replace.txt deleted file mode 100644 index b2abc79c2d76b..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-replace.txt +++ /dev/null @@ -1,33 +0,0 @@ -## REPLACE - -The `REPLACE` function substitutes any match of a regular expression within a string with a replacement string. - -### Syntax - -`REPLACE(string, regex, newString)` - -#### Parameters - -- `string`: String expression. -- `regex`: Regular expression. -- `newString`: Replacement string. - -### Examples - -Here are a couple of examples of how to use the `REPLACE` function in ES|QL queries: - -```esql -ROW str = "Hello World" -| EVAL str = REPLACE(str, "World", "Universe") -| KEEP str -``` - -In this example, the `REPLACE` function is used to replace any occurrence of the word "World" with the word "Universe" in the string "Hello World". - -```esql -ROW str = "Elasticsearch is awesome" -| EVAL str = REPLACE(str, "awesome", "fantastic") -| KEEP str -``` - -In this example, the `REPLACE` function is used to replace the word "awesome" with "fantastic" in the string "Elasticsearch is awesome". \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-right.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-right.txt deleted file mode 100644 index c3e2fd9303e68..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-right.txt +++ /dev/null @@ -1,33 +0,0 @@ -## RIGHT - -The `RIGHT` function in ES|QL is used to extract a substring from a string, starting from the right. The number of characters to return is specified by the `length` parameter. - -### Syntax - -`RIGHT(string, length)` - -#### Parameters - -- `string`: The string from which to return a substring. -- `length`: The number of characters to return. - -### Examples - -Here are a couple of examples of how to use the `RIGHT` function in ES|QL: - -```esql -FROM employees -| KEEP last_name -| EVAL right = RIGHT(last_name, 3) -| SORT last_name ASC -| LIMIT 5 -``` - -In this example, the `RIGHT` function is used to extract the last three characters from the `last_name` field of each record in the `employees` index. The resulting substring is then stored in a new field called `right`. The query then sorts the results in ascending order by `last_name` and limits the output to the first 5 records. - -```esql -FROM logs-* -| EVAL file_extension = RIGHT(file_name, 3) -``` - -In this second example, the `RIGHT` function is used to extract the file extension from a `file_name` field in a `logs-*` index. The resulting substring is stored in a new field called `file_extension`. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-round.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-round.txt deleted file mode 100644 index aaef24b83a673..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-round.txt +++ /dev/null @@ -1,31 +0,0 @@ -## ROUND - -The `ROUND` function in ES|QL is used to round a number to a specified number of decimal places. By default, it rounds to 0 decimal places, returning the nearest integer. If the precision is a negative number, it rounds to the number of digits left of the decimal point. - -### Syntax: - -`ROUND(number, decimals)` - -#### Parameters: - -- `number`: The numeric value to round. If null, the function returns null. -- `decimals`: The number of decimal places to round to. Defaults to 0. If null, the function returns null. - -### Examples: - -Here are a couple of examples of how to use the `ROUND` function in ES|QL queries: - -```esql -FROM employees -| KEEP first_name, last_name, height -| EVAL height_ft = ROUND(height * 3.281, 1) -``` - -In this example, the `ROUND` function is used to round the result of the multiplication of the `height` field and `3.281` to `1` decimal place. The result is stored in the `height_ft` field. - -```esql -FROM sales_data -| EVAL rounded_sales = ROUND(sales * 1.2) -``` - -In this second example, the `ROUND` function is used to round the result of the multiplication of the `sales` field and `1.2` to the nearest integer (since no decimal places are specified). The result is stored in the `rounded_sales` field. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-row.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-row.txt deleted file mode 100644 index 48aaa65963786..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-row.txt +++ /dev/null @@ -1,42 +0,0 @@ -## ROW - -The `ROW` command in ES|QL is used to produce a row with one or more columns with specified values. This can be particularly useful for testing purposes. - -### Syntax - -The syntax for the `ROW` command is as follows: - -``` -ROW column1 = value1[, ..., columnN = valueN] -``` - -#### Parameters - -- `columnX`: The name of the column. -- `valueX`: The value for the column. This can be a literal, an expression, or a function. - -### Examples - -Here are some examples of how the `ROW` command can be used in ES|QL: - -1. Creating a row with specified values: - - ```esql -ROW a = 1, b = "two", c = null -``` - -2. Using square brackets to create multi-value columns: - - ```esql -ROW a = [2, 1] -``` - -3. Using functions within the `ROW` command: - - ```esql -ROW a = ROUND(1.23, 0) -``` - -### Limitations - -There are no known limitations for the `ROW` command in ES|QL. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-rtrim.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-rtrim.txt deleted file mode 100644 index 9e92e45629f70..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-rtrim.txt +++ /dev/null @@ -1,23 +0,0 @@ -## RTRIM - -The `RTRIM` function in ES|QL is used to remove trailing whitespaces from a string. If the string expression is null, the function will return null. - -### Examples - -Here are a couple of examples of how you can use the `RTRIM` function in ES|QL queries: - -```esql -ROW message = " some text " -| EVAL message = RTRIM(message) -| EVAL message = CONCAT("'", message, "'") -``` - -In this example, the `RTRIM` function is used to remove trailing whitespaces from the `message` string. The `CONCAT` function is then used to concatenate the modified `message` string with single quotes. - -```esql -ROW color = " red " -| EVAL color = RTRIM(color) -| EVAL color = CONCAT("'", color, "'") -``` - -In this second example, the `RTRIM` function is used to remove trailing whitespaces from the `color` string. The `CONCAT` function is then used to concatenate the modified `color` string with single quotes. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-show.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-show.txt deleted file mode 100644 index 80f92061f141f..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-show.txt +++ /dev/null @@ -1,19 +0,0 @@ -## SHOW - -The `SHOW` command in ES|QL is used to return information about the deployment and its capabilities. Currently, the only supported item for this command is `INFO`, which returns the deployment’s version, build date, and hash. - -### Examples - -Here are some examples of how to use the `SHOW` command in ES|QL: - -1. To get the deployment's version, build date, and hash: - -```esql -SHOW INFO -``` - -Please note that the `SHOW` command can only be used with `INFO` as its parameter. Any other parameters will not be recognized by the command. - -### Limitations - -Currently, the `SHOW` command only supports `INFO` as its parameter. It does not support any other parameters or options. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-signum.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-signum.txt deleted file mode 100644 index bab747ef9b185..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-signum.txt +++ /dev/null @@ -1,29 +0,0 @@ -## SIGNUM - -The `SIGNUM` function in ES|QL returns the sign of a given number. It returns `-1` for negative numbers, `0` for `0`, and `1` for positive numbers. - -### Syntax - -The syntax for the `SIGNUM` function is as follows: - -`SIGNUM(number)` - -Here, `number` is a numeric expression. If `null`, the function returns `null`. - -### Examples - -Here are a couple of examples of how you can use the `SIGNUM` function in ES|QL: - -```esql -ROW d = 100.0 -| EVAL s = SIGNUM(d) -``` - -In this example, the `SIGNUM` function is used to determine the sign of the number `100.0`. Since `100.0` is a positive number, the function returns `1`. - -```esql -ROW d = -50.0 -| EVAL s = SIGNUM(d) -``` - -In this example, the `SIGNUM` function is used to determine the sign of the number `-50.0`. Since `-50.0` is a negative number, the function returns `-1`. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-sin.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-sin.txt deleted file mode 100644 index 730df58969234..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-sin.txt +++ /dev/null @@ -1,21 +0,0 @@ -## SIN - -The `SIN` function in ES|QL is used to calculate the sine of an angle. The angle should be provided in radians. If the provided angle is null, the function will return null. - -### Examples - -Here are a couple of examples of how you can use the `SIN` function in ES|QL: - -```esql -ROW a=1.8 -| EVAL sin = SIN(a) -``` - -In this example, the `SIN` function is used to calculate the sine of the angle `1.8` radians. The result is stored in the `sin` variable. - -```esql -ROW a=3.14 -| EVAL sin_value = SIN(a) -``` - -In this second example, the `SIN` function is used to calculate the sine of the angle `3.14` radians (approximately equal to π, the angle for a half circle in the unit circle). The result is stored in the `sin_value` variable. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-sinh.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-sinh.txt deleted file mode 100644 index 62a1d8e089b06..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-sinh.txt +++ /dev/null @@ -1,21 +0,0 @@ -## SINH - -The `SINH` function in ES|QL returns the hyperbolic sine of an angle. The angle should be provided in radians. If the provided angle is null, the function will return null. - -### Examples - -Here are a couple of examples of how you can use the `SINH` function in ES|QL: - -```esql -ROW a=1.8 -| EVAL sinh = SINH(a) -``` - -In this example, the `SINH` function is used to calculate the hyperbolic sine of the angle `1.8` radians. - -```esql -ROW a=3.14 -| EVAL sinh_value = SINH(a) -``` - -In this second example, the `SINH` function is used to calculate the hyperbolic sine of the angle `3.14` radians. The result is stored in the `sinh_value` variable. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-sort.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-sort.txt deleted file mode 100644 index 68bc7790a033c..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-sort.txt +++ /dev/null @@ -1,45 +0,0 @@ -## SORT - -The `SORT` command in ES|QL is a processing command that sorts a table based on one or more columns. The default sort order is ascending, but you can specify an explicit sort order using `ASC` or `DESC`. - -In cases where two rows have the same sort key, they are considered equal. However, you can provide additional sort expressions to act as tie breakers. When sorting on multivalued columns, the lowest value is used when sorting in ascending order and the highest value when sorting in descending order. - -By default, null values are treated as being larger than any other value. This means that with an ascending sort order, null values are sorted last, and with a descending sort order, null values are sorted first. You can change this by providing `NULLS FIRST` or `NULLS LAST`. - -### Examples - -Here are some examples of how to use the `SORT` command in ES|QL: - -1. Sorting by height in ascending order (default): - - ```esql -FROM employees -| KEEP first_name, last_name, height -| SORT height -``` - -2. Explicitly sorting in descending order with `DESC`: - - ```esql -FROM employees -| KEEP first_name, last_name, height -| SORT height DESC -``` - -3. Providing additional sort expressions to act as tie breakers: - - ```esql -FROM employees -| KEEP first_name, last_name, height -| SORT height DESC, first_name ASC -``` - -4. Sorting null values first using `NULLS FIRST`: - - ```esql -FROM employees -| KEEP first_name, last_name, height -| SORT first_name ASC NULLS FIRST -``` - -Please note that the `SORT` command does not support sorting on spatial types (`geo_point`, `geo_shape`, `cartesian_point`, `cartesian_shape`). \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-split.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-split.txt deleted file mode 100644 index 658c161292b64..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-split.txt +++ /dev/null @@ -1,30 +0,0 @@ -## SPLIT - -The `SPLIT` function in ES|QL is used to split a single valued string into multiple strings based on a specified delimiter. - -### Syntax - -`SPLIT(string, delim)` - -#### Parameters - -- `string`: This is the string expression that you want to split. If null, the function returns null. -- `delim`: This is the delimiter that will be used to split the string. Only single byte delimiters are currently supported. - -### Examples - -Here are a couple of examples of how you can use the `SPLIT` function in ES|QL: - -```esql -ROW words="foo;bar;baz;qux;quux;corge" -| EVAL word = SPLIT(words, ";") -``` - -In this example, the string "foo;bar;baz;qux;quux;corge" is split into multiple strings using the semicolon (;) as the delimiter. - -```esql -ROW data="John,Doe,30" -| EVAL details = SPLIT(data, ",") -``` - -In this second example, the string "John,Doe,30" is split into multiple strings using the comma (,) as the delimiter. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-sqrt.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-sqrt.txt deleted file mode 100644 index 6e1820242b468..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-sqrt.txt +++ /dev/null @@ -1,29 +0,0 @@ -## SQRT - -The `SQRT` function in ES|QL is used to calculate the square root of a number. The input can be any numeric value and the return value is always a double. If the input is a negative number or infinity, the function returns null. - -### Syntax - -`SQRT(number)` - -#### Parameters - -`number`: Numeric expression. If null, the function returns null. - -### Examples - -Here are a couple of examples of how to use the `SQRT` function in ES|QL: - -```esql -ROW d = 100.0 -| EVAL s = SQRT(d) -``` - -In this example, the `SQRT` function is used to calculate the square root of 100. The result is stored in the variable `s`. - -```esql -ROW d = 16.0 -| EVAL s = SQRT(d) -``` - -In this example, the `SQRT` function is used to calculate the square root of 16. The result is stored in the variable `s`. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-st_centroid_agg.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-st_centroid_agg.txt deleted file mode 100644 index f76bbdbf54e8f..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-st_centroid_agg.txt +++ /dev/null @@ -1,21 +0,0 @@ -## ST_CENTROID_AGG - -ST_CENTROID_AGG is a function that calculates the spatial centroid over a field with spatial point geometry type. This functionality is currently in technical preview and may be changed or removed in a future release. - -### Examples - -Here are a couple of examples of full ES|QL queries using the ST_CENTROID_AGG function: - -```esql -FROM airports -| STATS centroid = ST_CENTROID_AGG(location) -``` - -In this example, the ST_CENTROID_AGG function is used to calculate the spatial centroid over the 'location' field from the 'airports' index. - -```esql -FROM geo_data -| STATS geo_centroid = ST_CENTROID_AGG(geo_point) -``` - -In this second example, the ST_CENTROID_AGG function is used to calculate the spatial centroid over the 'geo_point' field from the 'geo_data' index. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-st_contains.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-st_contains.txt deleted file mode 100644 index da7c244d57d76..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-st_contains.txt +++ /dev/null @@ -1,34 +0,0 @@ -## ST_CONTAINS - -ST_CONTAINS is a function in ES|QL that checks whether the first geometry contains the second geometry. This function is the inverse of the ST_WITHIN function. - -### Syntax - -The syntax for the ST_CONTAINS function is as follows: - -`ST_CONTAINS(geomA, geomB)` - -#### Parameters - -- `geomA`: An expression of type geo_point, cartesian_point, geo_shape or cartesian_shape. If null, the function returns null. -- `geomB`: An expression of type geo_point, cartesian_point, geo_shape or cartesian_shape. If null, the function returns null. The second parameter must also have the same coordinate system as the first. This means it is not possible to combine geo_* and cartesian_* parameters. - -### Examples - -Here are a couple of examples of how to use the ST_CONTAINS function in ES|QL queries: - -```esql -FROM airport_city_boundaries -| WHERE ST_CONTAINS(city_boundary, TO_GEOSHAPE("POLYGON((109.35 18.3, 109.45 18.3, 109.45 18.4, 109.35 18.4, 109.35 18.3))")) -| KEEP abbrev, airport, region, city, city_location -``` - -In this example, the ST_CONTAINS function is used to check if the `city_boundary` contains the specified polygon. The query then keeps the `abbrev`, `airport`, `region`, `city`, and `city_location` fields. - -```esql -FROM geo_shapes -| WHERE ST_CONTAINS(shape_field, TO_GEOSHAPE("POINT(10 20)")) -| KEEP id, name, shape_field -``` - -In this second example, the ST_CONTAINS function is used to check if the `shape_field` contains the specified point. The query then keeps the `id`, `name`, and `shape_field` fields. diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-st_disjoint.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-st_disjoint.txt deleted file mode 100644 index c1b8cc165ac7d..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-st_disjoint.txt +++ /dev/null @@ -1,31 +0,0 @@ -## ST_DISJOINT - -ST_DISJOINT is a function in ES|QL that checks whether two geometries or geometry columns are disjoint. In other words, it verifies if the two geometries do not intersect at any point. This function is the inverse of the ST_INTERSECTS function. In mathematical terms, if A and B are two geometries, they are disjoint if their intersection is an empty set (A ⋂ B = ∅). - -### Syntax - -`ST_DISJOINT(geomA, geomB)` - -#### Parameters - -- `geomA`: An expression of type geo_point, cartesian_point, geo_shape, or cartesian_shape. If null, the function returns null. -- `geomB`: An expression of type geo_point, cartesian_point, geo_shape, or cartesian_shape. If null, the function returns null. The second parameter must also have the same coordinate system as the first. This means it is not possible to combine geo_* and cartesian_* parameters. - -### Examples - -Here are a couple of examples of how to use the ST_DISJOINT function in ES|QL queries: - -```esql -FROM airport_city_boundaries -| WHERE ST_DISJOINT(city_boundary, TO_GEOSHAPE("POLYGON((-10 -60, 120 -60, 120 60, -10 60, -10 -60))")) -| KEEP abbrev, airport, region, city, city_location -``` - -In this example, the query checks if the city_boundary is disjoint from the specified polygon. If they are disjoint, the query returns the abbrev, airport, region, city, and city_location fields. - -```esql -FROM geo_shapes -| WHERE ST_DISJOINT(shape1, shape2) -``` - -In this example, the query checks if shape1 and shape2 are disjoint. If they are, the query returns all the fields of the matching documents. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-st_intersects.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-st_intersects.txt deleted file mode 100644 index 1682fcaccc014..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-st_intersects.txt +++ /dev/null @@ -1,30 +0,0 @@ -## ST_INTERSECTS - -The `ST_INTERSECTS` function returns `true` if two geometries intersect. They intersect if they have any point in common, including their interior points (points along lines or within polygons). This is the inverse of the `ST_DISJOINT` function. In mathematical terms: `ST_Intersects(A, B) ⇔ A ⋂ B ≠ ∅`. - -### Syntax - -`ST_INTERSECTS(geomA, geomB)` - -#### Parameters - -- `geomA`: Expression of type `geo_point`, `cartesian_point`, `geo_shape` or `cartesian_shape`. If `null`, the function returns `null`. -- `geomB`: Expression of type `geo_point`, `cartesian_point`, `geo_shape` or `cartesian_shape`. If `null`, the function returns `null`. The second parameter must also have the same coordinate system as the first. This means it is not possible to combine `geo_*` and `cartesian_*` parameters. - -### Examples - -Here are a couple of examples of how to use the `ST_INTERSECTS` function in ES|QL queries: - -```esql -FROM airports -| WHERE ST_INTERSECTS(location, TO_GEOSHAPE("POLYGON((42 14, 43 14, 43 15, 42 15, 42 14))")) -``` - -In this example, the `ST_INTERSECTS` function is used to find airports that are located within a specific polygon. - -```esql -FROM geo_shapes -| WHERE ST_INTERSECTS(shape_field, TO_GEOSHAPE("POINT(42 14)")) -``` - -In this second example, the `ST_INTERSECTS` function is used to find geo shapes that intersect with a specific point. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-st_within.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-st_within.txt deleted file mode 100644 index 027f5b1eae393..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-st_within.txt +++ /dev/null @@ -1,34 +0,0 @@ -## ST_WITHIN - -ST_WITHIN is a function in ES|QL that checks whether the first geometry is within the second geometry. This function is the inverse of the ST_CONTAINS function. - -### Syntax - -The syntax for the ST_WITHIN function is as follows: - -`ST_WITHIN(geomA, geomB)` - -#### Parameters - -- `geomA`: This is an expression of type geo_point, cartesian_point, geo_shape, or cartesian_shape. If null, the function returns null. -- `geomB`: This is an expression of type geo_point, cartesian_point, geo_shape, or cartesian_shape. If null, the function returns null. The second parameter must also have the same coordinate system as the first. This means it is not possible to combine geo_* and cartesian_* parameters. - -### Examples - -Here are a couple of examples of how to use the ST_WITHIN function in ES|QL: - -```esql -FROM airport_city_boundaries -| WHERE ST_WITHIN(city_boundary, TO_GEOSHAPE("POLYGON((109.1 18.15, 109.6 18.15, 109.6 18.65, 109.1 18.65, 109.1 18.15))")) -| KEEP abbrev, airport, region, city, city_location -``` - -In this example, the ST_WITHIN function is used to check if the `city_boundary` is within the specified polygon. The query then keeps the `abbrev`, `airport`, `region`, `city`, and `city_location` fields from the `airport_city_boundaries` index. - -```esql -FROM my_index -| WHERE ST_WITHIN(my_geo_point, TO_GEOSHAPE("POLYGON((10 10, 20 20, 30 30, 10 10))")) -| KEEP field1, field2 -``` - -In this second example, the ST_WITHIN function is used to check if the `my_geo_point` field is within the specified polygon. The query then keeps the `field1` and `field2` fields from the `my_index` index. diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-st_x.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-st_x.txt deleted file mode 100644 index 820ec0176ad9d..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-st_x.txt +++ /dev/null @@ -1,21 +0,0 @@ -## ST_X - -The `ST_X` function is used to extract the x coordinate from a provided point. If the point is of type `geo_point`, this is equivalent to extracting the longitude value. - -### Examples - -Here are a couple of examples of how you can use the `ST_X` function in ES|QL queries: - -```esql -ROW point = TO_GEOPOINT("POINT(42.97109629958868 14.7552534006536)") -| EVAL x = ST_X(point) -``` - -In this example, the `ST_X` function is used to extract the x coordinate (or longitude) from a `geo_point` that is created using the `TO_GEOPOINT` function. - -```esql -ROW point = TO_GEOPOINT("POINT(50.8503 4.3517)") -| EVAL x = ST_X(point) -``` - -In this second example, the `ST_X` function is used to extract the x coordinate (or longitude) from a different `geo_point`. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-st_y.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-st_y.txt deleted file mode 100644 index 4e7daf35db08d..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-st_y.txt +++ /dev/null @@ -1,26 +0,0 @@ -## ST_Y - -The `ST_Y` function extracts the y coordinate from the supplied point. If the points is of type `geo_point` this is equivalent to extracting the latitude value. - -### Syntax - -`ST_Y(point)` - -### Parameters - -- `point`: Expression of type `geo_point` or `cartesian_point`. If null, the function returns null. - -### Examples - -Here are a couple of examples of how you can use the `ST_Y` function in ES|QL queries: - -```esql -ROW point = TO_GEOPOINT("POINT(42.97109629958868 14.7552534006536)") -| EVAL y = ST_Y(point) -``` - -```esql -FROM geo_data -| EVAL latitude = ST_Y(location) -| WHERE latitude > 50 -``` \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-starts_with.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-starts_with.txt deleted file mode 100644 index bee2a57300a25..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-starts_with.txt +++ /dev/null @@ -1,33 +0,0 @@ -## STARTS_WITH - -The `STARTS_WITH` function in ES|QL is used to check if a keyword string starts with another string. It returns a boolean value indicating the result of this comparison. - -### Syntax - -The syntax for using the `STARTS_WITH` function is as follows: - -`STARTS_WITH(str, prefix)` - -#### Parameters - -- `str`: This is a string expression. If null, the function returns null. -- `prefix`: This is another string expression. If null, the function returns null. - -### Examples - -Here are a couple of examples showing how to use the `STARTS_WITH` function in ES|QL queries: - -```esql -FROM employees -| KEEP last_name -| EVAL ln_S = STARTS_WITH(last_name, "B") -``` - -In this example, the `STARTS_WITH` function is used to check if the `last_name` of employees starts with the letter "B". The result is stored in the `ln_S` field. - -```esql -FROM logs-* -| WHERE STARTS_WITH(log_message, "ERROR") -``` - -In this second example, the `STARTS_WITH` function is used in a `WHERE` clause to filter out log messages that start with the word "ERROR". \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-stats.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-stats.txt deleted file mode 100644 index 4369353daa3cb..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-stats.txt +++ /dev/null @@ -1,92 +0,0 @@ -# STATS - -The `STATS` command in ES|QL is a processing command that groups rows according to a common value and calculates one or more aggregated values over the grouped rows. If `BY` is omitted, the output table contains exactly one row with the aggregations applied over the entire dataset. - -The following aggregation functions are supported: - -- `AVG` -- `COUNT` -- `COUNT_DISTINCT` -- `MAX` -- `MEDIAN` -- `MEDIAN_ABSOLUTE_DEVIATION` -- `MIN` -- `PERCENTILE` -- `ST_CENTROID_AGG` (This functionality is in technical preview and may be changed or removed in a future release) -- `SUM` -- `VALUES` - -It's important to note that `STATS` without any groups is much faster than adding a group. Grouping on a single expression is currently much more optimized than grouping on many expressions. - -## Examples - -Here are some examples of how you can use the `STATS` command in ES|QL: - -1. Calculating a statistic and grouping by the values of another column: - - ```esql -FROM employees -| STATS count = COUNT(emp_no) BY languages -| SORT languages -``` - -2. Omitting `BY` returns one row with the aggregations applied over the entire dataset: - - ```esql -FROM employees -| STATS avg_lang = AVG(languages) -``` - -3. It’s possible to calculate multiple values: - - ```esql -FROM employees -| STATS avg_lang = AVG(languages), max_lang = MAX(languages) -``` - -4. It’s also possible to group by multiple values (only supported for long and keyword family fields): - - ```esql -FROM employees -| EVAL hired = DATE_FORMAT("YYYY", hire_date) -| STATS avg_salary = AVG(salary) BY hired, languages.long -| EVAL avg_salary = ROUND(avg_salary) -| SORT hired, languages.long -``` - -5. Both the aggregating functions and the grouping expressions accept other functions. This is useful for using `STATS...BY` on multivalue columns. For example, to calculate the average salary change, you can use `MV_AVG` to first average the multiple values per employee, and use the result with the `AVG` function: - - ```esql -FROM employees -| STATS avg_salary_change = ROUND(AVG(MV_AVG(salary_change)), 10) -``` - -6. An example of grouping by an expression is grouping employees on the first letter of their last name: - - ```esql -FROM employees -| STATS my_count = COUNT() BY LEFT(last_name, 1) -| SORT `LEFT(last_name, 1)` -``` - -7. Specifying the output column name is optional. If not specified, the new column name is equal to the expression. The following query returns a column named `AVG(salary)`: - - ```esql -FROM employees -| STATS AVG(salary) -| EVAL avg_salary_rounded = ROUND(`AVG(salary)`) -``` - -## Limitations - -- `STATS` does not support configurations where the `_source` field is disabled. -- Full-text search is not supported. -- `text` fields behave like `keyword` fields. -- Time series data streams are not supported. -- Date math expressions work well when the leftmost expression is a datetime. -- Enrich limitations: The ES|QL `ENRICH` command only supports enrich policies of type `match`. Furthermore, `ENRICH` only supports enriching on a column of type `keyword`. -- Dissect limitations: The `DISSECT` command does not support reference keys. -- Grok limitations: The `GROK` command does not support configuring custom patterns, or multiple patterns. The `GROK` command is not subject to Grok watchdog settings. -- Multivalue limitations: ES|QL supports multivalued fields, but functions return `null` when applied to a multivalued field, unless documented otherwise. -- Timezone support: ES|QL only supports the UTC timezone. -- Kibana limitations: The user interface to filter data is not enabled when Discover is in ES|QL mode. To filter data, write a query that uses the `WHERE` command instead. Discover shows no more than 10,000 rows. This limit only applies to the number of rows that are retrieved by the query and displayed in Discover. Queries and aggregations run on the full data set. Discover shows no more than 50 columns. If a query returns more than 50 columns, Discover only shows the first 50. CSV export from Discover shows no more than 10,000 rows. This limit only applies to the number of rows that are retrieved by the query and displayed in Discover. Queries and aggregations run on the full data set. Querying many indices at once without any filters can cause an error in kibana which looks like `[esql] > Unexpected error from Elasticsearch: The content length (536885793) is bigger than the maximum allowed string (536870888)`. The response from ES|QL is too long. Use `DROP` or `KEEP` to limit the number of fields returned. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-substring.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-substring.txt deleted file mode 100644 index 53aedd96c3466..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-substring.txt +++ /dev/null @@ -1,41 +0,0 @@ -## SUBSTRING - -The `SUBSTRING` function in ES|QL is used to extract a specific portion of a string. It is specified by a start position and an optional length. If the length is not provided, the function returns all positions after the start. - -### Syntax: - -`SUBSTRING(string, start, [length])` - -#### Parameters: - -- `string`: The string expression from which to extract the substring. If null, the function returns null. -- `start`: The starting position for the substring. -- `length`: The length of the substring from the start position. This is optional; if omitted, all positions after start are returned. - -### Examples: - -Here are a couple of examples of how to use the `SUBSTRING` function in ES|QL: - -1. Extracting the first three characters of every last name: - - ```esql -FROM employees -| KEEP last_name -| EVAL ln_sub = SUBSTRING(last_name, 1, 3) -``` - -2. Extracting the last three characters of every last name: - - ```esql -FROM employees -| KEEP last_name -| EVAL ln_sub = SUBSTRING(last_name, -3, 3) -``` - -3. Extracting all characters except for the first: - - ```esql -FROM employees -| KEEP last_name -| EVAL ln_sub = SUBSTRING(last_name, 2) -``` \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-sum.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-sum.txt deleted file mode 100644 index 2e555961c553f..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-sum.txt +++ /dev/null @@ -1,21 +0,0 @@ -## SUM - -The `SUM` function in ES|QL is used to calculate the sum of a numeric expression. - -### Examples - -Here are a couple of examples of how you can use the `SUM` function in ES|QL: - -1. To calculate the sum of a field named `languages` in an index named `employees`, you can use the following query: - -```esql -FROM employees -| STATS SUM(languages) -``` - -2. You can also use the `SUM` function with other functions like `MV_MAX`. In the following example, the `MV_MAX` function is applied to each row of the `salary_change` field to get the maximum salary change for each employee. The `SUM` function then calculates the total of these maximum salary changes: - -```esql -FROM employees -| STATS total_salary_changes = SUM(MV_MAX(salary_change)) -``` \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-syntax.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-syntax.txt deleted file mode 100644 index dac9e93b7ea14..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-syntax.txt +++ /dev/null @@ -1,121 +0,0 @@ -# ES|QL Syntax - -ES|QL (Elasticsearch Query Language) uses a simple yet powerful syntax that allows you to filter, transform, and analyze data stored in Elasticsearch. The syntax is composed of a source command followed by an optional series of processing commands, separated by a pipe character (`|`). - -## Basic Syntax - -An ES|QL query typically looks like this: - -source-command -| processing-command1 -| processing-command2 - -The result of a query is the table produced by the final processing command. For an overview of all supported commands, functions, and operators, refer to Commands and Functions and operators. - -You can also write an ES|QL query as a single line. For example: - -source-command -| processing-command1 -| processing-command2 - -## Identifiers - -Identifiers in ES|QL need to be quoted with backticks (```) if they don’t start with a letter, `_` or `@` or if any of the other characters is not a letter, number, or `_`. For example: - -```esql -FROM index -| KEEP `1.field` -``` - -When referencing a function alias that itself uses a quoted identifier, the backticks of the quoted identifier need to be escaped with another backtick. For example: - -```esql -FROM index -| STATS COUNT(`1.field`) -| EVAL my_count = `COUNT(``1.field``)` -``` - -## Literals - -ES|QL currently supports numeric and string literals. - -### String Literals - -A string literal is a sequence of unicode characters delimited by double quotes (`"`). If the literal string itself contains quotes, these need to be escaped (`\\"`). ES|QL also supports the triple-quotes (`"""`) delimiter, for convenience. Special characters CR, LF and TAB can be provided with the usual escaping: `\r`, `\n`, `\t`, respectively. - -```esql -FROM index -| WHERE first_name == "Georgi" -``` - -### Numerical Literals - -The numeric literals are accepted in decimal and in the scientific notation with the exponent marker (`e` or `E`), starting either with a digit, decimal point `.` or the negative sign `-`. The integer numeric literals are implicitly converted to the `integer`, `long` or the `double` type, whichever can first accommodate the literal’s value. The floating point literals are implicitly converted the `double` type. - -1969 -- integer notation -3.14 -- decimal notation -.1234 -- decimal notation starting with decimal point -4E5 -- scientific notation (with exponent marker) -1.2e-3 -- scientific notation with decimal point --.1e2 -- scientific notation starting with the negative sign - -## Comments - -ES|QL uses C++ style comments: double slash `//` for single line comments and `/*` and `*/` for block comments. - -```esql -// Query the employees index -FROM employees -| WHERE height > 2 -``` - -## Timespan Literals - -Datetime intervals and timespans can be expressed using timespan literals. Timespan literals are a combination of a number and a qualifier. These qualifiers are supported: `millisecond`/`milliseconds`/`ms`, `second`/`seconds`/`sec`/`s`, `minute`/`minutes`/`min`, `hour`/`hours`/`h`, `day`/`days`/`d`, `week`/`weeks`/`w`, `month`/`months`/`mo`, `quarter`/`quarters`/`q`, `year`/`years`/`yr`/`y`. Timespan literals are not whitespace sensitive. - -1day -1 day -1 day - -## Example Queries with Timespan Literals - -Here are some example queries using timespan literals: - -1. Querying data from the last 7 days: - -```esql -FROM logs -| WHERE @timestamp >= NOW() - 7d -``` - -2. Aggregating data on an hourly basis for the past 24 hours: - -```esql -FROM logs -| STATS COUNT(*) BY timestamp = BUCKET(@timestamp, 1h) -| WHERE timestamp >= NOW() - 24h -``` - -3. Finding the average response time per minute for the last hour: - -```esql -FROM logs -| STATS AVG(response_time) BY minute = BUCKET(@timestamp, 1m) -| WHERE @timestamp >= NOW() - 1h -``` - -4. Aggregating data on a weekly basis for the past year: - -```esql -FROM logs -| STATS COUNT(*) BY week = BUCKET(@timestamp, 1w) -| WHERE @timestamp >= NOW() - 1y -``` - -5. Finding the maximum response time per second for the last minute: - -```esql -FROM logs -| STATS MAX(response_time) BY second = BUCKET(@timestamp, 1s) -| WHERE @timestamp >= NOW() - 1m -``` diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-tan.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-tan.txt deleted file mode 100644 index 7f1280c10f536..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-tan.txt +++ /dev/null @@ -1,29 +0,0 @@ -## TAN - -The `TAN` function in ES|QL is used to calculate the Tangent of an angle. The angle should be provided in radians. - -### Syntax - -The syntax for using the `TAN` function is as follows: - -`TAN(angle)` - -Here, `angle` is the angle in radians for which you want to calculate the Tangent. If `angle` is null, the function will return null. - -### Examples - -Here are a couple of examples showing how to use the `TAN` function in ES|QL: - -```esql -ROW a=1.8 -| EVAL tan = TAN(a) -``` - -In this example, the `TAN` function is used to calculate the Tangent of the angle `1.8` radians. - -```esql -ROW a=3.14 -| EVAL tan = TAN(a) -``` - -In this example, the `TAN` function is used to calculate the Tangent of the angle `3.14` radians. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-tanh.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-tanh.txt deleted file mode 100644 index d1412a7016bd6..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-tanh.txt +++ /dev/null @@ -1,29 +0,0 @@ -## TANH - -The `TANH` function in ES|QL returns the Tangent hyperbolic function of an angle. The angle should be provided in radians. If the angle is null, the function will return null. - -### Syntax - -`TANH(angle)` - -#### Parameters - -- `angle`: An angle, in radians. If null, the function returns null. - -### Examples - -Here are a couple of examples of how to use the `TANH` function in ES|QL: - -```esql -ROW a=1.8 -| EVAL tanh = TANH(a) -``` - -In this example, the `TANH` function is used to calculate the Tangent hyperbolic function of the angle `1.8` radians. - -```esql -ROW a=3.14 -| EVAL tanh_result = TANH(a) -``` - -In this second example, the `TANH` function is used to calculate the Tangent hyperbolic function of the angle `3.14` radians. The result is stored in the `tanh_result` variable. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-tau.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-tau.txt deleted file mode 100644 index c7e0a5211d48f..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-tau.txt +++ /dev/null @@ -1,20 +0,0 @@ -## TAU - -TAU function in ES|QL returns the ratio of a circle’s circumference to its radius. - -### Examples - -Here are a couple of examples of how to use the TAU function in ES|QL: - -```esql -ROW TAU() -``` - -In this example, the TAU function is used to return the ratio of a circle’s circumference to its radius. - -```esql -FROM my-index -| EVAL tau_ratio = TAU() -``` - -In this example, the TAU function is used within an EVAL function to create a new column `tau_ratio` in the result set, which contains the ratio of a circle’s circumference to its radius. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_base64.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_base64.txt deleted file mode 100644 index 4069711b9524f..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_base64.txt +++ /dev/null @@ -1,29 +0,0 @@ -## TO_BASE64 - -The `TO_BASE64` function in ES|QL is used to encode a string to a base64 string. - -### Syntax - -`TO_BASE64(string)` - -#### Parameters - -- `string`: The string you want to encode. - -### Examples - -Here are a couple of examples of how you can use the `TO_BASE64` function in ES|QL: - -```esql -ROW a = "elastic" -| EVAL e = TO_BASE64(a) -``` - -In this example, the string "elastic" is encoded to a base64 string. - -```esql -ROW b = "Elasticsearch Query Language" -| EVAL encoded = TO_BASE64(b) -``` - -In this example, the string "Elasticsearch Query Language" is encoded to a base64 string. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_boolean.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_boolean.txt deleted file mode 100644 index 34868efa9ccab..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_boolean.txt +++ /dev/null @@ -1,29 +0,0 @@ -## TO_BOOLEAN - -The `TO_BOOLEAN` function converts an input value to a boolean value. A string value of true will be case-insensitive converted to the Boolean true. For anything else, including the empty string, the function will return false. The numerical value of 0 will be converted to false, anything else will be converted to true. - -### Syntax - -`TO_BOOLEAN(field)` - -#### Parameters - -- `field`: Input value. The input can be a single- or multi-valued column or an expression. - -### Examples - -Here are a couple of examples of full ES|QL queries using the `TO_BOOLEAN` function: - -```esql -ROW str = ["true", "TRuE", "false", "", "yes", "1"] -| EVAL bool = TO_BOOLEAN(str) -``` - -In this example, the `TO_BOOLEAN` function is used to convert a list of string values to boolean. The resulting `bool` column will contain boolean values corresponding to the input strings. - -```esql -ROW str = ["0", "1", "2", "-1", "0.5"] -| EVAL bool = TO_BOOLEAN(str) -``` - -In this second example, the `TO_BOOLEAN` function is used to convert a list of numeric strings to boolean. The resulting `bool` column will contain boolean values corresponding to the input strings. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_cartesianpoint.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_cartesianpoint.txt deleted file mode 100644 index 35310b547f32d..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_cartesianpoint.txt +++ /dev/null @@ -1,22 +0,0 @@ -## TO_CARTESIANPOINT - -The `TO_CARTESIANPOINT` function converts an input value to a `cartesian_point` value. This conversion will only be successful if the input string respects the WKT Point format. - -### Examples - -Here are a couple of examples of how you can use the `TO_CARTESIANPOINT` function in ES|QL queries: - -```esql -ROW wkt = ["POINT(4297.11 -1475.53)", "POINT(7580.93 2272.77)"] -| MV_EXPAND wkt -| EVAL pt = TO_CARTESIANPOINT(wkt) -``` - -In this example, the `TO_CARTESIANPOINT` function is used to convert the values in the `wkt` field (which are in WKT Point format) to `cartesian_point` values. The `MV_EXPAND` function is used to expand the multi-valued `wkt` field into individual rows, and then the `TO_CARTESIANPOINT` function is applied to each row. - -```esql -ROW wkt = "POINT(4297.11 -1475.53)" -| EVAL pt = TO_CARTESIANPOINT(wkt) -``` - -In this second example, the `TO_CARTESIANPOINT` function is used to convert a single WKT Point string to a `cartesian_point` value. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_datetime.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_datetime.txt deleted file mode 100644 index b23b9cb6934bd..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_datetime.txt +++ /dev/null @@ -1,21 +0,0 @@ -## TO_DATETIME - -The `TO_DATETIME` function converts an input value to a date value. A string will only be successfully converted if it’s respecting the format `yyyy-MM-dd'T'HH:mm:ss.SSS'Z'`. To convert dates in other formats, use `DATE_PARSE`. - -### Examples - -Here are a couple of examples of how to use the `TO_DATETIME` function in ES|QL queries: - -```esql -ROW string = ["1953-09-02T00:00:00.000Z", "1964-06-02T00:00:00.000Z", "1964-06-02 00:00:00"] -| EVAL datetime = TO_DATETIME(string) -``` - -In this example, the last value in the source multi-valued field has not been converted. This is because if the date format is not respected, the conversion will result in a null value. When this happens a Warning header is added to the response. The header will provide information on the source of the failure. - -```esql -ROW int = [0, 1] -| EVAL dt = TO_DATETIME(int) -``` - -In this example, if the input parameter is of a numeric type, its value will be interpreted as milliseconds since the Unix epoch. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_degrees.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_degrees.txt deleted file mode 100644 index d82a65148947f..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_degrees.txt +++ /dev/null @@ -1,29 +0,0 @@ -## TO_DEGREES - -The `TO_DEGREES` function in ES|QL is used to convert a number in radians to degrees. - -### Syntax - -`TO_DEGREES(number)` - -#### Parameters - -- `number`: This is the input value. It can be a single or multi-valued column or an expression. - -### Examples - -Here are a couple of examples of how you can use the `TO_DEGREES` function in ES|QL: - -```esql -ROW rad = [1.57, 3.14, 4.71] -| EVAL deg = TO_DEGREES(rad) -``` - -In this example, the `TO_DEGREES` function is used to convert the values in the `rad` array from radians to degrees. - -```esql -FROM my_index -| EVAL angle_deg = TO_DEGREES(angle_rad) -``` - -In this example, the `TO_DEGREES` function is used to convert the values in the `angle_rad` field from radians to degrees and the result is stored in the `angle_deg` field. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_double.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_double.txt deleted file mode 100644 index 22567ab279190..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_double.txt +++ /dev/null @@ -1,21 +0,0 @@ -## TO_DOUBLE - -TO_DOUBLE function converts an input value to a double value. If the input parameter is of a date type, its value will be interpreted as milliseconds since the Unix epoch, converted to double. Boolean true will be converted to double 1.0, false to 0.0. - -### Examples - -Here are a couple of examples of how to use the `TO_DOUBLE` function in ES|QL: - -```esql -ROW str1 = "5.20128E11" -| EVAL dbl = TO_DOUBLE("520128000000"), dbl1 = TO_DOUBLE(str1) -``` - -In this example, the string "5.20128E11" is converted to a double value. - -```esql -ROW str2 = "foo" -| EVAL dbl2 = TO_DOUBLE(str2) -``` - -In this example, the string "foo" cannot be converted to a double value, resulting in a null value. A warning header is added to the response indicating the source of the failure. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_geopoint.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_geopoint.txt deleted file mode 100644 index e3db60a8fcf5d..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_geopoint.txt +++ /dev/null @@ -1,21 +0,0 @@ -## TO_GEOPOINT - -The `TO_GEOPOINT` function in ES|QL is used to convert an input value to a `geo_point` value. This function is successful in conversion only if the input string respects the WKT (Well-Known Text) Point format. - -### Examples - -Here are a couple of examples of how you can use the `TO_GEOPOINT` function in your ES|QL queries: - -```esql -ROW wkt = "POINT(42.97109630194 14.7552534413725)" -| EVAL pt = TO_GEOPOINT(wkt) -``` - -In this example, the `TO_GEOPOINT` function is used to convert the WKT representation of a point to a `geo_point` value. - -```esql -ROW wkt = "POINT(34.052235 -118.243683)" -| EVAL location = TO_GEOPOINT(wkt) -``` - -In this second example, the `TO_GEOPOINT` function is used to convert the WKT representation of a different point to a `geo_point` value. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_geoshape.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_geoshape.txt deleted file mode 100644 index d3b6bb198040f..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_geoshape.txt +++ /dev/null @@ -1,21 +0,0 @@ -## TO_GEOSHAPE - -The `TO_GEOSHAPE` function in ES|QL is used to convert an input value to a `geo_shape` value. The conversion will be successful only if the input string respects the Well-Known Text (WKT) format. - -### Examples - -Here are a couple of examples of how you can use the `TO_GEOSHAPE` function in your ES|QL queries: - -```esql -ROW wkt = "POLYGON ((30 10, 40 40, 20 40, 10 20, 30 10))" -| EVAL geom = TO_GEOSHAPE(wkt) -``` - -In this example, the `TO_GEOSHAPE` function is used to convert a WKT representation of a polygon into a `geo_shape` value. - -```esql -ROW wkt = "POINT (30 10)" -| EVAL geom = TO_GEOSHAPE(wkt) -``` - -In this second example, the `TO_GEOSHAPE` function is used to convert a WKT representation of a point into a `geo_shape` value. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_integer.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_integer.txt deleted file mode 100644 index f0881133c7d8a..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_integer.txt +++ /dev/null @@ -1,24 +0,0 @@ -## TO_INTEGER - -The `TO_INTEGER` function converts an input value to an integer value. If the input parameter is of a date type, its value will be interpreted as milliseconds since the Unix epoch, converted to integer. Boolean `true` will be converted to integer `1`, `false` to `0`. - -### Examples - -Here are a couple of examples of full ES|QL queries using the `TO_INTEGER` function: - -```esql -ROW long = [5013792, 2147483647, 501379200000] -| EVAL int = TO_INTEGER(long) -``` - -In this example, the last value of the multi-valued field cannot be converted as an integer. When this happens, the result is a null value. A Warning header is added to the response providing information on the source of the failure: - -```esql -"Line 1:61: evaluation of [TO_INTEGER(long)] failed, treating result as null. Only first 20 failures recorded." -``` - -A following header will contain the failure reason and the offending value: - -``` -"org.elasticsearch.xpack.esql.core.InvalidArgumentException: [501379200000] out of [integer] range" -``` diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_ip.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_ip.txt deleted file mode 100644 index ddd8e0eeaee39..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_ip.txt +++ /dev/null @@ -1,22 +0,0 @@ -## TO_IP - -The `TO_IP` function in ES|QL is used to convert an input string to an IP value. - -### Examples - -Here are a couple of examples of how you can use the `TO_IP` function in your ES|QL queries: - -```esql -ROW str1 = "1.1.1.1" -| EVAL ip1 = TO_IP(str1) -| WHERE CIDR_MATCH(ip1, "1.0.0.0/8") -``` - -In this example, the `TO_IP` function is used to convert the string "1.1.1.1" to an IP value. The `WHERE` clause then uses the `CIDR_MATCH` function to check if the IP value falls within the specified CIDR range. - -```esql -ROW str2 = "foo" -| EVAL ip2 = TO_IP(str2) -``` - -In this second example, the `TO_IP` function attempts to convert the string "foo" to an IP value. However, since "foo" is not a valid IP string literal, the function returns a null value and a warning is added to the response header. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_long.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_long.txt deleted file mode 100644 index 4e23ae659f17a..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_long.txt +++ /dev/null @@ -1,21 +0,0 @@ -## TO_LONG - -The `TO_LONG` function converts an input value to a long value. If the input parameter is of a date type, its value will be interpreted as milliseconds since the Unix epoch, converted to long. Boolean true will be converted to long 1, false to 0. - -### Examples - -Here are a couple of examples of how you can use the `TO_LONG` function in ES|QL queries: - -```esql -ROW str1 = "2147483648" -| EVAL long1 = TO_LONG(str1) -``` - -In this example, the string "2147483648" is converted to a long value. - -```esql -ROW str2 = "2147483648.2", str3 = "foo" -| EVAL long2 = TO_LONG(str2), long3 = TO_LONG(str3) -``` - -In this example, the string "2147483648.2" is converted to a long value. However, the string "foo" cannot be converted to a long value, resulting in a null value and a warning header added to the response. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_lower.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_lower.txt deleted file mode 100644 index ccfdf623a83bf..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_lower.txt +++ /dev/null @@ -1,29 +0,0 @@ -## TO_LOWER - -The `TO_LOWER` function in ES|QL is used to convert an input string to lower case. - -### Syntax - -`TO_LOWER(str)` - -#### Parameters - -- `str`: String expression. If null, the function returns null. - -### Examples - -Here are a couple of examples of how you can use the `TO_LOWER` function in ES|QL queries: - -```esql -ROW message = "HELLO WORLD" -| EVAL lower_message = TO_LOWER(message) -``` - -In this example, the `TO_LOWER` function is used to convert the string "HELLO WORLD" to lower case. The result would be "hello world". - -```esql -ROW name = "John Doe" -| EVAL lower_name = TO_LOWER(name) -``` - -In this example, the `TO_LOWER` function is used to convert the string "John Doe" to lower case. The result would be "john doe". \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_radians.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_radians.txt deleted file mode 100644 index dbf3ef1e2aa62..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_radians.txt +++ /dev/null @@ -1,29 +0,0 @@ -## TO_RADIANS - -The `TO_RADIANS` function in ES|QL is used to convert a number in degrees to radians. - -### Syntax - -`TO_RADIANS(number)` - -#### Parameters - -- `number`: This is the input value. It can be a single or multi-valued column or an expression. - -### Examples - -Here are a couple of examples of how you can use the `TO_RADIANS` function in ES|QL: - -```esql -ROW deg = [90.0, 180.0, 270.0] -| EVAL rad = TO_RADIANS(deg) -``` - -In this example, the `TO_RADIANS` function is used to convert an array of degree values into radians. - -```esql -ROW deg = 45 -| EVAL rad = TO_RADIANS(deg) -``` - -In this example, the `TO_RADIANS` function is used to convert a single degree value into radians. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_string.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_string.txt deleted file mode 100644 index ed7aad31edd8b..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_string.txt +++ /dev/null @@ -1,21 +0,0 @@ -## TO_STRING - -The `TO_STRING` function in ES|QL is used to convert an input value into a string. The input can be a single or multi-valued column or an expression. - -### Examples - -Here are a couple of examples of how you can use the `TO_STRING` function in your ES|QL queries: - -```esql -ROW a=10 -| EVAL j = TO_STRING(a) -``` - -In this example, the function is used to convert the numeric value `10` into a string. - -```esql -ROW a=[10, 9, 8] -| EVAL j = TO_STRING(a) -``` - -In this example, the function is used to convert the values in a multi-valued field into strings. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_unsigned_long.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_unsigned_long.txt deleted file mode 100644 index afba5be08f5cd..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_unsigned_long.txt +++ /dev/null @@ -1,21 +0,0 @@ -## TO_UNSIGNED_LONG - -The `TO_UNSIGNED_LONG` function converts an input value to an unsigned long value. If the input parameter is of a date type, its value will be interpreted as milliseconds since the Unix epoch, converted to unsigned long. Boolean true will be converted to unsigned long 1, false to 0. - -### Examples - -Here are a couple of examples of full ES|QL queries using the `TO_UNSIGNED_LONG` function: - -```esql -ROW str1 = "2147483648", str2 = "2147483648.2", str3 = "foo" -| EVAL long1 = TO_UNSIGNED_LONG(str1), long2 = TO_ULONG(str2), long3 = TO_UL(str3) -``` - -In this example, the `TO_UNSIGNED_LONG` function is used to convert string values to unsigned long. Note that the last conversion of the string isn’t possible. When this happens, the result is a null value. - -```esql -ROW date = "2022-01-01T00:00:00Z" -| EVAL timestamp = TO_UNSIGNED_LONG(date) -``` - -In this example, the `TO_UNSIGNED_LONG` function is used to convert a date string to an unsigned long value, representing the milliseconds since the Unix epoch. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_upper.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_upper.txt deleted file mode 100644 index 9f703c69c167e..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_upper.txt +++ /dev/null @@ -1,34 +0,0 @@ -## TO_UPPER - -The `TO_UPPER` function in ES|QL is used to convert an input string to upper case. - -### Syntax - -`TO_UPPER(str)` - -#### Parameters - -- `str`: This is a string expression. If null, the function returns null. - -### Description - -The function returns a new string representing the input string converted to upper case. - -### Examples - -Here are a couple of examples of full ES|QL queries using the `TO_UPPER` function: - -```esql -ROW message = "Hello World" -| EVAL upper_message = TO_UPPER(message) -``` - -In this example, the `TO_UPPER` function is used to convert the string "Hello World" to upper case. - -```esql -FROM employees -| EVAL upper_last_name = TO_UPPER(last_name) -| KEEP emp_no, upper_last_name -``` - -In this example, the `TO_UPPER` function is used to convert the `last_name` field of each record in the `employees` index to upper case. The query then keeps the `emp_no` and the upper case `last_name` for each record. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_version.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_version.txt deleted file mode 100644 index b560d15edc942..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-to_version.txt +++ /dev/null @@ -1,19 +0,0 @@ -## TO_VERSION - -TO_VERSION function converts an input string to a version value. - -### Examples - -Here are a couple of examples of how you can use the TO_VERSION function in ES|QL queries: - -```esql -ROW v = TO_VERSION("1.2.3") -``` - -In this example, the TO_VERSION function is used to convert the string "1.2.3" to a version value. - -```esql -ROW v = TO_VERSION("2.3.4") -``` - -In this example, the TO_VERSION function is used to convert the string "2.3.4" to a version value. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-trim.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-trim.txt deleted file mode 100644 index faaad56be6f8b..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-trim.txt +++ /dev/null @@ -1,30 +0,0 @@ -## TRIM - -The `TRIM` function in ES|QL is used to remove leading and trailing whitespaces from a string. If the string expression is null, the function will return null. - -### Syntax - -`TRIM(string)` - -#### Parameters - -`string`: A string expression. If null, the function returns null. - -### Examples - -Here are a couple of examples of how you can use the `TRIM` function in ES|QL: - -```esql -ROW message = " some text ", color = " red " -| EVAL message = TRIM(message) -| EVAL color = TRIM(color) -``` - -In this example, the `TRIM` function is used to remove the leading and trailing whitespaces from the `message` and `color` strings. - -```esql -ROW name = " John Doe " -| EVAL trimmed_name = TRIM(name) -``` - -In this second example, the `TRIM` function is used to remove the leading and trailing whitespaces from the `name` string. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-values.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-values.txt deleted file mode 100644 index edc92663a03b8..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-values.txt +++ /dev/null @@ -1,24 +0,0 @@ -## VALUES - -The `VALUES` function in ES|QL is used to return all values in a group as a multivalued field. The order of the returned values isn’t guaranteed. If you need the values returned in order, you can use `MV_SORT`. This function can use a significant amount of memory and ES|QL doesn’t yet grow aggregations beyond memory. So this aggregation will work until it is used to collect more values than can fit into memory. Once it collects too many values it will fail the query with a Circuit Breaker Error. - -### Syntax - -`VALUES(expression)` - -Where `expression` is an expression of any type except `geo_point`, `cartesian_point`, `geo_shape`, or `cartesian_shape`. - -### Examples - -Here are a couple of examples of how you can use the `VALUES` function in ES|QL queries: - -```esql -FROM employees -| EVAL first_letter = SUBSTRING(first_name, 0, 1) -| STATS first_name = MV_SORT(VALUES(first_name)) BY first_letter -| SORT first_letter -``` - -In this example, the `VALUES` function is used to return all values of the `first_name` field in a group as a multivalued field. The `MV_SORT` function is then used to sort these values. - -Please note that this function is in technical preview and may be changed or removed in a future release. It is not recommended to use `VALUES` on production environments. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-where.txt b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-where.txt deleted file mode 100644 index 3c8e9ab1bd14b..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/esql_docs/esql-where.txt +++ /dev/null @@ -1,65 +0,0 @@ -## WHERE - -The `WHERE` command in ES|QL is a processing command that produces a table containing all the rows from the input table for which the provided condition evaluates to true. This command is particularly useful in filtering data based on specific conditions. - -The `WHERE` command supports various functions and operators, including date math for retrieving data from a specific time range, `LIKE` and `RLIKE` for filtering data based on string patterns, and the `IN` operator for testing whether a field or expression equals an element in a list of literals, fields, or expressions. - -However, it's important to note that the `WHERE` command has certain limitations. For instance, it does not support configurations where the `_source` field is disabled. Also, full-text search is not yet supported because of the way ES|QL treats `text` values. - -### Syntax: - -`WHERE expression` - -#### Parameters: - -- `expression`: A boolean expression. - -### Examples: - -Here are some examples of how the `WHERE` command can be used in different scenarios: - -1. Filtering employees who are still hired: - - ```esql -FROM employees -| KEEP first_name, last_name, still_hired -| WHERE still_hired == true -``` - -2. Retrieving the last hour of logs: - - ```esql -FROM sample_data -| WHERE @timestamp > NOW() - 1 hour -``` - -3. Filtering employees based on the length of their first name: - - ```esql -FROM employees -| KEEP first_name, last_name, height -| WHERE LENGTH(first_name) < 4 -``` - -4. Filtering data based on string patterns using `LIKE`: - - ```esql -FROM employees -| WHERE first_name LIKE "?b*" -| KEEP first_name, last_name -``` - -5. Filtering data based on string patterns using `RLIKE`: - - ```esql -FROM employees -| WHERE first_name RLIKE ".leja.*" -| KEEP first_name, last_name -``` - -6. Using the `IN` operator to test whether a field or expression equals an element in a list: - - ```esql -ROW a = 1, b = 4, c = 3 -| WHERE c-a IN (3, b / 2, a) -``` \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/get_errors_with_commands.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/get_errors_with_commands.ts index b25c79161f79b..636a37ba14fe2 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/get_errors_with_commands.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/get_errors_with_commands.ts @@ -6,7 +6,7 @@ */ import type { EditorError, ESQLMessage } from '@kbn/esql-ast'; -import { splitIntoCommands } from './correct_common_esql_mistakes'; +import { splitIntoCommands } from '@kbn/inference-plugin/common'; export function getErrorsWithCommands(query: string, errors: Array) { const asCommands = splitIntoCommands(query); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/index.ts index 797d85fc10341..13c2bee278c8a 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/index.ts @@ -5,73 +5,29 @@ * 2.0. */ -import Fs from 'fs'; -import { keyBy, mapValues, once, pick } from 'lodash'; -import pLimit from 'p-limit'; -import Path from 'path'; -import { lastValueFrom, startWith } from 'rxjs'; -import { promisify } from 'util'; -import { FunctionVisibility, MessageRole } from '@kbn/observability-ai-assistant-plugin/common'; +import { isChatCompletionChunkEvent, isOutputEvent } from '@kbn/inference-plugin/common'; +import { naturalLanguageToEsql } from '@kbn/inference-plugin/server'; import { - VisualizeESQLUserIntention, - VISUALIZE_ESQL_USER_INTENTIONS, -} from '@kbn/observability-ai-assistant-plugin/common/functions/visualize_esql'; -import { - concatenateChatCompletionChunks, - ConcatenatedMessage, -} from '@kbn/observability-ai-assistant-plugin/common/utils/concatenate_chat_completion_chunks'; -import { emitWithConcatenatedMessage } from '@kbn/observability-ai-assistant-plugin/common/utils/emit_with_concatenated_message'; + FunctionVisibility, + MessageAddEvent, + MessageRole, + StreamingChatResponseEventType, +} from '@kbn/observability-ai-assistant-plugin/common'; import { createFunctionResponseMessage } from '@kbn/observability-ai-assistant-plugin/common/utils/create_function_response_message'; +import { map } from 'rxjs'; +import { v4 } from 'uuid'; import type { FunctionRegistrationParameters } from '..'; -import { correctCommonEsqlMistakes } from './correct_common_esql_mistakes'; import { runAndValidateEsqlQuery } from './validate_esql_query'; -import { INLINE_ESQL_QUERY_REGEX } from './constants'; +import { convertMessagesForInference } from '../../../common/convert_messages_for_inference'; export const QUERY_FUNCTION_NAME = 'query'; export const EXECUTE_QUERY_NAME = 'execute_query'; -const readFile = promisify(Fs.readFile); -const readdir = promisify(Fs.readdir); - -const loadSystemMessage = once(async () => { - const data = await readFile(Path.join(__dirname, './system_message.txt')); - return data.toString('utf-8'); -}); - -const loadEsqlDocs = once(async () => { - const dir = Path.join(__dirname, './esql_docs'); - const files = (await readdir(dir)).filter((file) => Path.extname(file) === '.txt'); - - if (!files.length) { - return {}; - } - - const limiter = pLimit(10); - return keyBy( - await Promise.all( - files.map((file) => - limiter(async () => { - const data = (await readFile(Path.join(dir, file))).toString('utf-8'); - const filename = Path.basename(file, '.txt'); - - const keyword = filename - .replace('esql-', '') - .replace('agg-', '') - .replaceAll('-', '_') - .toUpperCase(); - - return { - keyword: keyword === 'STATS_BY' ? 'STATS' : keyword, - data, - }; - }) - ) - ), - 'keyword' - ); -}); - -export function registerQueryFunction({ functions, resources }: FunctionRegistrationParameters) { +export function registerQueryFunction({ + functions, + resources, + pluginsStart, +}: FunctionRegistrationParameters) { functions.registerInstruction(({ availableFunctionNames }) => availableFunctionNames.includes(QUERY_FUNCTION_NAME) ? `You MUST use the "${QUERY_FUNCTION_NAME}" function when the user wants to: @@ -97,8 +53,16 @@ export function registerQueryFunction({ functions, resources }: FunctionRegistra functions.registerFunction( { name: EXECUTE_QUERY_NAME, - visibility: FunctionVisibility.UserOnly, - description: 'Display the results of an ES|QL query.', + visibility: FunctionVisibility.Internal, + description: `Execute a generated ES|QL query on behalf of the user. The results + will be returned to you. + + You must use this function if the user is asking for the result of a query, + such as a metric or list of things, but does not want to visualize it in + a table or chart. You do NOT need to ask permission to execute the query + after generating it, use the "${EXECUTE_QUERY_NAME}" function directly instead. + + Do not use when the user just asks for an example.`, parameters: { type: 'object', properties: { @@ -137,345 +101,83 @@ export function registerQueryFunction({ functions, resources }: FunctionRegistra functions.registerFunction( { name: QUERY_FUNCTION_NAME, - description: `This function generates, executes and/or visualizes a query based on the user's request. It also explains how ES|QL works and how to convert queries from one language to another. Make sure you call one of the get_dataset functions first if you need index or field names. This function takes no input.`, + description: `This function generates, executes and/or visualizes a query + based on the user's request. It also explains how ES|QL works and how to + convert queries from one language to another. Make sure you call one of + the get_dataset functions first if you need index or field names. This + function takes no input.`, visibility: FunctionVisibility.AssistantOnly, }, - async ({ messages, chat }, signal) => { - const [systemMessage, esqlDocs] = await Promise.all([loadSystemMessage(), loadEsqlDocs()]); - - const withEsqlSystemMessage = (message?: string) => [ - { - '@timestamp': new Date().toISOString(), - message: { role: MessageRole.System, content: `${systemMessage}\n${message ?? ''}` }, - }, - // remove the query function request - ...messages.filter((msg) => msg.message.role !== MessageRole.System), - ]; - - const userQuestion = messages - .concat() - .reverse() - .find((message) => message.message.role === MessageRole.User && !message.message.name); + async ({ messages, connectorId }, signal) => { + const esqlFunctions = functions + .getFunctions() + .filter( + (fn) => + fn.definition.name === EXECUTE_QUERY_NAME || fn.definition.name === 'visualize_query' + ) + .map((fn) => fn.definition); + + const actions = functions.getActions(); + + const events$ = naturalLanguageToEsql({ + client: pluginsStart.inference.getClient({ request: resources.request }), + connectorId, + messages: convertMessagesForInference( + // remove system message and query function request + messages.filter((message) => message.message.role !== MessageRole.System).slice(0, -1) + ), + logger: resources.logger, + tools: Object.fromEntries( + actions + .concat(esqlFunctions) + .map((fn) => [fn.name, { description: fn.description, schema: fn.parameters }]) + ), + }); - const abbreviatedUserQuestion = userQuestion!.message.content!.substring(0, 50); + const chatMessageId = v4(); - const source$ = ( - await chat('classify_esql', { - messages: withEsqlSystemMessage().concat( - createFunctionResponseMessage({ - name: QUERY_FUNCTION_NAME, + return events$.pipe( + map((event) => { + if (isOutputEvent(event)) { + return createFunctionResponseMessage({ content: {}, - }).message, - { - '@timestamp': new Date().toISOString(), + name: QUERY_FUNCTION_NAME, + data: event.output, + }); + } + if (isChatCompletionChunkEvent(event)) { + return { + id: chatMessageId, + type: StreamingChatResponseEventType.ChatCompletionChunk, message: { - role: MessageRole.User, - content: `Use the classify_esql tool attached to this conversation - to classify the user's request in the user message before this ("${abbreviatedUserQuestion}..."). - and get more information about specific functions and commands - you think are candidates for answering the question. - - Examples for functions and commands: - Do you need to group data? Request \`STATS\`. - Extract data? Request \`DISSECT\` AND \`GROK\`. - Convert a column based on a set of conditionals? Request \`EVAL\` and \`CASE\`. - - ONLY use ${VisualizeESQLUserIntention.executeAndReturnResults} if you are absolutely sure - it is executable. If one of the get_dataset_info functions were not called before, OR if - one of the get_dataset_info functions returned no data, opt for an explanation only and - mention that there is no data for these indices. You can still use - ${VisualizeESQLUserIntention.generateQueryOnly} and generate an example ES|QL query. - - For determining the intention of the user, the following options are available: - - ${VisualizeESQLUserIntention.generateQueryOnly}: the user only wants to generate the query, - but not run it, or they ask a general question about ES|QL. - - ${VisualizeESQLUserIntention.executeAndReturnResults}: the user wants to execute the query, - and have the assistant return/analyze/summarize the results. they don't need a - visualization. - - ${VisualizeESQLUserIntention.visualizeAuto}: The user wants to visualize the data from the - query, but wants us to pick the best visualization type, or their preferred - visualization is unclear. - - These intentions will display a specific visualization: - ${VisualizeESQLUserIntention.visualizeBar} - ${VisualizeESQLUserIntention.visualizeDonut} - ${VisualizeESQLUserIntention.visualizeHeatmap} - ${VisualizeESQLUserIntention.visualizeLine} - ${VisualizeESQLUserIntention.visualizeArea} - ${VisualizeESQLUserIntention.visualizeTable} - ${VisualizeESQLUserIntention.visualizeTagcloud} - ${VisualizeESQLUserIntention.visualizeTreemap} - ${VisualizeESQLUserIntention.visualizeWaffle} - ${VisualizeESQLUserIntention.visualizeXy} - - Some examples: - - "I want a query that ..." => ${VisualizeESQLUserIntention.generateQueryOnly} - "... Just show me the query" => ${VisualizeESQLUserIntention.generateQueryOnly} - "Create a query that ..." => ${VisualizeESQLUserIntention.generateQueryOnly} - - "Show me the avg of x" => ${VisualizeESQLUserIntention.executeAndReturnResults} - "Show me the results of y" => ${VisualizeESQLUserIntention.executeAndReturnResults} - "Display the sum of z" => ${VisualizeESQLUserIntention.executeAndReturnResults} - - "Show me the avg of x over time" => ${VisualizeESQLUserIntention.visualizeAuto} - "I want a bar chart of ... " => ${VisualizeESQLUserIntention.visualizeBar} - "I want to see a heat map of ..." => ${VisualizeESQLUserIntention.visualizeHeatmap} - `, + content: event.content, }, - } - ), - signal, - functions: [ - { - name: 'classify_esql', - description: `Use this function to determine: - - what ES|QL functions and commands are candidates for answering the user's question - - whether the user has requested a query, and if so, it they want it to be executed, or just shown. - - All parameters are required. Make sure the functions and commands you request are available in the - system message. - `, - parameters: { - type: 'object', - properties: { - commands: { - type: 'array', - items: { - type: 'string', - }, - description: - 'A list of processing or source commands that are referenced in the list of commands in this conversation', - }, - functions: { - type: 'array', - items: { - type: 'string', - }, - description: - 'A list of functions that are referenced in the list of functions in this conversation', - }, - intention: { - type: 'string', - description: `What the user\'s intention is.`, - enum: VISUALIZE_ESQL_USER_INTENTIONS, - }, - }, - required: ['commands', 'functions', 'intention'], - }, - }, - ], - functionCall: 'classify_esql', - }) - ).pipe(concatenateChatCompletionChunks()); - - const response = await lastValueFrom(source$); - - if (!response.message.function_call.arguments) { - resources.logger.debug( - () => - `LLM should have called "classify_esql", but instead responded with the following message: ${JSON.stringify( - response.message - )}` - ); - throw new Error( - 'LLM did not call classify_esql function during query generation, execute the "query" function and try again' - ); - } - - const args = JSON.parse(response.message.function_call.arguments) as { - commands?: string[]; - functions?: string[]; - intention: VisualizeESQLUserIntention; - }; - - const keywords = [ - ...(args.commands ?? []), - ...(args.functions ?? []), - 'SYNTAX', - 'OVERVIEW', - 'OPERATORS', - ].map((keyword) => keyword.toUpperCase()); - - const messagesToInclude = mapValues(pick(esqlDocs, keywords), ({ data }) => data); - - let userIntentionMessage: string; - - switch (args.intention) { - case VisualizeESQLUserIntention.executeAndReturnResults: - userIntentionMessage = `When you generate a query, it will automatically be executed and its results returned to you. The user does not need to do anything for this.`; - break; - - case VisualizeESQLUserIntention.generateQueryOnly: - userIntentionMessage = `Any generated query will not be executed automatically, the user needs to do this themselves.`; - break; - - default: - userIntentionMessage = `The generated query will automatically be visualized to the user, displayed below your message. The user does not need to do anything for this.`; - break; - } - - const queryFunctionResponseMessage = createFunctionResponseMessage({ - name: QUERY_FUNCTION_NAME, - content: {}, - data: { - // add the included docs for debugging - documentation: { - intention: args.intention, - keywords, - files: messagesToInclude, - }, - }, - }); + }; + } - const esqlResponse$ = await chat('answer_esql_question', { - messages: [ - ...withEsqlSystemMessage().concat(queryFunctionResponseMessage.message), - { - '@timestamp': new Date().toISOString(), - message: { - role: MessageRole.Assistant, - content: '', - function_call: { - name: 'get_esql_info', - arguments: JSON.stringify(args), + const fnCall = event.toolCalls[0] + ? { + name: event.toolCalls[0].function.name, + arguments: JSON.stringify(event.toolCalls[0].function.arguments), trigger: MessageRole.Assistant as const, - }, - }, - }, - { - '@timestamp': new Date().toISOString(), - message: { - role: MessageRole.User, - name: 'get_esql_info', - content: JSON.stringify({ - documentation: messagesToInclude, - }), - }, - }, - { - '@timestamp': new Date().toISOString(), - message: { - role: MessageRole.Assistant, - content: 'Thank you for providing the ES|QL info. What can I help you with?', - }, - }, - { - '@timestamp': new Date().toISOString(), - message: { - role: MessageRole.User, - content: `Answer the user's question that was previously asked ("${abbreviatedUserQuestion}...") using the attached documentation. Take into account any previous errors from the \`${EXECUTE_QUERY_NAME}\` or \`visualize_query\` function. - - Format any ES|QL query as follows: - \`\`\`esql - - \`\`\` - - Respond in plain text. Do not attempt to use a function. - - You must use commands and functions for which you have requested documentation. - - ${ - args.intention !== VisualizeESQLUserIntention.generateQueryOnly - ? `DO NOT UNDER ANY CIRCUMSTANCES generate more than a single query. - If multiple queries are needed, do it as a follow-up step. Make this clear to the user. For example: - - Human: plot both yesterday's and today's data. - - Assistant: Here's how you can plot yesterday's data: - \`\`\`esql - - \`\`\` - - Let's see that first. We'll look at today's data next. - - Human: - - Assistant: Let's look at today's data: - - \`\`\`esql - - \`\`\` - ` - : '' - } - - ${userIntentionMessage} - - DO NOT UNDER ANY CIRCUMSTANCES use commands or functions that are not a capability of ES|QL - as mentioned in the system message and documentation. When converting queries from one language - to ES|QL, make sure that the functions are available and documented in ES|QL. - E.g., for SPL's LEN, use LENGTH. For IF, use CASE. - - `, - }, - }, - ], - signal, - functions: functions.getActions(), - }); - - return esqlResponse$.pipe( - emitWithConcatenatedMessage(async (msg) => { - msg.message.content = msg.message.content.replaceAll( - INLINE_ESQL_QUERY_REGEX, - (_match, query) => { - const correction = correctCommonEsqlMistakes(query); - if (correction.isCorrection) { - resources.logger.debug( - `Corrected query, from: \n${correction.input}\nto:\n${correction.output}` - ); } - return '```esql\n' + correction.output + '\n```'; - } - ); - - if (msg.message.function_call.name) { - return msg; - } + : undefined; - const esqlQuery = msg.message.content.match( - new RegExp(INLINE_ESQL_QUERY_REGEX, 'ms') - )?.[1]; - - let functionCall: ConcatenatedMessage['message']['function_call'] | undefined; - - if ( - !args.intention || - !esqlQuery || - args.intention === VisualizeESQLUserIntention.generateQueryOnly - ) { - functionCall = undefined; - } else if (args.intention === VisualizeESQLUserIntention.executeAndReturnResults) { - functionCall = { - name: EXECUTE_QUERY_NAME, - arguments: JSON.stringify({ query: esqlQuery }), - trigger: MessageRole.Assistant as const, - }; - } else { - functionCall = { - name: 'visualize_query', - arguments: JSON.stringify({ query: esqlQuery, intention: args.intention }), - trigger: MessageRole.Assistant as const, - }; - } - - return { - ...msg, + const messageAddEvent: MessageAddEvent = { + type: StreamingChatResponseEventType.MessageAdd, + id: chatMessageId, message: { - ...msg.message, - ...(functionCall - ? { - function_call: functionCall, - } - : {}), + '@timestamp': new Date().toISOString(), + message: { + content: event.content, + role: MessageRole.Assistant, + function_call: fnCall, + }, }, }; - }), - startWith(queryFunctionResponseMessage) + + return messageAddEvent; + }) ); } ); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/validate_esql_query.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/validate_esql_query.ts index 0b72dfd1de32d..ac26846f940e6 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/validate_esql_query.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/validate_esql_query.ts @@ -11,7 +11,7 @@ import type { ElasticsearchClient } from '@kbn/core/server'; import { ESQLSearchResponse, ESQLRow } from '@kbn/es-types'; import { esFieldTypeToKibanaFieldType } from '@kbn/field-types'; import { DatatableColumn, DatatableColumnType } from '@kbn/expressions-plugin/common'; -import { splitIntoCommands } from './correct_common_esql_mistakes'; +import { splitIntoCommands } from '@kbn/inference-plugin/common'; export async function runAndValidateEsqlQuery({ query, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/types.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/types.ts index 30cce3f0b37b3..320459d203f9e 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/types.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/types.ts @@ -35,6 +35,7 @@ import type { import type { CloudSetup, CloudStart } from '@kbn/cloud-plugin/server'; import type { SecurityPluginSetup, SecurityPluginStart } from '@kbn/security-plugin/server'; import type { ObservabilityPluginSetup } from '@kbn/observability-plugin/server'; +import type { InferenceServerStart, InferenceServerSetup } from '@kbn/inference-plugin/server'; // eslint-disable-next-line @typescript-eslint/no-empty-interface export interface ObservabilityAIAssistantAppServerStart {} @@ -53,6 +54,7 @@ export interface ObservabilityAIAssistantAppPluginStartDependencies { dataViews: DataViewsServerPluginStart; cloud?: CloudStart; serverless?: ServerlessPluginStart; + inference: InferenceServerStart; } export interface ObservabilityAIAssistantAppPluginSetupDependencies { @@ -68,4 +70,5 @@ export interface ObservabilityAIAssistantAppPluginSetupDependencies { observability: ObservabilityPluginSetup; cloud?: CloudSetup; serverless?: ServerlessPluginSetup; + inference: InferenceServerSetup; } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/tsconfig.json b/x-pack/plugins/observability_solution/observability_ai_assistant_app/tsconfig.json index 282363e50ec3f..bc92d37d3cd70 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/tsconfig.json +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/tsconfig.json @@ -70,7 +70,8 @@ "@kbn/cloud-plugin", "@kbn/observability-plugin", "@kbn/esql-datagrid", - "@kbn/alerting-comparators" + "@kbn/alerting-comparators", + "@kbn/inference-plugin" ], "exclude": ["target/**/*"] }