From 1eae619beafd7b443bc9722ae44cc882fca7f755 Mon Sep 17 00:00:00 2001 From: Drew Tate Date: Thu, 21 Mar 2024 23:01:48 +0100 Subject: [PATCH] [ES|QL] grammar sync job (#178347) ## Summary Introduces a CI job to check for changes to the Elasticsearch grammar. Part of https://github.com/elastic/kibana/issues/178262 The first time this job runs, it will result in a PR to update the grammar because of formatting differences. That should be merged. Then, it will only create a PR when something has changed on the Elasticsearch side. --------- Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .buildkite/pipelines/esql_grammar_sync.yml | 6 + .buildkite/scripts/steps/esql_grammar_sync.sh | 128 ++++++++++++++++++ .github/CODEOWNERS | 2 + .../kbn-monaco/src/esql/antlr/esql_parser.g4 | 2 +- 4 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 .buildkite/pipelines/esql_grammar_sync.yml create mode 100644 .buildkite/scripts/steps/esql_grammar_sync.sh diff --git a/.buildkite/pipelines/esql_grammar_sync.yml b/.buildkite/pipelines/esql_grammar_sync.yml new file mode 100644 index 0000000000000..f516d61ad6b4a --- /dev/null +++ b/.buildkite/pipelines/esql_grammar_sync.yml @@ -0,0 +1,6 @@ +steps: + - command: .buildkite/scripts/steps/esql_grammar_sync.sh + label: ES|QL Grammar Sync + timeout_in_minutes: 10 + agents: + queue: n2-2-spot diff --git a/.buildkite/scripts/steps/esql_grammar_sync.sh b/.buildkite/scripts/steps/esql_grammar_sync.sh new file mode 100644 index 0000000000000..f0f3ad80e05e3 --- /dev/null +++ b/.buildkite/scripts/steps/esql_grammar_sync.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +set -euo pipefail + +synchronize_lexer_grammar () { + license_header="$1" + source_file="$PARENT_DIR/elasticsearch/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4" + destination_file="./packages/kbn-monaco/src/esql/antlr/esql_lexer.g4" + + # Copy the file + cp "$source_file" "$destination_file" + + # Insert the license header + temp_file=$(mktemp) + printf "%s\n\n// DO NOT MODIFY THIS FILE BY HAND. IT IS MANAGED BY A CI JOB.\n\n%s" "$license_header" "$(cat $destination_file)" > "$temp_file" + mv "$temp_file" "$destination_file" + + # Replace the line containing "lexer grammar" with "lexer grammar esql_lexer;" + sed -i -e 's/lexer grammar.*$/lexer grammar esql_lexer;/' "$destination_file" + + # Insert "options { caseInsensitive = true; }" one line below + sed -i -e '/lexer grammar esql_lexer;/a\ + options { caseInsensitive = true; }' "$destination_file" + + echo "File copied and modified successfully." +} + +synchronize_parser_grammar () { + license_header="$1" + source_file="$PARENT_DIR/elasticsearch/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4" + destination_file="./packages/kbn-monaco/src/esql/antlr/esql_parser.g4" + + # Copy the file + cp "$source_file" "$destination_file" + + # Insert the license header + temp_file=$(mktemp) + printf "%s\n\n// DO NOT MODIFY THIS FILE BY HAND. IT IS MANAGED BY A CI JOB.\n\n%s" "$license_header" "$(cat ${destination_file})" > "$temp_file" + mv "$temp_file" "$destination_file" + + # Replace the line containing "parser grammar" with "parser grammar esql_parser;" + sed -i -e 's/parser grammar.*$/parser grammar esql_parser;/' "$destination_file" + + # Replace options {tokenVocab=EsqlBaseLexer;} with options {tokenVocab=esql_lexer;} + sed -i -e 's/options {tokenVocab=EsqlBaseLexer;}/options {tokenVocab=esql_lexer;}/' "$destination_file" + + echo "File copied and modified successfully." +} + +report_main_step () { + echo "--- $1" +} + +main () { + cd "$PARENT_DIR" + + report_main_step "Cloning repositories" + + rm -rf elasticsearch + git clone https://github.com/elastic/elasticsearch --depth 1 + + rm -rf open-source + git clone https://github.com/elastic/open-source --depth 1 + + cd "$KIBANA_DIR" + + license_header=$(cat "$PARENT_DIR/open-source/legal/elastic-license-2.0-header.txt") + + report_main_step "Synchronizing lexer grammar..." + synchronize_lexer_grammar "$license_header" + + report_main_step "Synchronizing parser grammar..." + synchronize_parser_grammar "$license_header" + + # Check for differences + set +e + git diff --exit-code --quiet "$destination_file" + if [ $? -eq 0 ]; then + echo "No differences found. Our work is done here." + exit + fi + set -e + + report_main_step "Differences found. Checking for an existing pull request." + + KIBANA_MACHINE_USERNAME="kibanamachine" + git config --global user.name "$KIBANA_MACHINE_USERNAME" + git config --global user.email '42973632+kibanamachine@users.noreply.github.com' + + PR_TITLE='[ES|QL] Update grammars' + PR_BODY='This PR updates the ES|QL grammars (lexer and parser) to match the latest version in Elasticsearch.' + + # Check if a PR already exists + pr_search_result=$(gh pr list --search "$PR_TITLE" --state open --author "$KIBANA_MACHINE_USERNAME" --limit 1 --json title -q ".[].title") + + if [ "$pr_search_result" == "$PR_TITLE" ]; then + echo "PR already exists. Exiting." + exit + fi + + echo "No existing PR found. Proceeding." + + report_main_step "Building ANTLR artifacts." + + # Bootstrap Kibana + .buildkite/scripts/bootstrap.sh + + # Build ANTLR stuff + cd ./packages/kbn-monaco/src + yarn build:antlr4:esql + + # Make a commit + BRANCH_NAME="esql_grammar_sync_$(date +%s)" + + git checkout -b "$BRANCH_NAME" + + git add -A + git commit -m "Update ES|QL grammars" + + report_main_step "Changes committed. Creating pull request." + + git remote add kibanamachine https://github.com/kibanamachine/kibana.git + git push kibanamachine "$BRANCH_NAME" + + # Create a PR + gh pr create --draft --title "$PR_TITLE" --body "$PR_BODY" --base main --head "kibanamachine:${BRANCH_NAME}" --label 'release_note:skip' --label 'Team:Visualizations' +} + +main diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ef24822f6ce5a..8f50910f78f03 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1175,6 +1175,8 @@ x-pack/plugins/observability_solution/infra/server/lib/alerting @elastic/obs-ux- /.bazelversion @elastic/kibana-operations /WORKSPACE.bazel @elastic/kibana-operations /.buildkite/ @elastic/kibana-operations +/.buildkite/scripts/steps/esql_grammar_sync.sh @elastic/kibana-visualizations +/.buildkite/pipelines/esql_grammar_sync.yml @elastic/kibana-visualizations /kbn_pm/ @elastic/kibana-operations /x-pack/dev-tools @elastic/kibana-operations /catalog-info.yaml @elastic/kibana-operations @elastic/kibana-tech-leads diff --git a/packages/kbn-monaco/src/esql/antlr/esql_parser.g4 b/packages/kbn-monaco/src/esql/antlr/esql_parser.g4 index ede884d6311d7..0c7cf96f5a2e1 100644 --- a/packages/kbn-monaco/src/esql/antlr/esql_parser.g4 +++ b/packages/kbn-monaco/src/esql/antlr/esql_parser.g4 @@ -259,4 +259,4 @@ enrichCommand enrichWithClause : (newName=qualifiedNamePattern ASSIGN)? enrichField=qualifiedNamePattern - ; + ; \ No newline at end of file