Merge branch 'main' into apm-downsample

danielmitterdorfer · Jan 26, 2024 · 72f8cf7 · 72f8cf7
2 parents 6b78ddc + d876ec7
commit 72f8cf7
Show file tree

Hide file tree

Showing 26 changed files with 2,262 additions and 766 deletions.
diff --git a/docs/changelog/103656.yaml b/docs/changelog/103656.yaml
@@ -0,0 +1,5 @@
+pr: 103656
+summary: "ESQL: add =~ operator (case insensitive equality)"
+area: ES|QL
+type: feature
+issues: []
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec
@@ -315,7 +315,7 @@ from employees | where birth_date > now() | sort emp_no asc | keep emp_no, birth
 emp_no:integer  | birth_date:date
 ;
 
-autoBucketYearInAgg
+autoBucketYearInAgg#[skip:-8.12.99, reason:date type is supported in 8.13]
 FROM employees
 | WHERE hire_date >= "1999-01-01T00:00:00Z"
 | EVAL bucket = AUTO_BUCKET(hire_date, 5, "1999-01-01T00:00:00Z", NOW())
@@ -917,7 +917,7 @@ FROM employees
 //end::docsAutoBucketWeeklyHistogram-result[]
 ;
 
-docsAutoBucketLast24hr
+docsAutoBucketLast24hr#[skip:-8.12.99, reason:date type is supported in 8.13]
 //tag::docsAutoBucketLast24hr[]
 FROM sample_data 
 | WHERE @timestamp >= NOW() - 1 day and @timestamp < NOW()
@@ -929,7 +929,7 @@ FROM sample_data
    COUNT(*):long    |    bucket:date
 ;
 
-docsGettingStartedAutoBucket
+docsGettingStartedAutoBucket#[skip:-8.12.99, reason:date type is supported in 8.13]
 // tag::gs-auto_bucket[]
 FROM sample_data
 | KEEP @timestamp

diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ignore-case.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ignore-case.csv-spec
@@ -0,0 +1,135 @@
+
+simpleFilter#[skip:-8.12.99, reason:case insensitive operators implemented in v 8.13]
+from employees | where first_name =~ "mary" | keep emp_no, first_name, last_name;
+
+emp_no:integer | first_name:keyword | last_name:keyword
+10011          | Mary               | Sluis
+;
+
+
+simpleFilterUpper#[skip:-8.12.99, reason:case insensitive operators implemented in v 8.13]
+from employees | where first_name =~ "MARY" | keep emp_no, first_name, last_name;
+
+emp_no:integer | first_name:keyword | last_name:keyword
+10011          | Mary               | Sluis
+;
+
+
+simpleFilterPartial#[skip:-8.12.99, reason:case insensitive operators implemented in v 8.13]
+from employees | where first_name =~ "mar" | keep emp_no, first_name, last_name;
+
+emp_no:integer | first_name:keyword | last_name:keyword
+;
+
+
+mixedConditionsAnd#[skip:-8.12.99, reason:case insensitive operators implemented in v 8.13]
+from employees | where first_name =~ "mary" AND emp_no == 10011 | keep emp_no, first_name, last_name;
+
+emp_no:integer | first_name:keyword | last_name:keyword
+10011          | Mary               | Sluis
+;
+
+
+mixedConditionsOr#[skip:-8.12.99, reason:case insensitive operators implemented in v 8.13]
+from employees | where first_name =~ "mary" OR emp_no == 10001 | keep emp_no, first_name, last_name |sort emp_no;
+
+emp_no:integer | first_name:keyword | last_name:keyword
+10001          | Georgi             | Facello
+10011          | Mary               | Sluis
+;
+
+
+evalEquals#[skip:-8.12.99, reason:case insensitive operators implemented in v 8.13]
+from employees | where emp_no == 10001 
+| eval a = first_name =~ "georgi", b = first_name == "georgi", c = first_name =~ "GEORGI", d = first_name =~ "Geor", e = first_name =~ "GeoRgI" 
+| keep emp_no, first_name, a, b, c, d, e;
+
+emp_no:integer | first_name:keyword | a:boolean | b:boolean | c:boolean | d:boolean | e:boolean
+10001          | Georgi             | true      | false     | true      | false     | true
+;
+
+
+//waiting for final decisions on supporting generic expressions on the right
+//https://github.com/elastic/elasticsearch/issues/103599
+constantsAndFolding-Ignore
+row name = "foobar" | where "FoObAr" =~ name;
+
+name:keyword
+foobar
+;
+
+
+noWildcardSimple#[skip:-8.12.99, reason:case insensitive operators implemented in v 8.13]
+row name = "foobar" | where name =~ "FoOb*";
+
+name:keyword
+;
+
+
+noWildcard#[skip:-8.12.99, reason:case insensitive operators implemented in v 8.13]
+from employees | where first_name =~ "Georg*" | sort emp_no | keep emp_no, first_name;
+
+emp_no:integer | first_name:keyword
+;
+
+
+noWildcardSingle#[skip:-8.12.99, reason:case insensitive operators implemented in v 8.13]
+from employees | where first_name =~ "Georg?" | sort emp_no | keep emp_no, first_name;
+
+emp_no:integer | first_name:keyword
+;
+
+
+//waiting for final decisions on supporting generic expressions on the right
+//https://github.com/elastic/elasticsearch/issues/103599
+fieldRight-Ignore
+from employees | where "Guoxiang" =~ first_name | keep emp_no, first_name;
+
+emp_no:integer  | first_name:keyword
+10015           | Guoxiang
+;
+
+
+expressionsRight#[skip:-8.12.99, reason:case insensitive operators implemented in v 8.13]
+from employees | where first_name =~ concat("Tzv","ETAN") | keep emp_no, first_name;
+
+emp_no:integer  | first_name:keyword
+10007           | Tzvetan
+;
+
+
+expressionsLeft#[skip:-8.12.99, reason:case insensitive operators implemented in v 8.13]
+from employees | where concat(first_name, "_foo") =~ "TzvETAN_fOo" | keep emp_no, first_name;
+
+emp_no:integer  | first_name:keyword
+10007           | Tzvetan
+;
+
+
+//waiting for final decisions on supporting generic expressions on the right
+//https://github.com/elastic/elasticsearch/issues/103599
+expressionsLeftRight-Ignore
+from employees | where substring(first_name, 1, 2) =~ substring(last_name, -2) | keep emp_no, first_name, last_name | sort emp_no;
+
+emp_no:integer  | first_name:keyword    | last_name:keyword
+10055           | Georgy                | Dredge
+10091           | Amabile               | Gomatam
+;
+
+
+multiValuesExcluded#[skip:-8.12.99, reason:case insensitive operators implemented in v 8.13]
+row a = ["Foo", "Bar"] | where a =~ "foo";
+
+a:keyword
+;
+
+
+multiValuesPushedDownExcluded#[skip:-8.12.99, reason:case insensitive operators implemented in v 8.13]
+from employees | where job_positions =~ "reporting analyst" | sort emp_no | keep emp_no, job_positions;
+warning:Line 1:24: evaluation of [job_positions =~ \"reporting analyst\"] failed, treating result as null. Only first 20 failures recorded.
+warning:Line 1:24: java.lang.IllegalArgumentException: single-value function encountered multi-value
+
+emp_no:integer  | job_positions:keyword
+10013           | Reporting Analyst
+10026           | Reporting Analyst
+;
diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4
@@ -130,6 +130,7 @@ RP : ')';
 TRUE : 'true';
 
 EQ  : '==';
+CIEQ  : '=~';
 NEQ : '!=';
 LT  : '<';
 LTE : '<=';

diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens
@@ -0,0 +1,166 @@
+DISSECT=1
+DROP=2
+ENRICH=3
+EVAL=4
+EXPLAIN=5
+FROM=6
+GROK=7
+INLINESTATS=8
+KEEP=9
+LIMIT=10
+MV_EXPAND=11
+PROJECT=12
+RENAME=13
+ROW=14
+SHOW=15
+SORT=16
+STATS=17
+WHERE=18
+UNKNOWN_CMD=19
+LINE_COMMENT=20
+MULTILINE_COMMENT=21
+WS=22
+EXPLAIN_WS=23
+EXPLAIN_LINE_COMMENT=24
+EXPLAIN_MULTILINE_COMMENT=25
+PIPE=26
+STRING=27
+INTEGER_LITERAL=28
+DECIMAL_LITERAL=29
+BY=30
+AND=31
+ASC=32
+ASSIGN=33
+COMMA=34
+DESC=35
+DOT=36
+FALSE=37
+FIRST=38
+LAST=39
+LP=40
+IN=41
+IS=42
+LIKE=43
+NOT=44
+NULL=45
+NULLS=46
+OR=47
+PARAM=48
+RLIKE=49
+RP=50
+TRUE=51
+EQ=52
+CIEQ=53
+NEQ=54
+LT=55
+LTE=56
+GT=57
+GTE=58
+PLUS=59
+MINUS=60
+ASTERISK=61
+SLASH=62
+PERCENT=63
+OPENING_BRACKET=64
+CLOSING_BRACKET=65
+UNQUOTED_IDENTIFIER=66
+QUOTED_IDENTIFIER=67
+EXPR_LINE_COMMENT=68
+EXPR_MULTILINE_COMMENT=69
+EXPR_WS=70
+METADATA=71
+FROM_UNQUOTED_IDENTIFIER=72
+FROM_LINE_COMMENT=73
+FROM_MULTILINE_COMMENT=74
+FROM_WS=75
+UNQUOTED_ID_PATTERN=76
+PROJECT_LINE_COMMENT=77
+PROJECT_MULTILINE_COMMENT=78
+PROJECT_WS=79
+AS=80
+RENAME_LINE_COMMENT=81
+RENAME_MULTILINE_COMMENT=82
+RENAME_WS=83
+ON=84
+WITH=85
+ENRICH_POLICY_NAME=86
+ENRICH_LINE_COMMENT=87
+ENRICH_MULTILINE_COMMENT=88
+ENRICH_WS=89
+ENRICH_FIELD_LINE_COMMENT=90
+ENRICH_FIELD_MULTILINE_COMMENT=91
+ENRICH_FIELD_WS=92
+MVEXPAND_LINE_COMMENT=93
+MVEXPAND_MULTILINE_COMMENT=94
+MVEXPAND_WS=95
+INFO=96
+FUNCTIONS=97
+SHOW_LINE_COMMENT=98
+SHOW_MULTILINE_COMMENT=99
+SHOW_WS=100
+COLON=101
+SETTING=102
+SETTING_LINE_COMMENT=103
+SETTTING_MULTILINE_COMMENT=104
+SETTING_WS=105
+'dissect'=1
+'drop'=2
+'enrich'=3
+'eval'=4
+'explain'=5
+'from'=6
+'grok'=7
+'inlinestats'=8
+'keep'=9
+'limit'=10
+'mv_expand'=11
+'project'=12
+'rename'=13
+'row'=14
+'show'=15
+'sort'=16
+'stats'=17
+'where'=18
+'|'=26
+'by'=30
+'and'=31
+'asc'=32
+'='=33
+','=34
+'desc'=35
+'.'=36
+'false'=37
+'first'=38
+'last'=39
+'('=40
+'in'=41
+'is'=42
+'like'=43
+'not'=44
+'null'=45
+'nulls'=46
+'or'=47
+'?'=48
+'rlike'=49
+')'=50
+'true'=51
+'=='=52
+'=~'=53
+'!='=54
+'<'=55
+'<='=56
+'>'=57
+'>='=58
+'+'=59
+'-'=60
+'*'=61
+'/'=62
+'%'=63
+']'=65
+'metadata'=71
+'as'=80
+'on'=84
+'with'=85
+'info'=96
+'functions'=97
+':'=101
diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4
@@ -225,7 +225,7 @@ string
     ;
 
 comparisonOperator
-    : EQ | NEQ | LT | LTE | GT | GTE
+    : EQ | CIEQ | NEQ | LT | LTE | GT | GTE
     ;
 
 explainCommand