semantic_text as string type in ES|QL - support for functions and ope…

…rators (#115243)

* fix tests

* Add CSV tests

* Add function tests

* Refactor tests

* spotless

* Use DataType.stringTypes() where possible

* Add tests for conditional functions and expressions

* Fix tests after merge

* Reorder semantic_text evaluators and tests

* Re-ordered two more places for SEMANTIC_TEXT after TEXT

---------

Co-authored-by: Elastic Machine <[email protected]>
Co-authored-by: Craig Taverner <[email protected]>

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java

@@ -370,6 +370,9 @@ public static boolean isUnsupported(DataType from) {
     }
 
     public static boolean isString(DataType t) {
+        if (EsqlCorePlugin.SEMANTIC_TEXT_FEATURE_FLAG.isEnabled() && t == SEMANTIC_TEXT) {
+            return true;
+        }
         return t == KEYWORD || t == TEXT;
     }
 
@@ -585,7 +588,7 @@ static Builder builder() {
     }
 
     public DataType noText() {
-        return this == TEXT ? KEYWORD : this;
+        return isString(this) ? KEYWORD : this;
     }
 
     /**

x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-semantic_text.json

@@ -68,6 +68,10 @@
     },
     "value": {
       "type": "long"
+    },
+    "st_base64": {
+      "type": "semantic_text",
+      "inference_id": "test_sparse_inference"
     }
   }
 }

x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv

@@ -1,4 +1,4 @@
-_id:keyword,semantic_text_field:semantic_text,st_bool:semantic_text,st_cartesian_point:semantic_text,st_cartesian_shape:semantic_text,st_datetime:semantic_text,st_double:semantic_text,st_geopoint:semantic_text,st_geoshape:semantic_text,st_integer:semantic_text,st_ip:semantic_text,st_long:semantic_text,st_unsigned_long:semantic_text,st_version:semantic_text,st_multi_value:semantic_text,st_unicode:semantic_text,host:keyword,description:text,value:long
-1,live long and prosper,false,"POINT(4297.11 -1475.53)",,1953-09-02T00:00:00.000Z,5.20128E11,"POINT(42.97109630194 14.7552534413725)","POLYGON ((30 10\, 40 40\, 20 40\, 10 20\, 30 10))",23,1.1.1.1,2147483648,2147483648,1.2.3,["Hello there!", "This is a random value", "for testing purposes"],你吃饭了吗,"host1","some description1",1001
-2,all we have to decide is what to do with the time that is given to us,true,"POINT(7580.93 2272.77)",,2023-09-24T15:57:00.000Z,4541.11,"POINT(37.97109630194 21.7552534413725)","POLYGON ((30 10\, 40 40\, 20 40\, 10 20\, 30 10))",122,1.1.2.1,123,2147483648.2,9.0.0,["nice to meet you", "bye bye!"],["谢谢", "对不起我的中文不好"],"host2","some description2",1002
-3,be excellent to each other,,,,,,,,,,,,,,,"host3","some description3",1003
+_id:keyword,semantic_text_field:semantic_text,st_bool:semantic_text,st_cartesian_point:semantic_text,st_cartesian_shape:semantic_text,st_datetime:semantic_text,st_double:semantic_text,st_geopoint:semantic_text,st_geoshape:semantic_text,st_integer:semantic_text,st_ip:semantic_text,st_long:semantic_text,st_unsigned_long:semantic_text,st_version:semantic_text,st_multi_value:semantic_text,st_unicode:semantic_text,host:keyword,description:text,value:long,st_base64:semantic_text
+1,live long and prosper,false,"POINT(4297.11 -1475.53)",,1953-09-02T00:00:00.000Z,5.20128E11,"POINT(42.97109630194 14.7552534413725)","POLYGON ((30 10\, 40 40\, 20 40\, 10 20\, 30 10))",23,1.1.1.1,2147483648,2147483648,1.2.3,["Hello there!", "This is a random value", "for testing purposes"],你吃饭了吗,"host1","some description1",1001,ZWxhc3RpYw==
+2,all we have to decide is what to do with the time that is given to us,true,"POINT(7580.93 2272.77)",,2023-09-24T15:57:00.000Z,4541.11,"POINT(37.97109630194 21.7552534413725)","POLYGON ((30 10\, 40 40\, 20 40\, 10 20\, 30 10))",122,1.1.2.1,123,2147483648.2,9.0.0,["nice to meet you", "bye bye!"],["谢谢", "对不起我的中文不好"],"host2","some description2",1002,aGVsbG8=
+3,be excellent to each other,,,,,,,,,,,,,,,"host3","some description3",1003,