illegal_argument_exception when using scripted fields.

[stacey-gammon]

There is a bug in the Kibana Repo that looks like it might be exposing an ES bug. Could someone take a look? elastic/kibana#8404

Here is the relevant request/response that is triggering the error, copied from the above issue:

POST logstash-0/_search
{  
   "size":500,
   "sort":[  
      {  
         "@timestamp":{  
            "order":"desc",
            "unmapped_type":"boolean"
         }
      }
   ],
   "highlight":{  
      "pre_tags":[  
         "@kibana-highlighted-field@"
      ],
      "post_tags":[  
         "@/kibana-highlighted-field@"
      ],
      "fields":{  
         "*":{  

         }
      },
      "require_field_match":false,
      "fragment_size":2147483647
   },
   "query":{  
      "bool":{  
         "must":[  
            {  
               "query_string":{  
                  "query":"*",
                  "analyze_wildcard":true
               }
            },
            {  
               "script":{  
                  "script":{  
                     "inline":"(-1 * doc['bytes'].value) == value",
                     "lang":"painless",
                     "params":{  
                        "value":0
                     }
                  }
               }
            },
            {  
               "script":{  
                  "script":{  
                     "inline":"(-1 * doc['bytes'].value) == value",
                     "lang":"painless",
                     "params":{  
                        "value":-2724
                     }
                  }
               }
            },
            {  
               "range":{  
                  "@timestamp":{  
                     "gte":1474479018497,
                     "lte":1474479918497,
                     "format":"epoch_millis"
                  }
               }
            }
         ],
         "must_not":[  

         ]
      }
   },
   "aggs":{  
      "2":{  
         "date_histogram":{  
            "field":"@timestamp",
            "interval":"30s",
            "time_zone":"America/New_York",
            "min_doc_count":1
         }
      }
   },
   "stored_fields":[  
      "*"
   ],
   "_source":true,
   "script_fields":{  
      "inverse bytes":{  
         "script":{  
            "inline":"-1 * doc['bytes'].value",
            "lang":"painless"
         }
      }
   },
   "docvalue_fields":[  
      "utc_time",
      "relatedContent.article:modified_time",
      "@timestamp",
      "relatedContent.article:published_time"
   ]
}

Response:

{  
   "error":{  
      "root_cause":[  
         {  
            "type":"script_exception",
            "reason":"compile error",
            "script_stack":[  
               "... * doc['bytes'].value) == value",
               "                             ^---- HERE"
            ],
            "script":"(-1 * doc['bytes'].value) == value",
            "lang":"painless"
         }
      ],
      "type":"search_phase_execution_exception",
      "reason":"all shards failed",
      "phase":"query_fetch",
      "grouped":true,
      "failed_shards":[  
         {  
            "shard":0,
            "index":"logstash-0",
            "node":"apPgvtcuTbSA2V4HWRUWww",
            "reason":{  
               "type":"query_shard_exception",
               "reason":"failed to create query: {\n  \"bool\" : {\n    \"must\" : [\n      {\n        \"query_string\" : {\n          \"query\" : \"*\",\n          \"fields\" : [ ],\n          \"use_dis_max\" : true,\n          \"tie_breaker\" : 0.0,\n          \"default_operator\" : \"or\",\n          \"auto_generate_phrase_queries\" : false,\n          \"max_determined_states\" : 10000,\n          \"lowercase_expanded_terms\" : true,\n          \"enable_position_increment\" : true,\n          \"fuzziness\" : \"AUTO\",\n          \"fuzzy_prefix_length\" : 0,\n          \"fuzzy_max_expansions\" : 50,\n          \"phrase_slop\" : 0,\n          \"analyze_wildcard\" : true,\n          \"locale\" : \"und\",\n          \"escape\" : false,\n          \"boost\" : 1.0\n        }\n      },\n      {\n        \"script\" : {\n          \"script\" : {\n            \"inline\" : \"(-1 * doc['bytes'].value) == value\",\n            \"lang\" : \"painless\",\n            \"params\" : {\n              \"value\" : 0\n            }\n          },\n          \"boost\" : 1.0\n        }\n      },\n      {\n        \"script\" : {\n          \"script\" : {\n            \"inline\" : \"(-1 * doc['bytes'].value) == value\",\n            \"lang\" : \"painless\",\n            \"params\" : {\n              \"value\" : -2724\n            }\n          },\n          \"boost\" : 1.0\n        }\n      },\n      {\n        \"range\" : {\n          \"@timestamp\" : {\n            \"from\" : 1474479018497,\n            \"to\" : 1474479918497,\n            \"include_lower\" : true,\n            \"include_upper\" : true,\n            \"format\" : \"epoch_millis\",\n            \"boost\" : 1.0\n          }\n        }\n      }\n    ],\n    \"disable_coord\" : false,\n    \"adjust_pure_negative\" : true,\n    \"boost\" : 1.0\n  }\n}",
               "index_uuid":"2Gr8UT6HTTCyaJWywRhrCQ",
               "index":"logstash-0",
               "caused_by":{  
                  "type":"script_exception",
                  "reason":"compile error",
                  "caused_by":{  
                     "type":"illegal_argument_exception",
                     "reason":"Variable [value] is not defined."
                  },
                  "script_stack":[  
                     "... * doc['bytes'].value) == value",
                     "                             ^---- HERE"
                  ],
                  "script":"(-1 * doc['bytes'].value) == value",
                  "lang":"painless"
               }
            }
         }
      ],
      "caused_by":{  
         "type":"query_shard_exception",
         "reason":"failed to create query: {\n  \"bool\" : {\n    \"must\" : [\n      {\n        \"query_string\" : {\n          \"query\" : \"*\",\n          \"fields\" : [ ],\n          \"use_dis_max\" : true,\n          \"tie_breaker\" : 0.0,\n          \"default_operator\" : \"or\",\n          \"auto_generate_phrase_queries\" : false,\n          \"max_determined_states\" : 10000,\n          \"lowercase_expanded_terms\" : true,\n          \"enable_position_increment\" : true,\n          \"fuzziness\" : \"AUTO\",\n          \"fuzzy_prefix_length\" : 0,\n          \"fuzzy_max_expansions\" : 50,\n          \"phrase_slop\" : 0,\n          \"analyze_wildcard\" : true,\n          \"locale\" : \"und\",\n          \"escape\" : false,\n          \"boost\" : 1.0\n        }\n      },\n      {\n        \"script\" : {\n          \"script\" : {\n            \"inline\" : \"(-1 * doc['bytes'].value) == value\",\n            \"lang\" : \"painless\",\n            \"params\" : {\n              \"value\" : 0\n            }\n          },\n          \"boost\" : 1.0\n        }\n      },\n      {\n        \"script\" : {\n          \"script\" : {\n            \"inline\" : \"(-1 * doc['bytes'].value) == value\",\n            \"lang\" : \"painless\",\n            \"params\" : {\n              \"value\" : -2724\n            }\n          },\n          \"boost\" : 1.0\n        }\n      },\n      {\n        \"range\" : {\n          \"@timestamp\" : {\n            \"from\" : 1474479018497,\n            \"to\" : 1474479918497,\n            \"include_lower\" : true,\n            \"include_upper\" : true,\n            \"format\" : \"epoch_millis\",\n            \"boost\" : 1.0\n          }\n        }\n      }\n    ],\n    \"disable_coord\" : false,\n    \"adjust_pure_negative\" : true,\n    \"boost\" : 1.0\n  }\n}",
         "index_uuid":"2Gr8UT6HTTCyaJWywRhrCQ",
         "index":"logstash-0",
         "caused_by":{  
            "type":"script_exception",
            "reason":"compile error",
            "caused_by":{  
               "type":"illegal_argument_exception",
               "reason":"Variable [value] is not defined."
            },
            "script_stack":[  
               "... * doc['bytes'].value) == value",
               "                             ^---- HERE"
            ],
            "script":"(-1 * doc['bytes'].value) == value",
            "lang":"painless"
         }
      }
   },
   "status":400
}

[clintongormley]

@stacey-gammon In painless, params need to be referenced as params.value, not just value

[stacey-gammon]

Thanks for the update @clintongormley. To confirm, this is not the case with groovy as the selected language? That one appears to break when I use params.value, but not if I just use value.

Error: Request to Elasticsearch failed: {"error":{"root_cause":[{"type":"script_exception","reason":"Error evaluating (doc[\"bytes\"].value * -1) == params.value","script_stack":[],"script":"","lang":"groovy"}],"type":"search_phase_execution_exception","reason":"all shards failed","phase":"query_fetch","grouped":true,"failed_shards":[{"shard":0,"index":"logstash-0","node":"eNSJAi1hRPW6RmxR14LeUA","reason":{"type":"script_exception","reason":"Error evaluating (doc[\"bytes\"].value * -1) == params.value","caused_by":{"type":"missing_property_exception","reason":"No such property: params for class: fd86f5f683b7e517cd8443c5efe4d70aab2aa216"},"script_stack":[],"script":"","lang":"groovy"}}],"caused_by":{"type":"script_exception","reason":"Error evaluating (doc[\"bytes\"].value * -1) == params.value","caused_by":{"type":"missing_property_exception","reason":"No such property: params for class: fd86f5f683b7e517cd8443c5efe4d70aab2aa216"},"script_stack":[],"script":"","lang":"groovy"}},"status":500}
    at https://localhost:5601/zfr/bundles/kibana.bundle.js?v=8467:74572:37
    at Function.Promise.try (https://localhost:5601/zfr/bundles/commons.bundle.js?v=8467:67701:21)
    at https://localhost:5601/zfr/bundles/commons.bundle.js?v=8467:67670:29
    at Array.map (native)
    at Function.Promise.map (https://localhost:5601/zfr/bundles/commons.bundle.js?v=8467:67669:29)
    at callResponseHandlers (https://localhost:5601/zfr/bundles/kibana.bundle.js?v=8467:74544:21)
    at https://localhost:5601/zfr/bundles/kibana.bundle.js?v=8467:73933:15
    at processQueue (https://localhost:5601/zfr/bundles/commons.bundle.js?v=8467:37125:29)
    at https://localhost:5601/zfr/bundles/commons.bundle.js?v=8467:37141:28
    at Scope.$eval (https://localhost:5601/zfr/bundles/commons.bundle.js?v=8467:38369:29)

[rjernst]

That is correct. Groovy dumps all the params into local variables. Painless has them in the params var.

[nik9000]

That is correct. Groovy dumps all the params into local variables. Painless has them in the params var.

I think this is a thing we'll be able to improve on with #20621 but that isn't ready. This level of clarity is about the best we'll get with painless for the next while. Painless is marked experimental because we're really trying to make improve on it constantly. It is the default because groovy security is a nightmare and we want off of groovy as fast as we can....

[rjernst]

I think this is a thing we'll be able to improve on with #20621

Yes and no. Params from the user (passed in through the runtime api) will stay in params. But things that each context provides (eg _source for scripted fields) will be accessible as a local variable again once we have contexts done.