support reference_time for duckling

_pytest/test_components.py

@@ -63,7 +63,7 @@ def test_all_arguments_can_be_satisfied_during_parse(component_class, default_co
     """Check that `parse` method parameters can be filled filled from the context. Similar to `pipeline_init` test."""
 
     # All available context arguments that will ever be generated during parse
-    context_arguments = {"text": None}
+    context_arguments = {"text": None, "time": None}
     for clz in registry.component_classes:
         for ctx_arg in clz.context_provides.get("pipeline_init", []):
             context_arguments[ctx_arg] = None

_pytest/test_emulators.py

@@ -8,7 +8,7 @@ def test_luis_request():
     from rasa_nlu.emulators.luis import LUISEmulator
     em = LUISEmulator()
     norm = em.normalise_request_json({"q": ["arb text"]})
-    assert norm == {"text": "arb text", "model": "default"}
+    assert norm == {"text": "arb text", "model": "default", "time": None}
 
 
 def test_luis_response():
@@ -78,7 +78,7 @@ def test_wit_request():
     from rasa_nlu.emulators.wit import WitEmulator
     em = WitEmulator()
     norm = em.normalise_request_json({"q": ["arb text"]})
-    assert norm == {"text": "arb text", "model": "default"}
+    assert norm == {"text": "arb text", "model": "default", "time": None}
 
 
 def test_wit_response():
@@ -109,7 +109,7 @@ def test_api_request():
     from rasa_nlu.emulators.api import ApiEmulator
     em = ApiEmulator()
     norm = em.normalise_request_json({"q": ["arb text"]})
-    assert norm == {"text": "arb text", "model": "default"}
+    assert norm == {"text": "arb text", "model": "default", "time": None}
 
 
 def test_api_response():
@@ -159,10 +159,10 @@ def test_dummy_request():
     from rasa_nlu.emulators import NoEmulator
     em = NoEmulator()
     norm = em.normalise_request_json({"q": ["arb text"]})
-    assert norm == {"text": "arb text", "model": "default"}
+    assert norm == {"text": "arb text", "model": "default", "time": None}
 
-    norm = em.normalise_request_json({"q": ["arb text"], "model": "specific"})
-    assert norm == {"text": "arb text", "model": "specific"}
+    norm = em.normalise_request_json({"q": ["arb text"], "model": "specific", "time": "1499279161658"})
+    assert norm == {"text": "arb text", "model": "specific", "time": "1499279161658"}
 
 
 def test_dummy_response():

_pytest/test_interpreter.py

@@ -35,7 +35,7 @@ def test_samples(pipeline_template, component_builder):
     ]
 
     for text, gold in samples:
-        result = interpreter.parse(text)
+        result = interpreter.parse(text, time=None)
         assert result['text'] == text, \
             "Wrong text for sample '{}'".format(text)
         assert result['intent']['name'] in available_intents, \

_pytest/test_train.py

@@ -26,7 +26,7 @@ def test_train_model(pipeline_template, component_builder):
     assert trained.pipeline
     loaded = utilities.load_interpreter_for_model(_config, persisted_path, component_builder)
     assert loaded.pipeline
-    assert loaded.parse("hello") is not None
+    assert loaded.parse("hello", time=None) is not None
 
 
 @slowtest
@@ -37,7 +37,7 @@ def test_train_model_noents(component_builder):
     assert trained.pipeline
     loaded = utilities.load_interpreter_for_model(_config, persisted_path, component_builder)
     assert loaded.pipeline
-    assert loaded.parse("hello") is not None
+    assert loaded.parse("hello", time=None) is not None
 
 
 @slowtest
@@ -48,7 +48,7 @@ def test_train_model_multithread(component_builder):
     assert trained.pipeline
     loaded = utilities.load_interpreter_for_model(_config, persisted_path, component_builder)
     assert loaded.pipeline
-    assert loaded.parse("hello") is not None
+    assert loaded.parse("hello", time=None) is not None
 
 
 def test_train_model_empty_pipeline(component_builder):
@@ -80,7 +80,7 @@ def test_load_and_persist_without_train(component_builder):
     persisted_path = trainer.persist(_config['path'], persistor, model_name=_config['name'])
     loaded = utilities.load_interpreter_for_model(_config, persisted_path, component_builder)
     assert loaded.pipeline
-    assert loaded.parse("hello") is not None
+    assert loaded.parse("hello", time=None) is not None
 
 
 def test_train_with_empty_data(component_builder):
@@ -91,4 +91,4 @@ def test_train_with_empty_data(component_builder):
     persisted_path = trainer.persist(_config['path'], persistor, model_name=_config['name'])
     loaded = utilities.load_interpreter_for_model(_config, persisted_path, component_builder)
     assert loaded.pipeline
-    assert loaded.parse("hello") is not None
+    assert loaded.parse("hello", time=None) is not None

rasa_nlu/components.py

@@ -152,7 +152,7 @@ def validate_arguments(pipeline, config, allow_empty_pipeline=False):
             raise Exception("Failed to validate at component '{}'. {}".format(component.name, e))
 
     # Reset context to test processing phase and prepare for training phase
-    context = {"entities": [], "text": None}
+    context = {"entities": [], "text": None, "time": None}
     context.update(after_init_context)
 
     for component in pipeline:

rasa_nlu/data_router.py

@@ -178,7 +178,7 @@ def parse(self, data):
                 raise InvalidModelError("No model found with alias '{}'. Error: {}".format(alias, e))
 
         model = self.model_store[alias]
-        response = model.parse(data['text'])
+        response = model.parse(data['text'], data.get('time', None))
         if self.responses:
             log = {"user_input": response, "model": alias, "time": datetime.datetime.now().isoformat()}
             self.responses.info(json.dumps(log, sort_keys=True))

rasa_nlu/emulators/__init__.py

@@ -26,6 +26,7 @@ def normalise_request_json(self, data):
             _data["model"] = data["model"][0]
         else:
             _data["model"] = data["model"]
+        _data['time'] = data["time"] if "time" in data else None
         return _data
 
     def normalise_response_json(self, data):

rasa_nlu/extractors/duckling_extractor.py

@@ -6,7 +6,9 @@
 import os
 import io
 import json
+import logging
 import typing
+import datetime
 from typing import Any
 from typing import Dict
 from typing import List
@@ -78,12 +80,34 @@ def cache_key(cls, model_metadata):
 
         return cls.name + "-" + model_metadata.language
 
-    def process(self, text, entities):
+    def pipeline_init(self, language):
+        # type: (Text, Text) -> None
+        from duckling import DucklingWrapper
+
+        if self.duckling is None:
+            try:
+                self.duckling = DucklingWrapper(language=language)  # languages in duckling are eg "de$core"
+            except ValueError as e:     # pragma: no cover
+                raise Exception("Duckling error. {}".format(e))
+
+    def process(self, text, entities, time):
         # type: (Text, List[Dict[Text, Any]]) -> Dict[Text, Any]
 
         extracted = []
         if self.duckling is not None:
-            matches = self.duckling.parse(text)
+            ref_time = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S+00:00')
+            if time is not None:
+                # check if time given is valid
+                try:
+                    ref_time = datetime.datetime.utcfromtimestamp(int(time)/1000.0).strftime('%Y-%m-%dT%H:%M:%S+00:00')
+                    logging.debug(
+                        "Passing reference time {} to duckling".format(ref_time))
+                except:
+                    logging.warning(
+                        "Could not parse timestamp {}. "
+                        "Instead current UTC time {} will be passed to duckling".format(time, ref_time))
+
+            matches = self.duckling.parse(text, reference_time=ref_time)
             relevant_matches = [match for match in matches if match["dim"] in self.dimensions]
             for match in relevant_matches:
                 entity = {"start": match["start"],

rasa_nlu/model.py

@@ -230,7 +230,7 @@ def __init__(self, pipeline, context, config, meta=None):
         self.meta = meta
         self.output_attributes = [output for component in pipeline for output in component.output_provides]
 
-    def parse(self, text):
+    def parse(self, text, time):
         # type: (Text) -> Dict[Text, Any]
         """Parse the input text, classify it and return an object containing its intent and entities."""
 
@@ -245,6 +245,7 @@ def parse(self, text):
 
         current_context.update({
             "text": text,
+            "time": time
         })
 
         for component in self.pipeline: