Success markers telemetry

changelog/10065.misc.md

@@ -0,0 +1 @@
+Add telemetry to markers extraction.

docs/docs/telemetry/events.json

@@ -3,6 +3,7 @@
     "Model Training",
     "Model Testing",
     "Model Serving",
+    "Markers Extraction",
     "Data Handling",
     "Miscellaneous"
   ],
@@ -422,6 +423,81 @@
         "num_synonyms",
         "num_regexes"
       ]
+    },
+    "Markers Extraction Initiated": {
+      "description": "Triggered when markers extraction has been initiated.",
+      "type": "object",
+      "section": "Markers Extraction",
+      "properties": {
+        "strategy": {
+          "type": "string",
+          "description": "Strategy to use when selecting trackers to extract from."
+        },
+        "only_extract": {
+          "type": "boolean",
+          "description": "Indicates if path to write out statistics hasn't been specified."
+        },
+        "seed": {
+          "type": ["string", "null"],
+          "description": "The seed to initialise the random number generator for use with the 'sample' strategy."
+        },
+        "count": {
+          "type": ["string", "null"],
+          "description": "Number of trackers to extract from (for any strategy except 'all')."
+        }
+      },
+      "additionalProperties": false,
+      "required": [
+        "strategy",
+        "only_extract",
+        "seed",
+        "count"
+      ]
+    },
+    "Markers Extracted": {
+      "description": "Triggered when markers have been extracted.",
+      "type": "object",
+      "section": "Markers Extraction",
+      "properties": {
+        "trackers_count": {
+          "type": "integer",
+          "description": "Number of processed trackers."
+        }
+      },
+      "additionalProperties": false,
+      "required": [
+        "trackers_count"
+      ]
+    },
+    "Markers Parsed": {
+      "description": "Triggered when markers have been successfully parsed.",
+      "type": "object",
+      "section": "Markers Extraction",
+      "properties": {
+        "marker_count": {
+          "type": "integer",
+          "description": "Number of parsed markers."
+        }
+      },
+      "additionalProperties": false,
+      "required": [
+        "marker_count"
+      ]
+    },
+    "Markers Stats Computed": {
+      "description": "Triggered when marker stats has been computed.",
+      "type": "object",
+      "section": "Markers Extraction",
+      "properties": {
+        "trackers_count": {
+          "type": "integer",
+          "description": "Number of processed trackers."
+        }
+      },
+      "additionalProperties": false,
+      "required": [
+        "trackers_count"
+      ]
     }
   }
 }

rasa/cli/arguments/evaluate.py

@@ -13,6 +13,7 @@ def set_markers_arguments(parser: argparse.ArgumentParser) -> None:
     parser.add_argument(
         "--config",
         default="markers.yml",
+        nargs="?",
         type=str,
         help="The config file(s) containing marker definitions. This can be a single "
         "YAML file, or a directory that contains several files with marker "

rasa/cli/evaluate.py

@@ -2,6 +2,7 @@
 from typing import List, Text, Optional
 from pathlib import Path
 
+from rasa import telemetry
 from rasa.core.utils import AvailableEndpoints
 from rasa.core.tracker_store import TrackerStore
 from rasa.core.evaluation.marker_tracker_loader import MarkerTrackerLoader
@@ -131,6 +132,13 @@ def _run_markers(
             computed per session will be stored in
             '<path-to-stats-folder>/statistics-per-session.csv'.
     """
+    telemetry.track_markers_extraction_initiated(
+        strategy=strategy,
+        only_extract=stats_file_prefix is not None,
+        seed=seed,
+        count=count,
+    )
+
     domain = Domain.load(domain_path) if domain_path else None
     markers = Marker.from_path(config)
     if domain and not markers.validate_against_domain(domain):
@@ -139,6 +147,10 @@ def _run_markers(
             "Please see errors listed above and fix before running again."
         )
 
+    # Subtract one to remove the virtual OR over all markers
+    num_markers = len(markers) - 1
+    telemetry.track_markers_parsed_count(num_markers)
+
     tracker_loader = _create_tracker_loader(endpoint_config, strategy, count, seed)
 
     def _append_suffix(path: Optional[Path], suffix: Text) -> Optional[Path]:

rasa/core/evaluation/marker_base.py

@@ -26,6 +26,7 @@
 from rasa.shared.data import is_likely_yaml_file
 from rasa.shared.exceptions import InvalidConfigException, RasaException
 from rasa.shared.core.events import ActionExecuted, UserUttered, Event
+from rasa import telemetry
 from rasa.shared.core.domain import Domain
 from rasa.shared.core.trackers import DialogueStateTracker
 from rasa.utils.io import WriteRow
@@ -608,6 +609,9 @@ def evaluate_trackers(
             if tracker:
                 tracker_result = self.evaluate_events(tracker.events)
                 processed_trackers[tracker.sender_id] = tracker_result
+
+        processed_trackers_count = len(processed_trackers)
+        telemetry.track_markers_extracted(processed_trackers_count)
         Marker._save_results(output_file, processed_trackers)
 
         # Compute and write statistics if requested.
@@ -622,6 +626,8 @@ def evaluate_trackers(
                         session_idx=session_idx,
                         meta_data_on_relevant_events_per_marker=session_result,
                     )
+
+            telemetry.track_markers_stats_computed(processed_trackers_count)
             if overall_stats_file:
                 stats.overall_statistic_to_csv(path=overall_stats_file)
             if session_stats_file:

rasa/telemetry.py

@@ -94,6 +94,10 @@
 TELEMETRY_VISUALIZATION_STARTED_EVENT = "Story Visualization Started"
 TELEMETRY_TEST_CORE_EVENT = "Model Core Tested"
 TELEMETRY_TEST_NLU_EVENT = "Model NLU Tested"
+TELEMETRY_MARKERS_EXTRACTION_INITIATED_EVENT = "Markers Extraction Initiated"
+TELEMETRY_MARKERS_EXTRACTED_EVENT = "Markers Extracted"
+TELEMETRY_MARKERS_STATS_COMPUTED_EVENT = "Markers Stats Computed"
+TELEMETRY_MARKERS_PARSED_COUNT = "Markers Parsed"
 
 # used to calculate the context on the first call and cache it afterwards
 TELEMETRY_CONTEXT = None
@@ -1005,3 +1009,64 @@ def track_nlu_model_test(test_data: "TrainingData") -> None:
             "num_regexes": len(test_data.regex_features),
         },
     )
+
+
+@ensure_telemetry_enabled
+def track_markers_extraction_initiated(
+    strategy: Text, only_extract: bool, seed: Optional[int], count: Optional[int],
+) -> None:
+    """Track when a user tries to extract success markers.
+
+    Args:
+        strategy: The strategy the user is using for tracker selection
+        only_extract: Indicates if the user is only extracting markers or also
+                      producing stats
+        seed: (Optional) The seed used if strategy is 'sample' and the user selects
+              one
+        count: (Optional) The number of trackers the user is trying to select.
+    """
+    _track(
+        TELEMETRY_MARKERS_EXTRACTION_INITIATED_EVENT,
+        {
+            "strategy": strategy,
+            "only_extract": only_extract,
+            "seed": seed,
+            "count": count,
+        },
+    )
+
+
+@ensure_telemetry_enabled
+def track_markers_extracted(trackers_count: int) -> None:
+    """Track when markers have been extracted by a user.
+
+    Args:
+        trackers_count: The actual number of trackers processed
+    """
+    _track(
+        TELEMETRY_MARKERS_EXTRACTED_EVENT, {"trackers_count": trackers_count},
+    )
+
+
+@ensure_telemetry_enabled
+def track_markers_stats_computed(trackers_count: int) -> None:
+    """Track when stats over markers have been computed by a user.
+
+    Args:
+        trackers_count: The actual number of trackers processed
+    """
+    _track(
+        TELEMETRY_MARKERS_STATS_COMPUTED_EVENT, {"trackers_count": trackers_count},
+    )
+
+
+@ensure_telemetry_enabled
+def track_markers_parsed_count(marker_count: int) -> None:
+    """Track when markers have been successfully parsed from config.
+
+    Args:
+        marker_count: The number of markers found in the config
+    """
+    _track(
+        TELEMETRY_MARKERS_PARSED_COUNT, {"marker_count": marker_count},
+    )

tests/test_telemetry.py

@@ -81,10 +81,16 @@ async def test_events_schema(
 
     telemetry.track_nlu_model_test(TrainingData())
 
+    telemetry.track_markers_evaluation_initiated("all", False, None, None)
+
+    telemetry.track_markers_extracted(1)
+
+    telemetry.track_markers_stats_computed(1)
+
     pending = asyncio.all_tasks() - initial
     await asyncio.gather(*pending)
 
-    assert mock.call_count == 15
+    assert mock.call_count == 18
 
     for args, _ in mock.call_args_list:
         event = args[0]