Feature/mean_reciprocal_rank

CHANGELOG.md

@@ -9,3 +9,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## Unreleased
 
 ### Added
+
+- Mean Reciprocal Rank metric

Makefile

@@ -24,6 +24,7 @@ SOURCES=rectools
 TESTS=tests
 REPORTS=reports
 
+
 # Installation
 
 reports:
@@ -111,6 +112,7 @@ coverage: .venv reports
 	${COVERAGE} html -d ${REPORTS}/coverage_html
 	${COVERAGE} xml -o ${REPORTS}/coverage.xml -i
 
+
 # Generalization
 
 .autoformat: .isort_fix .autopep8_fix .black_fix
@@ -122,8 +124,6 @@ lint: .venv .lint
 .test: .pytest .doctest
 test: .venv .test
 
-test_metrics: .venv .test_metrics
-
 
 # Cleaning
 

rectools/metrics/__init__.py

@@ -26,6 +26,7 @@
 `metrics.Accuracy`
 `metrics.MAP`
 `metrics.NDCG`
+`metrics.MRR`
 `metrics.MeanInvUserFreq`
 `metrics.IntraListDiversity`
 `metrics.Serendipity`
@@ -46,7 +47,7 @@
 )
 from .diversity import IntraListDiversity
 from .novelty import MeanInvUserFreq
-from .ranking import MAP, NDCG
+from .ranking import MAP, MRR, NDCG
 from .scoring import calc_metrics
 from .serendipity import Serendipity
 
@@ -56,6 +57,7 @@
     "Accuracy",
     "MAP",
     "NDCG",
+    "MRR",
     "MeanInvUserFreq",
     "IntraListDiversity",
     "Serendipity",

rectools/metrics/ranking.py

@@ -189,7 +189,7 @@ def fit(cls, merged: pd.DataFrame, k_max: int) -> MAPFitted:
             prec_at_k_csr = sparse.csr_matrix(np.array([]).reshape(0, 0))
             return MAPFitted(prec_at_k_csr, users, np.array([]))
 
-        n_relevant_items = merged.groupby(Columns.User)[Columns.Item].agg("size")[users].values
+        n_relevant_items = merged.groupby(Columns.User, sort=False)[Columns.Item].agg("size")[users].values
 
         user_to_idx_map = pd.Series(np.arange(users.size), index=users)
         df_prepared = merged.query(f"{Columns.Rank} <= @k_max")
@@ -407,22 +407,134 @@ def calc_per_user_from_merged(self, merged: pd.DataFrame) -> pd.Series:
         idcg = (1 / log_at_base(np.arange(1, self.k + 1) + 1, self.log_base)).sum()
         ndcg = (
             pd.DataFrame({Columns.User: merged[Columns.User], "__ndcg": dcg / idcg})
-            .groupby(Columns.User)["__ndcg"]
+            .groupby(Columns.User, sort=False)["__ndcg"]
             .sum()
             .rename(None)
         )
         return ndcg
 
 
-RankingMetric = tp.Union[NDCG, MAP]
+class MRR(_RankingMetric):
+    r"""
+    Mean Reciprocal Rank at k (MRR@k).
+
+    MRR calculates as mean value of reciprocal rank
+    of first relevant recommendation among all users.
+
+    Estimates relevance of recommendations taking in account their order.
+
+    .. math::
+        MRR@K = \frac{1}{|U|} \sum_{i=1}^{|U|} \frac{1}{rank_{i}}
+
+    where
+        - :math:`{|U|}` is a number of unique users;
+        - :math:`rank_{i}` is a rank of first relevant recommendation
+          starting from ``1``.
+
+    If a user doesn't have any relevant recommendation then his metric value will be ``0``.
+
+    Parameters
+    ----------
+     k : int
+        Number of items at the top of recommendations list that will be used to calculate metric.
+
+    Examples
+    --------
+    >>> reco = pd.DataFrame(
+    ...     {
+    ...         Columns.User: [1, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4],
+    ...         Columns.Item: [7, 8, 1, 2, 2, 1, 3, 4, 7, 8, 3],
+    ...         Columns.Rank: [1, 2, 1, 2, 1, 2, 3, 4, 1, 2, 3],
+    ...     }
+    ... )
+    >>> interactions = pd.DataFrame(
+    ...     {
+    ...         Columns.User: [1, 1, 2, 3, 3, 3, 4, 4, 4],
+    ...         Columns.Item: [1, 2, 1, 1, 3, 4, 1, 2, 3],
+    ...     }
+    ... )
+    >>> # Here
+    >>> #    - for user ``1`` we return non-relevant recommendations;
+    >>> #    - for user ``2`` we return 2 items and relevant is first;
+    >>> #    - for user ``3`` we return 4 items, 2nd, 3rd and 4th are relevant;
+    >>> #    - for user ``4`` we return 3 items and relevant is last;
+    >>> MRR(k=1).calc_per_user(reco, interactions).values
+    array([0., 1., 0., 0.])
+    >>> MRR(k=3).calc_per_user(reco, interactions).values
+    array([0.        , 1.        , 0.5       , 0.33333333])
+    """
+
+    def calc_per_user(self, reco: pd.DataFrame, interactions: pd.DataFrame) -> pd.Series:
+        """
+        Calculate metric values for all users.
+
+        Parameters
+        ----------
+        reco : pd.DataFrame
+            Recommendations table with columns `Columns.User`, `Columns.Item`, `Columns.Rank`.
+        interactions : pd.DataFrame
+            Interactions table with columns `Columns.User`, `Columns.Item`.
+
+        Returns
+        -------
+        pd.Series
+            Values of metric (index - user id, values - metric value for every user).
+        """
+        self._check(reco, interactions=interactions)
+        merged_reco = merge_reco(reco, interactions)
+        return self.calc_per_user_from_merged(merged_reco)
+
+    def calc_per_user_from_merged(self, merged: pd.DataFrame) -> pd.Series:
+        """
+        Calculate metric values for all users from merged recommendations.
+
+        Parameters
+        ----------
+        merged : pd.DataFrame
+            Result of merging recommendations and interactions tables.
+            Can be obtained using `merge_reco` function.
+
+        Returns
+        -------
+        pd.Series
+            Values of metric (index - user id, values - metric value for every user).
+        """
+        cutted_rank = np.where(merged[Columns.Rank] <= self.k, merged[Columns.Rank], np.nan)
+        min_rank_per_user = (
+            pd.DataFrame({Columns.User: merged[Columns.User], "__cutted_rank": cutted_rank})
+            .groupby(Columns.User, sort=False)["__cutted_rank"]
+            .min()
+        )
+        return (1.0 / min_rank_per_user).fillna(0).rename(None)
+
+    def calc_from_merged(self, merged: pd.DataFrame) -> float:
+        """
+        Calculate metric value from merged recommendations.
+
+        Parameters
+        ----------
+        merged : pd.DataFrame
+            Result of merging recommendations and interactions tables.
+            Can be obtained using `merge_reco` function.
+
+        Returns
+        -------
+        float
+            Value of metric (average between users).
+        """
+        per_user = self.calc_per_user_from_merged(merged)
+        return per_user.mean()
+
+
+RankingMetric = tp.Union[NDCG, MAP, MRR]
 
 
 def calc_ranking_metrics(
     metrics: tp.Dict[str, RankingMetric],
     merged: pd.DataFrame,
 ) -> tp.Dict[str, float]:
     """
-    Calculate any ranking metrics (MAP and NDCG for now).
+    Calculate any ranking metrics (MAP, NDCG and MRR for now).
 
     Works with pre-prepared data.
 
@@ -431,7 +543,7 @@ def calc_ranking_metrics(
 
     Parameters
     ----------
-    metrics : dict(str -> (MAP | NDCG))
+    metrics : dict(str -> (MAP | NDCG | MRR))
         Dict of metric objects to calculate,
         where key is metric name and value is metric object.
     merged : pd.DataFrame
@@ -446,12 +558,11 @@ def calc_ranking_metrics(
     """
     results = {}
 
-    # NDCG
-    ndcg_metrics: tp.Dict[str, NDCG] = select_by_type(metrics, NDCG)
-    for name, ndcg_metric in ndcg_metrics.items():
-        results[name] = ndcg_metric.calc_from_merged(merged)
+    for ranking_metric_cls in [NDCG, MRR]:
+        ranking_metrics: tp.Dict[str, tp.Union[NDCG, MRR]] = select_by_type(metrics, ranking_metric_cls)
+        for name, metric in ranking_metrics.items():
+            results[name] = metric.calc_from_merged(merged)
 
-    # MAP
     map_metrics: tp.Dict[str, MAP] = select_by_type(metrics, MAP)
     if map_metrics:
         k_max = max(metric.k for metric in map_metrics.values())

tests/metrics/test_ranking.py

@@ -21,7 +21,7 @@
 import pytest
 
 from rectools import Columns
-from rectools.metrics.ranking import MAP, NDCG
+from rectools.metrics.ranking import MAP, MRR, NDCG
 
 EMPTY_INTERACTIONS = pd.DataFrame(columns=[Columns.User, Columns.Item], dtype=int)
 
@@ -102,12 +102,6 @@ class TestNDCG:
         ),
     )
     def test_calc(self, k: int, expected_ndcg: tp.List[float]) -> None:
-        pd.DataFrame(
-            {
-                "user": [10, 20, 30, 30, 30, 40, 50, 50, 50, 50],
-                "rank": [15, np.nan, 1, 2, 3, 1, np.nan, 3, 7, 9],
-            }
-        )
         reco = pd.DataFrame(
             {
                 Columns.User: [1, 2, 3, 3, 3, 4, 5, 5, 5, 5, 6],
@@ -137,3 +131,65 @@ def test_when_no_interactions(self) -> None:
         metric = NDCG(k=3)
         pd.testing.assert_series_equal(metric.calc_per_user(reco, EMPTY_INTERACTIONS), expected_metric_per_user)
         assert np.isnan(metric.calc(reco, EMPTY_INTERACTIONS))
+
+
+class TestMRR:
+    @pytest.mark.parametrize(
+        "k,expected_mrr",
+        (
+            (1, [0, 0, 1, 1, 0]),
+            (3, [0, 0, 1, 1, 1 / 3]),
+        ),
+    )
+    def test_calc(self, k: int, expected_mrr: tp.List[float]) -> None:
+        reco = pd.DataFrame(
+            {
+                Columns.User: [1, 2, 3, 3, 3, 4, 5, 5, 5, 5],
+                Columns.Item: [1, 2, 1, 2, 3, 1, 1, 2, 3, 5],
+                Columns.Rank: [9, 1, 1, 2, 3, 1, 3, 7, 9, 1],
+            }
+        )
+        interactions = pd.DataFrame(
+            {
+                Columns.User: [1, 2, 3, 3, 3, 4, 5, 5, 5, 5],
+                Columns.Item: [1, 1, 1, 2, 3, 1, 1, 2, 3, 4],
+            }
+        )
+
+        metric = MRR(k=k)
+        expected_metric_per_user = pd.Series(
+            expected_mrr,
+            index=pd.Series([1, 2, 3, 4, 5], name=Columns.User),
+            dtype=float,
+        )
+        pd.testing.assert_series_equal(metric.calc_per_user(reco, interactions), expected_metric_per_user)
+        assert np.allclose(metric.calc(reco, interactions), expected_metric_per_user.mean())
+
+    def test_when_no_interactions(self) -> None:
+        reco = pd.DataFrame([[1, 1, 1], [2, 1, 1]], columns=[Columns.User, Columns.Item, Columns.Rank])
+        expected_metric_per_user = pd.Series(index=pd.Series(name=Columns.User, dtype=int), dtype=np.float64)
+        metric = MRR(k=3)
+        pd.testing.assert_series_equal(metric.calc_per_user(reco, EMPTY_INTERACTIONS), expected_metric_per_user)
+        assert np.isnan(metric.calc(reco, EMPTY_INTERACTIONS))
+
+    def test_when_duplicates_in_interactions(self) -> None:
+        reco = pd.DataFrame(
+            {
+                Columns.User: [1, 1, 1, 2, 2, 2],
+                Columns.Item: [1, 2, 3, 1, 2, 3],
+                Columns.Rank: [1, 2, 3, 4, 5, 6],
+            }
+        )
+        interactions = pd.DataFrame(
+            {
+                Columns.User: [1, 1, 1, 2, 2, 2],
+                Columns.Item: [1, 2, 1, 1, 2, 3],
+            }
+        )
+        metric = MRR(k=3)
+        expected_metric_per_user = pd.Series(
+            [1, 0],
+            index=pd.Series([1, 2], name=Columns.User),
+            dtype=float,
+        )
+        pd.testing.assert_series_equal(metric.calc_per_user(reco, interactions), expected_metric_per_user)

tests/metrics/test_scoring.py

@@ -20,6 +20,7 @@
 from rectools import Columns
 from rectools.metrics import (
     MAP,
+    MRR,
     NDCG,
     Accuracy,
     IntraListDiversity,
@@ -73,6 +74,7 @@ def test_success(self) -> None:
             "map@1": MAP(k=1),
             "map@2": MAP(k=2),
             "ndcg@1": NDCG(k=1, log_base=3),
+            "mrr@1": MRR(k=1),
             "miuf": MeanInvUserFreq(k=3),
             "ild": IntraListDiversity(k=3, distance_calculator=self.calculator),
             "serendipity": Serendipity(k=3),
@@ -88,6 +90,7 @@ def test_success(self) -> None:
             "map@1": 0.125,
             "map@2": 0.375,
             "ndcg@1": 0.25,
+            "mrr@1": 0.25,
             "miuf": 0.125,
             "ild": 0.25,
             "serendipity": 0,