From 853c25e8023bf877f28cda4dade0694d0299a83e Mon Sep 17 00:00:00 2001 From: Garrett Wu <6505921+GarrettWu@users.noreply.github.com> Date: Wed, 3 Apr 2024 10:33:45 -0700 Subject: [PATCH] feat: add ml.metrics.mean_squared_error (#559) * feat: add ml.metrics.mean_squared_error * fix docs * fix docs --- bigframes/ml/metrics/__init__.py | 2 + bigframes/ml/metrics/_metrics.py | 14 ++ tests/system/small/ml/test_metrics.py | 121 ++++++++---------- .../sklearn/metrics/_regression.py | 27 ++++ 4 files changed, 95 insertions(+), 69 deletions(-) diff --git a/bigframes/ml/metrics/__init__.py b/bigframes/ml/metrics/__init__.py index 6b0a243426..e79b46877b 100644 --- a/bigframes/ml/metrics/__init__.py +++ b/bigframes/ml/metrics/__init__.py @@ -18,6 +18,7 @@ auc, confusion_matrix, f1_score, + mean_squared_error, precision_score, r2_score, recall_score, @@ -35,5 +36,6 @@ "confusion_matrix", "precision_score", "f1_score", + "mean_squared_error", "pairwise", ] diff --git a/bigframes/ml/metrics/_metrics.py b/bigframes/ml/metrics/_metrics.py index ee86798b33..542e6300a8 100644 --- a/bigframes/ml/metrics/_metrics.py +++ b/bigframes/ml/metrics/_metrics.py @@ -335,3 +335,17 @@ def f1_score( f1_score.__doc__ = inspect.getdoc(vendored_metrics_classification.f1_score) + + +def mean_squared_error( + y_true: Union[bpd.DataFrame, bpd.Series], + y_pred: Union[bpd.DataFrame, bpd.Series], +) -> float: + y_true_series, y_pred_series = utils.convert_to_series(y_true, y_pred) + + return (y_pred_series - y_true_series).pow(2).sum() / len(y_true_series) + + +mean_squared_error.__doc__ = inspect.getdoc( + vendored_metrics_regression.mean_squared_error +) diff --git a/tests/system/small/ml/test_metrics.py b/tests/system/small/ml/test_metrics.py index b40982e282..c4c7eb4b88 100644 --- a/tests/system/small/ml/test_metrics.py +++ b/tests/system/small/ml/test_metrics.py @@ -19,7 +19,8 @@ import pytest import sklearn.metrics as sklearn_metrics # type: ignore -import bigframes.ml.metrics +import bigframes +from bigframes.ml import metrics def test_r2_score_perfect_fit(session): @@ -32,9 +33,7 @@ def test_r2_score_perfect_fit(session): df = session.read_pandas(pd_df) assert ( - bigframes.ml.metrics.r2_score( - df[["y_true_arbitrary_name"]], df[["y_pred_arbitrary_name"]] - ) + metrics.r2_score(df[["y_true_arbitrary_name"]], df[["y_pred_arbitrary_name"]]) == 1.0 ) @@ -43,7 +42,7 @@ def test_r2_score_bad_fit(session): pd_df = pd.DataFrame({"y_true": [1, 2, 3, 4, 5], "y_pred": [5, 4, 3, 2, 1]}) df = session.read_pandas(pd_df) - assert bigframes.ml.metrics.r2_score(df[["y_true"]], df[["y_pred"]]) == -3.0 + assert metrics.r2_score(df[["y_true"]], df[["y_pred"]]) == -3.0 def test_r2_score_force_finite(session): @@ -56,23 +55,21 @@ def test_r2_score_force_finite(session): ) df = session.read_pandas(pd_df) - assert bigframes.ml.metrics.r2_score( + assert metrics.r2_score( df[["y_true"]], df[["y_pred_1"]], force_finite=False ) == float("-inf") - assert bigframes.ml.metrics.r2_score(df[["y_true"]], df[["y_pred_1"]]) == 0.0 + assert metrics.r2_score(df[["y_true"]], df[["y_pred_1"]]) == 0.0 assert math.isnan( - bigframes.ml.metrics.r2_score( - df[["y_true"]], df[["y_pred_2"]], force_finite=False - ) + metrics.r2_score(df[["y_true"]], df[["y_pred_2"]], force_finite=False) ) - assert bigframes.ml.metrics.r2_score(df[["y_true"]], df[["y_pred_2"]]) == 1.0 + assert metrics.r2_score(df[["y_true"]], df[["y_pred_2"]]) == 1.0 def test_r2_score_ok_fit_matches_sklearn(session): pd_df = pd.DataFrame({"y_true": [1, 2, 3, 4, 5], "y_pred": [2, 3, 4, 3, 6]}) df = session.read_pandas(pd_df) - bf_result = bigframes.ml.metrics.r2_score(df[["y_true"]], df[["y_pred"]]) + bf_result = metrics.r2_score(df[["y_true"]], df[["y_pred"]]) sklearn_result = sklearn_metrics.r2_score(pd_df[["y_true"]], pd_df[["y_pred"]]) assert math.isclose(bf_result, sklearn_result) @@ -81,7 +78,7 @@ def test_r2_score_series(session): pd_df = pd.DataFrame({"y_true": [1, 7, 3, 2, 5], "y_pred": [1, 7, 3, 2, 5]}) df = session.read_pandas(pd_df) - assert bigframes.ml.metrics.r2_score(df["y_true"], df["y_pred"]) == 1.0 + assert metrics.r2_score(df["y_true"], df["y_pred"]) == 1.0 def test_accuracy_score_perfect_fit(session): @@ -94,7 +91,7 @@ def test_accuracy_score_perfect_fit(session): df = session.read_pandas(pd_df) assert ( - bigframes.ml.metrics.accuracy_score( + metrics.accuracy_score( df[["y_true_arbitrary_name"]], df[["y_pred_arbitrary_name"]] ) == 1.0 @@ -105,26 +102,21 @@ def test_accuracy_score_bad_fit(session): pd_df = pd.DataFrame({"y_true": [0, 2, 1, 3, 4], "y_pred": [0, 1, 2, 3, 4]}) df = session.read_pandas(pd_df) - assert bigframes.ml.metrics.accuracy_score(df[["y_true"]], df[["y_pred"]]) == 0.6 + assert metrics.accuracy_score(df[["y_true"]], df[["y_pred"]]) == 0.6 def test_accuracy_score_not_normailze(session): pd_df = pd.DataFrame({"y_true": [0, 2, 1, 3, 4], "y_pred": [0, 1, 2, 3, 4]}) df = session.read_pandas(pd_df) - assert ( - bigframes.ml.metrics.accuracy_score( - df[["y_true"]], df[["y_pred"]], normalize=False - ) - == 3 - ) + assert metrics.accuracy_score(df[["y_true"]], df[["y_pred"]], normalize=False) == 3 def test_accuracy_score_fit_matches_sklearn(session): pd_df = pd.DataFrame({"y_true": [1, 2, 3, 4, 5], "y_pred": [2, 3, 4, 3, 6]}) df = session.read_pandas(pd_df) - bf_result = bigframes.ml.metrics.accuracy_score(df[["y_true"]], df[["y_pred"]]) + bf_result = metrics.accuracy_score(df[["y_true"]], df[["y_pred"]]) sklearn_result = sklearn_metrics.accuracy_score( pd_df[["y_true"]], pd_df[["y_pred"]] ) @@ -135,7 +127,7 @@ def test_accuracy_score_series(session): pd_df = pd.DataFrame({"y_true": [1, 7, 3, 2, 5], "y_pred": [1, 7, 3, 2, 5]}) df = session.read_pandas(pd_df) - assert bigframes.ml.metrics.accuracy_score(df["y_true"], df["y_pred"]) == 1.0 + assert metrics.accuracy_score(df["y_true"], df["y_pred"]) == 1.0 def test_roc_curve_binary_classification_prediction_returns_expected(session): @@ -158,7 +150,7 @@ def test_roc_curve_binary_classification_prediction_returns_expected(session): ) df = session.read_pandas(pd_df) - fpr, tpr, thresholds = bigframes.ml.metrics.roc_curve( + fpr, tpr, thresholds = metrics.roc_curve( df[["y_true_arbitrary_name"]], df[["y_score_arbitrary_name"]], drop_intermediate=False, @@ -219,7 +211,7 @@ def test_roc_curve_binary_classification_prediction_matches_sklearn(session): ) df = session.read_pandas(pd_df) - fpr, tpr, thresholds = bigframes.ml.metrics.roc_curve( + fpr, tpr, thresholds = metrics.roc_curve( df[["y_true"]], df[["y_score"]], drop_intermediate=False ) expected_fpr, expected_tpr, expected_thresholds = sklearn_metrics.roc_curve( @@ -259,7 +251,7 @@ def test_roc_curve_binary_classification_decision_returns_expected(session): ) df = session.read_pandas(pd_df) - fpr, tpr, thresholds = bigframes.ml.metrics.roc_curve( + fpr, tpr, thresholds = metrics.roc_curve( df[["y_true"]], df[["y_score"]], drop_intermediate=False ) @@ -314,7 +306,7 @@ def test_roc_curve_binary_classification_decision_matches_sklearn(session): ) df = session.read_pandas(pd_df) - fpr, tpr, thresholds = bigframes.ml.metrics.roc_curve( + fpr, tpr, thresholds = metrics.roc_curve( df[["y_true"]], df[["y_score"]], drop_intermediate=False ) expected_fpr, expected_tpr, expected_thresholds = sklearn_metrics.roc_curve( @@ -350,7 +342,7 @@ def test_roc_curve_binary_classification_prediction_series(session): ) df = session.read_pandas(pd_df) - fpr, tpr, thresholds = bigframes.ml.metrics.roc_curve( + fpr, tpr, thresholds = metrics.roc_curve( df["y_true"], df["y_score"], drop_intermediate=False ) @@ -420,7 +412,7 @@ def test_roc_auc_score_returns_expected(session): ) df = session.read_pandas(pd_df) - score = bigframes.ml.metrics.roc_auc_score( + score = metrics.roc_auc_score( df[["y_true_arbitrary_name"]], df[["y_score_arbitrary_name"]] ) @@ -436,7 +428,7 @@ def test_roc_auc_score_returns_matches_sklearn(session): ) df = session.read_pandas(pd_df) - score = bigframes.ml.metrics.roc_auc_score(df[["y_true"]], df[["y_score"]]) + score = metrics.roc_auc_score(df[["y_true"]], df[["y_score"]]) expected_score = sklearn_metrics.roc_auc_score( pd_df[["y_true"]], pd_df[["y_score"]] ) @@ -453,7 +445,7 @@ def test_roc_auc_score_series(session): ) df = session.read_pandas(pd_df) - score = bigframes.ml.metrics.roc_auc_score(df["y_true"], df["y_score"]) + score = metrics.roc_auc_score(df["y_true"], df["y_score"]) assert score == 0.625 @@ -462,33 +454,33 @@ def test_auc_invalid_x_size(session): pd_df = pd.DataFrame({"x_arbitrary_name": [0], "y_arbitrary_name": [0]}) df = session.read_pandas(pd_df) with pytest.raises(ValueError): - bigframes.ml.metrics.auc(df[["x_arbitrary_name"]], df[["y_arbitrary_name"]]) + metrics.auc(df[["x_arbitrary_name"]], df[["y_arbitrary_name"]]) def test_auc_nondecreasing_x(session): pd_df = pd.DataFrame({"x": [0, 0, 0.5, 0.5, 1], "y": [0, 0.5, 0.5, 1, 1]}) df = session.read_pandas(pd_df) - assert bigframes.ml.metrics.auc(df[["x"]], df[["y"]]) == 0.75 + assert metrics.auc(df[["x"]], df[["y"]]) == 0.75 def test_auc_nonincreasing_x(session): pd_df = pd.DataFrame({"x": [0, 0, -0.5, -0.5, -1], "y": [0, 0.5, 0.5, 1, 1]}) df = session.read_pandas(pd_df) - assert bigframes.ml.metrics.auc(df[["x"]], df[["y"]]) == 0.75 + assert metrics.auc(df[["x"]], df[["y"]]) == 0.75 def test_auc_nonincreasing_x_negative(session): pd_df = pd.DataFrame({"x": [0, 0, -0.5, -0.5, -1], "y": [0, -0.5, -0.5, -1, -1]}) df = session.read_pandas(pd_df) - assert bigframes.ml.metrics.auc(df[["x"]], df[["y"]]) == -0.75 + assert metrics.auc(df[["x"]], df[["y"]]) == -0.75 def test_auc_series(session): pd_df = pd.DataFrame({"x": [0, 0, 0.5, 0.5, 1], "y": [0, 0.5, 0.5, 1, 1]}) df = session.read_pandas(pd_df) - assert bigframes.ml.metrics.auc(df["x"], df["y"]) == 0.75 + assert metrics.auc(df["x"], df["y"]) == 0.75 def test_confusion_matrix(session): @@ -499,7 +491,7 @@ def test_confusion_matrix(session): } ).astype("Int64") df = session.read_pandas(pd_df) - confusion_matrix = bigframes.ml.metrics.confusion_matrix( + confusion_matrix = metrics.confusion_matrix( df[["y_true_arbitrary_name"]], df[["y_pred_arbitrary_name"]] ) expected_pd_df = pd.DataFrame( @@ -522,9 +514,7 @@ def test_confusion_matrix_column_index(session): } ).astype("Int64") df = session.read_pandas(pd_df) - confusion_matrix = bigframes.ml.metrics.confusion_matrix( - df[["y_true"]], df[["y_pred"]] - ) + confusion_matrix = metrics.confusion_matrix(df[["y_true"]], df[["y_pred"]]) expected_pd_df = ( pd.DataFrame( {1: [1, 0, 1, 0], 2: [0, 0, 2, 0], 3: [0, 0, 0, 0], 4: [0, 1, 0, 1]} @@ -545,9 +535,7 @@ def test_confusion_matrix_matches_sklearn(session): } ).astype("Int64") df = session.read_pandas(pd_df) - confusion_matrix = bigframes.ml.metrics.confusion_matrix( - df[["y_true"]], df[["y_pred"]] - ) + confusion_matrix = metrics.confusion_matrix(df[["y_true"]], df[["y_pred"]]) expected_confusion_matrix = sklearn_metrics.confusion_matrix( pd_df[["y_true"]], pd_df[["y_pred"]] ) @@ -565,9 +553,7 @@ def test_confusion_matrix_str_matches_sklearn(session): } ).astype("str") df = session.read_pandas(pd_df) - confusion_matrix = bigframes.ml.metrics.confusion_matrix( - df[["y_true"]], df[["y_pred"]] - ) + confusion_matrix = metrics.confusion_matrix(df[["y_true"]], df[["y_pred"]]) expected_confusion_matrix = sklearn_metrics.confusion_matrix( pd_df[["y_true"]], pd_df[["y_pred"]] ) @@ -588,7 +574,7 @@ def test_confusion_matrix_series(session): } ).astype("Int64") df = session.read_pandas(pd_df) - confusion_matrix = bigframes.ml.metrics.confusion_matrix(df["y_true"], df["y_pred"]) + confusion_matrix = metrics.confusion_matrix(df["y_true"], df["y_pred"]) expected_pd_df = pd.DataFrame( { 0: [2, 0, 1], @@ -609,7 +595,7 @@ def test_recall_score(session): } ).astype("Int64") df = session.read_pandas(pd_df) - recall = bigframes.ml.metrics.recall_score( + recall = metrics.recall_score( df[["y_true_arbitrary_name"]], df[["y_pred_arbitrary_name"]], average=None ) expected_values = [1.000000, 0.000000, 0.666667] @@ -627,9 +613,7 @@ def test_recall_score_matches_sklearn(session): } ).astype("Int64") df = session.read_pandas(pd_df) - recall = bigframes.ml.metrics.recall_score( - df[["y_true"]], df[["y_pred"]], average=None - ) + recall = metrics.recall_score(df[["y_true"]], df[["y_pred"]], average=None) expected_values = sklearn_metrics.recall_score( pd_df[["y_true"]], pd_df[["y_pred"]], average=None ) @@ -646,9 +630,7 @@ def test_recall_score_str_matches_sklearn(session): } ).astype("str") df = session.read_pandas(pd_df) - recall = bigframes.ml.metrics.recall_score( - df[["y_true"]], df[["y_pred"]], average=None - ) + recall = metrics.recall_score(df[["y_true"]], df[["y_pred"]], average=None) expected_values = sklearn_metrics.recall_score( pd_df[["y_true"]], pd_df[["y_pred"]], average=None ) @@ -665,7 +647,7 @@ def test_recall_score_series(session): } ).astype("Int64") df = session.read_pandas(pd_df) - recall = bigframes.ml.metrics.recall_score(df["y_true"], df["y_pred"], average=None) + recall = metrics.recall_score(df["y_true"], df["y_pred"], average=None) expected_values = [1.000000, 0.000000, 0.666667] expected_index = [0, 1, 2] expected_recall = pd.Series(expected_values, index=expected_index) @@ -681,7 +663,7 @@ def test_precision_score(session): } ).astype("Int64") df = session.read_pandas(pd_df) - precision_score = bigframes.ml.metrics.precision_score( + precision_score = metrics.precision_score( df[["y_true_arbitrary_name"]], df[["y_pred_arbitrary_name"]], average=None ) expected_values = [0.666667, 0.000000, 0.666667] @@ -701,7 +683,7 @@ def test_precision_score_matches_sklearn(session): } ).astype("Int64") df = session.read_pandas(pd_df) - precision_score = bigframes.ml.metrics.precision_score( + precision_score = metrics.precision_score( df[["y_true"]], df[["y_pred"]], average=None ) expected_values = sklearn_metrics.precision_score( @@ -722,7 +704,7 @@ def test_precision_score_str_matches_sklearn(session): } ).astype("str") df = session.read_pandas(pd_df) - precision_score = bigframes.ml.metrics.precision_score( + precision_score = metrics.precision_score( df[["y_true"]], df[["y_pred"]], average=None ) expected_values = sklearn_metrics.precision_score( @@ -743,9 +725,7 @@ def test_precision_score_series(session): } ).astype("Int64") df = session.read_pandas(pd_df) - precision_score = bigframes.ml.metrics.precision_score( - df["y_true"], df["y_pred"], average=None - ) + precision_score = metrics.precision_score(df["y_true"], df["y_pred"], average=None) expected_values = [0.666667, 0.000000, 0.666667] expected_index = [0, 1, 2] expected_precision = pd.Series(expected_values, index=expected_index) @@ -763,7 +743,7 @@ def test_f1_score(session): } ).astype("Int64") df = session.read_pandas(pd_df) - f1_score = bigframes.ml.metrics.f1_score( + f1_score = metrics.f1_score( df[["y_true_arbitrary_name"]], df[["y_pred_arbitrary_name"]], average=None ) expected_values = [0.8, 0.000000, 0.666667] @@ -781,9 +761,7 @@ def test_f1_score_matches_sklearn(session): } ).astype("Int64") df = session.read_pandas(pd_df) - f1_score = bigframes.ml.metrics.f1_score( - df[["y_true"]], df[["y_pred"]], average=None - ) + f1_score = metrics.f1_score(df[["y_true"]], df[["y_pred"]], average=None) expected_values = sklearn_metrics.f1_score( pd_df[["y_true"]], pd_df[["y_pred"]], average=None ) @@ -800,9 +778,7 @@ def test_f1_score_str_matches_sklearn(session): } ).astype("str") df = session.read_pandas(pd_df) - f1_score = bigframes.ml.metrics.f1_score( - df[["y_true"]], df[["y_pred"]], average=None - ) + f1_score = metrics.f1_score(df[["y_true"]], df[["y_pred"]], average=None) expected_values = sklearn_metrics.f1_score( pd_df[["y_true"]], pd_df[["y_pred"]], average=None ) @@ -819,9 +795,16 @@ def test_f1_score_series(session): } ).astype("Int64") df = session.read_pandas(pd_df) - f1_score = bigframes.ml.metrics.f1_score(df["y_true"], df["y_pred"], average=None) + f1_score = metrics.f1_score(df["y_true"], df["y_pred"], average=None) expected_values = [0.8, 0.000000, 0.666667] expected_index = [0, 1, 2] expected_f1 = pd.Series(expected_values, index=expected_index) pd.testing.assert_series_equal(f1_score, expected_f1, check_index_type=False) + + +def test_mean_squared_error(session: bigframes.Session): + pd_df = pd.DataFrame({"y_true": [3, -0.5, 2, 7], "y_pred": [2.5, 0.0, 2, 8]}) + df = session.read_pandas(pd_df) + mse = metrics.mean_squared_error(df["y_true"], df["y_pred"]) + assert mse == 0.375 diff --git a/third_party/bigframes_vendored/sklearn/metrics/_regression.py b/third_party/bigframes_vendored/sklearn/metrics/_regression.py index be531a9b1c..c3e579bd29 100644 --- a/third_party/bigframes_vendored/sklearn/metrics/_regression.py +++ b/third_party/bigframes_vendored/sklearn/metrics/_regression.py @@ -64,3 +64,30 @@ def r2_score(y_true, y_pred, force_finite=True) -> float: float: The :math:`R^2` score. """ raise NotImplementedError(constants.ABSTRACT_METHOD_ERROR_MESSAGE) + + +def mean_squared_error(y_true, y_pred) -> float: + """Mean squared error regression loss. + + **Examples:** + + >>> import bigframes.pandas as bpd + >>> import bigframes.ml.metrics + >>> bpd.options.display.progress_bar = None + + >>> y_true = bpd.DataFrame([3, -0.5, 2, 7]) + >>> y_pred = bpd.DataFrame([2.5, 0.0, 2, 8]) + >>> mse = bigframes.ml.metrics.mean_squared_error(y_true, y_pred) + >>> mse + 0.375 + + Args: + y_true (Series or DataFrame of shape (n_samples,)): + Ground truth (correct) target values. + y_pred (Series or DataFrame of shape (n_samples,)): + Estimated target values. + + Returns: + float: Mean squared error. + """ + raise NotImplementedError(constants.ABSTRACT_METHOD_ERROR_MESSAGE)