Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add precision_recall_fscore_support function #186

Merged
merged 6 commits into from
Oct 17, 2023
Merged

Add precision_recall_fscore_support function #186

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
cc965d5
Add precision_recall_fscore_support function
0urobor0s Oct 10, 2023
16fd842
Remove beta from opts in defn context
0urobor0s Oct 12, 2023
9271b3b
Fix opts and improve docs
0urobor0s Oct 12, 2023
f40789a
Require beta for fbeta_score
0urobor0s Oct 12, 2023
92710d6
Have beta as function argument
0urobor0s Oct 16, 2023
88582f2
Use rank instead of size
0urobor0s Oct 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
166 changes: 144 additions & 22 deletions lib/scholar/metrics/classification.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,67 @@ defmodule Scholar.Metrics.Classification do
]
]

fbeta_score_schema = f1_score_schema
fbeta_score_schema =
general_schema ++
[
average: [
type: {:in, [:micro, :macro, :weighted, :none]},
default: :none,
doc: """
This determines the type of averaging performed on the data.

* `:macro` - Calculate metrics for each label, and find their unweighted mean.
This does not take label imbalance into account.

* `:weighted` - Calculate metrics for each label, and find their average weighted by
support (the number of true instances for each label).

* `:micro` - Calculate metrics globally by counting the total true positives,
false negatives and false positives.

* `:none` - The F-score values for each class are returned.
"""
],
beta: [
type: {:custom, Scholar.Options, :beta, []},
required: true,
doc: """
Determines the weight of recall in the combined score.
For values of `beta` > 1 it gives more weight to recall, while `beta` < 1 favors precision.
"""
]
]

precision_recall_fscore_support_schema =
general_schema ++
[
average: [
type: {:in, [:micro, :macro, :weighted, :none]},
default: :none,
doc: """
This determines the type of averaging performed on the data.

* `:macro` - Calculate metrics for each label, and find their unweighted mean.
This does not take label imbalance into account.

* `:weighted` - Calculate metrics for each label, and find their average weighted by
support (the number of true instances for each label).

* `:micro` - Calculate metrics globally by counting the total true positives,
false negatives and false positives.

* `:none` - The F-score values for each class are returned.
"""
],
beta: [
type: {:custom, Scholar.Options, :beta, []},
default: 1,
doc: """
Determines the weight of recall in the combined score.
For values of `beta` > 1 it gives more weight to recall, while `beta` < 1 favors precision.
"""
]
]

confusion_matrix_schema =
general_schema ++
Expand Down Expand Up @@ -167,6 +227,9 @@ defmodule Scholar.Metrics.Classification do
@cohen_kappa_schema NimbleOptions.new!(cohen_kappa_schema)
@fbeta_score_schema NimbleOptions.new!(fbeta_score_schema)
@f1_score_schema NimbleOptions.new!(f1_score_schema)
@precision_recall_fscore_support_schema NimbleOptions.new!(
precision_recall_fscore_support_schema
)
@brier_score_loss_schema NimbleOptions.new!(brier_score_loss_schema)
@accuracy_schema NimbleOptions.new!(accuracy_schema)
@top_k_accuracy_score_schema NimbleOptions.new!(top_k_accuracy_score_schema)
Expand Down Expand Up @@ -603,58 +666,56 @@ defmodule Scholar.Metrics.Classification do

iex> y_true = Nx.tensor([0, 1, 1, 1, 1, 0, 2, 1, 0, 1], type: :u32)
iex> y_pred = Nx.tensor([0, 2, 1, 1, 2, 2, 2, 0, 0, 1], type: :u32)
iex> Scholar.Metrics.Classification.fbeta_score(y_true, y_pred, Nx.u32(1), num_classes: 3)
iex> Scholar.Metrics.Classification.fbeta_score(y_true, y_pred, beta: Nx.u32(1), num_classes: 3)
#Nx.Tensor<
f32[3]
[0.6666666865348816, 0.6666666865348816, 0.4000000059604645]
>
iex> Scholar.Metrics.Classification.fbeta_score(y_true, y_pred, Nx.u32(2), num_classes: 3)
iex> Scholar.Metrics.Classification.fbeta_score(y_true, y_pred, beta: Nx.u32(2), num_classes: 3)
#Nx.Tensor<
f32[3]
[0.6666666865348816, 0.5555555820465088, 0.625]
>
iex> Scholar.Metrics.Classification.fbeta_score(y_true, y_pred, Nx.f32(0.5), num_classes: 3)
iex> Scholar.Metrics.Classification.fbeta_score(y_true, y_pred, beta: Nx.f32(0.5), num_classes: 3)
#Nx.Tensor<
f32[3]
[0.6666666865348816, 0.8333333134651184, 0.29411765933036804]
>
iex> Scholar.Metrics.Classification.fbeta_score(y_true, y_pred, Nx.u32(2), num_classes: 3, average: :macro)
iex> Scholar.Metrics.Classification.fbeta_score(y_true, y_pred, beta: Nx.u32(2), num_classes: 3, average: :macro)
#Nx.Tensor<
f32
0.6157407760620117
>
iex> Scholar.Metrics.Classification.fbeta_score(y_true, y_pred, Nx.u32(2), num_classes: 3, average: :weighted)
iex> Scholar.Metrics.Classification.fbeta_score(y_true, y_pred, beta: Nx.u32(2), num_classes: 3, average: :weighted)
#Nx.Tensor<
f32
0.5958333611488342
>
iex> Scholar.Metrics.Classification.fbeta_score(y_true, y_pred, Nx.f32(0.5), num_classes: 3, average: :micro)
iex> Scholar.Metrics.Classification.fbeta_score(y_true, y_pred, beta: Nx.f32(0.5), num_classes: 3, average: :micro)
#Nx.Tensor<
f32
0.6000000238418579
>
iex> Scholar.Metrics.Classification.fbeta_score(Nx.tensor([1, 0, 1, 0]), Nx.tensor([0, 1, 0, 1]), Nx.tensor(0.5), num_classes: 2, average: :none)
iex> Scholar.Metrics.Classification.fbeta_score(Nx.tensor([1, 0, 1, 0]), Nx.tensor([0, 1, 0, 1]), beta: Nx.tensor(0.5), num_classes: 2, average: :none)
#Nx.Tensor<
f32[2]
[0.0, 0.0]
>
iex> Scholar.Metrics.Classification.fbeta_score(Nx.tensor([1, 0, 1, 0]), Nx.tensor([0, 1, 0, 1]), 0.5, num_classes: 2, average: :none)
iex> Scholar.Metrics.Classification.fbeta_score(Nx.tensor([1, 0, 1, 0]), Nx.tensor([0, 1, 0, 1]), beta: 0.5, num_classes: 2, average: :none)
#Nx.Tensor<
f32[2]
[0.0, 0.0]
>
"""
deftransform fbeta_score(y_true, y_pred, beta, opts \\ []) do
fbeta_score_n(y_true, y_pred, beta, NimbleOptions.validate!(opts, @fbeta_score_schema))
deftransform fbeta_score(y_true, y_pred, opts \\ []) do
opts = NimbleOptions.validate!(opts, @fbeta_score_schema)
{beta, opts} = Keyword.pop(opts, :beta)
fbeta_score_n(y_true, y_pred, beta, opts)
end

defnp fbeta_score_n(y_true, y_pred, beta, opts) do
check_shape(y_pred, y_true)
num_classes = check_num_classes(opts[:num_classes])
average = opts[:average]

{_precision, _recall, per_class_fscore} =
precision_recall_fscore_n(y_true, y_pred, beta, num_classes, average)
{_precision, _recall, per_class_fscore, _support} =
precision_recall_fscore_support_n(y_true, y_pred, beta, opts)

per_class_fscore
end
Expand All @@ -677,7 +738,66 @@ defmodule Scholar.Metrics.Classification do
end
end

defnp precision_recall_fscore_n(y_true, y_pred, beta, num_classes, average) do
@doc """
Calculates precision, recall, F-score and support for each
class. It also supports a `beta` argument which weights
recall more than precision by it's value.

## Options

#{NimbleOptions.docs(@precision_recall_fscore_support_schema)}

## Examples

iex> y_true = Nx.tensor([0, 1, 1, 1, 1, 0, 2, 1, 0, 1], type: :u32)
iex> y_pred = Nx.tensor([0, 2, 1, 1, 2, 2, 2, 0, 0, 1], type: :u32)
iex> Scholar.Metrics.Classification.precision_recall_fscore_support(y_true, y_pred, num_classes: 3)
{Nx.f32([0.6666666865348816, 1.0, 0.25]),
Nx.f32([0.6666666865348816, 0.5, 1.0]),
Nx.f32([0.6666666865348816, 0.6666666865348816, 0.4000000059604645]),
Nx.u64([3, 6, 1])}
iex> Scholar.Metrics.Classification.precision_recall_fscore_support(y_true, y_pred, num_classes: 3, average: :macro)
{Nx.f32([0.6666666865348816, 1.0, 0.25]),
Nx.f32([0.6666666865348816, 0.5, 1.0]),
Nx.f32(0.5777778029441833),
Nx.Constants.nan()}
iex> Scholar.Metrics.Classification.precision_recall_fscore_support(y_true, y_pred, num_classes: 3, average: :weighted)
{Nx.f32([0.6666666865348816, 1.0, 0.25]),
Nx.f32([0.6666666865348816, 0.5, 1.0]),
Nx.f32(0.6399999856948853),
Nx.Constants.nan()}
iex> Scholar.Metrics.Classification.precision_recall_fscore_support(y_true, y_pred, num_classes: 3, average: :micro)
{Nx.f32(0.6000000238418579),
Nx.f32(0.6000000238418579),
Nx.f32(0.6000000238418579),
Nx.Constants.nan()}

iex> y_true = Nx.tensor([1, 0, 1, 0], type: :u32)
iex> y_pred = Nx.tensor([0, 1, 0, 1], type: :u32)
iex> opts = [beta: 2, num_classes: 2, average: :none]
iex> Scholar.Metrics.Classification.precision_recall_fscore_support(y_true, y_pred, opts)
{Nx.f32([0.0, 0.0]),
Nx.f32([0.0, 0.0]),
Nx.f32([0.0, 0.0]),
Nx.u64([2, 2])}
"""
deftransform precision_recall_fscore_support(y_true, y_pred, opts) do
opts = NimbleOptions.validate!(opts, @precision_recall_fscore_support_schema)
{beta, opts} = Keyword.pop(opts, :beta)

precision_recall_fscore_support_n(
y_true,
y_pred,
beta,
opts
)
end

defnp precision_recall_fscore_support_n(y_true, y_pred, beta, opts) do
check_shape(y_pred, y_true)
num_classes = check_num_classes(opts[:num_classes])
average = opts[:average]

confusion_matrix = confusion_matrix(y_true, y_pred, num_classes: num_classes)
{true_positive, false_positive, false_negative} = fbeta_score_v(confusion_matrix, average)

Expand All @@ -700,13 +820,15 @@ defmodule Scholar.Metrics.Classification do

case average do
:none ->
{precision, recall, per_class_fscore}
support = (y_true == Nx.iota({num_classes, 1})) |> Nx.sum(axes: [1])

{precision, recall, per_class_fscore, support}

:micro ->
{precision, recall, per_class_fscore}
{precision, recall, per_class_fscore, Nx.Constants.nan()}
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Any idea on what to add here? The first try was to use :none, but it seems atoms can't be returned from defn.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Using NaNs seems reasonable

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would personally prefer to have something more clear that this value contains "nothing" useful. But with the current constraints I thought this was ok(ish). Maybe a new type/struct could be created for these cases?


:macro ->
{precision, recall, Nx.mean(per_class_fscore)}
{precision, recall, Nx.mean(per_class_fscore), Nx.Constants.nan()}

:weighted ->
support = (y_true == Nx.iota({num_classes, 1})) |> Nx.sum(axes: [1])
Expand All @@ -716,7 +838,7 @@ defmodule Scholar.Metrics.Classification do
|> safe_division(Nx.sum(support))
|> Nx.sum()

{precision, recall, per_class_fscore}
{precision, recall, per_class_fscore, Nx.Constants.nan()}
end
end

Expand Down
8 changes: 8 additions & 0 deletions lib/scholar/options.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,4 +100,12 @@ defmodule Scholar.Options do
{:error,
"expected metric to be a :cosine or tuple {:minkowski, p} where p is a positive number or :infinity, got: #{inspect(metric)}"}
end

def beta(beta) do
if (is_number(beta) and beta >= 0) or (Nx.is_tensor(beta) and Nx.size(beta) == 1) do
0urobor0s marked this conversation as resolved.
Show resolved Hide resolved
{:ok, beta}
else
{:error, "expect 'beta' to be in the range [0, inf]"}
end
end
end
4 changes: 2 additions & 2 deletions test/scholar/metrics/classification_test.exs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ defmodule Scholar.Metrics.ClassificationTest do
beta = Nx.tensor(:infinity)
y_true = Nx.tensor([0, 0, 0, 0, 0, 1, 1, 1, 1, 1], type: :u32)
y_pred = Nx.tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1], type: :u32)
fbeta_scores = Classification.fbeta_score(y_true, y_pred, beta, num_classes: 2)
fbeta_scores = Classification.fbeta_score(y_true, y_pred, beta: beta, num_classes: 2)

assert_all_close(fbeta_scores, Classification.recall(y_true, y_pred, num_classes: 2))
end
Expand All @@ -29,7 +29,7 @@ defmodule Scholar.Metrics.ClassificationTest do
beta = 0
y_true = Nx.tensor([0, 0, 0, 0, 0, 1, 1, 1, 1, 1], type: :u32)
y_pred = Nx.tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1], type: :u32)
fbeta_scores = Classification.fbeta_score(y_true, y_pred, beta, num_classes: 2)
fbeta_scores = Classification.fbeta_score(y_true, y_pred, beta: beta, num_classes: 2)

assert_all_close(fbeta_scores, Classification.precision(y_true, y_pred, num_classes: 2))
end
Expand Down
Loading