Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix DetailedPRCurve examples for classification tasks #744

Merged
merged 4 commits into from
Sep 5, 2024
Merged

Fix DetailedPRCurve examples for classification tasks #744

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2c4a430
add additional checks to make sure we aren't returning any examples w…
ntlind Sep 4, 2024
c33ae51
fix edge case where some true negative examples weren't included in o…
ntlind Sep 5, 2024
2f84210
remove pdb call
ntlind Sep 5, 2024
f7f2885
Merge branch 'main' into fix_737
ntlind Sep 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
130 changes: 130 additions & 0 deletions core/tests/conftest_inputs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3238,6 +3238,136 @@ def multiclass_pr_curve_predictions():
]


@pytest.fixture
def multiclass_pr_curve_check_zero_count_examples_groundtruths():
return [
schemas.GroundTruth(
datum=schemas.Datum(
uid="uid0",
metadata={
"height": 900,
"width": 300,
},
),
annotations=[
schemas.Annotation(
labels=[
schemas.Label(key="k", value="ant"),
],
),
],
),
]


@pytest.fixture
def multiclass_pr_curve_check_zero_count_examples_predictions():
return [
schemas.Prediction(
datum=schemas.Datum(
uid="uid0",
metadata={
"height": 900,
"width": 300,
},
),
annotations=[
schemas.Annotation(
labels=[
schemas.Label(key="k", value="ant", score=0.15),
schemas.Label(key="k", value="bee", score=0.48),
schemas.Label(key="k", value="cat", score=0.37),
],
)
],
),
]


@pytest.fixture
def multiclass_pr_curve_check_true_negatives_groundtruths():
return [
schemas.GroundTruth(
datum=schemas.Datum(
uid="uid0",
metadata={
"height": 900,
"width": 300,
},
),
annotations=[
schemas.Annotation(
labels=[
schemas.Label(key="dataset1", value="ant"),
],
),
],
),
schemas.GroundTruth(
datum=schemas.Datum(
uid="uid1",
metadata={
"height": 900,
"width": 300,
},
),
annotations=[
schemas.Annotation(
labels=[
schemas.Label(key="dataset2", value="egg"),
],
),
],
),
]


@pytest.fixture
def multiclass_pr_curve_check_true_negatives_predictions():
return [
schemas.Prediction(
datum=schemas.Datum(
uid="uid0",
metadata={
"height": 900,
"width": 300,
},
),
annotations=[
schemas.Annotation(
labels=[
schemas.Label(key="dataset1", value="ant", score=0.15),
schemas.Label(key="dataset1", value="bee", score=0.48),
schemas.Label(key="dataset1", value="cat", score=0.37),
],
)
],
),
schemas.Prediction(
datum=schemas.Datum(
uid="uid1",
metadata={
"height": 900,
"width": 300,
},
),
annotations=[
schemas.Annotation(
labels=[
schemas.Label(key="dataset2", value="egg", score=0.15),
schemas.Label(
key="dataset2", value="milk", score=0.48
),
schemas.Label(
key="dataset2", value="flour", score=0.37
),
],
)
],
),
]


@pytest.fixture
def evaluate_detection_false_negatives_single_image_baseline_inputs():
groundtruths = [
Expand Down
33 changes: 33 additions & 0 deletions core/tests/conftest_outputs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2892,6 +2892,39 @@ def detailed_curve_examples_output():
("datum2",),
("datum0",),
},
# check cases where we shouldn't have any examples since the count is zero
("bee", 0.3, "fn", "misclassifications"): set(),
("dog", 0.1, "tn", "all"): set(),
}

return expected_outputs


@pytest.fixture
def detailed_curve_examples_check_zero_count_examples_output():
expected_outputs = {
("ant", 0.05, "fp", "misclassifications"): 0,
("ant", 0.95, "tn", "all"): 0,
("bee", 0.2, "fn", "misclassifications"): 0,
("cat", 0.2, "fn", "misclassifications"): 0,
}

return expected_outputs


@pytest.fixture
def detailed_curve_examples_check_true_negatives_output():
expected_outputs = {
("bee", 0.05, "tn", "all"): {
("uid1",),
},
("bee", 0.15, "tn", "all"): {
("uid1",),
},
("bee", 0.95, "tn", "all"): {
("uid1",),
("uid0",),
},
}

return expected_outputs
Expand Down
52 changes: 52 additions & 0 deletions core/tests/functional-tests/test_classification.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -601,8 +601,14 @@ def _get_specific_keys_from_pr_output(output_dict):

def test_detailed_curve_examples(
multiclass_pr_curve_groundtruths: list,
multiclass_pr_curve_check_zero_count_examples_groundtruths: list,
multiclass_pr_curve_check_true_negatives_groundtruths: list,
multiclass_pr_curve_predictions: list,
multiclass_pr_curve_check_zero_count_examples_predictions: list,
multiclass_pr_curve_check_true_negatives_predictions: list,
detailed_curve_examples_output: dict,
detailed_curve_examples_check_zero_count_examples_output: dict,
detailed_curve_examples_check_true_negatives_output: dict,
):
"""Test that we get back the right examples in DetailedPRCurves."""

Expand All @@ -624,3 +630,49 @@ def test_detailed_curve_examples(
)
== expected
)

# test additional cases to make sure that we aren't returning examples where count == 0
eval_job = evaluate_classification(
groundtruths=multiclass_pr_curve_check_zero_count_examples_groundtruths,
predictions=multiclass_pr_curve_check_zero_count_examples_predictions,
metrics_to_return=[enums.MetricType.DetailedPrecisionRecallCurve],
)
output = eval_job.metrics[0]["value"]

for (
key,
expected,
) in detailed_curve_examples_check_zero_count_examples_output.items():
assert (
len(
output[key[0]][key[1]][key[2]]["observations"][key[3]][
"examples"
]
)
== expected
)
assert (
output[key[0]][key[1]][key[2]]["observations"][key[3]]["count"]
) == 0

# test additional cases to make sure that we're getting back enough true negative examples
eval_job = evaluate_classification(
groundtruths=multiclass_pr_curve_check_true_negatives_groundtruths,
predictions=multiclass_pr_curve_check_true_negatives_predictions,
metrics_to_return=[enums.MetricType.DetailedPrecisionRecallCurve],
pr_curve_max_examples=5,
)
output = eval_job.metrics[0]["value"]

for (
key,
expected,
) in detailed_curve_examples_check_true_negatives_output.items():
assert (
set(
output[key[0]][key[1]][key[2]]["observations"][key[3]][
"examples"
]
)
== expected
)
Loading
Loading