Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add precision classification metric #2293

Merged
merged 31 commits into from
Nov 20, 2024
+694 −1
Merged

Add precision classification metric #2293

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
c73438f
Implement confusion matrix and precision, first draft
Aug 21, 2024
63f4a1d
Implement confusion matrix
Sep 9, 2024
b9d71b6
format :D
Sep 9, 2024
eac29aa
add agg type to cm, reformat debug representation add testing.
Sep 20, 2024
59db68b
formating and tiny refactor
Sep 21, 2024
4261bd8
add ClassificationMetric trait, rename variables and types, move test…
Sep 21, 2024
5431a2f
change unwrap to expect
Sep 21, 2024
fd2e585
update book
Sep 21, 2024
56965e8
remove unused code
Sep 22, 2024
419438a
changes to make reusing code easier
Sep 22, 2024
dfac847
format :D
Sep 22, 2024
ea4b29c
change to static data tests
Sep 24, 2024
e23aa7b
remove classification metric trait, add auxiliary code for classific…
Oct 14, 2024
60a246b
move classification objects to classification.rs, use rstest, remove …
Oct 21, 2024
c145531
review docstring, add top_k for multiclass tasks.
Oct 23, 2024
0c984c4
move class averaging and metric computation to metric implementation,…
Oct 25, 2024
b0a2939
change struct and var names
Oct 25, 2024
f18e321
Merge branch 'main' into add-to-metrics
Oct 26, 2024
386802c
rename params, enforce nonzero for top_k param, optimize one_hot for …
Oct 30, 2024
b525527
add adaptor por classification input, correct one hot function
Nov 1, 2024
ff7611a
define default for ClassReduction, derive new for Precision metric wi…
Nov 8, 2024
4cbcff2
Merge branch 'main' into add-to-metrics
Nov 8, 2024
eeab0d3
expose PrecisionMetric, change metric initialization
Nov 8, 2024
aea207f
check one_hot input tensor has more than 1 classes and correct it's i…
Nov 16, 2024
410f273
Merge branch 'main' into add-to-metrics
Nov 16, 2024
746fa9d
implement adaptor for MultilabelClassificationOutput and Classificati…
Nov 16, 2024
7428b86
change with_top_k to take usize
Nov 18, 2024
58e1902
Merge branch 'main' into add-to-metrics
Nov 18, 2024
d598f00
Add precision config for binary, multiclass and multilabel
laggui Nov 18, 2024
1542ee9
Fix dummy_classification_input
laggui Nov 18, 2024
03ebe1d
make PrecisionMetric public
Nov 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
move class averaging and metric computation to metric implementation,… 
… make dummy data more predictable and add tests for top_k > 1
  • Loading branch information
Tiago Sanona committed Oct 25, 2024
commit 0c984c4ace2643434c9047009b398bd202c9e97a
89 changes: 49 additions & 40 deletions crates/burn-train/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,7 @@ pub(crate) type TestDevice = burn_ndarray::NdArrayDevice;
#[cfg(test)]
pub(crate) mod tests {
use crate::{metric::classification::ClassificationInput, TestBackend, TestDevice};
use burn_core::{
prelude::{Bool, Tensor},
tensor::Distribution,
};
use burn_core::prelude::{Bool, Tensor};
use std::default::Default;

/// Probability of tp before adding errors
Expand All @@ -53,49 +50,61 @@ pub(crate) mod tests {
pub fn dummy_classification_input(
classification_type: &ClassificationType,
) -> ClassificationInput<TestBackend> {
let (real_targets, prediction_targets) = match classification_type {
let (targets, predictions) = match classification_type {
ClassificationType::Binary => {
let real_targets = Tensor::<TestBackend, 2, Bool>::from_data(
[[0], [1], [0], [0], [1]],
&TestDevice::default(),
);

let prediction_targets = Tensor::<TestBackend, 2>::from_data(
[[0], [0], [1], [0], [1]],
&TestDevice::default(),
);
(real_targets, prediction_targets)
(
Tensor::<TestBackend, 2, Bool>::from_data(
[[0], [1], [0], [0], [1]],
&TestDevice::default(),
),
Tensor::<TestBackend, 2>::from_data(
[[0.3], [0.2], [0.7], [0.1], [0.55]],
//[[0], [0], [1], [0], [1]] with threshold=0.5
&TestDevice::default(),
),
)
}
ClassificationType::Multiclass => {
let real_targets = Tensor::<TestBackend, 2, Bool>::from_data(
[[0, 1, 0], [1, 0, 0], [0, 0, 1], [0, 0, 1], [1, 0, 0]],
&TestDevice::default(),
);

let prediction_targets = Tensor::<TestBackend, 2>::from_data(
[[0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 0, 1], [1, 0, 0]],
&TestDevice::default(),
);
(real_targets, prediction_targets)
(
Tensor::<TestBackend, 2, Bool>::from_data(
[[0, 1, 0], [1, 0, 0], [0, 0, 1], [0, 0, 1], [1, 0, 0]],
&TestDevice::default(),
),
Tensor::<TestBackend, 2>::from_data(
[
[0.2, 0.8, 0.0],
[0.3, 0.6, 0.1],
[0.7, 0.25, 0.05],
[0.1, 0.15, 0.8],
[0.9, 0.03, 0.07],
],
//[[0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 0, 1], [1, 0, 0]] with top_k=1
//[[1, 1, 0], [1, 1, 0], [1, 1, 0], [0, 1, 1], [1, 0, 1]] with top_k=2
&TestDevice::default(),
),
)
}
ClassificationType::Multilabel => {
let real_targets = Tensor::<TestBackend, 2, Bool>::from_data(
[[1, 1, 0], [1, 0, 1], [1, 1, 1], [0, 0, 1], [1, 0, 0]],
&TestDevice::default(),
);

let prediction_targets = Tensor::<TestBackend, 2>::from_data(
[[0, 1, 1], [0, 1, 0], [1, 1, 0], [1, 0, 1], [1, 0, 0]],
&TestDevice::default(),
);
(real_targets, prediction_targets)
(
Tensor::<TestBackend, 2, Bool>::from_data(
[[1, 1, 0], [1, 0, 1], [1, 1, 1], [0, 0, 1], [1, 0, 0]],
&TestDevice::default(),
),
Tensor::<TestBackend, 2>::from_data(
[
[0.1, 0.7, 0.6],
[0.3, 0.9, 0.05],
[0.8, 0.9, 0.4],
[0.7, 0.5, 0.9],
[1.0, 0.3, 0.2],
],
//[[0, 1, 1], [0, 1, 0], [1, 1, 0], [1, 0, 1], [1, 0, 0]] with threshold=0.5
&TestDevice::default(),
),
)
}
};
let predictions = prediction_targets
.random_like(Distribution::Uniform(0.0, THRESHOLD - 0.1))
.sub(prediction_targets.clone())
.abs();

ClassificationInput::new(predictions, real_targets)
ClassificationInput::new(predictions, targets)
}
}
169 changes: 71 additions & 98 deletions crates/burn-train/src/metric/confusion_stats.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
use super::classification::ClassAverageType;
use burn_core::prelude::{Backend, Bool, Int, Tensor};
use burn_core::tensor::cast::ToElement;
use std::fmt::{self, Debug};

#[derive(Clone)]
Expand All @@ -19,7 +18,7 @@ impl<B: Backend> Debug for ConfusionStats<B> {
};
let ratio_of_support_vec =
|metric: Tensor<B, 1>| to_vec(self.clone().ratio_of_support(metric));
f.debug_struct("ConfusionMatrix")
f.debug_struct("ConfusionStats")
.field("tp", &ratio_of_support_vec(self.clone().true_positive()))
.field("fp", &ratio_of_support_vec(self.clone().false_positive()))
.field("tn", &ratio_of_support_vec(self.clone().true_negative()))
Expand Down Expand Up @@ -68,24 +67,6 @@ impl<B: Backend> ConfusionStats<B> {
}
}

///convert to averaged metric, returns float
fn average(mut aggregated_metric: Tensor<B, 1>, class_average: ClassAverageType) -> f64 {
use ClassAverageType::*;
let avg_tensor = match class_average {
Micro => aggregated_metric,
Macro => {
if aggregated_metric.contains_nan().any().into_scalar() {
let nan_mask = aggregated_metric.is_nan();
aggregated_metric = aggregated_metric
.clone()
.select(0, nan_mask.bool_not().argwhere().squeeze(1))
}
aggregated_metric.mean()
}
};
avg_tensor.into_scalar().to_f64()
}

pub fn true_positive(self) -> Tensor<B, 1> {
Self::aggregate(self.confusion_classes.equal_elem(3), self.class_average)
}
Expand Down Expand Up @@ -121,14 +102,6 @@ impl<B: Backend> ConfusionStats<B> {
pub fn ratio_of_support(self, metric: Tensor<B, 1>) -> Tensor<B, 1> {
metric / self.clone().support()
}

pub fn precision(self) -> f64 {
let class_average = self.class_average;
Self::average(
self.clone().true_positive() / self.predicted_positive(),
class_average,
)
}
}

#[cfg(test)]
Expand Down Expand Up @@ -159,22 +132,22 @@ mod tests {
}

#[rstest]
#[case::binary_micro(Binary, Micro, [1].into())]
#[case::binary_macro(Binary, Macro, [1].into())]
#[case::multiclass_micro(Multiclass, Micro, [3].into())]
#[case::multiclass_macro(Multiclass, Macro, [1, 1, 1].into())]
#[case::multilabel_micro(Multilabel, Micro, [5].into())]
#[case::multilabel_macro(Multilabel, Macro, [2, 2, 1].into())]
#[case::binary_micro(Binary, Micro, Some(THRESHOLD), None, [1].into())]
#[case::binary_macro(Binary, Macro, Some(THRESHOLD), None, [1].into())]
#[case::multiclass_micro(Multiclass, Micro, None, Some(1), [3].into())]
#[case::multiclass_macro(Multiclass, Macro, None, Some(1), [1, 1, 1].into())]
#[case::multiclass_micro(Multiclass, Micro, None, Some(2), [4].into())]
#[case::multiclass_macro(Multiclass, Macro, None, Some(2), [2, 1, 1].into())]
#[case::multilabel_micro(Multilabel, Micro, Some(THRESHOLD), None, [5].into())]
#[case::multilabel_macro(Multilabel, Macro, Some(THRESHOLD), None, [2, 2, 1].into())]
fn test_true_positive(
#[case] class_type: ClassificationType,
#[case] avg_type: ClassAverageType,
#[case] threshold: Option<f64>,
#[case] top_k: Option<usize>,
#[case] expected: Vec<i64>,
) {
let (predictions, targets) = dummy_classification_input(&class_type).into();
let (threshold, top_k) = match class_type {
Multiclass => (None, Some(1)),
_ => (Some(THRESHOLD), None),
};
ConfusionStats::new(predictions, targets, threshold, top_k, avg_type)
.true_positive()
.int()
Expand All @@ -183,22 +156,22 @@ mod tests {
}

#[rstest]
#[case::binary_micro(Binary, Micro, [2].into())]
#[case::binary_macro(Binary, Macro, [2].into())]
#[case::multiclass_micro(Multiclass, Micro, [8].into())]
#[case::multiclass_macro(Multiclass, Macro, [2, 3, 3].into())]
#[case::multilabel_micro(Multilabel, Micro, [3].into())]
#[case::multilabel_macro(Multilabel, Macro, [0, 2, 1].into())]
#[case::binary_micro(Binary, Micro, Some(THRESHOLD), None, [2].into())]
#[case::binary_macro(Binary, Macro, Some(THRESHOLD), None, [2].into())]
#[case::multiclass_micro(Multiclass, Micro, None, Some(1), [8].into())]
#[case::multiclass_macro(Multiclass, Macro, None, Some(1), [2, 3, 3].into())]
#[case::multiclass_micro(Multiclass, Micro, None, Some(2), [4].into())]
#[case::multiclass_macro(Multiclass, Macro, None, Some(2), [1, 1, 2].into())]
#[case::multilabel_micro(Multilabel, Micro, Some(THRESHOLD), None, [3].into())]
#[case::multilabel_macro(Multilabel, Macro, Some(THRESHOLD), None, [0, 2, 1].into())]
fn test_true_negative(
#[case] class_type: ClassificationType,
#[case] avg_type: ClassAverageType,
#[case] threshold: Option<f64>,
#[case] top_k: Option<usize>,
#[case] expected: Vec<i64>,
) {
let (predictions, targets) = dummy_classification_input(&class_type).into();
let (threshold, top_k) = match class_type {
Multiclass => (None, Some(1)),
_ => (Some(THRESHOLD), None),
};
ConfusionStats::new(predictions, targets, threshold, top_k, avg_type)
.true_negative()
.int()
Expand All @@ -207,22 +180,22 @@ mod tests {
}

#[rstest]
#[case::binary_micro(Binary, Micro, [1].into())]
#[case::binary_macro(Binary, Macro, [1].into())]
#[case::multiclass_micro(Multiclass, Micro, [2].into())]
#[case::multiclass_macro(Multiclass, Macro, [1, 1, 0].into())]
#[case::multilabel_micro(Multilabel, Micro, [3].into())]
#[case::multilabel_macro(Multilabel, Macro, [1, 1, 1].into())]
#[case::binary_micro(Binary, Micro, Some(THRESHOLD), None, [1].into())]
#[case::binary_macro(Binary, Macro, Some(THRESHOLD), None, [1].into())]
#[case::multiclass_micro(Multiclass, Micro, None, Some(1), [2].into())]
#[case::multiclass_macro(Multiclass, Macro, None, Some(1), [1, 1, 0].into())]
#[case::multiclass_micro(Multiclass, Micro, None, Some(2), [6].into())]
#[case::multiclass_macro(Multiclass, Macro, None, Some(2), [2, 3, 1].into())]
#[case::multilabel_micro(Multilabel, Micro, Some(THRESHOLD), None, [3].into())]
#[case::multilabel_macro(Multilabel, Macro, Some(THRESHOLD), None, [1, 1, 1].into())]
fn test_false_positive(
#[case] class_type: ClassificationType,
#[case] avg_type: ClassAverageType,
#[case] threshold: Option<f64>,
#[case] top_k: Option<usize>,
#[case] expected: Vec<i64>,
) {
let (predictions, targets) = dummy_classification_input(&class_type).into();
let (threshold, top_k) = match class_type {
Multiclass => (None, Some(1)),
_ => (Some(THRESHOLD), None),
};
ConfusionStats::new(predictions, targets, threshold, top_k, avg_type)
.false_positive()
.int()
Expand All @@ -231,22 +204,22 @@ mod tests {
}

#[rstest]
#[case::binary_micro(Binary, Micro, [1].into())]
#[case::binary_macro(Binary, Macro, [1].into())]
#[case::multiclass_micro(Multiclass, Micro, [2].into())]
#[case::multiclass_macro(Multiclass, Macro, [1, 0, 1].into())]
#[case::multilabel_micro(Multilabel, Micro, [4].into())]
#[case::multilabel_macro(Multilabel, Macro, [2, 0, 2].into())]
#[case::binary_micro(Binary, Micro, Some(THRESHOLD), None, [1].into())]
#[case::binary_macro(Binary, Macro, Some(THRESHOLD), None, [1].into())]
#[case::multiclass_micro(Multiclass, Micro, None, Some(1), [2].into())]
#[case::multiclass_macro(Multiclass, Macro, None, Some(1), [1, 0, 1].into())]
#[case::multiclass_micro(Multiclass, Micro, None, Some(2), [1].into())]
#[case::multiclass_macro(Multiclass, Macro, None, Some(2), [0, 0, 1].into())]
#[case::multilabel_micro(Multilabel, Micro, Some(THRESHOLD), None, [4].into())]
#[case::multilabel_macro(Multilabel, Macro, Some(THRESHOLD), None, [2, 0, 2].into())]
fn test_false_negatives(
#[case] class_type: ClassificationType,
#[case] avg_type: ClassAverageType,
#[case] threshold: Option<f64>,
#[case] top_k: Option<usize>,
#[case] expected: Vec<i64>,
) {
let (predictions, targets) = dummy_classification_input(&class_type).into();
let (threshold, top_k) = match class_type {
Multiclass => (None, Some(1)),
_ => (Some(THRESHOLD), None),
};
ConfusionStats::new(predictions, targets, threshold, top_k, avg_type)
.false_negative()
.int()
Expand All @@ -255,22 +228,22 @@ mod tests {
}

#[rstest]
#[case::binary_micro(Binary, Micro, [2].into())]
#[case::binary_macro(Binary, Macro, [2].into())]
#[case::multiclass_micro(Multiclass, Micro, [5].into())]
#[case::multiclass_macro(Multiclass, Macro, [2, 1, 2].into())]
#[case::multilabel_micro(Multilabel, Micro, [9].into())]
#[case::multilabel_macro(Multilabel, Macro, [4, 2, 3].into())]
#[case::binary_micro(Binary, Micro, Some(THRESHOLD), None, [2].into())]
#[case::binary_macro(Binary, Macro, Some(THRESHOLD), None, [2].into())]
#[case::multiclass_micro(Multiclass, Micro, None, Some(1), [5].into())]
#[case::multiclass_macro(Multiclass, Macro, None, Some(1), [2, 1, 2].into())]
#[case::multiclass_micro(Multiclass, Micro, None, Some(2), [5].into())]
#[case::multiclass_macro(Multiclass, Macro, None, Some(2), [2, 1, 2].into())]
#[case::multilabel_micro(Multilabel, Micro, Some(THRESHOLD), None, [9].into())]
#[case::multilabel_macro(Multilabel, Macro, Some(THRESHOLD), None, [4, 2, 3].into())]
fn test_positive(
#[case] class_type: ClassificationType,
#[case] avg_type: ClassAverageType,
#[case] threshold: Option<f64>,
#[case] top_k: Option<usize>,
#[case] expected: Vec<i64>,
) {
let (predictions, targets) = dummy_classification_input(&class_type).into();
let (threshold, top_k) = match class_type {
Multiclass => (None, Some(1)),
_ => (Some(THRESHOLD), None),
};
ConfusionStats::new(predictions, targets, threshold, top_k, avg_type)
.positive()
.int()
Expand All @@ -279,22 +252,22 @@ mod tests {
}

#[rstest]
#[case::binary_micro(Binary, Micro, [3].into())]
#[case::binary_macro(Binary, Macro, [3].into())]
#[case::multiclass_micro(Multiclass, Micro, [10].into())]
#[case::multiclass_macro(Multiclass, Macro, [3, 4, 3].into())]
#[case::multilabel_micro(Multilabel, Micro, [6].into())]
#[case::multilabel_macro(Multilabel, Macro, [1, 3, 2].into())]
#[case::binary_micro(Binary, Micro, Some(THRESHOLD), None, [3].into())]
#[case::binary_macro(Binary, Macro, Some(THRESHOLD), None, [3].into())]
#[case::multiclass_micro(Multiclass, Micro, None, Some(1), [10].into())]
#[case::multiclass_macro(Multiclass, Macro, None, Some(1), [3, 4, 3].into())]
#[case::multiclass_micro(Multiclass, Micro, None, Some(2), [10].into())]
#[case::multiclass_macro(Multiclass, Macro, None, Some(2), [3, 4, 3].into())]
#[case::multilabel_micro(Multilabel, Micro, Some(THRESHOLD), None, [6].into())]
#[case::multilabel_macro(Multilabel, Macro, Some(THRESHOLD), None, [1, 3, 2].into())]
fn test_negative(
#[case] class_type: ClassificationType,
#[case] avg_type: ClassAverageType,
#[case] threshold: Option<f64>,
#[case] top_k: Option<usize>,
#[case] expected: Vec<i64>,
) {
let (predictions, targets) = dummy_classification_input(&class_type).into();
let (threshold, top_k) = match class_type {
Multiclass => (None, Some(1)),
_ => (Some(THRESHOLD), None),
};
ConfusionStats::new(predictions, targets, threshold, top_k, avg_type)
.negative()
.int()
Expand All @@ -303,22 +276,22 @@ mod tests {
}

#[rstest]
#[case::binary_micro(Binary, Micro, [2].into())]
#[case::binary_macro(Binary, Macro, [2].into())]
#[case::multiclass_micro(Multiclass, Micro, [5].into())]
#[case::multiclass_macro(Multiclass, Macro, [2, 2, 1].into())]
#[case::multilabel_micro(Multilabel, Micro, [8].into())]
#[case::multilabel_macro(Multilabel, Macro, [3, 3, 2].into())]
#[case::binary_micro(Binary, Micro, Some(THRESHOLD), None, [2].into())]
#[case::binary_macro(Binary, Macro, Some(THRESHOLD), None, [2].into())]
#[case::multiclass_micro(Multiclass, Micro, None, Some(1), [5].into())]
#[case::multiclass_macro(Multiclass, Macro, None, Some(1), [2, 2, 1].into())]
#[case::multiclass_micro(Multiclass, Micro, None, Some(2), [10].into())]
#[case::multiclass_macro(Multiclass, Macro, None, Some(2), [4, 4, 2].into())]
#[case::multilabel_micro(Multilabel, Micro, Some(THRESHOLD), None, [8].into())]
#[case::multilabel_macro(Multilabel, Macro, Some(THRESHOLD), None, [3, 3, 2].into())]
fn test_predicted_positive(
#[case] class_type: ClassificationType,
#[case] avg_type: ClassAverageType,
#[case] threshold: Option<f64>,
#[case] top_k: Option<usize>,
#[case] expected: Vec<i64>,
) {
let (predictions, targets) = dummy_classification_input(&class_type).into();
let (threshold, top_k) = match class_type {
Multiclass => (None, Some(1)),
_ => (Some(THRESHOLD), None),
};
ConfusionStats::new(predictions, targets, threshold, top_k, avg_type)
.predicted_positive()
.int()
Expand Down
Loading