From 2fb60f70e9a39926a5f8acadc131a5a1c02d7f26 Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Sun, 8 Apr 2018 21:42:30 -0700 Subject: [PATCH 01/10] "add metrics" --- python/paddle/fluid/metrics.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 python/paddle/fluid/metrics.py diff --git a/python/paddle/fluid/metrics.py b/python/paddle/fluid/metrics.py new file mode 100644 index 0000000000000..5196cc517da09 --- /dev/null +++ b/python/paddle/fluid/metrics.py @@ -0,0 +1,16 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Fluid Metrics +""" From 31ac9f4bfb23a5fe2fe46f1f3a37e288a4c2c18a Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Mon, 9 Apr 2018 01:28:15 -0700 Subject: [PATCH 02/10] "add fluid metrics" --- python/paddle/fluid/metrics.py | 156 +++++++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) diff --git a/python/paddle/fluid/metrics.py b/python/paddle/fluid/metrics.py index 5196cc517da09..fccf95102d029 100644 --- a/python/paddle/fluid/metrics.py +++ b/python/paddle/fluid/metrics.py @@ -14,3 +14,159 @@ """ Fluid Metrics """ +import numpy as np +import copy + + +def _is_numpy_(var): + return isinstance(var, (np.ndarray, np.generic)) + + +def _is_number_(var): + return isinstance(var, int) or isinstance(var, float) or (isinstance( + var, np.ndarray) and var.shape == (1, )) + + +def _is_number_or_matrix_(var): + return _is_number_(var) or isinstance(var, np.ndarray) + + +class MetricBase(object): + """ + """ + + def __init__(self, name=None, **kwargs): + self._name = name if name != None else self.__class__.__name__ + self._kwargs = kwargs + self.reset() + + def reset(self): + """ + states is the attributes who not has _ prefix. + reset the states of metrics. + """ + states = { + attr: value + for attr, value in self.__dict__.iteritems() + if not attr.startswith("_") + } + for attr, value in states: + if isinstance(value, int): + states[attr] = 0 + elif isinstance(value, float): + states[attr] = .0 + elif isinstance(value, (np.ndarray, np.generic)): + states[attr] = np.zeros_like(value) + else: + states[attr] = None + + def get_config(self): + states = { + attr: value + for attr, value in self.__dict__.iteritems() + if not attr.startswith("_") + } + config = copy.deepcopy(self._kwargs) + return config.update({ + "name": self._name, + "states": copy.deepcopy(states) + }) + + def update(self): + raise NotImplementedError() + + def eval(self): + raise NotImplementedError() + + +class CompositeMetric(MetricBase): + """ + Compute multiple metrics in each minibatch. + for example, merge F1, accuracy, recall into one Metric. + """ + + def __init__(self): + self._metrics = [] + + def add_metric(self, metric): + if not isinstance(metric, MetricBase): + raise ValueError("SubMetric should be inherit from MetricBase.") + self._metrics.append(metric) + + def eval(self): + ans = [] + for m in self._metrics: + ans.append(m.eval()) + return ans + + +class Accuracy(MetricBase): + def __init__(self): + self.value = .0 + self.weight = .0 + + def update(self, value, weight): + if not _is_number_or_matrix_(value): + raise ValueError( + "The 'value' must be a number(int, float) or a numpy ndarray.") + if not _is_number_(weight): + raise ValueError("The 'weight' must be a number(int, float).") + self.value += value * weight + self.weight += weight + + def eval(self): + if self.weight == 0: + raise ValueError( + "There is no data in Accuracy Metrics. Please check layers.accuracy output has added to Accuracy." + ) + return self.value / self.weight + + +class ChunkEvalutor(MetricBase): + def __init__(self): + self.num_infer_chunks = 0 + self.num_label_chunks = 0 + self.num_correct_chunks = 0 + + def update(self, precision, recall, f1_score, num_infer_chunks, + num_label_chunks, num_correct_chunks): + self.num_infer_chunks += num_infer_chunks + self.num_label_chunks += num_label_chunks + self.num_correct_chunks += num_correct_chunks + + def eval(self): + precision = float( + self.num_correct_chunks + ) / self.num_infer_chunks if self.num_infer_chunks else 0 + recall = float(self.num_correct_chunks + ) / self.num_label_chunks if self.num_label_chunks else 0 + f1_score = float(2 * precision * recall) / ( + precision + recall) if self.num_correct_chunks else 0 + return precision, recall, f1_score + + +class EditDistance(MetricBase): + def __init__(self): + self.total_distance = .0 + self.seq_num = 0 + self.instance_error = 0 + + def update(self, distances, seq_num): + if not _is_numpy_(distances): + raise ValueError("The 'distances' must be a numpy ndarray.") + if not _is_number_(seq_num): + raise ValueError("The 'seq_num' must be a number(int, float).") + seq_right_count = np.sum(distances == 0) + total_distance = np.sum(distances) + self.seq_num += seq_num + self.instance_error += seq_num - seq_right_count + self.total_distance += total_distance + + def eval(): + if self.seq_num == 0: + raise ValueError( + "There is no data in EditDistance Metric. Please check layers.edit_distance output has been added to EditDistance." + ) + avg_distance = self.total_distance / self.seq_num + avg_instance_error = self.instance_error / self.seq_num + return avg_distance, avg_instance_error From e6f621d73b20e8ea1d1897a51f1eb99251a142c4 Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Mon, 9 Apr 2018 01:59:03 -0700 Subject: [PATCH 03/10] "add import guards" --- python/paddle/fluid/__init__.py | 1 + python/paddle/fluid/average.py | 5 +++++ python/paddle/fluid/evaluator.py | 5 +++++ python/paddle/fluid/metrics.py | 11 +++++++++++ 4 files changed, 22 insertions(+) diff --git a/python/paddle/fluid/__init__.py b/python/paddle/fluid/__init__.py index f01d638efddd4..dea9d2372324b 100644 --- a/python/paddle/fluid/__init__.py +++ b/python/paddle/fluid/__init__.py @@ -29,6 +29,7 @@ import backward import regularizer import average +import metrics from param_attr import ParamAttr, WeightNormParamAttr from data_feeder import DataFeeder from core import LoDTensor, CPUPlace, CUDAPlace, CUDAPinnedPlace diff --git a/python/paddle/fluid/average.py b/python/paddle/fluid/average.py index ded6eb0859683..4a0f6c0f2c27e 100644 --- a/python/paddle/fluid/average.py +++ b/python/paddle/fluid/average.py @@ -22,6 +22,8 @@ wrappers of Python functions. """ +__all__ = ["WeightedAverage"] + def _is_number_(var): return isinstance(var, int) or isinstance(var, float) or (isinstance( @@ -34,6 +36,9 @@ def _is_number_or_matrix_(var): class WeightedAverage(object): def __init__(self): + warnings.warn( + "The %s is deprecated, please use fluid.metrics.Accuracy instead." % + (self.__class__.__name__), DeprecationWarning) self.reset() def reset(self): diff --git a/python/paddle/fluid/evaluator.py b/python/paddle/fluid/evaluator.py index 19e5b61b0b32a..ef7d3c111a418 100644 --- a/python/paddle/fluid/evaluator.py +++ b/python/paddle/fluid/evaluator.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import warnings import numpy as np import layers @@ -59,6 +60,10 @@ class Evaluator(object): """ def __init__(self, name, **kwargs): + warnings.warn( + "The %s is deprecated, because maintain a modified program inside evaluator cause bug easily, please use fluid.metrics.%s instead." + % (self.__class__.__name__, self.__class__.__name__), + DeprecationWarning) self.states = [] self.metrics = [] self.helper = LayerHelper(name, **kwargs) diff --git a/python/paddle/fluid/metrics.py b/python/paddle/fluid/metrics.py index fccf95102d029..345fbbc57315a 100644 --- a/python/paddle/fluid/metrics.py +++ b/python/paddle/fluid/metrics.py @@ -13,10 +13,21 @@ # limitations under the License. """ Fluid Metrics + +The metrics are accomplished via Python natively. """ import numpy as np import copy +__all__ = [ + 'MetricBase', + 'CompositeMetric', + 'Accuracy', + 'ChunkEvaluator', + 'EditDistance', + 'DetectionMAP', +] + def _is_numpy_(var): return isinstance(var, (np.ndarray, np.generic)) From 969b91d63173aa0c135e6082a53672fc1007afc4 Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Mon, 9 Apr 2018 19:23:14 -0700 Subject: [PATCH 04/10] "show warnings" --- python/paddle/fluid/average.py | 2 +- python/paddle/fluid/evaluator.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/python/paddle/fluid/average.py b/python/paddle/fluid/average.py index 4a0f6c0f2c27e..f6844db4b1170 100644 --- a/python/paddle/fluid/average.py +++ b/python/paddle/fluid/average.py @@ -38,7 +38,7 @@ class WeightedAverage(object): def __init__(self): warnings.warn( "The %s is deprecated, please use fluid.metrics.Accuracy instead." % - (self.__class__.__name__), DeprecationWarning) + (self.__class__.__name__), Warning) self.reset() def reset(self): diff --git a/python/paddle/fluid/evaluator.py b/python/paddle/fluid/evaluator.py index ef7d3c111a418..13475025b5c2a 100644 --- a/python/paddle/fluid/evaluator.py +++ b/python/paddle/fluid/evaluator.py @@ -62,8 +62,7 @@ class Evaluator(object): def __init__(self, name, **kwargs): warnings.warn( "The %s is deprecated, because maintain a modified program inside evaluator cause bug easily, please use fluid.metrics.%s instead." - % (self.__class__.__name__, self.__class__.__name__), - DeprecationWarning) + % (self.__class__.__name__, self.__class__.__name__), Warning) self.states = [] self.metrics = [] self.helper = LayerHelper(name, **kwargs) From b07aa10251cbc9568c624a4814799add523218fb Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Mon, 9 Apr 2018 19:35:03 -0700 Subject: [PATCH 05/10] "add demo" --- .../tests/book/test_label_semantic_roles.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/python/paddle/fluid/tests/book/test_label_semantic_roles.py b/python/paddle/fluid/tests/book/test_label_semantic_roles.py index c0a6df831acbf..117a071ab0fbf 100644 --- a/python/paddle/fluid/tests/book/test_label_semantic_roles.py +++ b/python/paddle/fluid/tests/book/test_label_semantic_roles.py @@ -181,13 +181,19 @@ def train(use_cuda, save_dirname=None, is_local=True): # add dependency track and move this config before optimizer crf_decode = fluid.layers.crf_decoding( input=feature_out, param_attr=fluid.ParamAttr(name='crfw')) - - chunk_evaluator = fluid.evaluator.ChunkEvaluator( + chunk_metrics = fluid.layers.chunk_eval( input=crf_decode, label=target, chunk_scheme="IOB", num_chunk_types=int(math.ceil((label_dict_len - 1) / 2.0))) + # chunk_evaluator = fluid.evaluator.ChunkEvaluator( + # input=crf_decode, + # label=target, + # chunk_scheme="IOB", + # num_chunk_types=int(math.ceil((label_dict_len - 1) / 2.0))) + chunk_evaluator = fluid.metrics.ChunkEvalutor() + train_data = paddle.batch( paddle.reader.shuffle( paddle.dataset.conll05.test(), buf_size=8192), @@ -213,14 +219,15 @@ def train_loop(main_program): start_time = time.time() batch_id = 0 for pass_id in xrange(PASS_NUM): - chunk_evaluator.reset(exe) + chunk_evaluator.reset() for data in train_data(): cost, precision, recall, f1_score = exe.run( main_program, feed=feeder.feed(data), - fetch_list=[avg_cost] + chunk_evaluator.metrics) + fetch_list=[avg_cost] + list(chunk_metrics)) + chunk_evaluator.update(*chunk_metrics) pass_precision, pass_recall, pass_f1_score = chunk_evaluator.eval( - exe) + ) if batch_id % 10 == 0: print("avg_cost:" + str(cost) + " precision:" + str( From d49c0e5c568837fc03df2a28de6701325364ca2b Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Mon, 9 Apr 2018 19:37:31 -0700 Subject: [PATCH 06/10] "fix ci" --- python/paddle/fluid/average.py | 1 + python/paddle/fluid/tests/book/test_label_semantic_roles.py | 6 ------ 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/python/paddle/fluid/average.py b/python/paddle/fluid/average.py index f6844db4b1170..6abe8233b07c4 100644 --- a/python/paddle/fluid/average.py +++ b/python/paddle/fluid/average.py @@ -13,6 +13,7 @@ # limitations under the License. import numpy as np +import warnings """ Class of all kinds of Average. diff --git a/python/paddle/fluid/tests/book/test_label_semantic_roles.py b/python/paddle/fluid/tests/book/test_label_semantic_roles.py index 117a071ab0fbf..41dbb25a9408f 100644 --- a/python/paddle/fluid/tests/book/test_label_semantic_roles.py +++ b/python/paddle/fluid/tests/book/test_label_semantic_roles.py @@ -187,13 +187,7 @@ def train(use_cuda, save_dirname=None, is_local=True): chunk_scheme="IOB", num_chunk_types=int(math.ceil((label_dict_len - 1) / 2.0))) - # chunk_evaluator = fluid.evaluator.ChunkEvaluator( - # input=crf_decode, - # label=target, - # chunk_scheme="IOB", - # num_chunk_types=int(math.ceil((label_dict_len - 1) / 2.0))) chunk_evaluator = fluid.metrics.ChunkEvalutor() - train_data = paddle.batch( paddle.reader.shuffle( paddle.dataset.conll05.test(), buf_size=8192), From 6e90d94c205f2ed3895d9f747f6ce4585fda9632 Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Mon, 9 Apr 2018 23:18:03 -0700 Subject: [PATCH 07/10] "add some details" --- benchmark/fluid/mnist.py | 7 +- python/paddle/fluid/metrics.py | 134 +++++++++++++++--- .../tests/book/test_label_semantic_roles.py | 11 +- 3 files changed, 125 insertions(+), 27 deletions(-) diff --git a/benchmark/fluid/mnist.py b/benchmark/fluid/mnist.py index 43866da9cb113..dc10ac2ec195a 100644 --- a/benchmark/fluid/mnist.py +++ b/benchmark/fluid/mnist.py @@ -139,9 +139,6 @@ def run_benchmark(model, args): # inference program inference_program = fluid.default_main_program().clone() - with fluid.program_guard(inference_program): - inference_program = fluid.io.get_inference_program( - target_vars=[batch_acc, batch_size_tensor]) # Optimization opt = fluid.optimizer.AdamOptimizer( @@ -161,7 +158,7 @@ def run_benchmark(model, args): train_reader = paddle.batch( paddle.dataset.mnist.train(), batch_size=args.batch_size) - accuracy = fluid.average.WeightedAverage() + accuracy = fluid.metrics.Accuracy() iters, num_samples, start_time = 0, 0, time.time() for pass_id in range(args.pass_num): accuracy.reset() @@ -184,7 +181,7 @@ def run_benchmark(model, args): "label": y_data}, fetch_list=[avg_cost, batch_acc, batch_size_tensor] ) # The accuracy is the accumulation of batches, but not the current batch. - accuracy.add(value=outs[1], weight=outs[2]) + accuracy.update(value=outs[1], weight=outs[2]) iters += 1 num_samples += len(y_data) loss = np.array(outs[0]) diff --git a/python/paddle/fluid/metrics.py b/python/paddle/fluid/metrics.py index 345fbbc57315a..39cb080851d77 100644 --- a/python/paddle/fluid/metrics.py +++ b/python/paddle/fluid/metrics.py @@ -44,13 +44,29 @@ def _is_number_or_matrix_(var): class MetricBase(object): """ + Base Class for all evaluators + + Args: + name(str): The name of evaluator. such as, "accuracy". Used for generate + temporary variable name. + Interface: + Note(*) : the states is the attributes who not has _ prefix. + + get_config(): print current states and configuration + reset(): clear the states. If the Metrics states type is not (int, float, np.ndarray), + Please override this method. + update(): update states at every minibatch + eval(): get metric evaluation in numpy type. """ - def __init__(self, name=None, **kwargs): - self._name = name if name != None else self.__class__.__name__ - self._kwargs = kwargs + def __init__(self, name, **kwargs): + self._name = str(name) if name != None else self.__class__.__name__ + self._kwargs = kwargs if kwargs != None else dict() self.reset() + def __str__(self): + return self._name + def reset(self): """ states is the attributes who not has _ prefix. @@ -61,15 +77,15 @@ def reset(self): for attr, value in self.__dict__.iteritems() if not attr.startswith("_") } - for attr, value in states: + for attr, value in states.iteritems(): if isinstance(value, int): - states[attr] = 0 + setattr(self, attr, 0) elif isinstance(value, float): - states[attr] = .0 + setattr(self, attr, .0) elif isinstance(value, (np.ndarray, np.generic)): - states[attr] = np.zeros_like(value) + setattr(self, attr, np.zeros_like(value)) else: - states[attr] = None + setattr(self, attr, None) def get_config(self): states = { @@ -78,10 +94,8 @@ def get_config(self): if not attr.startswith("_") } config = copy.deepcopy(self._kwargs) - return config.update({ - "name": self._name, - "states": copy.deepcopy(states) - }) + config.update({"name": self._name, "states": copy.deepcopy(states)}) + return config def update(self): raise NotImplementedError() @@ -96,7 +110,8 @@ class CompositeMetric(MetricBase): for example, merge F1, accuracy, recall into one Metric. """ - def __init__(self): + def __init__(self, name=None, **kwargs): + super(CompositeMetric, self).__init__(name, kwargs) self._metrics = [] def add_metric(self, metric): @@ -112,7 +127,26 @@ def eval(self): class Accuracy(MetricBase): - def __init__(self): + """ + Accumulate the accuracy from minibatches and compute the average accuracy + for every pass. + + Args: + name: the metrics name + + Example: + minibatch_accuracy = fluid.layers.accuracy(pred, label) + accuracy_evaluator = fluid.metrics.Accuracy() + for epoch in PASS_NUM: + accuracy_evaluator.reset() + for data in batches: + loss = exe.run(fetch_list=[cost, minibatch_accuracy]) + accuracy_evaluator.update(value=minibatch_accuracy, weight=batches) + accuracy = accuracy_evaluator.eval() + """ + + def __init__(self, name=None): + super(Accuracy, self).__init__(name) self.value = .0 self.weight = .0 @@ -134,7 +168,14 @@ def eval(self): class ChunkEvalutor(MetricBase): - def __init__(self): + """ + Accumulate counter numbers output by chunk_eval from mini-batches and + compute the precision recall and F1-score using the accumulated counter + numbers. + """ + + def __init__(self, name=None): + super(ChunkEvalutor, self).__init__(name) self.num_infer_chunks = 0 self.num_label_chunks = 0 self.num_correct_chunks = 0 @@ -157,7 +198,31 @@ def eval(self): class EditDistance(MetricBase): - def __init__(self): + """ + Accumulate edit distance sum and sequence number from mini-batches and + compute the average edit_distance and instance error of all batches. + + Args: + name: the metrics name + + Example: + edit_distance_metrics = fluid.layers.edit_distance(input, label) + distance_evaluator = fluid.metrics.EditDistance() + for epoch in PASS_NUM: + distance_evaluator.reset() + for data in batches: + loss = exe.run(fetch_list=[cost] + list(edit_distance_metrics)) + distance_evaluator.update(*edit_distance_metrics) + distance, instance_error = distance_evaluator.eval() + + In the above example: + 'distance' is the average of the edit distance in a pass. + 'instance_error' is the instance error rate in a pass. + + """ + + def __init__(self, name): + super(EditDistance, self).__init__(name) self.total_distance = .0 self.seq_num = 0 self.instance_error = 0 @@ -181,3 +246,40 @@ def eval(): avg_distance = self.total_distance / self.seq_num avg_instance_error = self.instance_error / self.seq_num return avg_distance, avg_instance_error + + +class DetectionMAP(MetricBase): + """ + Calculate the detection mean average precision (mAP). + + TODO (Dang Qingqing): update the following doc. + The general steps are as follows: + 1. calculate the true positive and false positive according to the input + of detection and labels. + 2. calculate mAP value, support two versions: '11 point' and 'integral'. + + Please get more information from the following articles: + https://sanchom.wordpress.com/tag/average-precision/ + https://arxiv.org/abs/1512.02325 + """ + + def __init__(self, name=None): + super(DetectionMAP, self).__init__(name) + # the current map value + self.value = .0 + + def update(self, value, weight): + if not _is_number_or_matrix_(value): + raise ValueError( + "The 'value' must be a number(int, float) or a numpy ndarray.") + if not _is_number_(weight): + raise ValueError("The 'weight' must be a number(int, float).") + self.value += value + self.weight += weight + + def eval(self): + if self.weight == 0: + raise ValueError( + "There is no data in DetectionMAP Metrics. Please check layers.detection_map output has added to DetectionMAP." + ) + return self.value / self.weight diff --git a/python/paddle/fluid/tests/book/test_label_semantic_roles.py b/python/paddle/fluid/tests/book/test_label_semantic_roles.py index 41dbb25a9408f..c0a6df831acbf 100644 --- a/python/paddle/fluid/tests/book/test_label_semantic_roles.py +++ b/python/paddle/fluid/tests/book/test_label_semantic_roles.py @@ -181,13 +181,13 @@ def train(use_cuda, save_dirname=None, is_local=True): # add dependency track and move this config before optimizer crf_decode = fluid.layers.crf_decoding( input=feature_out, param_attr=fluid.ParamAttr(name='crfw')) - chunk_metrics = fluid.layers.chunk_eval( + + chunk_evaluator = fluid.evaluator.ChunkEvaluator( input=crf_decode, label=target, chunk_scheme="IOB", num_chunk_types=int(math.ceil((label_dict_len - 1) / 2.0))) - chunk_evaluator = fluid.metrics.ChunkEvalutor() train_data = paddle.batch( paddle.reader.shuffle( paddle.dataset.conll05.test(), buf_size=8192), @@ -213,15 +213,14 @@ def train_loop(main_program): start_time = time.time() batch_id = 0 for pass_id in xrange(PASS_NUM): - chunk_evaluator.reset() + chunk_evaluator.reset(exe) for data in train_data(): cost, precision, recall, f1_score = exe.run( main_program, feed=feeder.feed(data), - fetch_list=[avg_cost] + list(chunk_metrics)) - chunk_evaluator.update(*chunk_metrics) + fetch_list=[avg_cost] + chunk_evaluator.metrics) pass_precision, pass_recall, pass_f1_score = chunk_evaluator.eval( - ) + exe) if batch_id % 10 == 0: print("avg_cost:" + str(cost) + " precision:" + str( From 8d36eba9fcecac56cebde1a71fe02e0e2f827076 Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Mon, 9 Apr 2018 23:19:35 -0700 Subject: [PATCH 08/10] "fix cci" --- python/paddle/fluid/metrics.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/paddle/fluid/metrics.py b/python/paddle/fluid/metrics.py index 39cb080851d77..43ceacd7ba8be 100644 --- a/python/paddle/fluid/metrics.py +++ b/python/paddle/fluid/metrics.py @@ -280,6 +280,7 @@ def update(self, value, weight): def eval(self): if self.weight == 0: raise ValueError( - "There is no data in DetectionMAP Metrics. Please check layers.detection_map output has added to DetectionMAP." + "There is no data in DetectionMAP Metrics. " + "Please check layers.detection_map output has added to DetectionMAP." ) return self.value / self.weight From 2f98b0fe641d402358bf9f2a5723fa05f966e963 Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Tue, 10 Apr 2018 00:54:44 -0700 Subject: [PATCH 09/10] "add demo Python" --- python/paddle/fluid/layers/metric.py | 28 ++++++++++ python/paddle/fluid/metrics.py | 80 ++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+) diff --git a/python/paddle/fluid/layers/metric.py b/python/paddle/fluid/layers/metric.py index 3d9157ad4ef93..3ddb9e2d32605 100644 --- a/python/paddle/fluid/layers/metric.py +++ b/python/paddle/fluid/layers/metric.py @@ -55,3 +55,31 @@ def accuracy(input, label, k=1, correct=None, total=None): "Total": [total], }) return acc_out + + +def auc(input, label, curve='ROC', num_thresholds=200): + helper = LayerHelper("auc", **locals()) + topk_out = helper.create_tmp_variable(dtype=input.dtype) + topk_indices = helper.create_tmp_variable(dtype="int64") + helper.append_op( + type="top_k", + inputs={"X": [input]}, + outputs={"Out": [topk_out], + "Indices": [topk_indices]}, + attrs={"k": k}) + auc_out = helper.create_tmp_variable(dtype="float32") + if correct is None: + correct = helper.create_tmp_variable(dtype="int64") + if total is None: + total = helper.create_tmp_variable(dtype="int64") + helper.append_op( + type="accuracy", + inputs={ + "Out": [topk_out], + "Indices": [topk_indices], + "Label": [label] + }, + attrs={"curve": curve, + "num_thresholds": num_thresholds}, + outputs={"AUC": [auc_out], }) + return auc_out diff --git a/python/paddle/fluid/metrics.py b/python/paddle/fluid/metrics.py index 43ceacd7ba8be..6c99c25a2844e 100644 --- a/python/paddle/fluid/metrics.py +++ b/python/paddle/fluid/metrics.py @@ -18,6 +18,7 @@ """ import numpy as np import copy +import warnings __all__ = [ 'MetricBase', @@ -284,3 +285,82 @@ def eval(self): "Please check layers.detection_map output has added to DetectionMAP." ) return self.value / self.weight + + +class Auc(MetricBase): + """ + Auc Metrics which adapts to binary classification. + Need to note that auc metrics compute the value via Python natively. + If you concern the speed, please use the fluid.layers.auc instead. + + The `auc` function creates four local variables, `true_positives`, + `true_negatives`, `false_positives` and `false_negatives` that are used to + compute the AUC. To discretize the AUC curve, a linearly spaced set of + thresholds is used to compute pairs of recall and precision values. The area + under the ROC-curve is therefore computed using the height of the recall + values by the false positive rate, while the area under the PR-curve is the + computed using the height of the precision values by the recall. + + Args: + name: metric name + curve: Specifies the name of the curve to be computed, 'ROC' [default] or + 'PR' for the Precision-Recall-curve. + num_thresholds: The number of thresholds to use when discretizing the roc + curve. + + "NOTE: only implement the ROC curve type via Python now." + """ + + def __init__(self, name, curve='ROC', num_thresholds=200): + super(MetricBase, self).__init__(name, curve, num_thresholds) + self._curve = curve + self._num_thresholds = num_thresholds + self._epsilon = 1e-6 + self.tp_list = np.ndarray((num_thresholds, )) + self.fn_list = np.ndarray((num_thresholds, )) + self.tn_list = np.ndarray((num_thresholds, )) + self.fp_list = np.ndarray((num_thresholds, )) + + def update(self, labels, predictions, axis=1): + if not _is_numpy_(labels): + raise ValueError("The 'labels' must be a numpy ndarray.") + if not _is_numpy_(predictions): + raise ValueError("The 'predictions' must be a numpy ndarray.") + + kepsilon = 1e-7 # to account for floating point imprecisions + thresholds = [(i + 1) * 1.0 / (num_thresholds - 1) + for i in range(num_thresholds - 2)] + thresholds = [0.0 - kepsilon] + thresholds + [1.0 + kepsilon] + + # caculate TP, FN, TN, FP count + for idx_thresh, thresh in enumerate(thresholds): + tp, fn, tn, fp = 0, 0, 0, 0 + for i, lbl in enumerate(labels): + if lbl: + if predictions[i, 0] >= thresh: + tp += 1 + else: + fn += 1 + else: + if predictions[i, 0] >= thresh: + fp += 1 + else: + tn += 1 + tp_list[idx_thresh] += tp + fn_list[idx_thresh] += fn + tn_list[idx_thresh] += tn + fp_list[idx_thresh] += fp + + def eval(self): + epsilon = self._epsilon + num_thresholds = self._num_thresholds + tpr = (tp_list.astype("float32") + epsilon) / ( + tp_list + fn_list + epsilon) + fpr = fp_list.astype("float32") / (fp_list + tn_list + epsilon) + rec = (tp_list.astype("float32") + epsilon) / ( + tp_list + fp_list + epsilon) + + x = fpr[:num_thresholds - 1] - fpr[1:] + y = (tpr[:num_thresholds - 1] + tpr[1:]) / 2.0 + auc_value = np.sum(x * y) + return auc_value From 773e688d3fe3303a617817e0dc395065ed7117b7 Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Tue, 10 Apr 2018 23:47:39 -0700 Subject: [PATCH 10/10] "add metrics" --- python/paddle/fluid/layers/metric.py | 9 ++++++++- python/paddle/fluid/metrics.py | 16 ++++++++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/python/paddle/fluid/layers/metric.py b/python/paddle/fluid/layers/metric.py index 3ddb9e2d32605..f66dccfa2d040 100644 --- a/python/paddle/fluid/layers/metric.py +++ b/python/paddle/fluid/layers/metric.py @@ -15,12 +15,13 @@ All layers just related to metric. """ +import warnings from ..layer_helper import LayerHelper from ..initializer import Normal, Constant from ..framework import Variable from ..param_attr import ParamAttr -__all__ = ['accuracy'] +__all__ = ['accuracy', 'auc'] def accuracy(input, label, k=1, correct=None, total=None): @@ -58,6 +59,12 @@ def accuracy(input, label, k=1, correct=None, total=None): def auc(input, label, curve='ROC', num_thresholds=200): + warnings.warn( + "This interface not recommended, fluid.layers.auc compute the auc at every minibatch, \ + but can not aggregate them and get the pass AUC, because pass \ + auc can not be averaged with weighted from the minibatch auc value. \ + Please use fluid.metrics.Auc, it can compute the auc value via Python natively, \ + which can get every minibatch and every pass auc value.", Warning) helper = LayerHelper("auc", **locals()) topk_out = helper.create_tmp_variable(dtype=input.dtype) topk_indices = helper.create_tmp_variable(dtype="int64") diff --git a/python/paddle/fluid/metrics.py b/python/paddle/fluid/metrics.py index 6c99c25a2844e..99a81c1d4244b 100644 --- a/python/paddle/fluid/metrics.py +++ b/python/paddle/fluid/metrics.py @@ -27,6 +27,7 @@ 'ChunkEvaluator', 'EditDistance', 'DetectionMAP', + 'Auc', ] @@ -181,8 +182,19 @@ def __init__(self, name=None): self.num_label_chunks = 0 self.num_correct_chunks = 0 - def update(self, precision, recall, f1_score, num_infer_chunks, - num_label_chunks, num_correct_chunks): + def update(self, num_infer_chunks, num_label_chunks, num_correct_chunks): + if not _is_number_or_matrix_(num_infer_chunks): + raise ValueError( + "The 'num_infer_chunks' must be a number(int, float) or a numpy ndarray." + ) + if not _is_number_or_matrix_(num_label_chunks): + raise ValueError( + "The 'num_label_chunks' must be a number(int, float) or a numpy ndarray." + ) + if not _is_number_or_matrix_(num_correct_chunks): + raise ValueError( + "The 'num_correct_chunks' must be a number(int, float) or a numpy ndarray." + ) self.num_infer_chunks += num_infer_chunks self.num_label_chunks += num_label_chunks self.num_correct_chunks += num_correct_chunks