Skip to content
This repository has been archived by the owner on Nov 17, 2023. It is now read-only.

Changes to mxnet.metric #18083

Merged
merged 27 commits into from
May 14, 2020
Merged

Changes to mxnet.metric #18083

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
f07d35e
finish 5 changes
Apr 15, 2020
575f23b
move metric.py to gluon, replace mx.metric with mx.gluon.metric in py…
acphile Apr 16, 2020
8992995
fix importError
acphile Apr 16, 2020
1b8f521
replace mx.metric with mx.gluon.metric in tests/python
acphile Apr 16, 2020
2ff2e38
remove global support
acphile Apr 20, 2020
c06f363
remove macro support
acphile Apr 20, 2020
6beba21
rewrite BinaryAccuracy
acphile Apr 20, 2020
b1fc42b
extend F1 to multiclass/multilabel
acphile Apr 21, 2020
4b091b0
add tests for new F1, remove global tests
acphile Apr 21, 2020
1dfe0e0
use mxnet.numpy instead of numpy
acphile Apr 22, 2020
083e85b
Merge remote-tracking branch 'upstream/master'
acphile Apr 24, 2020
59d98b3
fix sanity
acphile Apr 25, 2020
40e87e3
rewrite ce and ppl, improve some details
acphile Apr 27, 2020
5e153e1
use mxnet.numpy.float64
acphile Apr 27, 2020
bf68c6d
remove sklearn
acphile Apr 28, 2020
56b846e
remove reset_local() and get_global in other files
acphile Apr 29, 2020
8a437e9
fix test_mlp
acphile Apr 29, 2020
b7c2b3b
replace mx.metric with mx.gluon.metric in example
acphile Apr 29, 2020
ec615a5
fix context difference
acphile Apr 29, 2020
c4a3b67
Disable -DUSE_TVM_OP on GPU builds
leezu Apr 30, 2020
0456416
Fix disable tvm op for gpu runs
leezu Apr 30, 2020
2a80a0a
resolve conflicts
acphile May 6, 2020
8163fbb
use label.ctx in metric.py; remove gluoncv dependency in test_cvnets
acphile May 7, 2020
d53e6ef
fix sanity
acphile May 7, 2020
3adfa5e
Merge branch 'master' into master
leezu May 7, 2020
a2b0ffe
fix importError
acphile May 8, 2020
ef3058a
remove nose
acphile May 9, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 38 additions & 33 deletions python/mxnet/gluon/metric.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -420,8 +420,8 @@ def update(self, labels, preds):
labels, preds = check_label_shapes(labels, preds, True)

for label, pred_label in zip(labels, preds):
pred_label = pred_label.as_np_ndarray()
label = label.as_np_ndarray().as_in_ctx(pred_label.ctx)
pred_label = pred_label.as_np_ndarray().as_in_ctx(label.ctx)
label = label.as_np_ndarray()
if pred_label.shape != label.shape:
pred_label = pred_label.argmax(axis=self.axis)
pred_label = pred_label.astype('int32')
Expand Down Expand Up @@ -502,8 +502,8 @@ def update(self, labels, preds):
# we do not care about the order of top k elements. It is
# much faster, which is important since that computation is
# single-threaded due to Python GIL.
pred_label = numpy.argpartition(pred_label.as_np_ndarray().astype('float32'), -self.top_k)
label = label.as_np_ndarray().astype('int32').as_in_ctx(pred_label.ctx)
pred_label = numpy.argpartition(pred_label.as_np_ndarray().astype('float32'), -self.top_k).as_in_ctx(label.ctx)
label = label.as_np_ndarray().astype('int32')
check_label_shapes(label, pred_label)
num_samples = pred_label.shape[0]
num_dims = len(pred_label.shape)
Expand Down Expand Up @@ -594,10 +594,10 @@ def update_stats(self, label, pred):
pred : `NDArray`
Predicted values.
"""
pred = pred.as_np_ndarray()
label = label.as_np_ndarray().astype('int32').as_in_ctx(pred.ctx)
pred = pred.as_np_ndarray().as_in_ctx(label.ctx)
label = label.as_np_ndarray().astype('int32')
if self.class_type == "binary":
self._set(1, pred.ctx)
self._set(1, label.ctx)
if label.max() > 1:
raise ValueError("Wrong label for binary classification.")
if pred.shape == label.shape:
Expand All @@ -611,14 +611,14 @@ def update_stats(self, label, pred):

elif self.class_type == "multiclass":
num = pred.shape[-1]
self._set(num, pred.ctx)
self._set(num, label.ctx)
assert label.max() < num, "pred contains fewer classes than label!"
pred_label = one_hot(pred.argmax(axis=-1).reshape(-1), num)
label = one_hot(label.reshape(-1), num)

elif self.class_type == "multilabel":
num = pred.shape[-1]
self._set(num, pred.ctx)
self._set(num, label.ctx)
assert pred.shape == label.shape, \
"The shape of label should be same as that of prediction for multilabel classification."
pred_label = predict_with_threshold(pred, self.threshold).reshape(-1, num)
Expand Down Expand Up @@ -920,8 +920,8 @@ def update(self, labels, preds):
for label, pred_label in zip(labels, preds):
pred_label = predict_with_threshold(pred_label, self.threshold)

pred_label = pred_label.as_np_ndarray().astype('int32')
label = label.as_np_ndarray().astype('int32').as_in_ctx(pred_label.ctx)
pred_label = pred_label.as_np_ndarray().astype('int32').as_in_ctx(label.ctx)
label = label.as_np_ndarray().astype('int32')
# flatten before checking shapes to avoid shape miss match
label = label.reshape(-1)
pred_label = pred_label.reshape(-1)
Expand Down Expand Up @@ -1080,8 +1080,8 @@ def update(self, labels, preds):
labels, preds = check_label_shapes(labels, preds, True)

for label, pred in zip(labels, preds):
label = label.as_np_ndarray().as_in_ctx(pred.ctx)
pred = pred.as_np_ndarray()
label = label.as_np_ndarray()
pred = pred.as_np_ndarray().as_in_ctx(label.ctx)

num_inst = label.shape[0]
mae = numpy.abs(label - pred).reshape(num_inst, -1).mean(axis=-1).sum()
Expand Down Expand Up @@ -1139,8 +1139,8 @@ def update(self, labels, preds):
labels, preds = check_label_shapes(labels, preds, True)

for label, pred in zip(labels, preds):
label = label.as_np_ndarray().as_in_ctx(pred.ctx)
pred = pred.as_np_ndarray()
label = label.as_np_ndarray()
pred = pred.as_np_ndarray().as_in_ctx(label.ctx)

num_inst = label.shape[0]
mse = ((label - pred)**2.0).reshape(num_inst, -1).mean(axis=-1).sum()
Expand Down Expand Up @@ -1243,8 +1243,8 @@ def update(self, labels, preds):
labels, preds = check_label_shapes(labels, preds, True)

for label, pred in zip(labels, preds):
label = label.as_np_ndarray().as_in_ctx(pred.ctx)
pred = pred.as_np_ndarray()
label = label.as_np_ndarray()
pred = pred.as_np_ndarray().as_in_ctx(label.ctx)

label = label.reshape(label.shape[0], -1)
pred = pred.reshape(pred.shape[0], -1)
Expand Down Expand Up @@ -1310,8 +1310,8 @@ def update(self, labels, preds):
labels, preds = check_label_shapes(labels, preds, True)

for label, pred in zip(labels, preds):
label = label.as_np_ndarray().as_in_ctx(pred.ctx)
pred = pred.as_np_ndarray()
label = label.as_np_ndarray()
pred = pred.as_np_ndarray().as_in_ctx(label.ctx)

if len(label.shape) == 1:
label = label.reshape(1, label.shape[0])
Expand Down Expand Up @@ -1345,6 +1345,8 @@ class :math:`k`.

Parameters
----------
eps : float, default 1e-12
Use small constant for the case that predicted value is 0.
ignore_label : int or None, default None
Index of invalid label to ignore when
counting. By default, sets to -1.
Expand All @@ -1370,12 +1372,13 @@ class :math:`k`.
>>> print ce.get()
('cross-entropy', 0.57159948348999023)
"""
def __init__(self, ignore_label=None, axis=-1, name='cross-entropy',
def __init__(self, eps=1e-12, ignore_label=None, axis=-1, name='cross-entropy',
output_names=None, label_names=None):
super(CrossEntropy, self).__init__(
name, output_names=output_names, label_names=label_names)
self.ignore_label = ignore_label
self.axis = axis
self.eps = eps

def update(self, labels, preds):
"""Updates the internal evaluation result.
Expand All @@ -1395,15 +1398,15 @@ def update(self, labels, preds):
for label, pred in zip(labels, preds):
assert label.size == pred.size/pred.shape[-1], \
"shape mismatch: %s vs. %s"%(label.shape, pred.shape)
label = label.as_in_context(pred.ctx).reshape((label.size,))
pred = ndarray.pick(pred, label.astype(dtype='int32'), axis=self.axis)
label = label.reshape((label.size,))
pred = ndarray.pick(pred.as_in_context(label.ctx), label.astype(dtype='int32'), axis=self.axis)
label = label.as_np_ndarray()
pred = pred.as_np_ndarray()
if self.ignore_label is not None:
ignore = (label == self.ignore_label).astype(pred.dtype)
num -= ignore.sum()
pred = pred * (1 - ignore) + ignore
loss -= numpy.log(numpy.maximum(1e-12, pred)).sum()
loss -= numpy.log(numpy.maximum(self.eps, pred)).sum()
num += pred.size
self.sum_metric += loss
self.num_inst += num
Expand Down Expand Up @@ -1438,6 +1441,8 @@ class Perplexity(CrossEntropy):

Parameters
----------
eps : float, default 1e-12
Use small constant for the case that predicted value is 0.
ignore_label : int or None, default None
Index of invalid label to ignore when
counting. By default, sets to -1.
Expand All @@ -1463,10 +1468,10 @@ class Perplexity(CrossEntropy):
>>> print perp.get()
('Perplexity', 1.7710976285155853)
"""
def __init__(self, ignore_label=None, axis=-1, name='perplexity',
def __init__(self, eps=1e-12, ignore_label=None, axis=-1, name='perplexity',
output_names=None, label_names=None):
super(Perplexity, self).__init__(
name=name, ignore_label=ignore_label, axis=axis,
name=name, eps=eps, ignore_label=ignore_label, axis=axis,
output_names=output_names, label_names=label_names)

def get(self):
Expand Down Expand Up @@ -1535,8 +1540,8 @@ def update(self, labels, preds):
labels, preds = check_label_shapes(labels, preds, True)

for label, pred in zip(labels, preds):
label = label.as_np_ndarray().as_in_ctx(pred.ctx)
pred = pred.as_np_ndarray()
label = label.as_np_ndarray()
pred = pred.as_np_ndarray().as_in_ctx(label.ctx)

label = label.reshape(-1)
num_examples = pred.shape[0]
Expand Down Expand Up @@ -1622,8 +1627,8 @@ def update(self, labels, preds):
labels, preds = check_label_shapes(labels, preds, True)
for label, pred in zip(labels, preds):
check_label_shapes(label, pred, False, True)
label = label.as_np_ndarray().as_in_ctx(pred.ctx).reshape(-1).astype(numpy.float64)
pred = pred.as_np_ndarray().reshape(-1).astype(numpy.float64)
label = label.as_np_ndarray().reshape(-1).astype(numpy.float64)
pred = pred.as_np_ndarray().as_in_ctx(label.ctx).reshape(-1).astype(numpy.float64)

self.num_inst += 1
self._label_nums, self._mean_l, self._sse_l = \
Expand Down Expand Up @@ -1733,8 +1738,8 @@ def update(self, labels, preds):

# update the confusion matrix
for label, pred in zip(labels, preds):
label = label.astype('int32', copy=False).as_np_ndarray().as_in_ctx(pred.ctx)
pred = pred.as_np_ndarray()
label = label.astype('int32', copy=False).as_np_ndarray()
pred = pred.as_np_ndarray().as_in_ctx(label.ctx)
if pred.shape != label.shape:
pred = pred.argmax(axis=1).astype(label, copy=False)
else:
Expand Down Expand Up @@ -1872,8 +1877,8 @@ def update(self, labels, preds):
labels, preds = check_label_shapes(labels, preds, True)

for pred, label in zip(preds, labels):
label = label.as_np_ndarray().as_in_ctx(pred.ctx)
pred = pred.as_np_ndarray()
label = label.as_np_ndarray()
pred = pred.as_np_ndarray().as_in_ctx(label.ctx)

reval = self._feval(label, pred)
if isinstance(reval, tuple):
Expand Down
13 changes: 7 additions & 6 deletions tests/python/tensorrt/test_cvnets.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
# under the License.

import gc
import gluoncv
import mxnet as mx
import numpy as np

Expand All @@ -29,7 +28,12 @@
def get_classif_model(model_name, use_tensorrt, ctx=mx.gpu(0), batch_size=128):
mx.contrib.tensorrt.set_use_fp16(False)
h, w = 32, 32
net = gluoncv.model_zoo.get_model(model_name, pretrained=True)
model_url = "https://raw.githubusercontent.com/dmlc/web-data/master/gluoncv/models/"
Copy link
Contributor

@leezu leezu May 7, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please don't hardcode master in the URL here. The repository may change and will then break the CI. Instead, use the commit ID: https://raw.githubusercontent.com/dmlc/web-data/221ce5b7c6d5b0777a1e3471f7f03ff98da90a0a/gluoncv/models

param_file = "{}-0000.params".format(model_name)
symbol_file = "{}-symbol.json".format(model_name)
mx.test_utils.download("{}/{}".format(model_url, param_file), fname=param_file, overwrite=True)
mx.test_utils.download("{}/{}".format(model_url, symbol_file), fname=symbol_file, overwrite=True)
net = gluon.SymbolBlock.imports(symbol_file, ['data'], param_file)
net.hybridize()
net.forward(mx.nd.zeros((batch_size, 3, h, w)))
net.export(model_name)
Expand Down Expand Up @@ -130,10 +134,7 @@ def test_tensorrt_on_cifar_resnets(batch_size=32, tolerance=0.1, num_workers=1):
'cifar_resnet20_v2',
'cifar_resnet56_v2',
'cifar_resnet110_v2',
'cifar_wideresnet16_10',
'cifar_wideresnet28_10',
'cifar_wideresnet40_8',
'cifar_resnext29_16x64d'
'cifar_wideresnet16_10'
]

num_models = len(models)
Expand Down