From ac964b8ea333081cbc6ebcb0a13ff0062abef6bf Mon Sep 17 00:00:00 2001 From: Wang Yanzhang Date: Wed, 28 Feb 2018 13:20:23 -0500 Subject: [PATCH 1/2] fix: use mkl add with two ptrs. --- .../intel/analytics/bigdl/tensor/MklDnnTensor.scala | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/tensor/MklDnnTensor.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/tensor/MklDnnTensor.scala index 7abef319112..40425dc0f1a 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/tensor/MklDnnTensor.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/tensor/MklDnnTensor.scala @@ -149,12 +149,14 @@ class MklDnnTensor[T: ClassTag]( require(x.getTensorType == MklDnnType, "just support two dnn tensor add") // todo: mkl add val y = x.asInstanceOf[MklDnnTensor[T]] - ev.vAdd(this.nElement(), this.storage().array(), this.storageOffset() - 1, - y.storage().array(), y.storageOffset() - 1, - this.storage().array(), this.storageOffset() - 1) +// ev.vAdd(this.nElement(), this.storage().array(), this.storageOffset() - 1, +// y.storage().array(), y.storageOffset() - 1, +// this.storage().array(), this.storageOffset() - 1) + + Memory.SAdd(this.nElement(), this.ptr, 0, y.ptr, 0, this.ptr, 0) // sync from heap to native // todo: all operations based on heap must be sync to native - syncFromHeap() +// syncFromHeap() } this } From 2fadc601bf23a54fc1992fd1fdd441e5bdd1b9e5 Mon Sep 17 00:00:00 2001 From: Wang Yanzhang Date: Thu, 1 Mar 2018 15:02:42 -0500 Subject: [PATCH 2/2] fix: batch norm initialize methods --- .../nn/mkldnn/SpatialBatchNormalization.scala | 35 +++++++++- .../SpatialBatchNormalizationSpec.scala | 70 +++++++++---------- 2 files changed, 66 insertions(+), 39 deletions(-) diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/mkldnn/SpatialBatchNormalization.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/mkldnn/SpatialBatchNormalization.scala index b843ede51d6..5248af25b97 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/mkldnn/SpatialBatchNormalization.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/mkldnn/SpatialBatchNormalization.scala @@ -19,6 +19,7 @@ package com.intel.analytics.bigdl.nn.mkldnn import java.io.{IOException, ObjectInputStream} import com.intel.analytics.bigdl.mkl.{Memory, MklDnn} +import com.intel.analytics.bigdl.nn.{RandomUniform, VariableFormat, Zeros} import com.intel.analytics.bigdl.nn.abstractnn.{Initializable, TensorModule} import com.intel.analytics.bigdl.tensor._ import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric @@ -54,6 +55,33 @@ class SpatialBatchNormalization[T: ClassTag]( @transient var backwardReorderPrims: ArrayBuffer[Long] = ArrayBuffer.empty @transient var forwardPrimDesc = 0L + { + val wInit = RandomUniform(0, 1) + val bInit = Zeros + setInitMethod(wInit, bInit) + } + + override def reset(): Unit = { + val weightAndBias = all.view(Array(2, nOutput)) + if (initWeight != null) { + require(initWeight.size(1) == nOutput) + weightAndBias.select(1, 1).copy(initWeight) + } else { + val weight = weightAndBias.select(1, 1) + weightInitMethod.init(weight, VariableFormat.ONE_D) + } + + if (initBias != null) { + require(initBias.size(1) == nOutput) + weightAndBias.select(1, 2).copy(initBias) + } else { + val bias = weightAndBias.select(1, 2) + biasInitMethod.init(bias, VariableFormat.ONE_D) + } + + zeroGradParameters() + } + @throws(classOf[IOException]) private def readObject(in: ObjectInputStream): Unit = { in.defaultReadObject() @@ -428,14 +456,16 @@ class SpatialBatchNormalization[T: ClassTag]( require(initWeight.size(1) == nOutput) concat.select(1, 1).copy(initWeight) } else { - concat.select(1, 1).fill(ev.fromType(1)) + val weight = concat.select(1, 1) + weightInitMethod.init(weight, VariableFormat.ONE_D) } if (initBias != null) { require(initBias.size(1) == nOutput) concat.select(1, 2).copy(initBias) } else { - concat.select(1, 2).fill(ev.fromType(0)) + val bias = concat.select(1, 2) + biasInitMethod.init(bias, VariableFormat.ONE_D) } weightAndBias.copy(concat.view(Array(2 * nOutput))) @@ -445,6 +475,7 @@ class SpatialBatchNormalization[T: ClassTag]( override def zeroGradParameters(): Unit = { if (affine) { gradAll.zero() + diffAll.zero() } } diff --git a/spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/mkldnn/SpatialBatchNormalizationSpec.scala b/spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/mkldnn/SpatialBatchNormalizationSpec.scala index 7bedac2435b..ede42e32a02 100644 --- a/spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/mkldnn/SpatialBatchNormalizationSpec.scala +++ b/spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/mkldnn/SpatialBatchNormalizationSpec.scala @@ -26,15 +26,15 @@ import org.scalatest.{FlatSpec, Matchers} class SpatialBatchNormalizationSpec extends FlatSpec with Matchers { "bn updateOutput" should "work correctly" in { - val (batchSize, channel, height, width) = (2, 3, 4, 4) + val (batchSize, channel, height, width) = (4, 64, 112, 112) val epsilon = 1e-5 - val initWeight = Tensor(channel).rand() - val initBias = Tensor(channel).rand() + val initWeight = Tensor(channel).rand(-1, 1) + val initBias = Tensor(channel).fill(0) val bn = SpatialBatchNormalization(channel, epsilon, initWeight = initWeight, - initBias = initBias) - val input = Tensor(batchSize, channel, height, width).rand() + initBias = initBias).setShouldConvert(true) + val input = Tensor(batchSize, channel, height, width).rand(-1, 1) val output = bn.forward(input) @@ -42,19 +42,19 @@ class SpatialBatchNormalizationSpec extends FlatSpec with Matchers { initWeight = initWeight, initBias = initBias) val nnOutput = nnBn.forward(input) - DnnUtils.nearequals(output, nnOutput) + DnnUtils.nearequals(output, nnOutput) should be (true) } "bn updateOutput multi times" should "work correctly" in { val (batchSize, channel, height, width) = (2, 3, 4, 4) val epsilon = 1e-5 - val initWeight = Tensor(channel).rand() - val initBias = Tensor(channel).rand() + val initWeight = Tensor(channel).rand(-1, 1) + val initBias = Tensor(channel).rand(-1, 1) val bn = SpatialBatchNormalization(channel, epsilon, initWeight = initWeight, - initBias = initBias) - val input = Tensor(batchSize, channel, height, width).rand() + initBias = initBias).setShouldConvert(true) + val input = Tensor(batchSize, channel, height, width).rand(-1, 1) Utils.manyTimes(bn.forward(input))(10) @@ -63,18 +63,18 @@ class SpatialBatchNormalizationSpec extends FlatSpec with Matchers { Utils.manyTimes(nnBn.forward(input))(10) - DnnUtils.nearequals(bn.output.toTensor, nnBn.output.toTensor) + DnnUtils.nearequals(bn.output.toTensor, nnBn.output.toTensor) should be (true) } "bn backward" should "work correctly" in { - val (batchSize, channel, height, width) = (2, 3, 4, 4) + val (batchSize, channel, height, width) = (5, 64, 112, 112) val epsilon = 0.0f val initWeight = Tensor(channel).rand(-1, 1) val initBias = Tensor(channel).rand(-1, 1) val bn = SpatialBatchNormalization(channel, epsilon, initWeight = initWeight, - initBias = initBias) + initBias = initBias).setShouldConvert(true) val input = Tensor(batchSize, channel, height, width).rand(-1, 1) val gradOutput = Tensor().resizeAs(input).rand(-1, 1) @@ -84,12 +84,12 @@ class SpatialBatchNormalizationSpec extends FlatSpec with Matchers { bn.forward(input) nnBn.forward(input) - bn.output should be (nnBn.output) - + DnnUtils.nearequals(bn.output, nnBn.output) should be (true) val gradInput = bn.backward(input, gradOutput) val nnGradInput = nnBn.backward(input, gradOutput) - DnnUtils.nearequals(gradInput, nnGradInput.toTensor) + DnnUtils.nearequals(gradInput, nnGradInput.toTensor) should be (true) + DnnUtils.nearequals(bn.getParameters()._2, nnBn.getParameters()._2, 1e-4) should be (true) } "bn backward multi times" should "work correctly" in { @@ -114,6 +114,10 @@ class SpatialBatchNormalizationSpec extends FlatSpec with Matchers { Utils.manyTimes(nnBn.backward(input, gradOutput))(10) bn.gradInput shouldEqual nnBn.gradInput + bn.getParameters()._2 shouldEqual nnBn.getParameters()._2 + + DnnUtils.nearequals(bn.gradInput, nnBn.gradInput) should be (true) + DnnUtils.nearequals(bn.getParameters()._2, nnBn.getParameters()._2) should be (true) } "bn perf" should "work correctly" in { @@ -467,8 +471,8 @@ class SpatialBatchNormalizationSpec extends FlatSpec with Matchers { val (channel, height, width) = (64, 112, 112) val epsilon = 1e-3 - val initWeight = Tensor(channel).rand() - val initBias = Tensor(channel).rand() + val initWeight = Tensor(channel).rand(-1, 1) + val initBias = Tensor(channel).rand(-1, 1) val bn = SpatialBatchNormalization(channel, epsilon, initWeight = initWeight, initBias = initBias) @@ -476,33 +480,25 @@ class SpatialBatchNormalizationSpec extends FlatSpec with Matchers { initWeight = initWeight, initBias = initBias) for (batchSize <- Array(2, 3, 4, 2)) { - val input = Tensor(batchSize, channel, height, width).rand() + bn.zeroGradParameters() + nnBn.zeroGradParameters() - val (weight1, gradweight1) = bn.getParameters() - val (weight2, gradweight2) = nnBn.getParameters() + val (weight, gradWeight) = bn.getParameters() + val (nnWeight, nnGradWeight) = nnBn.getParameters() - DnnUtils.nearequals(weight1, weight2) should be(true) - DnnUtils.nearequals(gradweight1, gradweight2, 1e-4) should be(true) + val input = Tensor(batchSize, channel, height, width).rand(-1, 1) + val gradOutput = Tensor().resizeAs(input).rand(-1, 1) bn.forward(input) nnBn.forward(input) - DnnUtils.nearequals(bn.output, nnBn.output, 1e-4) should be(true) - - DnnUtils.nearequals(weight1, weight2) should be(true) - DnnUtils.nearequals(gradweight1, gradweight2, 1e-4) should be(true) - - val gradOutput = Tensor().resizeAs(input).rand() - bn.backward(input, gradOutput) - nnBn.backward(input, gradOutput) + DnnUtils.nearequals(bn.output, nnBn.output) should be (true) - DnnUtils.nearequals(bn.gradInput, nnBn.gradInput, 1e-4) should be(true) + val gradInput = bn.backward(input, gradOutput) + val nnGradInput = nnBn.backward(input, gradOutput) - - DnnUtils.nearequals(weight1, weight2) should be(true) - DnnUtils.nearequals(gradweight1, gradweight2, 1e-4) should be(true) - - println("=" * 120) + DnnUtils.nearequals(gradInput, nnGradInput.toTensor) should be (true) + DnnUtils.nearequals(bn.getParameters()._2, nnBn.getParameters()._2, 1e-3) should be (true) } } }