Skip to content

Commit

Permalink
Merge pull request #11 from i8run/open-source-mkl-dnn-for-cherry
Browse files Browse the repository at this point in the history 
fix: use mkl add with two ptrs.
  • Loading branch information
@zhangxiaoli73
zhangxiaoli73 authored Mar 1, 2018
2 parents 33ffc36 + 2fadc60 commit db7087a
Show file tree
Hide file tree
Showing 3 changed files with 72 additions and 43 deletions.
35 changes: 33 additions & 2 deletions spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/mkldnn/SpatialBatchNormalization.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package com.intel.analytics.bigdl.nn.mkldnn
import java.io.{IOException, ObjectInputStream}

import com.intel.analytics.bigdl.mkl.{Memory, MklDnn}
import com.intel.analytics.bigdl.nn.{RandomUniform, VariableFormat, Zeros}
import com.intel.analytics.bigdl.nn.abstractnn.{Initializable, TensorModule}
import com.intel.analytics.bigdl.tensor._
import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric
Expand Down Expand Up @@ -54,6 +55,33 @@ class SpatialBatchNormalization[T: ClassTag](
@transient var backwardReorderPrims: ArrayBuffer[Long] = ArrayBuffer.empty
@transient var forwardPrimDesc = 0L

{
val wInit = RandomUniform(0, 1)
val bInit = Zeros
setInitMethod(wInit, bInit)
}

override def reset(): Unit = {
val weightAndBias = all.view(Array(2, nOutput))
if (initWeight != null) {
require(initWeight.size(1) == nOutput)
weightAndBias.select(1, 1).copy(initWeight)
} else {
val weight = weightAndBias.select(1, 1)
weightInitMethod.init(weight, VariableFormat.ONE_D)
}

if (initBias != null) {
require(initBias.size(1) == nOutput)
weightAndBias.select(1, 2).copy(initBias)
} else {
val bias = weightAndBias.select(1, 2)
biasInitMethod.init(bias, VariableFormat.ONE_D)
}

zeroGradParameters()
}

@throws(classOf[IOException])
private def readObject(in: ObjectInputStream): Unit = {
in.defaultReadObject()
Expand Down Expand Up @@ -428,14 +456,16 @@ class SpatialBatchNormalization[T: ClassTag](
require(initWeight.size(1) == nOutput)
concat.select(1, 1).copy(initWeight)
} else {
concat.select(1, 1).fill(ev.fromType(1))
val weight = concat.select(1, 1)
weightInitMethod.init(weight, VariableFormat.ONE_D)
}

if (initBias != null) {
require(initBias.size(1) == nOutput)
concat.select(1, 2).copy(initBias)
} else {
concat.select(1, 2).fill(ev.fromType(0))
val bias = concat.select(1, 2)
biasInitMethod.init(bias, VariableFormat.ONE_D)
}

weightAndBias.copy(concat.view(Array(2 * nOutput)))
Expand All @@ -445,6 +475,7 @@ class SpatialBatchNormalization[T: ClassTag](
override def zeroGradParameters(): Unit = {
if (affine) {
gradAll.zero()
diffAll.zero()
}
}

Expand Down
10 changes: 6 additions & 4 deletions spark/dl/src/main/scala/com/intel/analytics/bigdl/tensor/MklDnnTensor.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,12 +149,14 @@ class MklDnnTensor[T: ClassTag](
require(x.getTensorType == MklDnnType, "just support two dnn tensor add")
// todo: mkl add
val y = x.asInstanceOf[MklDnnTensor[T]]
ev.vAdd(this.nElement(), this.storage().array(), this.storageOffset() - 1,
y.storage().array(), y.storageOffset() - 1,
this.storage().array(), this.storageOffset() - 1)
// ev.vAdd(this.nElement(), this.storage().array(), this.storageOffset() - 1,
// y.storage().array(), y.storageOffset() - 1,
// this.storage().array(), this.storageOffset() - 1)

Memory.SAdd(this.nElement(), this.ptr, 0, y.ptr, 0, this.ptr, 0)
// sync from heap to native
// todo: all operations based on heap must be sync to native
syncFromHeap()
// syncFromHeap()
}
this
}
Expand Down
70 changes: 33 additions & 37 deletions ...dl/src/test/scala/com/intel/analytics/bigdl/nn/mkldnn/SpatialBatchNormalizationSpec.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,35 +26,35 @@ import org.scalatest.{FlatSpec, Matchers}

class SpatialBatchNormalizationSpec extends FlatSpec with Matchers {
"bn updateOutput" should "work correctly" in {
val (batchSize, channel, height, width) = (2, 3, 4, 4)
val (batchSize, channel, height, width) = (4, 64, 112, 112)
val epsilon = 1e-5

val initWeight = Tensor(channel).rand()
val initBias = Tensor(channel).rand()
val initWeight = Tensor(channel).rand(-1, 1)
val initBias = Tensor(channel).fill(0)

val bn = SpatialBatchNormalization(channel, epsilon, initWeight = initWeight,
initBias = initBias)
val input = Tensor(batchSize, channel, height, width).rand()
initBias = initBias).setShouldConvert(true)
val input = Tensor(batchSize, channel, height, width).rand(-1, 1)

val output = bn.forward(input)

val nnBn = nn.SpatialBatchNormalization(channel, epsilon,
initWeight = initWeight, initBias = initBias)
val nnOutput = nnBn.forward(input)

DnnUtils.nearequals(output, nnOutput)
DnnUtils.nearequals(output, nnOutput) should be (true)
}

"bn updateOutput multi times" should "work correctly" in {
val (batchSize, channel, height, width) = (2, 3, 4, 4)
val epsilon = 1e-5

val initWeight = Tensor(channel).rand()
val initBias = Tensor(channel).rand()
val initWeight = Tensor(channel).rand(-1, 1)
val initBias = Tensor(channel).rand(-1, 1)

val bn = SpatialBatchNormalization(channel, epsilon, initWeight = initWeight,
initBias = initBias)
val input = Tensor(batchSize, channel, height, width).rand()
initBias = initBias).setShouldConvert(true)
val input = Tensor(batchSize, channel, height, width).rand(-1, 1)

Utils.manyTimes(bn.forward(input))(10)

Expand All @@ -63,18 +63,18 @@ class SpatialBatchNormalizationSpec extends FlatSpec with Matchers {

Utils.manyTimes(nnBn.forward(input))(10)

DnnUtils.nearequals(bn.output.toTensor, nnBn.output.toTensor)
DnnUtils.nearequals(bn.output.toTensor, nnBn.output.toTensor) should be (true)
}

"bn backward" should "work correctly" in {
val (batchSize, channel, height, width) = (2, 3, 4, 4)
val (batchSize, channel, height, width) = (5, 64, 112, 112)
val epsilon = 0.0f

val initWeight = Tensor(channel).rand(-1, 1)
val initBias = Tensor(channel).rand(-1, 1)

val bn = SpatialBatchNormalization(channel, epsilon, initWeight = initWeight,
initBias = initBias)
initBias = initBias).setShouldConvert(true)
val input = Tensor(batchSize, channel, height, width).rand(-1, 1)
val gradOutput = Tensor().resizeAs(input).rand(-1, 1)

Expand All @@ -84,12 +84,12 @@ class SpatialBatchNormalizationSpec extends FlatSpec with Matchers {
bn.forward(input)
nnBn.forward(input)

bn.output should be (nnBn.output)

DnnUtils.nearequals(bn.output, nnBn.output) should be (true)
val gradInput = bn.backward(input, gradOutput)
val nnGradInput = nnBn.backward(input, gradOutput)

DnnUtils.nearequals(gradInput, nnGradInput.toTensor)
DnnUtils.nearequals(gradInput, nnGradInput.toTensor) should be (true)
DnnUtils.nearequals(bn.getParameters()._2, nnBn.getParameters()._2, 1e-4) should be (true)
}

"bn backward multi times" should "work correctly" in {
Expand All @@ -114,6 +114,10 @@ class SpatialBatchNormalizationSpec extends FlatSpec with Matchers {
Utils.manyTimes(nnBn.backward(input, gradOutput))(10)

bn.gradInput shouldEqual nnBn.gradInput
bn.getParameters()._2 shouldEqual nnBn.getParameters()._2

DnnUtils.nearequals(bn.gradInput, nnBn.gradInput) should be (true)
DnnUtils.nearequals(bn.getParameters()._2, nnBn.getParameters()._2) should be (true)
}

"bn perf" should "work correctly" in {
Expand Down Expand Up @@ -467,42 +471,34 @@ class SpatialBatchNormalizationSpec extends FlatSpec with Matchers {
val (channel, height, width) = (64, 112, 112)
val epsilon = 1e-3

val initWeight = Tensor(channel).rand()
val initBias = Tensor(channel).rand()
val initWeight = Tensor(channel).rand(-1, 1)
val initBias = Tensor(channel).rand(-1, 1)

val bn = SpatialBatchNormalization(channel, epsilon, initWeight = initWeight,
initBias = initBias)
val nnBn = nn.SpatialBatchNormalization(channel, epsilon,
initWeight = initWeight, initBias = initBias)

for (batchSize <- Array(2, 3, 4, 2)) {
val input = Tensor(batchSize, channel, height, width).rand()
bn.zeroGradParameters()
nnBn.zeroGradParameters()

val (weight1, gradweight1) = bn.getParameters()
val (weight2, gradweight2) = nnBn.getParameters()
val (weight, gradWeight) = bn.getParameters()
val (nnWeight, nnGradWeight) = nnBn.getParameters()

DnnUtils.nearequals(weight1, weight2) should be(true)
DnnUtils.nearequals(gradweight1, gradweight2, 1e-4) should be(true)
val input = Tensor(batchSize, channel, height, width).rand(-1, 1)
val gradOutput = Tensor().resizeAs(input).rand(-1, 1)

bn.forward(input)
nnBn.forward(input)
DnnUtils.nearequals(bn.output, nnBn.output, 1e-4) should be(true)

DnnUtils.nearequals(weight1, weight2) should be(true)
DnnUtils.nearequals(gradweight1, gradweight2, 1e-4) should be(true)

val gradOutput = Tensor().resizeAs(input).rand()

bn.backward(input, gradOutput)
nnBn.backward(input, gradOutput)
DnnUtils.nearequals(bn.output, nnBn.output) should be (true)

DnnUtils.nearequals(bn.gradInput, nnBn.gradInput, 1e-4) should be(true)
val gradInput = bn.backward(input, gradOutput)
val nnGradInput = nnBn.backward(input, gradOutput)


DnnUtils.nearequals(weight1, weight2) should be(true)
DnnUtils.nearequals(gradweight1, gradweight2, 1e-4) should be(true)

println("=" * 120)
DnnUtils.nearequals(gradInput, nnGradInput.toTensor) should be (true)
DnnUtils.nearequals(bn.getParameters()._2, nnBn.getParameters()._2, 1e-3) should be (true)
}
}
}

0 comments on commit db7087a

Please sign in to comment.