Skip to content

Commit

Permalink
Merge pull request intel#4498 from dding3/move_visulazition
Browse files Browse the repository at this point in the history
Move visulazition from dllib.utils to dllib
dding3 authored Aug 18, 2021

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
2 parents 2113376 + f10b6b7 commit b1949ae
Showing 24 changed files with 723 additions and 18 deletions.
Original file line number Diff line number Diff line change
@@ -26,7 +26,7 @@ import com.intel.analytics.bigdl.dllib.optim._
import com.intel.analytics.bigdl.dllib.utils._
import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric._
import com.intel.analytics.bigdl.utils._
import com.intel.analytics.bigdl.dllib.utils.visualization.{TrainSummary, ValidationSummary}
import com.intel.analytics.bigdl.dllib.visualization.{TrainSummary, ValidationSummary}
import org.apache.log4j.{Level, Logger}
import org.apache.spark.SparkContext

Original file line number Diff line number Diff line change
@@ -27,7 +27,7 @@ import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric._
import com.intel.analytics.bigdl.dllib.utils.{T, Table}
import com.intel.analytics.bigdl.utils.{Engine, OptimizerV1, OptimizerV2}
import com.intel.analytics.bigdl.utils.LoggerFilter
import com.intel.analytics.bigdl.dllib.utils.visualization.{TrainSummary, ValidationSummary}
import com.intel.analytics.bigdl.dllib.visualization.{TrainSummary, ValidationSummary}
import org.apache.log4j.{Level, Logger}
import org.apache.spark.SparkContext

Original file line number Diff line number Diff line change
@@ -23,7 +23,7 @@ import com.intel.analytics.bigdl.dllib.optim._
import com.intel.analytics.bigdl.utils.{Engine, MklBlas, MklDnn, OptimizerV1, OptimizerV2}
import com.intel.analytics.bigdl.dllib.utils._
import com.intel.analytics.bigdl.utils.LoggerFilter
import com.intel.analytics.bigdl.dllib.utils.visualization.TrainSummary
import com.intel.analytics.bigdl.dllib.visualization.TrainSummary
import org.apache.log4j.{Level, Logger}
import org.apache.spark.SparkContext

Original file line number Diff line number Diff line change
@@ -28,7 +28,7 @@ import com.intel.analytics.bigdl.dllib.utils._
import com.intel.analytics.bigdl.dllib.utils.serializer._
import com.intel.analytics.bigdl.dllib.utils.serializer.converters.DataConverter
import com.intel.analytics.bigdl.dllib.utils.tf.Tensorflow
import com.intel.analytics.bigdl.dllib.utils.visualization.tensorboard.{FileWriter => TFFileWriter}
import com.intel.analytics.bigdl.dllib.visualization.tensorboard.{FileWriter => TFFileWriter}

import scala.collection.mutable
import scala.collection.mutable.ArrayBuffer
Original file line number Diff line number Diff line change
@@ -26,7 +26,7 @@ import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric
import com.intel.analytics.bigdl.dllib.utils.intermediate.IRGraph
import com.intel.analytics.bigdl.utils.{Engine, MklBlas, MklDnn}
import com.intel.analytics.bigdl.dllib.utils.{Table}
import com.intel.analytics.bigdl.dllib.utils.visualization.{TrainSummary, ValidationSummary}
import com.intel.analytics.bigdl.dllib.visualization.{TrainSummary, ValidationSummary}
import org.apache.spark.rdd.{RDD, ZippedPartitionsWithLocalityRDD}

import scala.reflect.ClassTag
Original file line number Diff line number Diff line change
@@ -26,7 +26,7 @@ import com.intel.analytics.bigdl.dllib.tensor.Tensor
import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric
import com.intel.analytics.bigdl.dllib.utils._
import com.intel.analytics.bigdl.dllib.utils.intermediate.{ConversionUtils, IRGraph}
import com.intel.analytics.bigdl.dllib.utils.visualization.{TrainSummary, ValidationSummary}
import com.intel.analytics.bigdl.dllib.visualization.{TrainSummary, ValidationSummary}
import com.intel.analytics.bigdl.utils._
import com.intel.analytics.bigdl.{Module, _}
import java.io.{File, FilenameFilter}
Original file line number Diff line number Diff line change
@@ -29,7 +29,7 @@ import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric
import com.intel.analytics.bigdl.utils._
import com.intel.analytics.bigdl.dllib.utils._
import com.intel.analytics.bigdl.dllib.utils.intermediate.ConversionUtils
import com.intel.analytics.bigdl.dllib.utils.visualization.{TrainSummary, ValidationSummary}
import com.intel.analytics.bigdl.dllib.visualization.{TrainSummary, ValidationSummary}
import com.intel.analytics.bigdl.{Module, _}
import org.apache.log4j.Logger
import org.apache.spark.rdd.RDD
Original file line number Diff line number Diff line change
@@ -26,7 +26,7 @@ import com.intel.analytics.bigdl.dllib.optim.parameters.{ConstantClippingProcess
L2NormClippingProcessor, ParameterProcessor}
import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric
import com.intel.analytics.bigdl.dllib.utils._
import com.intel.analytics.bigdl.dllib.utils.visualization.{TrainSummary, ValidationSummary}
import com.intel.analytics.bigdl.dllib.visualization.{TrainSummary, ValidationSummary}
import com.intel.analytics.bigdl.utils._
import org.apache.log4j.Logger
import org.apache.spark.rdd.RDD
Original file line number Diff line number Diff line change
@@ -27,7 +27,7 @@ import com.intel.analytics.bigdl.dllib.tensor.Tensor
import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric
import com.intel.analytics.bigdl.dllib.utils._
import com.intel.analytics.bigdl.utils._
import com.intel.analytics.bigdl.dllib.utils.visualization.{TrainSummary, ValidationSummary}
import com.intel.analytics.bigdl.dllib.visualization.{TrainSummary, ValidationSummary}
import com.intel.analytics.bigdl.{Module, _}
import java.io.{File, FilenameFilter}
import java.text.SimpleDateFormat
Original file line number Diff line number Diff line change
@@ -28,7 +28,7 @@ import com.intel.analytics.bigdl.dllib.optim.{Optimizer, _}
import com.intel.analytics.bigdl.dllib.tensor.{DenseType, SparseType, Storage, Tensor}
import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric
import com.intel.analytics.bigdl.dllib.utils.{Table, _}
import com.intel.analytics.bigdl.dllib.utils.visualization.{Summary, TrainSummary, ValidationSummary}
import com.intel.analytics.bigdl.dllib.visualization.{Summary, TrainSummary, ValidationSummary}
import com.intel.analytics.bigdl.utils._
import org.apache.spark.api.java.{JavaRDD, JavaSparkContext}
import org.apache.spark.rdd.RDD
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
# Visualization via Tensorboard

> This wheel distribution is provided by https://github.com/dmlc/tensorboard
> You can find the wheel repository at https://pypi.python.org/pypi/tensorboard
Please follow the instructions below to install TensorBoard; it has been tested on both Ubuntu and Mac OS. Please refer to the [Know Issues](https://github.com/122689305/BigDL/tree/readme/spark/dl/src/main/scala/com/intel/analytics/bigdl/visualization#known-issues) section for possible errors.

## Requirement

Python verison: 2.7, 3.4, 3.5, 3.6

Pip version >= 9.0.1

## Installation

### Python 2
```pip install tensorboard==1.0.0a4```
### Python 3
```pip3 install tensorboard==1.0.0a4```

## Known Issues

> #### 1. Issue: No compatible version of tensorboard
Solutions
* [Update](https://pip.pypa.io/en/stable/installing/) your pip version to the latest: https://pip.pypa.io/en/stable/installing/
*  Check whether your python support wide unicode if you use python 2.7
```
python -c 'import sys;print(sys.maxunicode)'
```
  It should output `1114111`

> #### 2. RuntimeError: module compiled against API version 0xa but this version of numpy is 0x9
  Check your python library path (sys.path) to see whether it includes numpy module

> #### 3. RuntimeError: Cannot load some specific libraries, like '_pywrap_tensorflow.so'.
  Set your 'PATH' environment variable so that `$ which python` outputs the path of your python that has installed tensorboard.
Original file line number Diff line number Diff line change
@@ -0,0 +1,186 @@
/*
* Copyright 2016 The BigDL Authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.intel.analytics.bigdl.dllib.visualization

import com.intel.analytics.bigdl.dllib.tensor.Tensor
import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric
import com.intel.analytics.bigdl.dllib.visualization.tensorboard.{FileWriter}
import org.tensorflow

import scala.reflect.ClassTag

/**
* Logger for tensorboard.
* Support scalar and histogram now.
* @param logDir
* @param appName
*/
abstract class Summary(
logDir: String,
appName: String) {
protected val writer: FileWriter

/**
* Add a scalar summary.
* @param tag tag name.
* @param value tag value.
* @param step current step.
* @return this
*/
def addScalar(
tag: String,
value: Float,
step: Long): this.type = {
writer.addSummary(
Summary.scalar(tag, value), step
)
this
}

/**
* Add a histogram summary.
* @param tag tag name.
* @param value a tensor.
* @param step current step.
* @return this
*/
def addHistogram[T: ClassTag](
tag: String,
value: Tensor[T],
step: Long)(implicit ev: TensorNumeric[T]): this.type = {
writer.addSummary(
Summary.histogram[T](tag, value), step
)
this
}

/**
* Read scalar values to an array of triple by tag name.
* First element of the triple is step, second is value, third is wallclocktime.
* @param tag tag name.
* @return an array of triple.
*/
def readScalar(tag: String): Array[(Long, Float, Double)]

/**
* Close this logger.
*/
def close(): Unit = {
writer.close()
}
}

object Summary {

/**
* Create a scalar summary.
* @param tag tag name
* @param scalar scalar value
* @return
*/
def scalar(tag: String, scalar : Float): tensorflow.framework.Summary = {
val v = tensorflow.framework.Summary.Value.newBuilder().setTag(tag).setSimpleValue(scalar)
tensorflow.framework.Summary.newBuilder().addValue(v).build()
}

private val limits = makeHistogramBuckets()

/**
* Create a histogram summary.
* @param tag tag name.
* @param values values.
* @return
*/
def histogram[T: ClassTag](
tag: String,
values: Tensor[T])(implicit ev: TensorNumeric[T]): tensorflow.framework.Summary = {
val counts = new Array[Int](limits.length)

var squares = 0.0
values.apply1{value =>
val v = ev.toType[Double](value)
squares += v * v
val index = bisectLeft(limits, v)
counts(index) += 1
value
}

val histogram = tensorflow.framework.HistogramProto.newBuilder()
.setMin(ev.toType[Double](values.min()))
.setMax(ev.toType[Double](values.max()))
.setNum(values.nElement())
.setSum(ev.toType[Double](values.sum()))
.setSumSquares(squares)

var i = 0
while (i < counts.length) {
if (counts(i) != 0) {
histogram.addBucket(counts(i))
histogram.addBucketLimit(limits(i))
}
i += 1
}
val v = tensorflow.framework.Summary.Value.newBuilder().setTag(tag).setHisto(histogram)
tensorflow.framework.Summary.newBuilder().addValue(v).build()
}

/**
* Find a bucket for x.
*/
private def bisectLeft(
a: Array[Double],
x: Double,
lo: Int = 0,
hi: Int = -1): Int = {
require(lo >= 0)
var high = if (hi == -1) {
a.length
} else {
hi
}
var low = lo

while (low < high) {
val mid = (low + high) / 2
if (a(mid) < x) {
low = mid + 1
} else {
high = mid
}
}
low
}

/**
* Create a histogram buckets.
* @return
*/
private def makeHistogramBuckets(): Array[Double] = {
var v = 1e-12
val buckets = new Array[Double](1549)
var i = 1
buckets(774) = 0.0
while (i <= 774) {
buckets(774 + i) = v
buckets(774 - i) = -v
v *= 1.1
i += 1
}
buckets
}

}
Loading

0 comments on commit b1949ae

Please sign in to comment.